From d556317ccc56abfd5734b7875cfe6c1032c09ae0 Mon Sep 17 00:00:00 2001 From: Paul Heinze Date: Thu, 18 Jul 2024 04:21:06 +0200 Subject: [PATCH] Add workflows for codeql and component testing (#350) * add workflows for codeql and component testing * remove codeql workflow * disable live tests during workflow * run action when pr is updated * remove duplicate key * remove type restriction * add cleanup to tests and improve logging * fix handling of pytest exit code * add manual installation of pytest if required * use unified env for python tests * update transformer version * try caching of virtual environments * disable client logging for tests * add exclusion checks for submodules * add KGQAnWrapper submodule * exit if external environment dir not possible --- .github/workflows/run-tests.yml | 51 ++++++++++++ .gitmodules | 3 + Qanary-component-QB-Python-KGQAnWrapper | 1 + .../pytest.ini | 2 +- .../pytest.ini | 2 +- qanary-component-MT-Python-MBart/pytest.ini | 2 +- .../requirements.txt | 4 +- qanary-component-MT-Python-NLLB/pytest.ini | 2 +- .../requirements.txt | 4 +- service_config/build_images.sh | 10 +++ service_config/build_python_images.sh | 4 +- service_config/test_java_components.sh | 18 ++++ service_config/test_python_components.sh | 83 +++++++++++++++++++ 13 files changed, 177 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/run-tests.yml create mode 100644 .gitmodules create mode 160000 Qanary-component-QB-Python-KGQAnWrapper create mode 100755 service_config/test_java_components.sh create mode 100755 service_config/test_python_components.sh diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 00000000..3b717908 --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,51 @@ +name: Qanary component test pipeline + +on: + pull_request: + branches: [master] + +jobs: + test-java: + runs-on: ubuntu-latest + env: + BABELFY_API_KEY : someapikey + CHATGPT_API_KEY : someapikey + DANDELION_API_KEY : someapikey + MEANINGCLOUD_API_KEY : someapikey + TAGME_API_KEY : someapikey + TEXTRAZOR_API_KEY : someapikey + OPENAI_API_KEY : someapikey + BABELFY_API_LIVE_TEST_ACTIVE : false + PLATYPUS_API_LIVE_TEST_ACTIVE : false + TEXTRAZOR_API_LIVE_TEST_ACTIVE : false + CHATGPT_API_LIVE_TEST_ACTIVE : false + DANDELION_API_LIVE_TEST_ACTIVE : false + MEANINGCLOUD_API_LIVE_TEST_ACTIVE : false + AGDISTIS_API_LIVE_TEST_ACTIVE : false + OPENAI_API_LIVE_TEST_ACTIVE : false + TAGME_API_LIVE_TEST_ACTIVE : false + steps: + - name: Configure java + uses: actions/setup-java@v4 + with: + distribution: 'corretto' + java-version: '17' + - uses: actions/checkout@v4 + - name: Test Java components + run: bash -c ./service_config/test_java_components.sh + test-python: + runs-on: ubuntu-latest + steps: + - name: Configure Python + uses: actions/setup-python@v4 + id: setup_python + with: + python-version: '3.12' + - uses: actions/checkout@v4 + - name: Cache virtual environments + uses: actions/cache@v4 + with: + key: environments-${{ runner.os }}-${{ steps.setup_python.outputs.python-version}}-${{ hashFiles('**/requirements.txt') }} + path: environments + - name: Test Python components + run: bash -c ./service_config/test_python_components.sh diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..3db8c1c9 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "Qanary-component-QB-Python-KGQAnWrapper"] + path = Qanary-component-QB-Python-KGQAnWrapper + url = https://github.com/WSE-research/Qanary-component-QB-Python-KGQAnWrapper.git diff --git a/Qanary-component-QB-Python-KGQAnWrapper b/Qanary-component-QB-Python-KGQAnWrapper new file mode 160000 index 00000000..1c1d4453 --- /dev/null +++ b/Qanary-component-QB-Python-KGQAnWrapper @@ -0,0 +1 @@ +Subproject commit 1c1d4453f80633956637e80d9c9dc634a7cf53b6 diff --git a/qanary-component-MT-Python-HelsinkiNLP/pytest.ini b/qanary-component-MT-Python-HelsinkiNLP/pytest.ini index 5fd73588..93627edc 100644 --- a/qanary-component-MT-Python-HelsinkiNLP/pytest.ini +++ b/qanary-component-MT-Python-HelsinkiNLP/pytest.ini @@ -1,5 +1,5 @@ [pytest] -log_cli = 1 +log_cli = 0 log_cli_level = INFO log_cli_format = %(asctime)s [%(levelname)8s] [%(filename)s:%(lineno)s] %(message)s log_cli_date_format=%Y-%m-%d %H:%M:%S diff --git a/qanary-component-MT-Python-LibreTranslate/pytest.ini b/qanary-component-MT-Python-LibreTranslate/pytest.ini index 3f65a4ac..49dd1c4f 100644 --- a/qanary-component-MT-Python-LibreTranslate/pytest.ini +++ b/qanary-component-MT-Python-LibreTranslate/pytest.ini @@ -1,5 +1,5 @@ [pytest] -log_cli = 1 +log_cli = 0 log_cli_level = INFO log_cli_format = %(asctime)s [%(levelname)8s] [%(filename)s:%(lineno)s] %(message)s log_cli_date_format=%Y-%m-%d %H:%M:%S diff --git a/qanary-component-MT-Python-MBart/pytest.ini b/qanary-component-MT-Python-MBart/pytest.ini index c0074e86..a51bc97c 100644 --- a/qanary-component-MT-Python-MBart/pytest.ini +++ b/qanary-component-MT-Python-MBart/pytest.ini @@ -1,5 +1,5 @@ [pytest] -log_cli = 1 +log_cli = 0 log_cli_level = INFO log_cli_format = %(asctime)s [%(levelname)8s] [%(filename)s:%(lineno)s] %(message)s log_cli_date_format=%Y-%m-%d %H:%M:%S diff --git a/qanary-component-MT-Python-MBart/requirements.txt b/qanary-component-MT-Python-MBart/requirements.txt index d4d20640..7e37256a 100644 --- a/qanary-component-MT-Python-MBart/requirements.txt +++ b/qanary-component-MT-Python-MBart/requirements.txt @@ -4,9 +4,9 @@ langid==1.1.6 mock==3.0.5 python-dotenv==0.21.1 qanary_helpers==0.2.2 -transformers==4.25.1 +transformers==4.41.0 sentencepiece==0.1.97 -torch==1.13.1 +torch==2.3.0 gunicorn==20.1.0 protobuf==3.20.* pytest diff --git a/qanary-component-MT-Python-NLLB/pytest.ini b/qanary-component-MT-Python-NLLB/pytest.ini index ff265966..4c87d9ab 100644 --- a/qanary-component-MT-Python-NLLB/pytest.ini +++ b/qanary-component-MT-Python-NLLB/pytest.ini @@ -1,5 +1,5 @@ [pytest] -log_cli = 1 +log_cli = 0 log_cli_level = INFO log_cli_format = %(asctime)s [%(levelname)8s] [%(filename)s:%(lineno)s] %(message)s log_cli_date_format=%Y-%m-%d %H:%M:%S diff --git a/qanary-component-MT-Python-NLLB/requirements.txt b/qanary-component-MT-Python-NLLB/requirements.txt index 267f0b6d..ec4a61a8 100644 --- a/qanary-component-MT-Python-NLLB/requirements.txt +++ b/qanary-component-MT-Python-NLLB/requirements.txt @@ -3,9 +3,9 @@ langdetect==1.0.9 mock==3.0.5 python-dotenv==0.21.1 qanary_helpers==0.2.2 -transformers==4.25.1 +transformers==4.41.0 sentencepiece==0.1.97 -torch==1.13.1 +torch==2.3.0 gunicorn==20.1.0 protobuf==3.20.* pytest diff --git a/service_config/build_images.sh b/service_config/build_images.sh index 32368f30..5f507044 100755 --- a/service_config/build_images.sh +++ b/service_config/build_images.sh @@ -1,5 +1,15 @@ #!/bin/bash +# ensure that submodules are not in maven reactor +submodules=$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }') +for submodule in $submodules; +do + if grep -qi $submodule "pom.xml"; then + echo "Submodules should be tested and built externally. Please remove \"${submodule}\" from the maven reactor list." + exit 4 + fi +done + # replace secrets if [ -z "$BABELFY_API_KEY" ] then diff --git a/service_config/build_python_images.sh b/service_config/build_python_images.sh index 2f445518..39fc19b4 100755 --- a/service_config/build_python_images.sh +++ b/service_config/build_python_images.sh @@ -1,6 +1,8 @@ #!/bin/bash -images=$(ls | grep -P "qanary-component.*Python-[a-zA-Z]+$") +# get a list of all Python component directories +# exclude submodules (external component repositories) +images=$(comm -3 <(ls | grep -P "[qQ]anary-component.*Python-[a-zA-Z]+$") <(git config --file .gitmodules --get-regexp path | awk '{ print $2 }')) for dir in $images do diff --git a/service_config/test_java_components.sh b/service_config/test_java_components.sh new file mode 100755 index 00000000..f2f39f70 --- /dev/null +++ b/service_config/test_java_components.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# ensure that submodules are not in maven reactor +submodules=$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }') +for submodule in $submodules; +do + if grep -qi $submodule "pom.xml"; then + echo "Submodules should be tested and built externally. Please remove \"${submodule}\" from the maven reactor list." + exit 4 + fi +done + +# test components +if ! mvn --batch-mode --no-transfer-progress test; +then + echo "Maven test failed" + exit 4 # stop if test fails +fi diff --git a/service_config/test_python_components.sh b/service_config/test_python_components.sh new file mode 100755 index 00000000..edd2d87b --- /dev/null +++ b/service_config/test_python_components.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +declare -A summary +failures=false +# get a list of all Python component directories +# exclude submodules (external component repositories) +components=$(comm -3 <(ls | grep -P "[qQ]anary-component.*Python-[a-zA-Z]+$") <(git config --file .gitmodules --get-regexp path | awk '{ print $2 }')) + +# create a super directory to hold virtual environments (for caching) + +if [ -d environments ]; then + echo "External evironment directory exists" +else + if mkdir environments; then + echo "External environment directory created" + else + echo "External environment directory could not be created" + exit 4 + fi +fi + +printf "Found Python components:\n\n${components}\n\n" + + +# iterate over components +for dir in $components +do + # iterate over Python components + name=$(echo ${dir} | tr "[:upper:]" "[:lower:]") + printf "\n\n===== ${name} =====\n\n" + + cd $dir + # setup virtual environment in external super directory + envname=../environments/${name} + python -m venv ${envname} + if source ${envname}/bin/activate; then + pip install -r requirements.txt + else + echo "Something went wrong trying to install requirements! Exiting ..." + exit 4 + fi + + # install pytest manually if not included in requirements + if ! pip show pytest; then + echo "Installing pytest manually..." + pip install pytest + fi + if ! pip show pytest-env; then + echo "Installing pytest-env manually..." + pip install pytest-env + fi + + # run tests + pytest + test_status=$? + # check exit codes + if [ $test_status -eq 0 ]; then # all tests successful + summary[${name}]="passed" + elif [ $test_status -eq 5 ]; then # no tests found + summary[${name}]="no tests" + else # tests failed or something else went wrong + summary[${name}]="failed" + failures=true + fi + + #pip freeze | xargs pip uninstall -y TODO: disabled because that would mess with caching + deactivate + rm -r ${envname} + + cd .. + +done + + +# print a summary +printf "\n\n===== SUMMARY =====\n\n" +for x in "${!summary[@]}"; do printf "%s\t:\t[%s]\n" "$x" "${summary[$x]}"; done | column -s$'\t' -t +if $failures; then + printf "\nSome tests failed!\n" + exit 4 +else + printf "\nTests succeeded\n" +fi