Add workflows for codeql and component testing (#350)

* add workflows for codeql and component testing

* remove codeql workflow

* disable live tests during workflow

* run action when pr is updated

* remove duplicate key

* remove type restriction

* add cleanup to tests and improve logging

* fix handling of pytest exit code

* add manual installation of pytest if required

* use unified env for python tests

* update transformer version

* try caching of virtual environments

* disable client logging for tests

* add exclusion checks for submodules

* add KGQAnWrapper submodule

* exit if external environment dir not possible

.github/workflows/run-tests.yml

@@ -0,0 +1,51 @@
+name: Qanary component test pipeline
+
+on:
+  pull_request:
+    branches: [master]
+
+jobs:
+  test-java:
+    runs-on: ubuntu-latest
+    env:
+      BABELFY_API_KEY : someapikey
+      CHATGPT_API_KEY : someapikey
+      DANDELION_API_KEY : someapikey
+      MEANINGCLOUD_API_KEY : someapikey
+      TAGME_API_KEY : someapikey
+      TEXTRAZOR_API_KEY : someapikey
+      OPENAI_API_KEY : someapikey
+      BABELFY_API_LIVE_TEST_ACTIVE : false
+      PLATYPUS_API_LIVE_TEST_ACTIVE : false
+      TEXTRAZOR_API_LIVE_TEST_ACTIVE : false
+      CHATGPT_API_LIVE_TEST_ACTIVE : false
+      DANDELION_API_LIVE_TEST_ACTIVE : false
+      MEANINGCLOUD_API_LIVE_TEST_ACTIVE : false
+      AGDISTIS_API_LIVE_TEST_ACTIVE : false
+      OPENAI_API_LIVE_TEST_ACTIVE : false
+      TAGME_API_LIVE_TEST_ACTIVE : false
+    steps:
+      - name: Configure java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'corretto'
+          java-version: '17'
+      - uses: actions/checkout@v4
+      - name: Test Java components
+        run: bash -c ./service_config/test_java_components.sh
+  test-python:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Configure Python
+        uses: actions/setup-python@v4
+        id: setup_python
+        with: 
+          python-version: '3.12'
+      - uses: actions/checkout@v4
+      - name: Cache virtual environments
+        uses: actions/cache@v4
+        with:
+          key: environments-${{ runner.os }}-${{ steps.setup_python.outputs.python-version}}-${{ hashFiles('**/requirements.txt') }}
+          path: environments
+      - name: Test Python components
+        run: bash -c ./service_config/test_python_components.sh

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "Qanary-component-QB-Python-KGQAnWrapper"]
+	path = Qanary-component-QB-Python-KGQAnWrapper
+	url = https://github.com/WSE-research/Qanary-component-QB-Python-KGQAnWrapper.git

qanary-component-MT-Python-HelsinkiNLP/pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-log_cli = 1
+log_cli = 0
 log_cli_level = INFO
 log_cli_format = %(asctime)s [%(levelname)8s] [%(filename)s:%(lineno)s] %(message)s 
 log_cli_date_format=%Y-%m-%d %H:%M:%S

qanary-component-MT-Python-LibreTranslate/pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-log_cli = 1
+log_cli = 0
 log_cli_level = INFO
 log_cli_format = %(asctime)s [%(levelname)8s] [%(filename)s:%(lineno)s] %(message)s 
 log_cli_date_format=%Y-%m-%d %H:%M:%S

qanary-component-MT-Python-MBart/pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-log_cli = 1
+log_cli = 0
 log_cli_level = INFO
 log_cli_format = %(asctime)s [%(levelname)8s] [%(filename)s:%(lineno)s] %(message)s 
 log_cli_date_format=%Y-%m-%d %H:%M:%S

qanary-component-MT-Python-MBart/requirements.txt

@@ -4,9 +4,9 @@ langid==1.1.6
 mock==3.0.5
 python-dotenv==0.21.1
 qanary_helpers==0.2.2
-transformers==4.25.1
+transformers==4.41.0
 sentencepiece==0.1.97
-torch==1.13.1
+torch==2.3.0
 gunicorn==20.1.0
 protobuf==3.20.*
 pytest

qanary-component-MT-Python-NLLB/pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-log_cli = 1
+log_cli = 0
 log_cli_level = INFO
 log_cli_format = %(asctime)s [%(levelname)8s] [%(filename)s:%(lineno)s] %(message)s 
 log_cli_date_format=%Y-%m-%d %H:%M:%S

qanary-component-MT-Python-NLLB/requirements.txt

@@ -3,9 +3,9 @@ langdetect==1.0.9
 mock==3.0.5
 python-dotenv==0.21.1
 qanary_helpers==0.2.2
-transformers==4.25.1
+transformers==4.41.0
 sentencepiece==0.1.97
-torch==1.13.1
+torch==2.3.0
 gunicorn==20.1.0
 protobuf==3.20.*
 pytest

service_config/build_images.sh

@@ -1,5 +1,15 @@
 #!/bin/bash
 
+# ensure that submodules are not in maven reactor
+submodules=$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }')
+for submodule in $submodules; 
+do
+  if grep -qi $submodule "pom.xml"; then
+    echo "Submodules should be tested and built externally. Please remove \"${submodule}\" from the maven reactor list."
+    exit 4
+  fi
+done
+
 # replace secrets
 if [ -z "$BABELFY_API_KEY" ]
 then

service_config/build_python_images.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
-images=$(ls | grep -P "qanary-component.*Python-[a-zA-Z]+$")
+# get a list of all Python component directories
+# exclude submodules (external component repositories)
+images=$(comm -3 <(ls | grep -P "[qQ]anary-component.*Python-[a-zA-Z]+$") <(git config --file .gitmodules --get-regexp path | awk '{ print $2 }'))
 
 for dir in $images
 do

service_config/test_java_components.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# ensure that submodules are not in maven reactor
+submodules=$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }')
+for submodule in $submodules; 
+do
+  if grep -qi $submodule "pom.xml"; then
+    echo "Submodules should be tested and built externally. Please remove \"${submodule}\" from the maven reactor list."
+    exit 4
+  fi
+done
+
+# test components
+if ! mvn --batch-mode --no-transfer-progress test;
+then
+  echo "Maven test failed"
+  exit 4 # stop if test fails
+fi 

service_config/test_python_components.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+declare -A summary
+failures=false 
+# get a list of all Python component directories
+# exclude submodules (external component repositories)
+components=$(comm -3 <(ls | grep -P "[qQ]anary-component.*Python-[a-zA-Z]+$") <(git config --file .gitmodules --get-regexp path | awk '{ print $2 }'))
+
+# create a super directory to hold virtual environments (for caching)
+
+if [ -d environments ]; then
+  echo "External evironment directory exists"
+else
+  if mkdir environments; then
+    echo "External environment directory created"
+  else
+    echo "External environment directory could not be created"
+    exit 4
+  fi
+fi
+
+printf "Found Python components:\n\n${components}\n\n"
+
+
+# iterate over components 
+for dir in $components
+do
+  # iterate over Python components 
+  name=$(echo ${dir} | tr "[:upper:]" "[:lower:]")
+  printf "\n\n===== ${name} =====\n\n"
+
+  cd $dir
+  # setup virtual environment in external super directory 
+  envname=../environments/${name}
+  python -m venv ${envname}
+  if source ${envname}/bin/activate; then
+    pip install -r requirements.txt 
+  else
+    echo "Something went wrong trying to install requirements! Exiting ..."
+    exit 4
+  fi
+
+  # install pytest manually if not included in requirements 
+  if ! pip show pytest; then
+    echo "Installing pytest manually..."
+    pip install pytest
+  fi 
+  if ! pip show pytest-env; then 
+    echo "Installing pytest-env manually..."
+    pip install pytest-env
+  fi
+
+  # run tests 
+  pytest
+  test_status=$?
+  # check exit codes
+  if [ $test_status -eq 0 ]; then # all tests successful 
+    summary[${name}]="passed"
+  elif [ $test_status -eq 5 ]; then # no tests found
+    summary[${name}]="no tests"
+  else # tests failed or something else went wrong
+    summary[${name}]="failed"
+    failures=true
+  fi 
+
+  #pip freeze | xargs pip uninstall -y TODO: disabled because that would mess with caching
+  deactivate
+  rm -r ${envname}
+
+  cd ..
+
+done
+
+
+# print a summary
+printf "\n\n===== SUMMARY =====\n\n"
+for x in "${!summary[@]}"; do printf "%s\t:\t[%s]\n" "$x" "${summary[$x]}"; done | column -s$'\t' -t
+if $failures; then
+  printf "\nSome tests failed!\n"
+  exit 4
+else
+  printf "\nTests succeeded\n"
+fi