From 1188639712cca3d9585da80a5f2070df079c8f77 Mon Sep 17 00:00:00 2001 From: Radu Carpa Date: Thu, 12 Oct 2023 20:13:58 +0200 Subject: [PATCH] Tests: reorganize test.sh by suite Make it very clear what code is executed for what suite. Avoid any confusion. Except for the multi_vo tests, where a little rework was needed, everything else is mostly just code movement to put everything under a unique if/else block. Remove rucio_syntax.cfg. It's identical to the sqlite one. --- .../extra/rucio_multi_vo_tst_postgres14.cfg | 3 + etc/docker/test/extra/rucio_syntax.cfg | 2 - tests/test_clients.py | 7 +- tools/run_multi_vo_tests_docker.sh | 2 +- tools/test/test.sh | 120 ++++++++---------- 5 files changed, 64 insertions(+), 70 deletions(-) delete mode 100644 etc/docker/test/extra/rucio_syntax.cfg diff --git a/etc/docker/test/extra/rucio_multi_vo_tst_postgres14.cfg b/etc/docker/test/extra/rucio_multi_vo_tst_postgres14.cfg index fac84d3a09..1af5e01ed6 100644 --- a/etc/docker/test/extra/rucio_multi_vo_tst_postgres14.cfg +++ b/etc/docker/test/extra/rucio_multi_vo_tst_postgres14.cfg @@ -14,3 +14,6 @@ max_overflow = 20 [policy] permission = generic_multi_vo schema = generic_multi_vo + +[alembic] +cfg = /opt/rucio/etc/multi_vo/tst/etc/alembic.ini diff --git a/etc/docker/test/extra/rucio_syntax.cfg b/etc/docker/test/extra/rucio_syntax.cfg deleted file mode 100644 index 7e514f8351..0000000000 --- a/etc/docker/test/extra/rucio_syntax.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[database] -default = sqlite:////tmp/rucio.db \ No newline at end of file diff --git a/tests/test_clients.py b/tests/test_clients.py index 3b3478ff50..8aad23db46 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -21,7 +21,7 @@ from rucio.client.baseclient import BaseClient from rucio.client.client import Client -from rucio.common.config import config_get, config_set +from rucio.common.config import config_get, config_set, Config from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupported, RucioException from rucio.common.utils import execute from tests.mocks.mock_http_server import MockServer @@ -142,7 +142,8 @@ def test_import_without_config_file(self, vo): thus have to disable the access to it (move it) and make sure to run the code in a different environment. """ - rename("/opt/rucio/etc/rucio.cfg", "/opt/rucio/etc/rucio.cfg.tmp") + configfile = Config().configfile + rename(configfile, f"{configfile}.tmp") try: exitcode, _, err = execute("python -c 'from rucio.client import Client'") print(exitcode, err) @@ -150,4 +151,4 @@ def test_import_without_config_file(self, vo): assert "RuntimeError: Could not load Rucio configuration file." not in err finally: # This is utterly important to not mess up the environment. - rename("/opt/rucio/etc/rucio.cfg.tmp", "/opt/rucio/etc/rucio.cfg") + rename(f"{configfile}.tmp", configfile) diff --git a/tools/run_multi_vo_tests_docker.sh b/tools/run_multi_vo_tests_docker.sh index f69ce2483a..9b139d2ee6 100755 --- a/tools/run_multi_vo_tests_docker.sh +++ b/tools/run_multi_vo_tests_docker.sh @@ -52,7 +52,7 @@ echo 'Removing old SQLite databases' rm -f /tmp/rucio.db echo 'Resetting database tables' -tools/reset_database.py +ALEMBIC_CONFIG="$RUCIO_HOME/etc/alembic.ini" tools/reset_database.py if [ $? != 0 ]; then echo 'Failed to reset the database!' exit 1 diff --git a/tools/test/test.sh b/tools/test/test.sh index 798ad0075f..fac52e8ff5 100755 --- a/tools/test/test.sh +++ b/tools/test/test.sh @@ -20,62 +20,49 @@ echo "* Using $(command -v python) $(python --version 2>&1) and $(command -v pip SOURCE_PATH=/usr/local/src/rucio CFG_PATH=/usr/local/src/rucio/etc/docker/test/extra/ -RUCIO_HOME=/opt/rucio +if [ -z "$RUCIO_HOME" ]; then + RUCIO_HOME=/opt/rucio +fi -# Prepare the test configurations -if [ "$SUITE" == "client" ] || [ "$SUITE" == "client_syntax" ]; then +function srchome() { + export RUCIO_HOME="$SOURCE_PATH" cd $SOURCE_PATH - cp etc/docker/test/extra/rucio_client.cfg etc/rucio.cfg - -elif [ "$SUITE" == "syntax" ] || [ "$SUITE" == "docs" ]; then - python3 $SOURCE_PATH/tools/merge_rucio_configs.py --use-env \ - -s "$CFG_PATH"/rucio_autotests_common.cfg "$CFG_PATH"/rucio_syntax.cfg \ - -d "$SOURCE_PATH"/etc/rucio.cfg - -elif [ "$SUITE" == "votest" ]; then - VOTEST_HELPER=$RUCIO_HOME/tools/test/votest_helper.py - VOTEST_CONFIG_FILE=$RUCIO_HOME/etc/docker/test/matrix_policy_package_tests.yml - echo "VOTEST: Overriding policy section in rucio.cfg" - python $VOTEST_HELPER --vo $POLICY --vo-config --file $VOTEST_CONFIG_FILE - echo "VOTEST: Restarting httpd to load config" - httpd -k restart - -elif [ "$SUITE" == "remote_dbs" ] || [ "$SUITE" == "multi_vo" ]; then - - if [ "$SUITE" == "multi_vo" ]; then - mkdir -p $RUCIO_HOME/etc/multi_vo/tst/etc - mkdir -p $RUCIO_HOME/etc/multi_vo/ts2/etc - python3 $SOURCE_PATH/tools/merge_rucio_configs.py --use-env \ - -s "$CFG_PATH"/rucio_autotests_common.cfg "$CFG_PATH"/rucio_multi_vo_tst_postgres14.cfg \ - -d $RUCIO_HOME/etc/multi_vo/tst/etc/rucio.cfg - python3 $SOURCE_PATH/tools/merge_rucio_configs.py --use-env \ - -s "$CFG_PATH"/rucio_autotests_common.cfg "$CFG_PATH"/rucio_multi_vo_ts2_postgres14.cfg \ - -d $RUCIO_HOME/etc/multi_vo/ts2/etc/rucio.cfg - fi +} +function wait_for_database() { echo 'Waiting for database to be ready' if ! python3 -c "from rucio.db.sqla.session import wait_for_database; wait_for_database()" then echo 'Cannot access database' exit 1 fi +} - httpd -k restart - -fi +if [ "$SUITE" == "client" ]; then + tools/run_tests.sh -i -# Run tests + cp "$SOURCE_PATH"/etc/docker/test/extra/rucio_client.cfg "$SOURCE_PATH"/etc/rucio.cfg + srchome + tools/pytest.sh -v --tb=short tests/test_clients.py tests/test_bin_rucio.py tests/test_module_import.py -function srchome() { - export RUCIO_HOME=$SOURCE_PATH - cd $SOURCE_PATH -} +elif [ "$SUITE" == "client_syntax" ]; then + srchome + CLIENT_BIN_FILES="bin/rucio bin/rucio-admin" + SYNTAX_PYLINT_ARGS="$(tools/test/ignoretool.py --pylint)" + SYNTAX_FLAKE_ARGS="$(tools/test/ignoretool.py --flake8) $CLIENT_BIN_FILES tests/test_clients.py tests/test_bin_rucio.py tests/test_module_import.py" + export SYNTAX_PYLINT_ARGS + export SYNTAX_PYLINT_BIN_ARGS="$CLIENT_BIN_FILES" + export SYNTAX_FLAKE_ARGS + tools/test/check_syntax.sh -if [ "$SUITE" == "syntax" ]; then +elif [ "$SUITE" == "syntax" ]; then srchome tools/test/check_syntax.sh elif [ "$SUITE" == "docs" ]; then + python3 $SOURCE_PATH/tools/merge_rucio_configs.py --use-env \ + -s "$CFG_PATH"/rucio_autotests_common.cfg "$CFG_PATH"/rucio_sqlite.cfg \ + -d "$SOURCE_PATH"/etc/rucio.cfg srchome export RUCIO_CLIENT_API_OUTPUT="rucio_client_api.md" export RUCIO_REST_API_OUTPUT="rucio_rest_api.md" @@ -87,37 +74,42 @@ elif [ "$SUITE" == "docs" ]; then tools/generate_rest_api_doc.py > $REST_API_DOC_FILENAME tools/test/check_rest_api_documentation.sh $REST_API_DOC_FILENAME +elif [ "$SUITE" == "votest" ]; then + VOTEST_HELPER=$RUCIO_HOME/tools/test/votest_helper.py + VOTEST_CONFIG_FILE=$RUCIO_HOME/etc/docker/test/matrix_policy_package_tests.yml + echo "VOTEST: Overriding policy section in rucio.cfg" + python $VOTEST_HELPER --vo "$POLICY" --vo-config --file $VOTEST_CONFIG_FILE + echo "VOTEST: Restarting httpd to load config" + httpd -k restart -elif [[ "$SUITE" =~ ^client.* ]]; then - if [ "$SUITE" == "client" ]; then - tools/run_tests.sh -i - fi + TESTS=$(python $VOTEST_HELPER --vo "$POLICY" --tests --file $VOTEST_CONFIG_FILE) + export TESTS + tools/run_tests.sh -p - srchome - if [ "$SUITE" == "client" ]; then - tools/pytest.sh -v --tb=short tests/test_clients.py tests/test_bin_rucio.py tests/test_module_import.py - elif [ "$SUITE" == "client_syntax" ]; then - CLIENT_BIN_FILES="bin/rucio bin/rucio-admin" - export SYNTAX_PYLINT_ARGS="$(tools/test/ignoretool.py --pylint)" - export SYNTAX_PYLINT_BIN_ARGS="$CLIENT_BIN_FILES" - export SYNTAX_FLAKE_ARGS="$(tools/test/ignoretool.py --flake8) $CLIENT_BIN_FILES tests/test_clients.py tests/test_bin_rucio.py tests/test_module_import.py" - tools/test/check_syntax.sh - fi +elif [ "$SUITE" == "multi_vo" ]; then + VO1_HOME="$RUCIO_HOME" + mkdir -p "$VO1_HOME/etc" + python3 $SOURCE_PATH/tools/merge_rucio_configs.py --use-env \ + -s "$CFG_PATH"/rucio_autotests_common.cfg "$CFG_PATH"/rucio_multi_vo_tst_postgres14.cfg \ + -d "$VO1_HOME"/etc/rucio.cfg + httpd -k restart + + VO2_HOME="$RUCIO_HOME/../ts2" + mkdir -p "$VO2_HOME/etc" + python3 $SOURCE_PATH/tools/merge_rucio_configs.py --use-env \ + -s "$CFG_PATH"/rucio_autotests_common.cfg "$CFG_PATH"/rucio_multi_vo_ts2_postgres14.cfg \ + -d "$VO2_HOME"/etc/rucio.cfg + + wait_for_database + + tools/run_multi_vo_tests_docker.sh + +elif [ "$SUITE" == "remote_dbs" ] || [ "$SUITE" == "sqlite" ]; then + wait_for_database -elif [ "$SUITE" == "remote_dbs" ] || [ "$SUITE" == "sqlite" ] || [ "$SUITE" == "py37py38" ]; then if [ -n "$TESTS" ]; then tools/run_tests.sh -p else tools/run_tests.sh fi - -elif [ "$SUITE" == "multi_vo" ]; then - tools/run_multi_vo_tests_docker.sh - -elif [ "$SUITE" == "votest" ]; then - RUCIO_HOME=/opt/rucio - VOTEST_HELPER=$RUCIO_HOME/tools/test/votest_helper.py - VOTEST_CONFIG_FILE=$RUCIO_HOME/etc/docker/test/matrix_policy_package_tests.yml - export TESTS=$(python $VOTEST_HELPER --vo $POLICY --tests --file $VOTEST_CONFIG_FILE) - tools/run_tests.sh -p fi