From b0d74e05ddf14004d4ddd764baf6289f0639aab3 Mon Sep 17 00:00:00 2001 From: Radu Carpa Date: Wed, 11 Oct 2023 14:55:36 +0200 Subject: [PATCH] Tests: merge before_script.sh into install_script.sh The content is now small enough that it doesn't justify having a separate script. The only tricky part was configuration of the oracle database. Luckily, the container which we use already supports mounting a shell script to be executed on initialization. It's enough to mount it at the correct location. --- etc/docker/dev/docker-compose.yml | 2 + tools/test/before_script.sh | 87 ------------------------------- tools/test/install_script.sh | 54 +++++++++++++++---- tools/test/run_tests.py | 13 ----- 4 files changed, 47 insertions(+), 109 deletions(-) delete mode 100755 tools/test/before_script.sh diff --git a/etc/docker/dev/docker-compose.yml b/etc/docker/dev/docker-compose.yml index 1904c88f913..dfc3530bbec 100644 --- a/etc/docker/dev/docker-compose.yml +++ b/etc/docker/dev/docker-compose.yml @@ -110,6 +110,8 @@ services: - processes=1000 - sessions=1105 - transactions=1215 + volumes: + - ../../../tools/test/oracle_setup.sh:/container-entrypoint-initdb.d/oracle_setup.sh:Z fts: image: docker.io/rucio/fts profiles: diff --git a/tools/test/before_script.sh b/tools/test/before_script.sh deleted file mode 100755 index 6e3dda29fa6..00000000000 --- a/tools/test/before_script.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -# -*- coding: utf-8 -*- -# Copyright European Organization for Nuclear Research (CERN) since 2012 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eo pipefail - -# Note: this file is not run when RUN_HTTPD is defined as false for the test suite. - -echo "* docker $CONTAINER_RUNTIME_ARGS info" -docker $CONTAINER_RUNTIME_ARGS info -echo -echo "* env" -env -echo - -RESTART_HTTPD=0 - -if [ $RDBMS == "oracle" ]; then - docker $CONTAINER_RUNTIME_ARGS cp tools/test/oracle_setup.sh ${CON_DB}:/ - docker $CONTAINER_RUNTIME_ARGS exec $CON_DB /oracle_setup.sh - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO python3 tools/merge_rucio_configs.py \ - -s /usr/local/src/rucio/etc/docker/test/extra/rucio_autotests_common.cfg \ - /usr/local/src/rucio/etc/docker/test/extra/rucio_oracle.cfg \ - --use-env \ - -d /opt/rucio/etc/rucio.cfg - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO cp /usr/local/src/rucio/etc/docker/test/extra/alembic_oracle.ini /opt/rucio/etc/alembic.ini - RESTART_HTTPD=1 - -elif [ $RDBMS == "mysql8" ]; then - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO python3 tools/merge_rucio_configs.py \ - -s /usr/local/src/rucio/etc/docker/test/extra/rucio_autotests_common.cfg \ - /usr/local/src/rucio/etc/docker/test/extra/rucio_mysql8.cfg \ - --use-env \ - -d /opt/rucio/etc/rucio.cfg - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO cp /usr/local/src/rucio/etc/docker/test/extra/alembic_mysql8.ini /opt/rucio/etc/alembic.ini - RESTART_HTTPD=1 - -elif [ $RDBMS == "postgres14" ]; then - if [ $SUITE == "multi_vo" ]; then - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO mkdir -p /opt/rucio/etc/multi_vo/tst/etc - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO mkdir -p /opt/rucio/etc/multi_vo/ts2/etc - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO python3 tools/merge_rucio_configs.py \ - -s /usr/local/src/rucio/etc/docker/test/extra/rucio_autotests_common.cfg \ - /usr/local/src/rucio/etc/docker/test/extra/rucio_multi_vo_tst_postgres14.cfg \ - --use-env \ - -d /opt/rucio/etc/multi_vo/tst/etc/rucio.cfg - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO python3 tools/merge_rucio_configs.py \ - -s /usr/local/src/rucio/etc/docker/test/extra/rucio_autotests_common.cfg \ - /usr/local/src/rucio/etc/docker/test/extra/rucio_multi_vo_ts2_postgres14.cfg \ - --use-env \ - -d /opt/rucio/etc/multi_vo/ts2/etc/rucio.cfg - fi - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO python3 tools/merge_rucio_configs.py \ - -s /usr/local/src/rucio/etc/docker/test/extra/rucio_autotests_common.cfg \ - /usr/local/src/rucio/etc/docker/test/extra/rucio_postgres14.cfg \ - --use-env \ - -d /opt/rucio/etc/rucio.cfg - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO cp /usr/local/src/rucio/etc/docker/test/extra/alembic_postgres14.ini /opt/rucio/etc/alembic.ini - RESTART_HTTPD=1 - -elif [ $RDBMS == "sqlite" ]; then - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO python3 tools/merge_rucio_configs.py \ - -s /usr/local/src/rucio/etc/docker/test/extra/rucio_autotests_common.cfg \ - /usr/local/src/rucio/etc/docker/test/extra/rucio_sqlite.cfg \ - --use-env \ - -d /opt/rucio/etc/rucio.cfg - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO cp /usr/local/src/rucio/etc/docker/test/extra/alembic_sqlite.ini /opt/rucio/etc/alembic.ini - RESTART_HTTPD=1 -fi - -if [ "$RESTART_HTTPD" == "1" ]; then - docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO httpd -k restart -fi - -docker $CONTAINER_RUNTIME_ARGS ps -a diff --git a/tools/test/install_script.sh b/tools/test/install_script.sh index 735449b1f9a..50dfffbe5ca 100755 --- a/tools/test/install_script.sh +++ b/tools/test/install_script.sh @@ -18,24 +18,60 @@ set -eo pipefail echo "* Using $(command -v python) $(python --version 2>&1) and $(command -v pip) $(pip --version 2>&1)" +SOURCE_PATH=/usr/local/src/rucio +CFG_PATH=/usr/local/src/rucio/etc/docker/test/extra/ +RUCIO_HOME=/opt/rucio + +generate_rucio_cfg(){ + local override=$1 + local destination=$2 + + python3 $SOURCE_PATH/tools/merge_rucio_configs.py \ + -s $CFG_PATH/rucio_autotests_common.cfg "$override" \ + --use-env \ + -d "$destination" +} + if [ "$SUITE" == "client" -o "$SUITE" == "client_syntax" ]; then - cd /usr/local/src/rucio + cd $SOURCE_PATH cp etc/docker/test/extra/rucio_client.cfg etc/rucio.cfg elif [ "$SUITE" == "syntax" -o "$SUITE" == "docs" ]; then - cd /usr/local/src/rucio - python3 tools/merge_rucio_configs.py \ - -s etc/docker/test/extra/rucio_autotests_common.cfg \ - etc/docker/test/extra/rucio_syntax.cfg \ - --use-env \ - -d etc/rucio.cfg + generate_rucio_cfg "$CFG_PATH"/rucio_syntax.cfg "$SOURCE_PATH"/etc/rucio.cfg elif [ "$SUITE" == "votest" ]; then - RUCIO_HOME=/opt/rucio VOTEST_HELPER=$RUCIO_HOME/tools/test/votest_helper.py VOTEST_CONFIG_FILE=$RUCIO_HOME/etc/docker/test/matrix_policy_package_tests.yml echo "VOTEST: Overriding policy section in rucio.cfg" python $VOTEST_HELPER --vo $POLICY --vo-config --file $VOTEST_CONFIG_FILE echo "VOTEST: Restarting httpd to load config" httpd -k restart -fi + +elif [ "$SUITE" == "remote_dbs" ] || [ "$SUITE" == "multi_vo" ]; then + + if [ $RDBMS == "oracle" ]; then + generate_rucio_cfg $CFG_PATH/rucio_oracle.cfg $RUCIO_HOME/etc/rucio.cfg + cp $CFG_PATH/alembic_oracle.ini $RUCIO_HOME/etc/alembic.ini + + elif [ $RDBMS == "mysql8" ]; then + generate_rucio_cfg $CFG_PATH/rucio_mysql8.cfg $RUCIO_HOME/etc/rucio.cfg + cp $CFG_PATH/alembic_mysql8.ini $RUCIO_HOME/etc/alembic.ini + + elif [ $RDBMS == "postgres14" ]; then + if [ $SUITE == "multi_vo" ]; then + mkdir -p $RUCIO_HOME/etc/multi_vo/tst/etc + mkdir -p $RUCIO_HOME/etc/multi_vo/ts2/etc + generate_rucio_cfg $CFG_PATH/rucio_multi_vo_tst_postgres14.cfg $RUCIO_HOME/etc/multi_vo/tst/etc/rucio.cfg + generate_rucio_cfg $CFG_PATH/rucio_multi_vo_ts2_postgres14.cfg $RUCIO_HOME/etc/multi_vo/ts2/etc/rucio.cfg + fi + generate_rucio_cfg $CFG_PATH/rucio_postgres14.cfg $RUCIO_HOME/etc/rucio.cfg + cp $CFG_PATH/alembic_postgres14.ini $RUCIO_HOME/etc/alembic.ini + fi + httpd -k restart + +elif [ "$SUITE" == "sqlite" ]; then + generate_rucio_cfg $CFG_PATH/rucio_sqlite.cfg $RUCIO_HOME/etc/rucio.cfg + cp $CFG_PATH/alembic_sqlite.ini $RUCIO_HOME/etc/alembic.ini + + httpd -k restart +fi \ No newline at end of file diff --git a/tools/test/run_tests.py b/tools/test/run_tests.py index 1e665e84d45..5df88606c86 100755 --- a/tools/test/run_tests.py +++ b/tools/test/run_tests.py @@ -326,19 +326,6 @@ def run_with_httpd( rdbms: f'{project}_{rdbms}_1', } - # Running before_script.sh - run( - './tools/test/before_script.sh', - env={ - **os.environ, - **caseenv, - **namespace_env, - "CONTAINER_RUNTIME_ARGS": ' '.join(namespace_args), - "CON_RUCIO": containers['rucio'], - "CON_DB": containers[rdbms], - }, - ) - # Running install_script.sh run('docker', *namespace_args, 'exec', containers['rucio'], './tools/test/install_script.sh')