diff --git a/lib/rucio/db/sqla/session.py b/lib/rucio/db/sqla/session.py index 22c7d9dbf81..4f8f2474df5 100644 --- a/lib/rucio/db/sqla/session.py +++ b/lib/rucio/db/sqla/session.py @@ -14,16 +14,19 @@ # limitations under the License. import copy +import logging import os import sys +from datetime import datetime, timedelta from functools import update_wrapper from inspect import isgeneratorfunction, getfullargspec from os.path import basename +from time import sleep from threading import Lock from typing import TYPE_CHECKING -from sqlalchemy import create_engine, event, MetaData -from sqlalchemy.exc import DatabaseError, DisconnectionError, OperationalError, TimeoutError +from sqlalchemy import create_engine, event, MetaData, text +from sqlalchemy.exc import DatabaseError, DisconnectionError, OperationalError, TimeoutError, SQLAlchemyError from sqlalchemy.orm import DeclarativeBase, sessionmaker, scoped_session, Session from sqlalchemy.pool import QueuePool, SingletonThreadPool, NullPool @@ -280,6 +283,27 @@ def get_session(): return session +def wait_for_database(timeout: int = 60, interval: int = 2, *, logger=logging.log): + """ Wait for the database for a specific amount of time """ + + end_time = datetime.utcnow() + timedelta(seconds=timeout) + while True: + try: + session = get_session() + if session.bind.dialect.name == 'oracle': + session.execute(text('select 1 from dual')) + else: + session.execute(text('select 1')) + session.close() + break + except SQLAlchemyError: + logger(logging.WARNING, 'Still waiting for database', exc_info=True) + if datetime.utcnow() >= end_time: + raise + + sleep(interval) + + def retry_if_db_connection_error(exception): """Return True if error in connecting to db.""" if isinstance(exception, (OperationalError, DatabaseException)): diff --git a/tools/run_tests.sh b/tools/run_tests.sh index 00a531ec6f0..7e29d960ad9 100755 --- a/tools/run_tests.sh +++ b/tools/run_tests.sh @@ -111,6 +111,13 @@ if test ${lint}; then fi fi +echo 'Waiting for database to be ready' +if ! python3 -c "from rucio.db.sqla.session import wait_for_database; wait_for_database()" +then + echo 'Cannot access database' + exit 1 +fi + if test ${keep_db}; then echo 'Keeping database tables' else diff --git a/tools/test/before_script.sh b/tools/test/before_script.sh index eb0d8b58b68..6e3dda29fa6 100755 --- a/tools/test/before_script.sh +++ b/tools/test/before_script.sh @@ -29,22 +29,7 @@ RESTART_HTTPD=0 if [ $RDBMS == "oracle" ]; then docker $CONTAINER_RUNTIME_ARGS cp tools/test/oracle_setup.sh ${CON_DB}:/ - date - ORACLE_STARTUP_STRING="DATABASE IS READY TO USE" - for i in {1..60}; do - sleep 2 - cont=$(bash -c 'docker '"$CONTAINER_RUNTIME_ARGS"' logs '"$CON_DB"' | grep "'"$ORACLE_STARTUP_STRING"'" | wc -l') - [ "$cont" -eq "1" ] && break - done - date - if [ "$cont" -ne "1" ]; then - echo "Oracle did not start up in time." - docker $CONTAINER_RUNTIME_ARGS logs $CON_DB || true - exit 1 - fi - sleep 3 docker $CONTAINER_RUNTIME_ARGS exec $CON_DB /oracle_setup.sh - sleep 3 docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO python3 tools/merge_rucio_configs.py \ -s /usr/local/src/rucio/etc/docker/test/extra/rucio_autotests_common.cfg \ /usr/local/src/rucio/etc/docker/test/extra/rucio_oracle.cfg \ @@ -54,17 +39,6 @@ if [ $RDBMS == "oracle" ]; then RESTART_HTTPD=1 elif [ $RDBMS == "mysql8" ]; then - date - for i in {1..30}; do - sleep 4 - cont=$(bash -c 'ping=`docker '"$CONTAINER_RUNTIME_ARGS"' exec '"$CON_DB"' mysqladmin --user=rucio --password=rucio ping`; echo $ping 1>&2; echo $ping | grep "mysqld is alive" 1>&2; echo $?') - [ "$cont" -eq "0" ] && break - done - date - if [ "$cont" -ne "0" ]; then - echo Could not connect to MySQL in time. - exit 1 - fi docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO python3 tools/merge_rucio_configs.py \ -s /usr/local/src/rucio/etc/docker/test/extra/rucio_autotests_common.cfg \ /usr/local/src/rucio/etc/docker/test/extra/rucio_mysql8.cfg \ @@ -74,17 +48,6 @@ elif [ $RDBMS == "mysql8" ]; then RESTART_HTTPD=1 elif [ $RDBMS == "postgres14" ]; then - date - for i in {1..30}; do - sleep 1 - cont=$(bash -c 'docker '"$CONTAINER_RUNTIME_ARGS"' exec '"$CON_DB"' pg_isready 1>&2; echo $?') - [ "$cont" -eq "0" ] && break - done - date - if [ "$cont" -ne "0" ]; then - echo Could not connect to Postgres in time. - exit 1 - fi if [ $SUITE == "multi_vo" ]; then docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO mkdir -p /opt/rucio/etc/multi_vo/tst/etc docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO mkdir -p /opt/rucio/etc/multi_vo/ts2/etc