Tests: use sqlalchemy to wait for database readiness

It's much more generic this way. This will probably allow to
completely get rid of the before_script.sh later.

lib/rucio/db/sqla/session.py

@@ -14,16 +14,19 @@
 # limitations under the License.
 
 import copy
+import logging
 import os
 import sys
+from datetime import datetime, timedelta
 from functools import update_wrapper
 from inspect import isgeneratorfunction, getfullargspec
 from os.path import basename
+from time import sleep
 from threading import Lock
 from typing import TYPE_CHECKING
 
-from sqlalchemy import create_engine, event, MetaData
-from sqlalchemy.exc import DatabaseError, DisconnectionError, OperationalError, TimeoutError
+from sqlalchemy import create_engine, event, MetaData, text
+from sqlalchemy.exc import DatabaseError, DisconnectionError, OperationalError, TimeoutError, SQLAlchemyError
 from sqlalchemy.orm import DeclarativeBase, sessionmaker, scoped_session, Session
 from sqlalchemy.pool import QueuePool, SingletonThreadPool, NullPool
 
@@ -280,6 +283,27 @@ def get_session():
     return session
 
 
+def wait_for_database(timeout: int = 60, interval: int = 2, *, logger=logging.log):
+    """ Wait for the database for a specific amount of time """
+
+    end_time = datetime.utcnow() + timedelta(seconds=timeout)
+    while True:
+        try:
+            session = get_session()
+            if session.bind.dialect.name == 'oracle':
+                session.execute(text('select 1 from dual'))
+            else:
+                session.execute(text('select 1'))
+            session.close()
+            break
+        except SQLAlchemyError:
+            logger(logging.WARNING, 'Still waiting for database', exc_info=True)
+            if datetime.utcnow() >= end_time:
+                raise
+
+        sleep(interval)
+
+
 def retry_if_db_connection_error(exception):
     """Return True if error in connecting to db."""
     if isinstance(exception, (OperationalError, DatabaseException)):

tools/test/before_script.sh

@@ -29,22 +29,7 @@ RESTART_HTTPD=0
 
 if [ $RDBMS == "oracle" ]; then
     docker $CONTAINER_RUNTIME_ARGS cp tools/test/oracle_setup.sh ${CON_DB}:/
-    date
-    ORACLE_STARTUP_STRING="DATABASE IS READY TO USE"
-    for i in {1..60}; do
-        sleep 2
-        cont=$(bash -c 'docker '"$CONTAINER_RUNTIME_ARGS"' logs '"$CON_DB"' | grep "'"$ORACLE_STARTUP_STRING"'" | wc -l')
-        [ "$cont" -eq "1" ] && break
-    done
-    date
-    if [ "$cont" -ne "1" ]; then
-        echo "Oracle did not start up in time."
-        docker $CONTAINER_RUNTIME_ARGS logs $CON_DB || true
-        exit 1
-    fi
-    sleep 3
     docker $CONTAINER_RUNTIME_ARGS exec $CON_DB /oracle_setup.sh
-    sleep 3
     docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO python3 tools/merge_rucio_configs.py \
         -s /usr/local/src/rucio/etc/docker/test/extra/rucio_autotests_common.cfg  \
            /usr/local/src/rucio/etc/docker/test/extra/rucio_oracle.cfg  \
@@ -54,17 +39,6 @@ if [ $RDBMS == "oracle" ]; then
     RESTART_HTTPD=1
 
 elif [ $RDBMS == "mysql8" ]; then
-    date
-    for i in {1..30}; do
-        sleep 4
-        cont=$(bash -c 'ping=`docker '"$CONTAINER_RUNTIME_ARGS"' exec '"$CON_DB"' mysqladmin --user=rucio --password=rucio ping`; echo $ping 1>&2; echo $ping | grep "mysqld is alive" 1>&2; echo $?')
-        [ "$cont" -eq "0" ] && break
-    done
-    date
-    if [ "$cont" -ne "0" ]; then
-        echo Could not connect to MySQL in time.
-        exit 1
-    fi
     docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO python3 tools/merge_rucio_configs.py \
         -s /usr/local/src/rucio/etc/docker/test/extra/rucio_autotests_common.cfg  \
            /usr/local/src/rucio/etc/docker/test/extra/rucio_mysql8.cfg  \
@@ -74,17 +48,6 @@ elif [ $RDBMS == "mysql8" ]; then
     RESTART_HTTPD=1
 
 elif [ $RDBMS == "postgres14" ]; then
-    date
-    for i in {1..30}; do
-        sleep 1
-        cont=$(bash -c 'docker '"$CONTAINER_RUNTIME_ARGS"' exec '"$CON_DB"' pg_isready 1>&2; echo $?')
-        [ "$cont" -eq "0" ] && break
-    done
-    date
-    if [ "$cont" -ne "0" ]; then
-        echo Could not connect to Postgres in time.
-        exit 1
-    fi
     if [ $SUITE == "multi_vo" ]; then
         docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO mkdir -p /opt/rucio/etc/multi_vo/tst/etc
         docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO mkdir -p /opt/rucio/etc/multi_vo/ts2/etc
@@ -117,6 +80,8 @@ elif [ $RDBMS == "sqlite" ]; then
     RESTART_HTTPD=1
 fi
 
+docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO python3 -c "from rucio.db.sqla.session import wait_for_database; wait_for_database()"
+
 if [ "$RESTART_HTTPD" == "1" ]; then
     docker $CONTAINER_RUNTIME_ARGS exec $CON_RUCIO httpd -k restart
 fi