Tests: use docker-compose in autotests

Remove the home-made suites.py which was manually starting the same docker containers, but with a slightly different configuration. The difference in configuration was not justified in any way. The relevant autotests will now use docker-compose to run database/elastic/activemq/influx. The same way it's done for integration tests. An override docker-compose file is used to set the container image and to disable the default 'ruciodb' postgres from running. This commit doesn't change the interface exposed by run_tests.py. In theory, donkeyrider should still run correctly for all those 1 person who actually use it, but will have a dependency on docker-compose, which wasn't the case until now. Rollback the oracle container version in docker-compose. Put there the version which was actually used in suites.py (18).
rcarpa · Oct 9, 2023 · ce4c079 · ce4c079
1 parent 7d232eb
commit ce4c079
Show file tree

Hide file tree

Showing 14 changed files with 146 additions and 357 deletions.
diff --git a/etc/docker/dev/docker-compose.autotests.yml b/etc/docker/dev/docker-compose.autotests.yml
diff --git a/etc/docker/dev/docker-compose.ports.yml b/etc/docker/dev/docker-compose.ports.yml
@@ -18,7 +18,7 @@ services:
   activemq:
     ports:
       - "127.0.0.1:61613:61613"
-  ruciomy8:
+  mysql8:
     ports:
       - "127.0.0.1:3308:3308"
   oracle:

diff --git a/etc/docker/dev/docker-compose.yml b/etc/docker/dev/docker-compose.yml
@@ -3,10 +3,13 @@ services:
   rucioclient:
     image: docker.io/rucio/rucio-dev:latest-alma9
     command: ["sleep", "infinity"]
+    profiles:
+      - client
     volumes:
       - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z
       - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z
       - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z
+      - ../../certs/rucio_ca.pem:/opt/rucio/etc/rucio_ca.pem:z
       - ../../certs/ruciouser.pem:/opt/rucio/etc/usercert.pem:z
       - ../../certs/ruciouser.key.pem:/opt/rucio/etc/userkey.pem:z
       - ../../certs/ruciouser.certkey.pem:/opt/rucio/etc/usercertkey.pem:z
@@ -26,6 +29,7 @@ services:
       - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z
       - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z
       - ../../certs/hostcert_rucio.key.pem:/etc/grid-security/hostkey.pem:z
+      - ../../certs/rucio_ca.pem:/opt/rucio/etc/rucio_ca.pem:z
       - ../../certs/ruciouser.pem:/opt/rucio/etc/usercert.pem:z
       - ../../certs/ruciouser.key.pem:/opt/rucio/etc/userkey.pem:z
       - ../../certs/ruciouser.certkey.pem:/opt/rucio/etc/usercertkey.pem:z
@@ -73,20 +77,32 @@ services:
       - ACTIVEMQ_CONFIG_SCHEDULERENABLED=true
       - ACTIVEMQ_USERS_hermes=supersecret
       - ACTIVEMQ_CONFIG_QUEUES_events=/queue/events'
-  ruciomy8:
+  postgres14:
+    image: docker.io/postgres:14
+    profiles:
+      - postgres14
+    environment:
+      - POSTGRES_USER=rucio
+      - POSTGRES_DB=rucio
+      - POSTGRES_PASSWORD=rucio
+    command: ["-c", "fsync=off","-c", "synchronous_commit=off","-c", "full_page_writes=off"]
+  mysql8:
     image: docker.io/mysql:8
     profiles:
-      - alldb
+      - mysql8
     environment:
       - MYSQL_USER=rucio
       - MYSQL_PASSWORD=rucio
       - MYSQL_ROOT_PASSWORD=rucio
       - MYSQL_DATABASE=rucio
       - MYSQL_TCP_PORT=3308
+    command:
+      - "--default-authentication-plugin=mysql_native_password"
+      - "--character-set-server=latin1"
   oracle:
-    image: docker.io/gvenzl/oracle-xe:21-slim
+    image: docker.io/gvenzl/oracle-xe:18.4.0
     profiles:
-      - alldb
+      - oracle
     environment:
       - ORACLE_PASSWORD=rucio
       - ORACLE_ALLOW_REMOTE=true

diff --git a/etc/docker/test/extra/alembic_mysql8.ini b/etc/docker/test/extra/alembic_mysql8.ini
@@ -24,8 +24,8 @@ script_location = lib/rucio/db/sqla/migrate_repo/
 # the 'revision' command, regardless of autogenerate
 # revision_environment = false
 
-sqlalchemy.url = mysql+pymysql://root:secret@mysql8/mysql
-version_table_schema = dev
+sqlalchemy.url = mysql+pymysql://rucio:rucio@mysql8:3308/rucio
+version_table_schema = rucio
 
 # Logging configuration
 [loggers]

diff --git a/etc/docker/test/extra/alembic_oracle.ini b/etc/docker/test/extra/alembic_oracle.ini
@@ -24,7 +24,7 @@ script_location = lib/rucio/db/sqla/migrate_repo/
 # the 'revision' command, regardless of autogenerate
 # revision_environment = false
 
-sqlalchemy.url = oracle://system:oracle@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=oracle)(PORT=1521))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=XE)))
+sqlalchemy.url = oracle://system:rucio@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=oracle)(PORT=1521))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=XE)))
 version_table_schema = SYSTEM
 
 # Logging configuration

diff --git a/etc/docker/test/extra/alembic_postgres14.ini b/etc/docker/test/extra/alembic_postgres14.ini
@@ -24,7 +24,7 @@ script_location = lib/rucio/db/sqla/migrate_repo/
 # the 'revision' command, regardless of autogenerate
 # revision_environment = false
 
-sqlalchemy.url = postgresql://postgres:secret@postgres14/postgres
+sqlalchemy.url = postgresql://rucio:rucio@postgres14/rucio
 version_table_schema = dev
 
 # Logging configuration

diff --git a/etc/docker/test/extra/rucio_multi_vo_ts2_postgres14.cfg b/etc/docker/test/extra/rucio_multi_vo_ts2_postgres14.cfg
@@ -20,7 +20,7 @@ request_retries = 3
 vo = testvo2
 
 [database]
-default = postgresql://postgres:secret@postgres14/postgres
+default = postgresql://rucio:rucio@postgres14/rucio
 schema = dev
 pool_recycle=3600
 echo=0

diff --git a/etc/docker/test/extra/rucio_multi_vo_tst_postgres14.cfg b/etc/docker/test/extra/rucio_multi_vo_tst_postgres14.cfg
@@ -20,7 +20,7 @@ request_retries = 3
 vo = testvo1
 
 [database]
-default = postgresql://postgres:secret@postgres14/postgres
+default = postgresql://rucio:rucio@postgres14/rucio
 schema = dev
 pool_recycle=3600
 echo=0

diff --git a/etc/docker/test/extra/rucio_mysql8.cfg b/etc/docker/test/extra/rucio_mysql8.cfg
@@ -17,8 +17,8 @@ account = root
 request_retries = 3
 
 [database]
-default = mysql+pymysql://root:secret@mysql8/mysql
-schema=dev
+default = mysql+pymysql://rucio:rucio@mysql8:3308/rucio
+schema=rucio
 pool_recycle=3600
 echo=0
 pool_reset_on_return=rollback

diff --git a/etc/docker/test/extra/rucio_oracle.cfg b/etc/docker/test/extra/rucio_oracle.cfg
@@ -17,7 +17,7 @@ account = root
 request_retries = 3
 
 [database]
-default = oracle://system:oracle@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=oracle)(PORT=1521))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=XE)))
+default = oracle://system:rucio@(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=oracle)(PORT=1521))(CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=XE)))
 schema=system
 pool_recycle=3600
 echo=0

diff --git a/etc/docker/test/extra/rucio_postgres14.cfg b/etc/docker/test/extra/rucio_postgres14.cfg
@@ -17,7 +17,7 @@ account = root
 request_retries = 3
 
 [database]
-default = postgresql://postgres:secret@postgres14/postgres
+default = postgresql://rucio:rucio@postgres14/rucio
 schema=dev
 pool_recycle=3600
 echo=0

diff --git a/tools/test/before_script.sh b/tools/test/before_script.sh
@@ -53,7 +53,7 @@ elif [ $RDBMS == "mysql8" ]; then
     date
     for i in {1..30}; do
         sleep 4
-        cont=$(bash -c 'ping=`docker '"$CONTAINER_RUNTIME_ARGS"' exec '"$CON_DB"' mysqladmin --user=root --password=secret ping`; echo $ping 1>&2; echo $ping | grep "mysqld is alive" 1>&2; echo $?')
+        cont=$(bash -c 'ping=`docker '"$CONTAINER_RUNTIME_ARGS"' exec '"$CON_DB"' mysqladmin --user=rucio --password=rucio ping`; echo $ping 1>&2; echo $ping | grep "mysqld is alive" 1>&2; echo $?')
         [ "$cont" -eq "0" ] && break
     done
     date

diff --git a/tools/test/run_tests.py b/tools/test/run_tests.py
@@ -15,6 +15,8 @@
 # limitations under the License.
 
 import json
+import io
+import itertools
 import multiprocessing
 import os
 import pathlib
@@ -24,11 +26,32 @@
 import time
 import traceback
 import uuid
-from collections.abc import Callable
 from datetime import datetime
-from typing import Optional
+from tempfile import NamedTemporaryFile
+from typing import Optional, Union, NoReturn
 
-from suites import run, Container, rdbms_container, services, CumulativeContextManager, service_hostnames, env_args
+import yaml
+
+
+def run(*args, check=True, return_stdout=False, env=None) -> Union[NoReturn, io.TextIOBase]:
+    kwargs = {'check': check, 'stdout': sys.stderr, 'stderr': subprocess.STDOUT}
+    if env is not None:
+        kwargs['env'] = env
+    if return_stdout:
+        kwargs['stderr'] = sys.stderr
+        kwargs['stdout'] = subprocess.PIPE
+    args = [str(a) for a in args]
+    print("** Running", " ".join(map(lambda a: repr(a) if ' ' in a else a, args)), kwargs, file=sys.stderr, flush=True)
+    proc = subprocess.run(args, **kwargs)
+    if return_stdout:
+        return proc.stdout
+
+
+def env_args(caseenv):
+    environment_args = list(itertools.chain(*map(lambda x: ('--env', f'{x[0]}={x[1]}'), caseenv.items())))
+    environment_args.append('--env')
+    environment_args.append('GITHUB_ACTIONS')
+    return environment_args
 
 
 def matches(small: dict, group: dict):
@@ -177,8 +200,6 @@ def run_case(caseenv, image, use_podman, use_namespace, use_httpd, copy_rucio_lo
             success = run_with_httpd(
                 caseenv=caseenv,
                 image=image,
-                use_podman=use_podman,
-                pod=pod,
                 namespace_args=namespace_args,
                 namespace_env=namespace_env,
                 copy_rucio_logs=copy_rucio_logs,
@@ -254,74 +275,84 @@ def run_test_directly(
 def run_with_httpd(
     caseenv: dict[str, str],
     image: str,
-    use_podman: bool,
-    pod: str,
     namespace_args: list[str],
     namespace_env: dict[str, str],
     copy_rucio_logs: bool,
     logs_dir: pathlib.Path,
     tests: list[str],
 ) -> bool:
-    pod_net_arg = ['--pod', pod] if use_podman else []
-    # Running rucio container from given image
-    with Container(image, runtime_args=namespace_args, run_args=pod_net_arg, environment=caseenv) as rucio_container:
+    compose_version = int(run('docker-compose', 'version', '--short', return_stdout=True).decode().split('.')[0])
+
+    with (NamedTemporaryFile() as compose_override_file):
+        compose_override_content = yaml.dump({
+            'services': {
+                'rucio': {
+                    'image': image,
+                    'environment': [f'{k}={v}' for k, v in caseenv.items()],
+                },
+                'ruciodb': {
+                    'profiles': ['donotstart'],
+                }
+            }
+        })
+        print("Overriding docker-compose configuration with: \n", compose_override_content)
+        with open(compose_override_file.name, 'w') as f:
+            f.write(compose_override_content)
+
+        rdbms = caseenv.get('RDBMS', '')
+        project = os.urandom(8).hex()
+        containers = {}
+        file_args = (
+            '--file', 'etc/docker/dev/docker-compose.yml',
+            '--file', compose_override_file.name,
+        )
         try:
-            network_arg = ('--network', 'container:' + rucio_container.cid)
-            container_run_args = pod_net_arg if use_podman else network_arg
-            additional_containers = []
-
-            def create_cnt(cnt_class: Callable) -> Container:
-                return cnt_class(
-                    runtime_args=namespace_args,
-                    run_args=container_run_args,
-                )
-
-            db_container = None
-            rdbms = caseenv.get('RDBMS', '')
-            if rdbms:
-                service_key = caseenv.get('SERVICES', 'default')
-                db_container_class = rdbms_container.get(rdbms, None)
-                if db_container_class:
-                    db_container = create_cnt(db_container_class)
-                    additional_containers.append(db_container)
-                additional_containers += list(map(create_cnt, services[service_key]))
-
-            with CumulativeContextManager(*additional_containers):
-                db_env = dict()
-                if db_container:
-                    db_env['CON_DB'] = db_container.cid
-
-                # Running before_script.sh
-                run(
-                    './tools/test/before_script.sh',
-                    env={
-                        **os.environ,
-                        **caseenv,
-                        **namespace_env,
-                        **db_env,
-                        "CONTAINER_RUNTIME_ARGS": ' '.join(namespace_args),
-                        "CON_RUCIO": rucio_container.cid,
-                    },
-                )
-
-                # register service hostnames
-                run('docker', *namespace_args, 'exec', rucio_container.cid, '/bin/sh', '-c', f'echo "127.0.0.1 {" ".join(service_hostnames)}" | tee -a /etc/hosts')
+            # Start docker compose
+            run('docker-compose', '-p', project, *file_args, '--profile', rdbms, 'up', '-d')
+
+            # Retrieve container names from docker compose
+            # or use pre-defined names for old, v1, docker-compose
+            if compose_version > 1:
+                containers = {
+                    c['Service']: c['Name']
+                    for c in json.loads(
+                        run('docker-compose', '-p', project, 'ps', '--format', 'json', return_stdout=True)
+                    )
+                }
+            else:
+                containers = {
+                    'rucio': f'{project}_rucio_1',
+                    rdbms: f'{project}_{rdbms}_1',
+                }
+
+            # Running before_script.sh
+            run(
+                './tools/test/before_script.sh',
+                env={
+                    **os.environ,
+                    **caseenv,
+                    **namespace_env,
+                    "CONTAINER_RUNTIME_ARGS": ' '.join(namespace_args),
+                    "CON_RUCIO": containers['rucio'],
+                    "CON_DB": containers[rdbms],
+                },
+            )
 
-                # Running install_script.sh
-                run('docker', *namespace_args, 'exec', rucio_container.cid, './tools/test/install_script.sh')
+            # Running install_script.sh
+            run('docker', *namespace_args, 'exec', containers['rucio'], './tools/test/install_script.sh')
 
-                # Running test.sh
-                if tests:
-                    tests_env = ('--env', 'TESTS=' + ' '.join(tests))
-                    tests_arg = ('-p', )
-                else:
-                    tests_env = ()
-                    tests_arg = ()
+            # Running test.sh
+            if tests:
+                tests_env = ('--env', 'TESTS=' + ' '.join(tests))
+                tests_arg = ('-p', )
+            else:
+                tests_env = ()
+                tests_arg = ()
 
-                run('docker', *namespace_args, 'exec', *tests_env, rucio_container.cid, './tools/test/test.sh', *tests_arg)
+            run('docker', *namespace_args, 'exec', *tests_env, containers['rucio'], './tools/test/test.sh', *tests_arg)
 
-                # if everything went through without an exception, mark this case as a success
-                return True
+            # if everything went through without an exception, mark this case as a success
+            return True
         except subprocess.CalledProcessError as error:
             print(
                 f"** Process '{error.cmd}' exited with code {error.returncode}",
@@ -330,22 +361,27 @@ def create_cnt(cnt_class: Callable) -> Container:
                 flush=True,
             )
         finally:
-            run('docker', *namespace_args, 'logs', rucio_container.cid, check=False)
-            if copy_rucio_logs:
-                try:
-                    if logs_dir.exists():
-                        shutil.rmtree(logs_dir)
-                    run('docker', *namespace_args, 'cp', f'{rucio_container.cid}:/var/log', str(logs_dir))
-                except Exception:
-                    print(
-                        "** Error on retrieving logs for",
-                        {**caseenv, "IMAGE": image},
-                        '\n',
-                        traceback.format_exc(),
-                        '\n**',
-                        file=sys.stderr,
-                        flush=True,
-                    )
+            if 'rucio' in containers:
+                run('docker', *namespace_args, 'logs', containers['rucio'], check=False)
+                if copy_rucio_logs:
+                    try:
+                        if logs_dir.exists():
+                            shutil.rmtree(logs_dir)
+                        run('docker', *namespace_args, 'cp', f'{containers["rucio"]}:/var/log', str(logs_dir))
+                    except Exception:
+                        print(
+                            "** Error on retrieving logs for",
+                            {**caseenv, "IMAGE": image},
+                            '\n',
+                            traceback.format_exc(),
+                            '\n**',
+                            file=sys.stderr,
+                            flush=True,
+                        )
+            additional_args = ()
+            if compose_version == 1:
+                additional_args = file_args
+            run('docker-compose', '-p', project, *additional_args, 'down', '-t', '30', check=False)
         return False