Tests: generate rucio.cfg on container startup

This will allow to easily create a test env for a specific database:
```
RDBMS=mysql8 docker-compose --file etc/docker/dev/docker-compose.yml --profile mysql8 up
```

Mount the rucio source into the rucio containers and use an alternative
entry-point script to generate the desired configuration before running
httpd.

Also copy the "default" rucio.cfg and alembic.ini from the containers
repository. This will reduce the dependency between the two
repositories.

Move the httpd restart command in run_tests.sh to fix the sqlite env.
Otherwise, httpd continues to use the old database even if we re-created
it a couple of lines above.

etc/docker/dev/docker-compose.yml

@@ -2,6 +2,7 @@ version: "3"
 services:
   rucioclient:
     image: docker.io/rucio/rucio-dev:latest-alma9
+    entrypoint: ["/rucio_source/etc/docker/dev/rucio_entrypoint.sh"]
     command: ["sleep", "infinity"]
     profiles:
       - client
@@ -19,11 +20,16 @@ services:
       - ../../../bin:/opt/rucio/bin:Z
       - ../../../lib:/opt/rucio/lib:Z
       - ../../../tests:/opt/rucio/tests:Z
+      - ../../../:/rucio_source:ro
     environment:
+      - RUCIO_SOURCE_DIR=/rucio_source
       - X509_USER_CERT=/opt/rucio/etc/usercert.pem
       - X509_USER_KEY=/opt/rucio/etc/userkey.pem
+      - RDBMS
   rucio:
     image: docker.io/rucio/rucio-dev:latest-alma9
+    entrypoint: ["/rucio_source/etc/docker/dev/rucio_entrypoint.sh"]
+    command: ["httpd","-D","FOREGROUND"]
     volumes:
       - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z
       - ../../certs/hostcert_rucio.pem:/etc/grid-security/hostcert.pem:z
@@ -38,9 +44,12 @@ services:
       - ../../../bin:/opt/rucio/bin:Z
       - ../../../lib:/opt/rucio/lib:Z
       - ../../../tests:/opt/rucio/tests:Z
+      - ../../../:/rucio_source:ro
     environment:
+      - RUCIO_SOURCE_DIR=/rucio_source
       - X509_USER_CERT=/opt/rucio/etc/usercert.pem
       - X509_USER_KEY=/opt/rucio/etc/userkey.pem
+      - RDBMS
   ruciodb:
     image: docker.io/postgres:14
     environment:

etc/docker/dev/rucio_entrypoint.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CFG_PATH="$RUCIO_SOURCE_DIR"/etc/docker/test/extra/
+if [ -z "$RUCIO_HOME" ]; then
+    RUCIO_HOME=/opt/rucio
+fi
+
+mkdir -p $RUCIO_HOME
+
+generate_rucio_cfg(){
+  	local override=$1
+  	local destination=$2
+
+    python3 $RUCIO_SOURCE_DIR/tools/merge_rucio_configs.py --use-env \
+        -s "$CFG_PATH"/rucio_autotests_common.cfg "$override" \
+        -d "$destination"
+}
+
+echo "Generating alembic.ini and rucio.cfg"
+
+if [ -z "$RDBMS" ]; then
+    cp "$CFG_PATH"/rucio_default.cfg $RUCIO_HOME/etc/rucio.cfg
+    cp "$CFG_PATH"/alembic_default.ini $RUCIO_HOME/etc/alembic.ini
+
+elif [ "$RDBMS" == "oracle" ]; then
+    generate_rucio_cfg "$CFG_PATH"/rucio_oracle.cfg $RUCIO_HOME/etc/rucio.cfg
+    cp "$CFG_PATH"/alembic_oracle.ini $RUCIO_HOME/etc/alembic.ini
+
+elif [ "$RDBMS" == "mysql8" ]; then
+    generate_rucio_cfg "$CFG_PATH"/rucio_mysql8.cfg $RUCIO_HOME/etc/rucio.cfg
+    cp "$CFG_PATH"/alembic_mysql8.ini $RUCIO_HOME/etc/alembic.ini
+
+elif [ "$RDBMS" == "sqlite" ]; then
+    generate_rucio_cfg "$CFG_PATH"/rucio_sqlite.cfg $RUCIO_HOME/etc/rucio.cfg
+    cp "$CFG_PATH"/alembic_sqlite.ini $RUCIO_HOME/etc/alembic.ini
+
+elif [ "$RDBMS" == "postgres14" ]; then
+    generate_rucio_cfg "$CFG_PATH"/rucio_postgres14.cfg $RUCIO_HOME/etc/rucio.cfg
+    cp "$CFG_PATH"/alembic_postgres14.ini $RUCIO_HOME/etc/alembic.ini
+
+fi
+
+exec "$@"

etc/docker/test/extra/alembic_default.ini

@@ -0,0 +1,64 @@
+# Copyright European Organization for Nuclear Research (CERN)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Authors:
+# - Vincent Garonne <[email protected]>, 2014
+# - Mario Lassnig <[email protected]>, 2019
+
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = lib/rucio/db/sqla/migrate_repo/
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+sqlalchemy.url = postgresql://rucio:secret@ruciodb/rucio
+version_table_schema = dev
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

etc/docker/test/extra/rucio_default.cfg

@@ -0,0 +1,187 @@
+[common]
+logdir = /var/log/rucio
+loglevel = DEBUG
+mailtemplatedir=/opt/rucio/etc/mail_templates
+
+[client]
+rucio_host = https://rucio:443
+auth_host = https://rucio:443
+auth_type = userpass
+username = ddmlab
+password = secret
+ca_cert = /etc/grid-security/certificates/5fca1cb1.0
+client_cert = /opt/rucio/etc/usercert.pem
+client_key = /opt/rucio/etc/userkey.pem
+client_x509_proxy = $X509_USER_PROXY
+account = root
+request_retries = 3
+
+[database]
+default = postgresql://rucio:secret@ruciodb/rucio
+schema = dev
+echo=0
+pool_recycle=3600
+pool_size=20
+max_overflow=20
+pool_reset_on_return=rollback
+
+[bootstrap]
+userpass_identity = ddmlab
+userpass_pwd = secret
+userpass_email = [email protected]
+
+# Default development client certificate from /opt/rucio/etc/usercert.pem
+x509_identity = /CN=Rucio User
+x509_email = [email protected]
+
+# Default Kerberos account
+gss_identity = [email protected]
+gss_email = [email protected]
+
+[monitor]
+carbon_server = graphite
+carbon_port = 8125
+user_scope = docker
+
+[conveyor]
+scheme = https,davs,gsiftp,root,srm,mock
+#scheme = https
+#user_transfers = cms
+#user_activities = ['dummy_user_activity']
+#hostcert = /etc/grid-security/hostcert.pem
+#hostkey = /etc/grid-security/hostkey.pem
+transfertool = fts3
+ftshosts = https://fts:8446
+cacert = /etc/grid-security/certificates/5fca1cb1.0
+usercert = /opt/rucio/etc/usercertkey.pem
+
+[messaging-fts3]
+port = 61613
+nonssl_port = 61613
+use_ssl = False
+ssl_key_file = /opt/rucio/etc/userkey.pem
+ssl_cert_file = /opt/rucio/etc/usercert.pem
+destination = /topic/transfer.fts_monitoring_complete
+username = receiver
+password = supersecret
+brokers = activemq
+voname = atlas
+
+[messaging-hermes]
+username = hermes
+password = supersecret
+port = 61613
+nonssl_port = 61613
+use_ssl = False
+destination = /queue/events
+ssl_key_file = /opt/rucio/etc/userkey.pem
+ssl_cert_file = /opt/rucio/etc/usercert.pem
+brokers = activemq
+voname = atlas
+email_from = Rucio <[email protected]>
+email_test = [email protected]
+
+[transmogrifier]
+maxdids = 100000
+
+[accounts]
+# These are accounts that can write into scopes owned by another account
+special_accounts = panda, tier0
+
+[trace]
+tracedir = /var/log/rucio/trace
+brokers=activemq
+port=61013
+username = username
+password = password
+topic = /topic/rucio.tracer
+
+[nongrid-trace]
+tracedir = /var/log/rucio/trace
+brokers=activemq
+port=61013
+username = username
+password = password
+topic = /topic/rucio.tracer
+
+[tracer-kronos]
+brokers=activemq
+port=61013
+ssl_key_file = /opt/rucio/etc/userkey.pem
+ssl_cert_file = /opt/rucio/etc/usercert.pem
+queue = /queue/Consumer.kronos.rucio.tracer
+prefetch_size = 10
+chunksize = 10
+subscription_id = rucio-tracer-listener
+use_ssl = False
+reconnect_attempts = 100
+excluded_usrdns = /DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=gangarbt/CN=722147/CN=Robot: Ganga Robot/CN=proxy
+username = username
+password = password
+dataset_wait = 60
+
+[injector]
+file = /opt/rucio/tools/test.file.1000
+bytes = 1000
+md5 = fd21ce524a9e45060fd3f62c4ef6a386
+adler32 = 52590737
+
+[alembic]
+cfg = /opt/rucio/etc/alembic.ini
+
+[messaging-cache]
+port = 61023
+ssl_key_file = /opt/rucio/etc/userkey.pem
+ssl_cert_file = /opt/rucio/etc/usercert.pem
+destination = /topic/rucio.cache
+brokers = activemq
+voname = atlas
+account = cache_mb
+
+[test]
+cacert = /etc/grid-security/certificates/5fca1cb1.0
+usercert = /opt/rucio/etc/usercert.pem
+userkey = /opt/rucio/etc/userkey.pem
+
+[nagios]
+proxy = /opt/rucio/etc/usercertkey.pem
+rfcproxy = /opt/rucio/etc/usercertkey.pem
+fts_servers = https://fts3:8446
+
+[auditor]
+cache = /opt/rucio/auditor-cache
+results = /opt/rucio/auditor-results
+
+[hermes]
+email_from = Rucio <[email protected]>
+email_test = [email protected]
+
+[c3po]
+placement_algorithm = t2_free_space
+elastic_url = http://elastic:9200
+redis_host = rucio
+redis_port = 6379
+
+[c3po-popularity]
+elastic_url = http://elastic:9200
+
+[c3po-site-mapper]
+panda_url = http://agis:80/request/pandaqueue/query/list/?json
+ddm_url = http://agis:80/request/ddmendpoint/query/list/?json
+
+[c3po-workload]
+panda_url = http://bigpanda:80/jobs/?category=analysis&jobstatus=running
+window = 604800
+
+[policy]
+permission = atlas
+schema = atlas
+lfn2pfn_algorithm_default = hash
+support = [email protected]
+support_rucio = https://github.com/rucio/rucio/issues/
+
+[credentials]
+gcs = /opt/rucio/etc/google-cloud-storage-test.json
+
+[api]
+endpoints = accountlimits, accounts, archives, auth, config, credentials, dids, dirac, export, heartbeats, identities, import, lifetime_exceptions, locks, meta, ping, redirect, replicas, requests, rses, rules, scopes, subscriptions, traces, vos

tools/run_tests.sh

@@ -60,9 +60,6 @@ done
 echo 'Clearing memcache'
 echo flush_all > /dev/tcp/127.0.0.1/11211
 
-echo 'Graceful restart of Apache'
-httpd -k graceful
-
 echo 'Update dependencies with pip'
 pip install --upgrade -r requirements.txt
 
@@ -128,7 +125,7 @@ else
 
     if [ -f /tmp/rucio.db ]; then
         echo 'Disable SQLite database access restriction'
-        chmod 777 /tmp/rucio.db
+        chmod 666 /tmp/rucio.db
     fi
 fi
 
@@ -141,6 +138,9 @@ if test ${alembic}; then
     fi
 fi
 
+echo 'Graceful restart of Apache'
+httpd -k graceful
+
 echo 'Bootstrapping tests'
 tools/bootstrap_tests.py
 if [ $? != 0 ]; then