From a3dc870ad82bbdcad915b9e78b7781e8965bd002 Mon Sep 17 00:00:00 2001 From: Radu Carpa Date: Thu, 11 Jan 2024 17:14:26 +0100 Subject: [PATCH] Tests: add test transfer with tokens Add required clients and scopes to indigoiam. I didn't try to make it work with keycloak yet. Configure xrootd to accept webdav requests, but I doubt it actually performs any authentication/authorisation of the tokens which are passed to it. Add a test which submits a transfer and verifies that the 'oauth2' authentication was actually used by fts. --- etc/docker/dev/docker-compose.yml | 21 ++++-- etc/docker/dev/fts/entrypoint.sh | 45 ++++++++++++ etc/docker/dev/fts/fts-diff-9.0.0.sql | 41 +++++++++++ etc/docker/dev/fts/fts3restconfig | 80 ++++++++++++++++++++++ etc/docker/dev/iam/indigoiam_db.sql | 24 +++++-- etc/docker/dev/web2/default-ssl.conf | 36 ++++++++++ etc/docker/dev/{xrd4 => xrd}/Authfile | 0 etc/docker/dev/{xrd4 => xrd}/entrypoint.sh | 3 + etc/docker/dev/{xrd4 => xrd}/scitokens.cfg | 2 +- etc/docker/dev/{xrd4 => xrd}/xrootd.cfg | 6 +- tests/test_conveyor.py | 47 +++++++++++++ tools/docker_activate_rses.sh | 11 ++- 12 files changed, 302 insertions(+), 14 deletions(-) create mode 100755 etc/docker/dev/fts/entrypoint.sh create mode 100644 etc/docker/dev/fts/fts-diff-9.0.0.sql create mode 100644 etc/docker/dev/fts/fts3restconfig create mode 100644 etc/docker/dev/web2/default-ssl.conf rename etc/docker/dev/{xrd4 => xrd}/Authfile (100%) rename etc/docker/dev/{xrd4 => xrd}/entrypoint.sh (83%) rename etc/docker/dev/{xrd4 => xrd}/scitokens.cfg (97%) rename etc/docker/dev/{xrd4 => xrd}/xrootd.cfg (90%) diff --git a/etc/docker/dev/docker-compose.yml b/etc/docker/dev/docker-compose.yml index d2409b4d7fc..9d01e9d6897 100644 --- a/etc/docker/dev/docker-compose.yml +++ b/etc/docker/dev/docker-compose.yml @@ -126,9 +126,15 @@ services: profiles: - storage volumes: + - ./fts/entrypoint.sh:/docker-entrypoint.sh:ro + - ./fts/fts-diff-9.0.0.sql:/tmp/fts-diff-9.0.0.sql:ro # TODO: remove this once fts fixes the code on their side + - ../../certs/rucio_ca.pem:/etc/pki/ca-trust/source/anchors/rucio_ca.pem:ro - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - ../../certs/hostcert_fts.pem:/etc/grid-security/hostcert.pem:Z - ../../certs/hostcert_fts.key.pem:/etc/grid-security/hostkey.pem:Z + - ./fts/fts3restconfig:/etc/fts3/fts3restconfig:ro + environment: + - REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt ulimits: nofile: soft: 10240 @@ -148,8 +154,11 @@ services: profiles: - storage environment: + - XRDHOST=xrd1 - XRDPORT=1094 volumes: + - ./xrd/entrypoint.sh:/docker-entrypoint.sh:ro + - ./xrd:/configs:ro - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - ../../certs/hostcert_xrd1.pem:/tmp/xrdcert.pem:Z - ../../certs/hostcert_xrd1.key.pem:/tmp/xrdkey.pem:Z @@ -162,8 +171,11 @@ services: profiles: - storage environment: + - XRDHOST=xrd2 - XRDPORT=1095 volumes: + - ./xrd/entrypoint.sh:/docker-entrypoint.sh:ro + - ./xrd:/configs:ro - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - ../../certs/hostcert_xrd2.pem:/tmp/xrdcert.pem:Z - ../../certs/hostcert_xrd2.key.pem:/tmp/xrdkey.pem:Z @@ -176,6 +188,7 @@ services: profiles: - storage environment: + - XRDHOST=xrd3 - XRDPORT=1096 volumes: - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z @@ -189,11 +202,10 @@ services: image: docker.io/rucio/xrootd profiles: - storage + environment: + - XRDHOST=xrd4 + - XRDPORT=1097 volumes: - - ./xrd4/xrootd.cfg:/etc/xrootd/xrdrucio.cfg:ro - - ./xrd4/entrypoint.sh:/docker-entrypoint.sh:ro - - ./xrd4/scitokens.cfg:/etc/xrootd/scitokens.cfg:ro - - ./xrd4/Authfile:/etc/xrootd/Authfile:ro - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z - ../../certs/hostcert_xrd4.pem:/tmp/xrdcert.pem:Z - ../../certs/hostcert_xrd4.key.pem:/tmp/xrdkey.pem:Z @@ -215,6 +227,7 @@ services: - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:ro - ../../certs/hostcert_web2.key.pem:/etc/grid-security/hostkey.pem:ro - ../../certs/hostcert_web2.pem:/etc/grid-security/hostcert.pem:Z + - ./web2/default-ssl.conf:/etc/apache2/sites-available/default-ssl.conf minio: image: docker.io/minio/minio profiles: diff --git a/etc/docker/dev/fts/entrypoint.sh b/etc/docker/dev/fts/entrypoint.sh new file mode 100755 index 00000000000..924348ae3b3 --- /dev/null +++ b/etc/docker/dev/fts/entrypoint.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# -*- coding: utf-8 -*- +# Copyright European Organization for Nuclear Research (CERN) since 2012 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# wait for MySQL readiness +/usr/local/bin/wait-for-it.sh -h ftsdb -p 3306 -t 3600 + +# TODO: remove following code once FTS fixes the schema on their side +rm -f /usr/share/fts-mysql/fts-schema-9.0.0.sql /usr/share/fts-mysql/fts-diff-9.0.0.sql +cp /tmp/fts-diff-9.0.0.sql /usr/share/fts-mysql/ + +# initialise / upgrade the database +/usr/share/fts/fts-database-upgrade.py -y + +# Configure the OIDC provider and the mapping and the mapping between the OIDC client id to the VO +# Note: 6fe2a9f5e8876772 is the "automatically generated" vo for the rucios test certificate. It will change if we re-generate the cert. +echo \ + "insert into t_token_provider (name,issuer,client_id,client_secret) values('indigoiam', 'https://indigoiam/', 'd6dad80f-11f7-4cf4-a4ef-fbd081ec7f98', 'AJWL5JZtM6I2iaj7XHYq98kPGo6-8Wde2ScSHJhHNvCLeKppTj9fBmeq2xGWi3RCFlj6cPJFjz-BxXIBva4kDYo');" \ + "insert into t_gridmap (dn,vo) values ('85e6f7a5-580b-4a1c-a6d2-39055143063d', '6fe2a9f5e8876772');" \ + | mysql -h ftsdb -u fts --password=fts fts + +# fix Apache configuration +/usr/bin/sed -i 's/Listen 80/#Listen 80/g' /etc/httpd/conf/httpd.conf +cp /opt/rh/httpd24/root/usr/lib64/httpd/modules/mod_rh-python36-wsgi.so /lib64/httpd/modules +cp /opt/rh/httpd24/root/etc/httpd/conf.modules.d/10-rh-python36-wsgi.conf /etc/httpd/conf.modules.d + +# Regenerate CA bundle in case new CAs where mounted into /etc/pki/ca-trust/source/anchors/ +update-ca-trust + +# startup the FTS services +/usr/sbin/fts_server # main FTS server daemonizes +/usr/sbin/fts_msg_bulk # daemon to send messages to activemq +/usr/sbin/httpd -DFOREGROUND # FTS REST frontend & FTSMON diff --git a/etc/docker/dev/fts/fts-diff-9.0.0.sql b/etc/docker/dev/fts/fts-diff-9.0.0.sql new file mode 100644 index 00000000000..8e4ce392aa1 --- /dev/null +++ b/etc/docker/dev/fts/fts-diff-9.0.0.sql @@ -0,0 +1,41 @@ +-- +-- FTS3 Schema 9.0.0 +-- [FTS-1928] REST API should accept and validate FTS submission token +-- + +CREATE TABLE `t_token_provider` ( + `name` varchar(255) NOT NULL, + `issuer` varchar(1024) NOT NULL, + `client_id` varchar(255) NOT NULL, + `client_secret` varchar(255) NOT NULL, + PRIMARY KEY (`issuer`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +CREATE TABLE `t_token` ( + `token_id` char(16) NOT NULL, + `access_token` longtext NOT NULL, + `refresh_token` longtext, + `issuer` varchar(1024) NOT NULL, + `scope` varchar(1024) NOT NULL, + `audience` varchar(1024) NOT NULL, + `retry_timestamp` timestamp NULL DEFAULT NULL, + `retry_delay_m` int unsigned NULL DEFAULT 0, + `attempts` int unsigned NULL DEFAULT 0, + PRIMARY KEY (`token_id`), + CONSTRAINT `fk_token_issuer` FOREIGN KEY (`issuer`) REFERENCES `t_token_provider` (`issuer`) ON DELETE RESTRICT ON UPDATE RESTRICT +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + +ALTER TABLE `t_file` + ADD COLUMN `src_token_id` char(16) DEFAULT NULL, + ADD COLUMN `dst_token_id` char(16) DEFAULT NULL, + MODIFY COLUMN `file_state` enum('STAGING','ARCHIVING','QOS_TRANSITION','QOS_REQUEST_SUBMITTED','STARTED','SUBMITTED','READY','ACTIVE','FINISHED','FAILED','CANCELED','NOT_USED','ON_HOLD','ON_HOLD_STAGING','FORCE_START','TOKEN_PREP') NOT NULL, + ADD CONSTRAINT `src_token_id` FOREIGN KEY (`src_token_id`) REFERENCES `t_token` (`token_id`) ON DELETE RESTRICT ON UPDATE RESTRICT, + ADD CONSTRAINT `dst_token_id` FOREIGN KEY (`dst_token_id`) REFERENCES `t_token` (`token_id`) ON DELETE RESTRICT ON UPDATE RESTRICT; + +ALTER TABLE `t_file_backup` + ADD COLUMN `src_token_id` char(16) DEFAULT NULL, + ADD COLUMN `dst_token_id` char(16) DEFAULT NULL, + MODIFY COLUMN `file_state` enum('STAGING','ARCHIVING','QOS_TRANSITION','QOS_REQUEST_SUBMITTED','STARTED','SUBMITTED','READY','ACTIVE','FINISHED','FAILED','CANCELED','NOT_USED','ON_HOLD','ON_HOLD_STAGING','FORCE_START','TOKEN_PREP') NOT NULL; + +INSERT INTO t_schema_vers (major, minor, patch, message) +VALUES (9, 0, 0, 'FTS-1925: Full OAuth2 capabilities in FTS for submission, transfers and tape operations'); diff --git a/etc/docker/dev/fts/fts3restconfig b/etc/docker/dev/fts/fts3restconfig new file mode 100644 index 00000000000..c01e58352c3 --- /dev/null +++ b/etc/docker/dev/fts/fts3restconfig @@ -0,0 +1,80 @@ +SiteName=DOCKER + +AuthorizedVO=* + +DbType=mysql +DbUserName=fts +DbPassword=fts +DbConnectString=ftsdb/fts + +#OpenID parameters +ValidateAccessTokenOffline=True +JWKCacheSeconds=86400 +TokenRefreshDaemonIntervalInSeconds=600 + +#The alias used for the FTS endpoint, will be published as such in the dashboard transfers UI http://dashb-wlcg-transfers.cern.ch/ui/ +Alias=rucio/fts + +MonitoringMessaging=false + +[sqlalchemy] +pool_timeout=10 +pool_size=10 + +[providers] +provider1 = https://indigoiam/ +provider1_ClientId = d6dad80f-11f7-4cf4-a4ef-fbd081ec7f98 +provider1_ClientSecret = AJWL5JZtM6I2iaj7XHYq98kPGo6-8Wde2ScSHJhHNvCLeKppTj9fBmeq2xGWi3RCFlj6cPJFjz-BxXIBva4kDYo + +[roles] +Public = vo:transfer;all:datamanagement +lcgadmin = all:config + +# Logging configuration +[loggers] +keys = root, routes, fts3rest, sqlalchemy + +[handlers] +keys = console, log_file + +[formatters] +keys = generic + +[logger_root] +level = INFO +handlers = log_file + +[logger_routes] +level = INFO +handlers = +qualname = routes.middleware +# "level = DEBUG" logs the route matched and routing variables. + +[logger_fts3rest] +level = INFO +handlers = +qualname = fts3rest + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine +# "level = INFO" logs SQL queries. +# "level = DEBUG" logs SQL queries and results. +# "level = WARN" logs neither. (Recommended for production systems.) + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[handler_log_file] +class = logging.FileHandler +args = ('/var/log/fts3rest/fts3rest.log', 'a') +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s,%(msecs)03d %(levelname)-5.5s [%(module)s] %(message)s +datefmt = %H:%M:%S diff --git a/etc/docker/dev/iam/indigoiam_db.sql b/etc/docker/dev/iam/indigoiam_db.sql index 05a05b4b0e1..902ecc289ae 100644 --- a/etc/docker/dev/iam/indigoiam_db.sql +++ b/etc/docker/dev/iam/indigoiam_db.sql @@ -507,7 +507,8 @@ LOCK TABLES `client_authority` WRITE; /*!40000 ALTER TABLE `client_authority` DISABLE KEYS */; INSERT INTO `client_authority` VALUES (2,'ROLE_CLIENT'), -(7,'ROLE_CLIENT'); +(7,'ROLE_CLIENT'), +(8,'ROLE_CLIENT'); /*!40000 ALTER TABLE `client_authority` ENABLE KEYS */; UNLOCK TABLES; @@ -627,7 +628,7 @@ CREATE TABLE `client_details` ( PRIMARY KEY (`id`), UNIQUE KEY `client_id` (`client_id`), KEY `cd_ci_idx` (`client_id`) -) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; +) ENGINE=InnoDB AUTO_INCREMENT=9 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -638,7 +639,8 @@ LOCK TABLES `client_details` WRITE; /*!40000 ALTER TABLE `client_details` DISABLE KEYS */; INSERT INTO `client_details` VALUES (2,NULL,0,0,1,600,'85e6f7a5-580b-4a1c-a6d2-39055143063d','AIYIneAVGs9PTVvQnxNGqDmh3rNTsyFOrrwRIqy1Zc6ngPN9hQe6I2VzDzN2uGLCPsvQI8nhYxf_V09NHk-yv7o',3600,2592000,NULL,'rucio','SECRET_BASIC',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,'2023-12-12 07:32:39',NULL,0,NULL,NULL,NULL,NULL,600), -(7,NULL,0,0,1,600,'9841f5c5-fb77-454c-a4a5-acd220e0faf2','ALORjpM78x3jvUzPZxJJw94Uu6tFu55dYf7NbQ97uNpbF-32Sxb0bprsUSqSrzWgZzLK64cqVlNwya6i3nvU_LU',3600,2592000,NULL,'web1','SECRET_BASIC',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,'2023-12-23 06:08:01',NULL,0,NULL,NULL,NULL,NULL,600); +(7,NULL,0,0,1,600,'9841f5c5-fb77-454c-a4a5-acd220e0faf2','ALORjpM78x3jvUzPZxJJw94Uu6tFu55dYf7NbQ97uNpbF-32Sxb0bprsUSqSrzWgZzLK64cqVlNwya6i3nvU_LU',3600,2592000,NULL,'web1','SECRET_BASIC',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,'2023-12-23 06:08:01',NULL,0,NULL,NULL,NULL,NULL,600), +(8,NULL,0,0,1,600,'d6dad80f-11f7-4cf4-a4ef-fbd081ec7f98','AJWL5JZtM6I2iaj7XHYq98kPGo6-8Wde2ScSHJhHNvCLeKppTj9fBmeq2xGWi3RCFlj6cPJFjz-BxXIBva4kDYo',3600,2592000,NULL,'fts3','SECRET_BASIC',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,'2024-01-11 15:15:44',NULL,0,NULL,NULL,NULL,NULL,600); /*!40000 ALTER TABLE `client_details` ENABLE KEYS */; UNLOCK TABLES; @@ -664,7 +666,11 @@ LOCK TABLES `client_grant_type` WRITE; INSERT INTO `client_grant_type` VALUES (2,'client_credentials'), (2,'urn:ietf:params:oauth:grant-type:token-exchange'), -(7,'authorization_code'); +(7,'authorization_code'), +(2,'refresh_token'), +(8,'client_credentials'), +(8,'refresh_token'), +(8,'urn:ietf:params:oauth:grant-type:token-exchange'); /*!40000 ALTER TABLE `client_grant_type` ENABLE KEYS */; UNLOCK TABLES; @@ -808,7 +814,15 @@ INSERT INTO `client_scope` VALUES (7,'openid'), (7,'profile'), (7,'email'), -(7,'web1'); +(7,'web1'), +(2,'fts'), +(2,'offline_access'), +(8,'openid'), +(8,'profile'), +(8,'email'), +(8,'offline_access'), +(8,'storage.read:/'), +(8,'storage.modify:/'); /*!40000 ALTER TABLE `client_scope` ENABLE KEYS */; UNLOCK TABLES; diff --git a/etc/docker/dev/web2/default-ssl.conf b/etc/docker/dev/web2/default-ssl.conf new file mode 100644 index 00000000000..547864c4e10 --- /dev/null +++ b/etc/docker/dev/web2/default-ssl.conf @@ -0,0 +1,36 @@ +ErrorLog /proc/self/fd/2 + + ServerName localhost + DocumentRoot /var/www/webdav/data/ + AllowEncodedSlashes on + + CustomLog /proc/self/fd/1 combined + + SSLEngine on + SSLCertificateFile /etc/grid-security/hostcert.pem + SSLCertificateKeyFile /etc/grid-security/hostkey.pem + SSLCACertificatePath /etc/grid-security/certificates/ + SSLVerifyClient optional + SSLVerifyDepth 10 + SSLOptions +StdEnvVars + SSLProtocol TLSv1.2 + SSLCipherSuite ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS + + OAuth2TokenVerify jwks_uri https://indigoiam/jwk jwks_uri.ssl_verify=false + + + Dav On + Options Indexes FollowSymLinks + + + AuthType oauth2 + Require oauth2_claim aud:web2 + + + Require all granted + + + Require all denied + + + diff --git a/etc/docker/dev/xrd4/Authfile b/etc/docker/dev/xrd/Authfile similarity index 100% rename from etc/docker/dev/xrd4/Authfile rename to etc/docker/dev/xrd/Authfile diff --git a/etc/docker/dev/xrd4/entrypoint.sh b/etc/docker/dev/xrd/entrypoint.sh similarity index 83% rename from etc/docker/dev/xrd4/entrypoint.sh rename to etc/docker/dev/xrd/entrypoint.sh index a02c9fd3499..e0453748475 100755 --- a/etc/docker/dev/xrd4/entrypoint.sh +++ b/etc/docker/dev/xrd/entrypoint.sh @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +sed "s/XRDPORT/$XRDPORT/g" /configs/xrootd.cfg > /etc/xrootd/xrdrucio.cfg +sed "s/XRDHOST/$XRDHOST/g" /configs/scitokens.cfg > /etc/xrootd/scitokens.cfg +cp /configs/Authfile /etc/xrootd/Authfile echo 'Fixing ownership and permissions' cp /tmp/xrdcert.pem /etc/grid-security/xrd/xrdcert.pem diff --git a/etc/docker/dev/xrd4/scitokens.cfg b/etc/docker/dev/xrd/scitokens.cfg similarity index 97% rename from etc/docker/dev/xrd4/scitokens.cfg rename to etc/docker/dev/xrd/scitokens.cfg index 53962986a00..428655e599a 100755 --- a/etc/docker/dev/xrd4/scitokens.cfg +++ b/etc/docker/dev/xrd/scitokens.cfg @@ -15,7 +15,7 @@ # limitations under the License. [Global] -audience = xrd4 +audience = XRDHOST onmissing = passthrough [Issuer IndigoIAM] diff --git a/etc/docker/dev/xrd4/xrootd.cfg b/etc/docker/dev/xrd/xrootd.cfg similarity index 90% rename from etc/docker/dev/xrd4/xrootd.cfg rename to etc/docker/dev/xrd/xrootd.cfg index 53f07cb703e..6f1df2d75e6 100755 --- a/etc/docker/dev/xrd4/xrootd.cfg +++ b/etc/docker/dev/xrd/xrootd.cfg @@ -18,6 +18,10 @@ all.export /rucio xrd.tls /etc/grid-security/xrd/xrdcert.pem /etc/grid-security/xrd/xrdkey.pem detail xrd.tlsca certfile /etc/grid-security/certificates/5fca1cb1.0 certdir /etc/grid-security/certificates/ proxies xrootd.tls all +if exec xrootd + xrd.protocol http:XRDPORT /usr/lib64/libXrdHttp.so + http.exthandler xrdtpc /usr/lib64/libXrdHttpTPC.so +fi acc.authdb /etc/xrootd/Authfile xrootd.seclib /usr/lib64/libXrdSec.so @@ -28,4 +32,4 @@ sec.protocol /usr/lib64 ztn -expiry required -maxsz 20k xrootd.chksum adler32 /usr/local/bin/xrdadler32.sh ofs.tpc autorm fcreds gsi =X509_USER_PROXY pgm /usr/bin/xrdcp --server -xrd.port 1097 +xrd.port XRDPORT diff --git a/tests/test_conveyor.py b/tests/test_conveyor.py index e6911941508..b9abea94deb 100644 --- a/tests/test_conveyor.py +++ b/tests/test_conveyor.py @@ -1578,6 +1578,53 @@ def on_submit(file): assert request['state'] == RequestState.FAILED +@skip_rse_tests_with_accounts +@pytest.mark.noparallel(groups=[NoParallelGroups.XRD, NoParallelGroups.SUBMITTER, NoParallelGroups.RECEIVER]) +@pytest.mark.parametrize("file_config_mock", [ + {"overrides": [('oidc', 'admin_issuer', 'indigoiam')]}, +], indirect=True) +def test_transfer_with_tokens(vo, did_factory, root_account, caches_mock, file_config_mock): + src_rse = 'XRD1' + src_rse_id = rse_core.get_rse_id(rse=src_rse, vo=vo) + dst_rse = 'XRD2' + dst_rse_id = rse_core.get_rse_id(rse=dst_rse, vo=vo) + all_rses = [src_rse_id, dst_rse_id] + + did = did_factory.upload_test_file(src_rse) + + rule_core.add_rule(dids=[did], account=root_account, copies=1, rse_expression=dst_rse, grouping='ALL', weight=None, lifetime=None, locked=False, subscription_id=None) + + received_messages = {} + + class ReceiverWrapper(Receiver): + """ + Wrap receiver to record the last handled message for each given request_id + """ + def _perform_request_update(self, msg, *, session=None, logger=logging.log): + ret = super()._perform_request_update(msg, session=session, logger=logger) + received_messages[msg['file_metadata']['request_id']] = msg + return ret + + with patch('rucio.daemons.conveyor.receiver.Receiver', ReceiverWrapper): + receiver_thread = threading.Thread(target=receiver, kwargs={'id_': 0, 'all_vos': True, 'total_threads': 1}) + receiver_thread.start() + try: + submitter(once=True, rses=[{'id': rse_id} for rse_id in all_rses], group_bulk=2, partition_wait_time=0, transfertype='single', filter_transfertool=None) + # Wait for the reception of the FTS Completion message for the submitted request + request = request_core.get_request_by_did(rse_id=dst_rse_id, **did) + for i in range(MAX_POLL_WAIT_SECONDS): + if request['id'] in received_messages: + break + if i == MAX_POLL_WAIT_SECONDS - 1: + assert False # Waited too long; fail the test + time.sleep(1) + assert received_messages[request['id']]['job_metadata']['auth_method'] == 'oauth2' + finally: + receiver_graceful_stop.set() + receiver_thread.join(timeout=5) + receiver_graceful_stop.clear() + + @pytest.mark.noparallel(groups=[NoParallelGroups.PREPARER]) @pytest.mark.parametrize("file_config_mock", [{ "overrides": [('conveyor', 'use_preparer', 'true')] diff --git a/tools/docker_activate_rses.sh b/tools/docker_activate_rses.sh index a27fbf2f43b..0f6cf21a803 100755 --- a/tools/docker_activate_rses.sh +++ b/tools/docker_activate_rses.sh @@ -44,8 +44,10 @@ rucio-admin rse add WEB1 rucio-admin rse add WEB2 # Add the protocol definitions for the storage servers -rucio-admin rse add-protocol --hostname xrd1 --scheme root --prefix //rucio --port 1094 --impl rucio.rse.protocols.xrootd.Default --domain-json '{"wan": {"read": 1, "write": 1, "delete": 1, "third_party_copy_read": 1, "third_party_copy_write": 1}, "lan": {"read": 1, "write": 1, "delete": 1}}' XRD1 -rucio-admin rse add-protocol --hostname xrd2 --scheme root --prefix //rucio --port 1095 --impl rucio.rse.protocols.xrootd.Default --domain-json '{"wan": {"read": 1, "write": 1, "delete": 1, "third_party_copy_read": 1, "third_party_copy_write": 1}, "lan": {"read": 1, "write": 1, "delete": 1}}' XRD2 +rucio-admin rse add-protocol --hostname xrd1 --scheme root --prefix //rucio --port 1094 --impl rucio.rse.protocols.xrootd.Default --domain-json '{"wan": {"read": 1, "write": 1, "delete": 1, "third_party_copy_read": 2, "third_party_copy_write": 2}, "lan": {"read": 1, "write": 1, "delete": 1}}' XRD1 +rucio-admin rse add-protocol --hostname xrd1 --scheme davs --prefix //rucio --port 1094 --impl rucio.rse.protocols.gfal.Default --domain-json '{"wan": {"read": 2, "write": 2, "delete": 2, "third_party_copy_read": 1, "third_party_copy_write": 1}, "lan": {"read": 2, "write": 2, "delete": 2}}' XRD1 +rucio-admin rse add-protocol --hostname xrd2 --scheme root --prefix //rucio --port 1095 --impl rucio.rse.protocols.xrootd.Default --domain-json '{"wan": {"read": 1, "write": 1, "delete": 1, "third_party_copy_read": 2, "third_party_copy_write": 2}, "lan": {"read": 1, "write": 1, "delete": 1}}' XRD2 +rucio-admin rse add-protocol --hostname xrd2 --scheme davs --prefix //rucio --port 1095 --impl rucio.rse.protocols.gfal.Default --domain-json '{"wan": {"read": 2, "write": 2, "delete": 2, "third_party_copy_read": 1, "third_party_copy_write": 1}, "lan": {"read": 2, "write": 2, "delete": 2}}' XRD2 rucio-admin rse add-protocol --hostname xrd3 --scheme root --prefix //rucio --port 1096 --impl rucio.rse.protocols.xrootd.Default --domain-json '{"wan": {"read": 1, "write": 1, "delete": 1, "third_party_copy_read": 1, "third_party_copy_write": 1}, "lan": {"read": 1, "write": 1, "delete": 1}}' XRD3 rucio-admin rse add-protocol --hostname xrd4 --scheme root --prefix //rucio --port 1097 --impl rucio.rse.protocols.xrootd.Default --domain-json '{"wan": {"read": 1, "write": 1, "delete": 1, "third_party_copy_read": 1, "third_party_copy_write": 1}, "lan": {"read": 1, "write": 1, "delete": 1}}' XRD4 rucio-admin rse add-protocol --hostname ssh1 --scheme scp --prefix /rucio --port 22 --impl rucio.rse.protocols.ssh.Default --domain-json '{"wan": {"read": 1, "write": 1, "delete": 1, "third_party_copy_read": 1, "third_party_copy_write": 1}, "lan": {"read": 1, "write": 1, "delete": 1}}' SSH1 @@ -60,7 +62,8 @@ rucio-admin rse set-attribute --rse XRD2 --key test_container_xrd --value True rucio-admin rse set-attribute --rse XRD3 --key test_container_xrd --value True rucio-admin rse set-attribute --rse XRD4 --key test_container_xrd --value True rucio-admin rse set-attribute --rse SSH1 --key test_container_ssh --value True -rucio-admin rse set-attribute --rse XRD4 --key oidc_support --value True +rucio-admin rse set-attribute --rse XRD1 --key oidc_support --value True +rucio-admin rse set-attribute --rse XRD2 --key oidc_support --value True rucio-admin rse set-attribute --rse WEB1 --key oidc_support --value True rucio-admin rse set-attribute --rse WEB2 --key oidc_support --value True rucio-admin rse set-attribute --rse WEB1 --key verify_checksum --value False @@ -93,6 +96,8 @@ rucio-admin rse add-distance --distance 1 --ranking 1 XRD3 XRD1 rucio-admin rse add-distance --distance 2 --ranking 2 XRD3 XRD2 rucio-admin rse add-distance --distance 3 --ranking 3 XRD3 XRD4 rucio-admin rse add-distance --distance 3 --ranking 3 XRD4 XRD3 +rucio-admin rse add-distance --distance 1 --ranking 1 WEB1 WEB2 +rucio-admin rse add-distance --distance 1 --ranking 1 WEB2 WEB1 # Indefinite limits for root rucio-admin account set-limits root XRD1 -1