Tests: add test transfer with tokens

Add required clients and scopes to indigoiam. I didn't try to make it work with keycloak yet. Configure xrootd to accept webdav requests, but I doubt it actually performs any authentication/authorisation of the tokens which are passed to it. Add a test which submits a transfer and verifies that the 'oauth2' authentication was actually used by fts.
rcarpa · Jan 12, 2024 · a3dc870 · a3dc870
1 parent e2afcee
commit a3dc870
Show file tree

Hide file tree

Showing 12 changed files with 302 additions and 14 deletions.
diff --git a/etc/docker/dev/docker-compose.yml b/etc/docker/dev/docker-compose.yml
@@ -126,9 +126,15 @@ services:
     profiles:
       - storage
     volumes:
+      - ./fts/entrypoint.sh:/docker-entrypoint.sh:ro
+      - ./fts/fts-diff-9.0.0.sql:/tmp/fts-diff-9.0.0.sql:ro  # TODO: remove this once fts fixes the code on their side
+      - ../../certs/rucio_ca.pem:/etc/pki/ca-trust/source/anchors/rucio_ca.pem:ro
       - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z
       - ../../certs/hostcert_fts.pem:/etc/grid-security/hostcert.pem:Z
       - ../../certs/hostcert_fts.key.pem:/etc/grid-security/hostkey.pem:Z
+      - ./fts/fts3restconfig:/etc/fts3/fts3restconfig:ro
+    environment:
+      - REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt
     ulimits:
       nofile:
         soft: 10240
@@ -148,8 +154,11 @@ services:
     profiles:
       - storage
     environment:
+      - XRDHOST=xrd1
       - XRDPORT=1094
     volumes:
+      - ./xrd/entrypoint.sh:/docker-entrypoint.sh:ro
+      - ./xrd:/configs:ro
       - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z
       - ../../certs/hostcert_xrd1.pem:/tmp/xrdcert.pem:Z
       - ../../certs/hostcert_xrd1.key.pem:/tmp/xrdkey.pem:Z
@@ -162,8 +171,11 @@ services:
     profiles:
       - storage
     environment:
+      - XRDHOST=xrd2
       - XRDPORT=1095
     volumes:
+      - ./xrd/entrypoint.sh:/docker-entrypoint.sh:ro
+      - ./xrd:/configs:ro
       - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z
       - ../../certs/hostcert_xrd2.pem:/tmp/xrdcert.pem:Z
       - ../../certs/hostcert_xrd2.key.pem:/tmp/xrdkey.pem:Z
@@ -176,6 +188,7 @@ services:
     profiles:
       - storage
     environment:
+      - XRDHOST=xrd3
       - XRDPORT=1096
     volumes:
       - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z
@@ -189,11 +202,10 @@ services:
     image: docker.io/rucio/xrootd
     profiles:
       - storage
+    environment:
+      - XRDHOST=xrd4
+      - XRDPORT=1097
     volumes:
-      - ./xrd4/xrootd.cfg:/etc/xrootd/xrdrucio.cfg:ro
-      - ./xrd4/entrypoint.sh:/docker-entrypoint.sh:ro
-      - ./xrd4/scitokens.cfg:/etc/xrootd/scitokens.cfg:ro
-      - ./xrd4/Authfile:/etc/xrootd/Authfile:ro
       - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:z
       - ../../certs/hostcert_xrd4.pem:/tmp/xrdcert.pem:Z
       - ../../certs/hostcert_xrd4.key.pem:/tmp/xrdkey.pem:Z
@@ -215,6 +227,7 @@ services:
       - ../../certs/rucio_ca.pem:/etc/grid-security/certificates/5fca1cb1.0:ro
       - ../../certs/hostcert_web2.key.pem:/etc/grid-security/hostkey.pem:ro
       - ../../certs/hostcert_web2.pem:/etc/grid-security/hostcert.pem:Z
+      - ./web2/default-ssl.conf:/etc/apache2/sites-available/default-ssl.conf
   minio:
     image: docker.io/minio/minio
     profiles:

diff --git a/etc/docker/dev/fts/entrypoint.sh b/etc/docker/dev/fts/entrypoint.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+# -*- coding: utf-8 -*-
+# Copyright European Organization for Nuclear Research (CERN) since 2012
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# wait for MySQL readiness
+/usr/local/bin/wait-for-it.sh -h ftsdb -p 3306 -t 3600
+
+# TODO: remove following code once FTS fixes the schema on their side
+rm -f /usr/share/fts-mysql/fts-schema-9.0.0.sql /usr/share/fts-mysql/fts-diff-9.0.0.sql
+cp /tmp/fts-diff-9.0.0.sql /usr/share/fts-mysql/
+
+# initialise / upgrade the database
+/usr/share/fts/fts-database-upgrade.py -y
+
+# Configure the OIDC provider and the mapping and the mapping between the OIDC client id to the VO
+# Note: 6fe2a9f5e8876772 is the "automatically generated" vo for the rucios test certificate. It will change if we re-generate the cert.
+echo \
+ "insert into t_token_provider (name,issuer,client_id,client_secret) values('indigoiam', 'https://indigoiam/', 'd6dad80f-11f7-4cf4-a4ef-fbd081ec7f98', 'AJWL5JZtM6I2iaj7XHYq98kPGo6-8Wde2ScSHJhHNvCLeKppTj9fBmeq2xGWi3RCFlj6cPJFjz-BxXIBva4kDYo');" \
+ "insert into t_gridmap (dn,vo) values ('85e6f7a5-580b-4a1c-a6d2-39055143063d', '6fe2a9f5e8876772');" \
+ | mysql -h ftsdb -u fts --password=fts fts
+
+# fix Apache configuration
+/usr/bin/sed -i 's/Listen 80/#Listen 80/g' /etc/httpd/conf/httpd.conf
+cp /opt/rh/httpd24/root/usr/lib64/httpd/modules/mod_rh-python36-wsgi.so /lib64/httpd/modules
+cp /opt/rh/httpd24/root/etc/httpd/conf.modules.d/10-rh-python36-wsgi.conf /etc/httpd/conf.modules.d
+
+# Regenerate CA bundle in case new CAs where mounted into /etc/pki/ca-trust/source/anchors/
+update-ca-trust
+
+# startup the FTS services
+/usr/sbin/fts_server               # main FTS server daemonizes
+/usr/sbin/fts_msg_bulk             # daemon to send messages to activemq
+/usr/sbin/httpd -DFOREGROUND       # FTS REST frontend & FTSMON
diff --git a/etc/docker/dev/fts/fts-diff-9.0.0.sql b/etc/docker/dev/fts/fts-diff-9.0.0.sql
@@ -0,0 +1,41 @@
+--
+-- FTS3 Schema 9.0.0
+-- [FTS-1928] REST API should accept and validate FTS submission token
+--
+
+CREATE TABLE `t_token_provider` (
+                                    `name` varchar(255) NOT NULL,
+                                    `issuer` varchar(1024) NOT NULL,
+                                    `client_id` varchar(255) NOT NULL,
+                                    `client_secret` varchar(255) NOT NULL,
+                                    PRIMARY KEY (`issuer`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+CREATE TABLE `t_token` (
+                           `token_id` char(16) NOT NULL,
+                           `access_token` longtext NOT NULL,
+                           `refresh_token` longtext,
+                           `issuer` varchar(1024) NOT NULL,
+                           `scope` varchar(1024) NOT NULL,
+                           `audience` varchar(1024) NOT NULL,
+                           `retry_timestamp` timestamp NULL DEFAULT NULL,
+                           `retry_delay_m` int unsigned NULL DEFAULT 0,
+                           `attempts` int unsigned NULL DEFAULT 0,
+                           PRIMARY KEY (`token_id`),
+                           CONSTRAINT `fk_token_issuer` FOREIGN KEY (`issuer`) REFERENCES `t_token_provider` (`issuer`) ON DELETE RESTRICT ON UPDATE RESTRICT
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+ALTER TABLE `t_file`
+    ADD COLUMN `src_token_id` char(16) DEFAULT NULL,
+  ADD COLUMN `dst_token_id` char(16) DEFAULT NULL,
+  MODIFY COLUMN `file_state` enum('STAGING','ARCHIVING','QOS_TRANSITION','QOS_REQUEST_SUBMITTED','STARTED','SUBMITTED','READY','ACTIVE','FINISHED','FAILED','CANCELED','NOT_USED','ON_HOLD','ON_HOLD_STAGING','FORCE_START','TOKEN_PREP') NOT NULL,
+  ADD CONSTRAINT `src_token_id` FOREIGN KEY (`src_token_id`) REFERENCES `t_token` (`token_id`) ON DELETE RESTRICT ON UPDATE RESTRICT,
+  ADD CONSTRAINT `dst_token_id` FOREIGN KEY (`dst_token_id`) REFERENCES `t_token` (`token_id`) ON DELETE RESTRICT ON UPDATE RESTRICT;
+
+ALTER TABLE `t_file_backup`
+    ADD COLUMN `src_token_id` char(16) DEFAULT NULL,
+  ADD COLUMN `dst_token_id` char(16) DEFAULT NULL,
+  MODIFY COLUMN `file_state` enum('STAGING','ARCHIVING','QOS_TRANSITION','QOS_REQUEST_SUBMITTED','STARTED','SUBMITTED','READY','ACTIVE','FINISHED','FAILED','CANCELED','NOT_USED','ON_HOLD','ON_HOLD_STAGING','FORCE_START','TOKEN_PREP') NOT NULL;
+
+INSERT INTO t_schema_vers (major, minor, patch, message)
+VALUES (9, 0, 0, 'FTS-1925: Full OAuth2 capabilities in FTS for submission, transfers and tape operations');
diff --git a/etc/docker/dev/fts/fts3restconfig b/etc/docker/dev/fts/fts3restconfig
@@ -0,0 +1,80 @@
+SiteName=DOCKER
+
+AuthorizedVO=*
+
+DbType=mysql
+DbUserName=fts
+DbPassword=fts
+DbConnectString=ftsdb/fts
+
+#OpenID parameters
+ValidateAccessTokenOffline=True
+JWKCacheSeconds=86400
+TokenRefreshDaemonIntervalInSeconds=600
+
+#The alias used for the FTS endpoint, will be published as such in the dashboard transfers UI http://dashb-wlcg-transfers.cern.ch/ui/
+Alias=rucio/fts
+
+MonitoringMessaging=false
+
+[sqlalchemy]
+pool_timeout=10
+pool_size=10
+
+[providers]
+provider1 = https://indigoiam/
+provider1_ClientId = d6dad80f-11f7-4cf4-a4ef-fbd081ec7f98
+provider1_ClientSecret = AJWL5JZtM6I2iaj7XHYq98kPGo6-8Wde2ScSHJhHNvCLeKppTj9fBmeq2xGWi3RCFlj6cPJFjz-BxXIBva4kDYo
+
+[roles]
+Public = vo:transfer;all:datamanagement
+lcgadmin = all:config
+
+# Logging configuration
+[loggers]
+keys = root, routes, fts3rest, sqlalchemy
+
+[handlers]
+keys = console, log_file
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = INFO
+handlers = log_file
+
+[logger_routes]
+level = INFO
+handlers =
+qualname = routes.middleware
+# "level = DEBUG" logs the route matched and routing variables.
+
+[logger_fts3rest]
+level = INFO
+handlers =
+qualname = fts3rest
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+# "level = INFO" logs SQL queries.
+# "level = DEBUG" logs SQL queries and results.
+# "level = WARN" logs neither.  (Recommended for production systems.)
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[handler_log_file]
+class = logging.FileHandler
+args = ('/var/log/fts3rest/fts3rest.log', 'a')
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s,%(msecs)03d %(levelname)-5.5s [%(module)s] %(message)s
+datefmt = %H:%M:%S
diff --git a/etc/docker/dev/iam/indigoiam_db.sql b/etc/docker/dev/iam/indigoiam_db.sql
@@ -507,7 +507,8 @@ LOCK TABLES `client_authority` WRITE;
 /*!40000 ALTER TABLE `client_authority` DISABLE KEYS */;
 INSERT INTO `client_authority` VALUES
 (2,'ROLE_CLIENT'),
-(7,'ROLE_CLIENT');
+(7,'ROLE_CLIENT'),
+(8,'ROLE_CLIENT');
 /*!40000 ALTER TABLE `client_authority` ENABLE KEYS */;
 UNLOCK TABLES;
 
@@ -627,7 +628,7 @@ CREATE TABLE `client_details` (
   PRIMARY KEY (`id`),
   UNIQUE KEY `client_id` (`client_id`),
   KEY `cd_ci_idx` (`client_id`)
-) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+) ENGINE=InnoDB AUTO_INCREMENT=9 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
 /*!40101 SET character_set_client = @saved_cs_client */;
 
 --
@@ -638,7 +639,8 @@ LOCK TABLES `client_details` WRITE;
 /*!40000 ALTER TABLE `client_details` DISABLE KEYS */;
 INSERT INTO `client_details` VALUES
 (2,NULL,0,0,1,600,'85e6f7a5-580b-4a1c-a6d2-39055143063d','AIYIneAVGs9PTVvQnxNGqDmh3rNTsyFOrrwRIqy1Zc6ngPN9hQe6I2VzDzN2uGLCPsvQI8nhYxf_V09NHk-yv7o',3600,2592000,NULL,'rucio','SECRET_BASIC',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,'2023-12-12 07:32:39',NULL,0,NULL,NULL,NULL,NULL,600),
-(7,NULL,0,0,1,600,'9841f5c5-fb77-454c-a4a5-acd220e0faf2','ALORjpM78x3jvUzPZxJJw94Uu6tFu55dYf7NbQ97uNpbF-32Sxb0bprsUSqSrzWgZzLK64cqVlNwya6i3nvU_LU',3600,2592000,NULL,'web1','SECRET_BASIC',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,'2023-12-23 06:08:01',NULL,0,NULL,NULL,NULL,NULL,600);
+(7,NULL,0,0,1,600,'9841f5c5-fb77-454c-a4a5-acd220e0faf2','ALORjpM78x3jvUzPZxJJw94Uu6tFu55dYf7NbQ97uNpbF-32Sxb0bprsUSqSrzWgZzLK64cqVlNwya6i3nvU_LU',3600,2592000,NULL,'web1','SECRET_BASIC',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,'2023-12-23 06:08:01',NULL,0,NULL,NULL,NULL,NULL,600),
+(8,NULL,0,0,1,600,'d6dad80f-11f7-4cf4-a4ef-fbd081ec7f98','AJWL5JZtM6I2iaj7XHYq98kPGo6-8Wde2ScSHJhHNvCLeKppTj9fBmeq2xGWi3RCFlj6cPJFjz-BxXIBva4kDYo',3600,2592000,NULL,'fts3','SECRET_BASIC',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,'2024-01-11 15:15:44',NULL,0,NULL,NULL,NULL,NULL,600);
 /*!40000 ALTER TABLE `client_details` ENABLE KEYS */;
 UNLOCK TABLES;
 
@@ -664,7 +666,11 @@ LOCK TABLES `client_grant_type` WRITE;
 INSERT INTO `client_grant_type` VALUES
 (2,'client_credentials'),
 (2,'urn:ietf:params:oauth:grant-type:token-exchange'),
-(7,'authorization_code');
+(7,'authorization_code'),
+(2,'refresh_token'),
+(8,'client_credentials'),
+(8,'refresh_token'),
+(8,'urn:ietf:params:oauth:grant-type:token-exchange');
 /*!40000 ALTER TABLE `client_grant_type` ENABLE KEYS */;
 UNLOCK TABLES;
 
@@ -808,7 +814,15 @@ INSERT INTO `client_scope` VALUES
 (7,'openid'),
 (7,'profile'),
 (7,'email'),
-(7,'web1');
+(7,'web1'),
+(2,'fts'),
+(2,'offline_access'),
+(8,'openid'),
+(8,'profile'),
+(8,'email'),
+(8,'offline_access'),
+(8,'storage.read:/'),
+(8,'storage.modify:/');
 /*!40000 ALTER TABLE `client_scope` ENABLE KEYS */;
 UNLOCK TABLES;
 

diff --git a/etc/docker/dev/web2/default-ssl.conf b/etc/docker/dev/web2/default-ssl.conf
@@ -0,0 +1,36 @@
+ErrorLog /proc/self/fd/2
+<VirtualHost *:443>
+  ServerName localhost
+  DocumentRoot /var/www/webdav/data/
+  AllowEncodedSlashes on
+
+  CustomLog /proc/self/fd/1 combined
+
+  SSLEngine on
+  SSLCertificateFile /etc/grid-security/hostcert.pem
+  SSLCertificateKeyFile /etc/grid-security/hostkey.pem
+  SSLCACertificatePath /etc/grid-security/certificates/
+  SSLVerifyClient optional
+  SSLVerifyDepth 10
+  SSLOptions +StdEnvVars
+  SSLProtocol TLSv1.2
+  SSLCipherSuite ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS
+
+  OAuth2TokenVerify jwks_uri https://indigoiam/jwk jwks_uri.ssl_verify=false
+
+  <Location />
+    Dav On
+    Options Indexes FollowSymLinks
+
+    <If "%{HTTP:Authorization} =~ m#^Bearer#i">
+      AuthType oauth2
+      Require oauth2_claim aud:web2
+    </If>
+    <ElseIf "%{SSL_CLIENT_VERIFY} == 'SUCCESS'">
+      Require all granted
+    </ElseIf>
+    <Else>
+      Require all denied
+    </Else>
+  </Location>
+</VirtualHost>
diff --git a/etc/docker/dev/xrd4/Authfile → etc/docker/dev/xrd/Authfile b/etc/docker/dev/xrd4/Authfile → etc/docker/dev/xrd/Authfile
diff --git a/etc/docker/dev/xrd4/entrypoint.sh → etc/docker/dev/xrd/entrypoint.sh b/etc/docker/dev/xrd4/entrypoint.sh → etc/docker/dev/xrd/entrypoint.sh
@@ -14,6 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+sed "s/XRDPORT/$XRDPORT/g" /configs/xrootd.cfg > /etc/xrootd/xrdrucio.cfg
+sed "s/XRDHOST/$XRDHOST/g" /configs/scitokens.cfg > /etc/xrootd/scitokens.cfg
+cp /configs/Authfile /etc/xrootd/Authfile
 
 echo 'Fixing ownership and permissions'
 cp /tmp/xrdcert.pem /etc/grid-security/xrd/xrdcert.pem

diff --git a/etc/docker/dev/xrd4/scitokens.cfg → etc/docker/dev/xrd/scitokens.cfg b/etc/docker/dev/xrd4/scitokens.cfg → etc/docker/dev/xrd/scitokens.cfg
@@ -15,7 +15,7 @@
 # limitations under the License.
 
 [Global]
-audience = xrd4
+audience = XRDHOST
 onmissing = passthrough
 
 [Issuer IndigoIAM]

diff --git a/etc/docker/dev/xrd4/xrootd.cfg → etc/docker/dev/xrd/xrootd.cfg b/etc/docker/dev/xrd4/xrootd.cfg → etc/docker/dev/xrd/xrootd.cfg
@@ -18,6 +18,10 @@ all.export /rucio
 xrd.tls /etc/grid-security/xrd/xrdcert.pem /etc/grid-security/xrd/xrdkey.pem detail
 xrd.tlsca certfile /etc/grid-security/certificates/5fca1cb1.0 certdir /etc/grid-security/certificates/ proxies
 xrootd.tls all
+if exec xrootd
+  xrd.protocol http:XRDPORT /usr/lib64/libXrdHttp.so
+  http.exthandler xrdtpc /usr/lib64/libXrdHttpTPC.so
+fi
 
 acc.authdb /etc/xrootd/Authfile
 xrootd.seclib /usr/lib64/libXrdSec.so
@@ -28,4 +32,4 @@ sec.protocol /usr/lib64 ztn -expiry required -maxsz 20k
 xrootd.chksum adler32 /usr/local/bin/xrdadler32.sh
 ofs.tpc autorm fcreds gsi =X509_USER_PROXY pgm /usr/bin/xrdcp --server
 
-xrd.port 1097
+xrd.port XRDPORT