From 860c88dc8213d6faf2d3e7501505a137492d65d4 Mon Sep 17 00:00:00 2001 From: Dimitrios Christidis Date: Mon, 11 Dec 2023 14:11:37 +0100 Subject: [PATCH] Deletion/Transfers: Make token support WebDAV-only #6412 There are some requirements in XRootD that might make token support more complex. It was agreed to focus on WebDAV for now and review XRootD after the Data Challenge 2024. This commits also applies a stronger restriction to the content of the `oidc_support` RSE attribute. --- lib/rucio/core/rse.py | 11 +++++------ lib/rucio/core/transfer.py | 13 +++++++++++++ lib/rucio/daemons/reaper/reaper.py | 2 +- lib/rucio/transfertool/fts3.py | 18 +----------------- 4 files changed, 20 insertions(+), 24 deletions(-) diff --git a/lib/rucio/core/rse.py b/lib/rucio/core/rse.py index 76a2de4bf3..d508c93960 100644 --- a/lib/rucio/core/rse.py +++ b/lib/rucio/core/rse.py @@ -17,7 +17,7 @@ from collections.abc import Iterator, Iterable from datetime import datetime from io import StringIO -from re import match, sub +from re import match from typing import Any, Generic, Optional, TypeVar, Union, TYPE_CHECKING import sqlalchemy @@ -1857,7 +1857,7 @@ def determine_audience_for_rse(rse_id: str) -> str: # that the protocol hostname be sufficient, but this may not come to pass. filtered_hostnames = {p['hostname'] for p in rse_protocols['protocols'] - if p['scheme'] in ('davs', 'root')} + if p['scheme'] == 'davs'} return ' '.join(sorted(filtered_hostnames)) @@ -1872,15 +1872,14 @@ def determine_scope_for_rse( rse_protocols = get_rse_protocols(rse_id) filtered_prefixes = set() for protocol in rse_protocols['protocols']: - # Skip protocol schemes which do not support tokens. - if protocol['scheme'] not in ('davs', 'root'): + # Token support is exclusive to WebDAV. + if protocol['scheme'] != 'davs': continue - # Squeeze leading double slashes, typically found in XRootD protocols. - prefix = sub('^//', '/', protocol['prefix']) # Remove base path from prefix. Storages typically map an issuer (i.e. # a VO) to a particular area. If so, then the path to that area acts as # a base which should be removed from the prefix (in order for '/' to # mean the entire resource associated with that issuer). + prefix = protocol['prefix'] if base_path := get_rse_attribute(rse_id, 'oidc_base_path'): prefix = prefix.removeprefix(base_path) filtered_prefixes.add(prefix) diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py index a11d94b74f..97c4579236 100644 --- a/lib/rucio/core/transfer.py +++ b/lib/rucio/core/transfer.py @@ -167,6 +167,19 @@ def use_ipv4(self): return self.dst.rse.attributes.get('use_ipv4', False) or any(src.rse.attributes.get('use_ipv4', False) for src in self.sources) + @property + def use_tokens(self) -> bool: + """Whether a transfer can be performed with tokens. + + In order to be so, all the involved RSEs must have it explicitly enabled + and the protocol being used must be WebDAV. + """ + for endpoint in [*self.sources, self.destination]: + if (endpoint.rse.attributes.get('oidc_support') is not True + or endpoint.scheme != 'davs'): + return False + return True + @staticmethod def __rewrite_source_url(source_url, source_sign_url, dest_sign_url, source_scheme): """ diff --git a/lib/rucio/daemons/reaper/reaper.py b/lib/rucio/daemons/reaper/reaper.py index 897166b6ec..c628690a9e 100644 --- a/lib/rucio/daemons/reaper/reaper.py +++ b/lib/rucio/daemons/reaper/reaper.py @@ -580,7 +580,7 @@ def _run_once(rses_to_process, chunk_size, greedy, scheme, try: rse.ensure_loaded(load_info=True, load_attributes=True) auth_token = None - if rse.attributes.get('oidc_support', False): + if rse.attributes.get('oidc_support') is True and scheme == 'davs': audience = config_get('reaper', 'oidc_audience', False) or determine_audience_for_rse(rse.id) # FIXME: At the time of writing, StoRM requires `storage.read` # in order to perform a stat operation. diff --git a/lib/rucio/transfertool/fts3.py b/lib/rucio/transfertool/fts3.py index ac4c36ad9e..2d2fbe5b0f 100644 --- a/lib/rucio/transfertool/fts3.py +++ b/lib/rucio/transfertool/fts3.py @@ -196,22 +196,6 @@ def _configured_source_strategy(activity: str, logger: Callable[..., Any]) -> st return activity_source_strategy.get(str(activity), default_source_strategy) -def oidc_supported(transfer_hop) -> bool: - """ - checking OIDC AuthN/Z support per destination and source RSEs; - - for oidc_support to be activated, all sources and the destination must explicitly support it - """ - # assumes use of boolean 'oidc_support' RSE attribute - if not transfer_hop.dst.rse.attributes.get('oidc_support', False): - return False - - for source in transfer_hop.sources: - if not source.rse.attributes.get('oidc_support', False): - return False - return True - - def _available_checksums( transfer: "DirectTransferDefinition", ) -> tuple[set[str], set[str]]: @@ -885,7 +869,7 @@ def submission_builder_for_path(cls, transfer_path, logger=logging.log): if sub_path: oidc_support = False - if all(oidc_supported(t) for t in sub_path): + if all(t.use_tokens for t in sub_path): logger(logging.DEBUG, 'OAuth2/OIDC available for transfer {}'.format([str(hop) for hop in sub_path])) oidc_support = True return sub_path, TransferToolBuilder(cls, external_host=fts_hosts[0], oidc_support=oidc_support, vo=vo)