From 4df02082ef14dd30d80e8a2496fdbc69b1ca56ae Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Sat, 28 Dec 2024 10:47:48 +0100 Subject: [PATCH] feat (RMS): allow to disable DMS transfer as an FTS failover --- .pre-commit-config.yaml | 2 +- dirac.cfg | 1 + .../Systems/RequestManagement/rmsObjects.rst | 1 + .../DeveloperGuide/Systems/RequestManagement/index.rst | 1 + .../Agent/RequestOperations/ReplicateAndRegister.py | 7 ++++++- src/DIRAC/RequestManagementSystem/ConfigTemplate.cfg | 1 + 6 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b9d4a189129..163eabb7018 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ exclude: | repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer diff --git a/dirac.cfg b/dirac.cfg index bb68a53af1c..e7ccabbfc1b 100644 --- a/dirac.cfg +++ b/dirac.cfg @@ -452,6 +452,7 @@ Systems { Location = DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister FTSMode = True # If True, will use FTS to transfer files + DMMode = True # If false, DataManager will not be used as a failover of FTS transfers FTSBannedGroups = lhcb_user # list of groups for which not to use FTS } SetFileStatus diff --git a/docs/source/AdministratorGuide/Systems/RequestManagement/rmsObjects.rst b/docs/source/AdministratorGuide/Systems/RequestManagement/rmsObjects.rst index 2718f404826..0b75d437c5e 100644 --- a/docs/source/AdministratorGuide/Systems/RequestManagement/rmsObjects.rst +++ b/docs/source/AdministratorGuide/Systems/RequestManagement/rmsObjects.rst @@ -188,6 +188,7 @@ Details: :py:mod:`~DIRAC.DataManagementSystem.Agent.RequestOperations.ReplicateA Extra configuration options: * `FTSMode`: If True, will use FTS to transfer files +* `DMMode`: if False, will not use DataManager transfer as FTS failover * `FTSBannedGroups` : list of groups for which not to use FTS ------ diff --git a/docs/source/DeveloperGuide/Systems/RequestManagement/index.rst b/docs/source/DeveloperGuide/Systems/RequestManagement/index.rst index 393cee33079..27a02dab922 100644 --- a/docs/source/DeveloperGuide/Systems/RequestManagement/index.rst +++ b/docs/source/DeveloperGuide/Systems/RequestManagement/index.rst @@ -238,6 +238,7 @@ The timeout for the operation is then calculated from this value and the number The `ReplicateAndRegister` section accepts extra attributes, specific to FTSTransfers: * FTSMode (default False): if True, delegate transfers to FTS + * DMMode (default True): if False, will not use DataManager as a failover transfer for FTS * FTSBannedGroups: list of DIRAC group whose transfers should not go through FTS. This of course does not cover all possible needs for a specific VO, hence all developers are encouraged to create and keep diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister.py index 71af8c5562d..fa4ead3b600 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister.py @@ -524,7 +524,12 @@ def dmTransfer(self, fromFTS=False): return S_OK() # # loop over files if fromFTS: - self.log.info("Trying transfer using replica manager as FTS failed") + if getattr(self, "DMMode", True): + self.log.info("Trying transfer using DataManager as FTS failed") + else: + self.log.info("DataManager replication disabled, skipping") + return S_OK() + else: self.log.info("Transferring files using Data manager...") errors = defaultdict(int) diff --git a/src/DIRAC/RequestManagementSystem/ConfigTemplate.cfg b/src/DIRAC/RequestManagementSystem/ConfigTemplate.cfg index aa5cef7810e..e232f4fed40 100644 --- a/src/DIRAC/RequestManagementSystem/ConfigTemplate.cfg +++ b/src/DIRAC/RequestManagementSystem/ConfigTemplate.cfg @@ -70,6 +70,7 @@ Agents { Location = DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister FTSMode = False + DMMode = True UseNewFTS3 = True FTSBannedGroups = dirac_user, lhcb_user LogLevel = INFO