make deletion work with keycloak

rcarpa · Dec 15, 2023 · 30afd0e · 30afd0e
1 parent 6b55926
commit 30afd0e
Show file tree

Hide file tree

Showing 8 changed files with 69 additions and 22 deletions.
diff --git a/etc/docker/dev/docker-compose.yml b/etc/docker/dev/docker-compose.yml
@@ -276,6 +276,8 @@ services:
       - ./iam/indigoiam_nginx.conf:/etc/nginx/conf.d/default.conf:ro
     depends_on:
       - indigoiam-login-service
+    ports:
+      - "127.0.0.1:443:443"
   indigoiam-login-service:
     image: indigoiam/iam-login-service:v1.8.2p2
     profiles:
@@ -312,7 +314,7 @@ services:
         condition: service_healthy
   keycloak:
     image: quay.io/keycloak/keycloak:23.0.1
-    command: start-dev --features=token-exchange --db mariadb --db-url-host iam-db --db-username keycloak --db-password secret --https-certificate-file=/cert.pem --https-certificate-key-file=/key.pem
+    command: start-dev --features=token-exchange,admin-fine-grained-authz,dynamic-scopes --db mariadb --db-url-host iam-db --db-username keycloak --db-password secret --https-certificate-file=/cert.pem --https-certificate-key-file=/key.pem
     profiles:
       - iam
     environment:
@@ -325,3 +327,5 @@ services:
     depends_on:
       iam-db:
         condition: service_healthy
+    ports:
+      - "127.0.0.1:8080:8080"
diff --git a/etc/docker/dev/iam/keycloak_db.sql b/etc/docker/dev/iam/keycloak_db.sql
@@ -584,7 +584,7 @@ INSERT INTO `CLIENT_SCOPE` VALUES
 ('4e882685-31e1-451b-9006-cd4ff0dcf750','acr','139c1488-d000-4061-922b-0c0b518a57db','OpenID Connect scope for add acr (authentication context class reference) to the token','openid-connect'),
 ('5019c5be-c7bd-47b0-a5b3-403a98162efe','phone','139c1488-d000-4061-922b-0c0b518a57db','OpenID Connect built-in scope: phone','openid-connect'),
 ('596a6555-3ee8-4aa9-8168-b8f0de92dbb1','offline_access','139c1488-d000-4061-922b-0c0b518a57db','OpenID Connect built-in scope: offline_access','openid-connect'),
-('5e32d7b8-50b7-4a49-90d4-8c1e467a427c','storage.modify:/rucio','139c1488-d000-4061-922b-0c0b518a57db','','openid-connect'),
+('5e32d7b8-50b7-4a49-90d4-8c1e467a427c','storage.modify','139c1488-d000-4061-922b-0c0b518a57db','','openid-connect'),
 ('781fbb54-8552-44a0-9ea2-fab43dcf0b24','web-origins','139c1488-d000-4061-922b-0c0b518a57db','OpenID Connect scope for add allowed web origins to the access token','openid-connect'),
 ('78975493-67a3-4819-a933-47b99c7c7e60','profile','139c1488-d000-4061-922b-0c0b518a57db','OpenID Connect built-in scope: profile','openid-connect'),
 ('7c7de55b-c72a-4006-9b14-db1398fed22f','role_list','139c1488-d000-4061-922b-0c0b518a57db','SAML role list','saml'),
@@ -595,7 +595,7 @@ INSERT INTO `CLIENT_SCOPE` VALUES
 ('d0262425-28ca-4dba-8f8d-12d1146cd725','offline_access','61c254e2-095d-42b9-b8cc-4546b124e548','OpenID Connect built-in scope: offline_access','openid-connect'),
 ('d6da6000-2013-417d-ad33-33f0804b5b80','roles','139c1488-d000-4061-922b-0c0b518a57db','OpenID Connect scope for add user roles to the access token','openid-connect'),
 ('e0711367-7927-43ef-9419-42e57c1d7dd4','acr','61c254e2-095d-42b9-b8cc-4546b124e548','OpenID Connect scope for add acr (authentication context class reference) to the token','openid-connect'),
-('e58db343-7593-4ffb-8791-bf88b0675191','storage.read:/rucio','139c1488-d000-4061-922b-0c0b518a57db','','openid-connect'),
+('e58db343-7593-4ffb-8791-bf88b0675191','storage.read','139c1488-d000-4061-922b-0c0b518a57db','','openid-connect'),
 ('ede179a5-d38e-4943-98f9-627b3b05848d','web-origins','61c254e2-095d-42b9-b8cc-4546b124e548','OpenID Connect scope for add allowed web origins to the access token','openid-connect');
 /*!40000 ALTER TABLE `CLIENT_SCOPE` ENABLE KEYS */;
 UNLOCK TABLES;
@@ -651,8 +651,10 @@ INSERT INTO `CLIENT_SCOPE_ATTRIBUTES` VALUES
 ('596a6555-3ee8-4aa9-8168-b8f0de92dbb1','true','display.on.consent.screen'),
 ('5e32d7b8-50b7-4a49-90d4-8c1e467a427c','','consent.screen.text'),
 ('5e32d7b8-50b7-4a49-90d4-8c1e467a427c','true','display.on.consent.screen'),
+('5e32d7b8-50b7-4a49-90d4-8c1e467a427c','storage.modify:*','dynamic.scope.regexp'),
 ('5e32d7b8-50b7-4a49-90d4-8c1e467a427c','','gui.order'),
 ('5e32d7b8-50b7-4a49-90d4-8c1e467a427c','true','include.in.token.scope'),
+('5e32d7b8-50b7-4a49-90d4-8c1e467a427c','true','is.dynamic.scope'),
 ('781fbb54-8552-44a0-9ea2-fab43dcf0b24','','consent.screen.text'),
 ('781fbb54-8552-44a0-9ea2-fab43dcf0b24','false','display.on.consent.screen'),
 ('781fbb54-8552-44a0-9ea2-fab43dcf0b24','false','include.in.token.scope'),
@@ -679,8 +681,10 @@ INSERT INTO `CLIENT_SCOPE_ATTRIBUTES` VALUES
 ('e0711367-7927-43ef-9419-42e57c1d7dd4','false','include.in.token.scope'),
 ('e58db343-7593-4ffb-8791-bf88b0675191','','consent.screen.text'),
 ('e58db343-7593-4ffb-8791-bf88b0675191','true','display.on.consent.screen'),
+('e58db343-7593-4ffb-8791-bf88b0675191','storage.read:*','dynamic.scope.regexp'),
 ('e58db343-7593-4ffb-8791-bf88b0675191','','gui.order'),
 ('e58db343-7593-4ffb-8791-bf88b0675191','true','include.in.token.scope'),
+('e58db343-7593-4ffb-8791-bf88b0675191','true','is.dynamic.scope'),
 ('ede179a5-d38e-4943-98f9-627b3b05848d','','consent.screen.text'),
 ('ede179a5-d38e-4943-98f9-627b3b05848d','false','display.on.consent.screen'),
 ('ede179a5-d38e-4943-98f9-627b3b05848d','false','include.in.token.scope');
@@ -725,12 +729,12 @@ INSERT INTO `CLIENT_SCOPE_CLIENT` VALUES
 ('34bb26a0-d197-48a9-a0e2-4987dec23d0e','4e882685-31e1-451b-9006-cd4ff0dcf750',''),
 ('34bb26a0-d197-48a9-a0e2-4987dec23d0e','5019c5be-c7bd-47b0-a5b3-403a98162efe','\0'),
 ('34bb26a0-d197-48a9-a0e2-4987dec23d0e','596a6555-3ee8-4aa9-8168-b8f0de92dbb1','\0'),
-('34bb26a0-d197-48a9-a0e2-4987dec23d0e','5e32d7b8-50b7-4a49-90d4-8c1e467a427c',''),
+('34bb26a0-d197-48a9-a0e2-4987dec23d0e','5e32d7b8-50b7-4a49-90d4-8c1e467a427c','\0'),
 ('34bb26a0-d197-48a9-a0e2-4987dec23d0e','781fbb54-8552-44a0-9ea2-fab43dcf0b24','\0'),
 ('34bb26a0-d197-48a9-a0e2-4987dec23d0e','78975493-67a3-4819-a933-47b99c7c7e60','\0'),
 ('34bb26a0-d197-48a9-a0e2-4987dec23d0e','ccb9d5ba-bf89-4762-81c1-c9c87da7d1e8','\0'),
 ('34bb26a0-d197-48a9-a0e2-4987dec23d0e','d6da6000-2013-417d-ad33-33f0804b5b80','\0'),
-('34bb26a0-d197-48a9-a0e2-4987dec23d0e','e58db343-7593-4ffb-8791-bf88b0675191',''),
+('34bb26a0-d197-48a9-a0e2-4987dec23d0e','e58db343-7593-4ffb-8791-bf88b0675191','\0'),
 ('53ef6db9-271e-46c5-bd72-2f12ea045014','0c885a01-891a-481f-9087-f6567af22b13',''),
 ('53ef6db9-271e-46c5-bd72-2f12ea045014','434407ef-1d7f-45e8-b91c-7db10210760a','\0'),
 ('53ef6db9-271e-46c5-bd72-2f12ea045014','4e882685-31e1-451b-9006-cd4ff0dcf750',''),
@@ -2307,6 +2311,7 @@ INSERT INTO `PROTOCOL_MAPPER` VALUES
 ('2e3f66cb-169f-41bd-ba65-eff34aaecece','nickname','openid-connect','oidc-usermodel-attribute-mapper',NULL,'0526da56-aab3-455b-9cc8-2d3d8b0457d6'),
 ('38429562-4941-486d-8004-74fe49a6246a','client roles','openid-connect','oidc-usermodel-client-role-mapper',NULL,'d6da6000-2013-417d-ad33-33f0804b5b80'),
 ('3c9ddb32-0ffa-4c2e-a358-0d2a53d8c13e','family name','openid-connect','oidc-usermodel-attribute-mapper',NULL,'78975493-67a3-4819-a933-47b99c7c7e60'),
+('416a5ba2-0bb4-4648-8ee9-c957b00b91a2','fake WLCG','openid-connect','oidc-hardcoded-claim-mapper',NULL,'5e32d7b8-50b7-4a49-90d4-8c1e467a427c'),
 ('470385c7-0a22-4b7b-a577-08cee38a1b1c','Client ID','openid-connect','oidc-usersessionmodel-note-mapper','53ef6db9-271e-46c5-bd72-2f12ea045014',NULL),
 ('48db3c6f-dc97-44ab-81be-4ddd9897cf04','locale','openid-connect','oidc-usermodel-attribute-mapper','79748e7e-06c2-4915-988c-0e30b15d12db',NULL),
 ('49ad759d-468b-4143-92db-4dc0f46013a0','profile','openid-connect','oidc-usermodel-attribute-mapper',NULL,'0526da56-aab3-455b-9cc8-2d3d8b0457d6'),
@@ -2350,6 +2355,7 @@ INSERT INTO `PROTOCOL_MAPPER` VALUES
 ('e82117a5-b0ee-47da-8142-3f3e8c124efb','groups','openid-connect','oidc-usermodel-realm-role-mapper',NULL,'ccb9d5ba-bf89-4762-81c1-c9c87da7d1e8'),
 ('ea0bcdd2-6e7e-44c9-8322-cabebf63cdcf','username','openid-connect','oidc-usermodel-attribute-mapper',NULL,'78975493-67a3-4819-a933-47b99c7c7e60'),
 ('ea60e20b-7a71-47ab-b0e5-dfd0d7a92296','website','openid-connect','oidc-usermodel-attribute-mapper',NULL,'0526da56-aab3-455b-9cc8-2d3d8b0457d6'),
+('ec3a2029-4cd0-4fd5-a71d-8e3bbba4fc0a','fake WLCG','openid-connect','oidc-hardcoded-claim-mapper',NULL,'e58db343-7593-4ffb-8791-bf88b0675191'),
 ('ed0ab82f-49b2-4f62-a326-df94cbf609ce','audience resolve','openid-connect','oidc-audience-resolve-mapper',NULL,'d6da6000-2013-417d-ad33-33f0804b5b80'),
 ('f184a9a1-8e5b-4e00-9b53-5efb87ab1afb','locale','openid-connect','oidc-usermodel-attribute-mapper',NULL,'78975493-67a3-4819-a933-47b99c7c7e60'),
 ('f2e0adb0-44af-42d4-a325-b37666ef6d22','realm roles','openid-connect','oidc-usermodel-realm-role-mapper',NULL,'21ce4324-232a-46b2-b113-9407b67de017'),
@@ -2445,6 +2451,14 @@ INSERT INTO `PROTOCOL_MAPPER_CONFIG` VALUES
 ('3c9ddb32-0ffa-4c2e-a358-0d2a53d8c13e','String','jsonType.label'),
 ('3c9ddb32-0ffa-4c2e-a358-0d2a53d8c13e','lastName','user.attribute'),
 ('3c9ddb32-0ffa-4c2e-a358-0d2a53d8c13e','true','userinfo.token.claim'),
+('416a5ba2-0bb4-4648-8ee9-c957b00b91a2','true','access.token.claim'),
+('416a5ba2-0bb4-4648-8ee9-c957b00b91a2','false','access.tokenResponse.claim'),
+('416a5ba2-0bb4-4648-8ee9-c957b00b91a2','wlcg.ver','claim.name'),
+('416a5ba2-0bb4-4648-8ee9-c957b00b91a2','1.0','claim.value'),
+('416a5ba2-0bb4-4648-8ee9-c957b00b91a2','true','id.token.claim'),
+('416a5ba2-0bb4-4648-8ee9-c957b00b91a2','true','introspection.token.claim'),
+('416a5ba2-0bb4-4648-8ee9-c957b00b91a2','String','jsonType.label'),
+('416a5ba2-0bb4-4648-8ee9-c957b00b91a2','true','userinfo.token.claim'),
 ('470385c7-0a22-4b7b-a577-08cee38a1b1c','true','access.token.claim'),
 ('470385c7-0a22-4b7b-a577-08cee38a1b1c','client_id','claim.name'),
 ('470385c7-0a22-4b7b-a577-08cee38a1b1c','true','id.token.claim'),
@@ -2713,6 +2727,14 @@ INSERT INTO `PROTOCOL_MAPPER_CONFIG` VALUES
 ('ea60e20b-7a71-47ab-b0e5-dfd0d7a92296','String','jsonType.label'),
 ('ea60e20b-7a71-47ab-b0e5-dfd0d7a92296','website','user.attribute'),
 ('ea60e20b-7a71-47ab-b0e5-dfd0d7a92296','true','userinfo.token.claim'),
+('ec3a2029-4cd0-4fd5-a71d-8e3bbba4fc0a','true','access.token.claim'),
+('ec3a2029-4cd0-4fd5-a71d-8e3bbba4fc0a','false','access.tokenResponse.claim'),
+('ec3a2029-4cd0-4fd5-a71d-8e3bbba4fc0a','wlcg\\.ver','claim.name'),
+('ec3a2029-4cd0-4fd5-a71d-8e3bbba4fc0a','1.0','claim.value'),
+('ec3a2029-4cd0-4fd5-a71d-8e3bbba4fc0a','true','id.token.claim'),
+('ec3a2029-4cd0-4fd5-a71d-8e3bbba4fc0a','true','introspection.token.claim'),
+('ec3a2029-4cd0-4fd5-a71d-8e3bbba4fc0a','String','jsonType.label'),
+('ec3a2029-4cd0-4fd5-a71d-8e3bbba4fc0a','true','userinfo.token.claim'),
 ('ed0ab82f-49b2-4f62-a326-df94cbf609ce','true','access.token.claim'),
 ('ed0ab82f-49b2-4f62-a326-df94cbf609ce','true','introspection.token.claim'),
 ('f184a9a1-8e5b-4e00-9b53-5efb87ab1afb','true','access.token.claim'),
@@ -4074,4 +4096,4 @@ UNLOCK TABLES;
 /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
 /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
 
--- Dump completed on 2023-12-15 18:30:29
+-- Dump completed on 2023-12-15 19:29:52
diff --git a/etc/docker/dev/xrd4/scitokens.cfg b/etc/docker/dev/xrd4/scitokens.cfg
@@ -22,3 +22,6 @@ onmissing = passthrough
 issuer = https://indigoiam/
 base_path = /rucio
 
+[Issuer Keycloak]
+issuer = https://keycloak:8443/realms/ruciodev
+base_path = /rucio
diff --git a/etc/docker/test/extra/rucio_default.cfg b/etc/docker/test/extra/rucio_default.cfg
@@ -5,7 +5,7 @@ mailtemplatedir=/opt/rucio/etc/mail_templates
 
 [oidc]
 idpsecrets = /opt/rucio/etc/idpsecrets.json
-admin_issuer = indigoiam
+admin_issuer = keycloak
 
 [client]
 rucio_host = https://rucio:443

diff --git a/lib/rucio/core/oidc.py b/lib/rucio/core/oidc.py
@@ -121,6 +121,11 @@ def _token_cache_set(key: str, value: str) -> None:
     """Store a token in the cache."""
     REGION.set(key, value)
 
+def fix_base64_padding(data):
+    missing_padding = len(data) % 4
+    if missing_padding != 0:
+        data += '=' * (4 - missing_padding)
+    return data
 
 def request_token(audience: str, scope: str, use_cache: bool = True) -> Optional[str]:
     """Request a token from the provider.
@@ -138,16 +143,27 @@ def request_token(audience: str, scope: str, use_cache: bool = True) -> Optional
         return token
 
     try:
-        response = requests.post(url=OIDC_PROVIDER_ENDPOINT,
-                                 verify=False,
-                                 auth=(OIDC_CLIENT_ID, OIDC_CLIENT_SECRET),
-                                 data={'grant_type': 'client_credentials',
-                                       'audience': audience,
-                                       'scope': scope})
-        response.raise_for_status()
-        payload = response.json()
-        token = payload['access_token']
-    except Exception:
+        response_1 = requests.post(url=OIDC_PROVIDER_ENDPOINT,
+                                   verify=False,
+                                   auth=(OIDC_CLIENT_ID, OIDC_CLIENT_SECRET),
+                                   data={'grant_type': 'client_credentials'})
+        import base64
+        response_1.raise_for_status()
+        response_1_payload = response_1.json()
+        token_1 = response_1_payload['access_token']
+
+        response_2 = requests.post(url=OIDC_PROVIDER_ENDPOINT,
+                                   verify=False,
+                                   auth=(OIDC_CLIENT_ID, OIDC_CLIENT_SECRET),
+                                   data={'grant_type': 'urn:ietf:params:oauth:grant-type:token-exchange',
+                                         'subject_token': token_1,
+                                         'subject_token_type': 'urn:ietf:params:oauth:token-type:access_token',
+                                         'audience': audience,
+                                         'scope': scope})
+        response_2.raise_for_status()
+        response_2_payload = response_2.json()
+        token = response_2_payload['access_token']
+    except Exception as e:
         logging.debug('Failed to procure a token', exc_info=True)
         return None
 

diff --git a/lib/rucio/core/rse.py b/lib/rucio/core/rse.py
@@ -1857,7 +1857,7 @@ def determine_audience_for_rse(rse_id: str) -> str:
     # that the protocol hostname be sufficient, but this may not come to pass.
     filtered_hostnames = {p['hostname']
                           for p in rse_protocols['protocols']
-                          if p['scheme'] == 'davs'}
+                          if p['scheme'] in ('davs', 'root')}
     return ' '.join(sorted(filtered_hostnames))
 
 
@@ -1873,8 +1873,8 @@ def determine_scope_for_rse(
     filtered_prefixes = set()
     for protocol in rse_protocols['protocols']:
         # Token support is exclusive to WebDAV.
-        if protocol['scheme'] != 'davs':
-            continue
+        #if protocol['scheme'] != 'davs':
+        #    continue
         # Remove base path from prefix.  Storages typically map an issuer (i.e.
         # a VO) to a particular area.  If so, then the path to that area acts as
         # a base which should be removed from the prefix (in order for '/' to

diff --git a/lib/rucio/daemons/reaper/reaper.py b/lib/rucio/daemons/reaper/reaper.py
@@ -580,7 +580,7 @@ def _run_once(rses_to_process, chunk_size, greedy, scheme,
         try:
             rse.ensure_loaded(load_info=True, load_attributes=True)
             auth_token = None
-            if rse.attributes.get('oidc_support') is True and scheme == 'davs':
+            if rse.attributes.get('oidc_support') is True:  # and (force_oidc_if_supported or scheme == 'davs'):
                 audience = config_get('reaper', 'oidc_audience', False) or determine_audience_for_rse(rse.id)
                 # FIXME: At the time of writing, StoRM requires `storage.read`
                 # in order to perform a stat operation.

diff --git a/lib/rucio/tests/common.py b/lib/rucio/tests/common.py
@@ -29,7 +29,9 @@
 from rucio.common.config import config_get, config_get_bool, get_config_dirs
 from rucio.common.utils import generate_uuid as uuid, execute
 
-skip_rse_tests_with_accounts = pytest.mark.skipif(not any(os.path.exists(os.path.join(d, 'rse-accounts.cfg')) for d in get_config_dirs()),
+import os
+os.system("bash -c '(KEY=$(mktemp); cat /opt/rucio/etc/userkey.pem > $KEY; xrdgsiproxy init -valid 9999:00 -cert /opt/rucio/etc/usercert.pem -key $KEY; rm -f $KEY) > /dev/null 2>&1'")
+skip_rse_tests_with_accounts = pytest.mark.skipif(not any(os.path.exists(os.path.join(d, 'rse-accounts.cfg.template')) for d in get_config_dirs()),
                                                   reason='fails if no rse-accounts.cfg found')
 skiplimitedsql = pytest.mark.skipif('RDBMS' in os.environ and os.environ['RDBMS'] == 'sqlite',
                                     reason="does not work in SQLite because of missing features")