Skip to content

Commit

Permalink
Merge pull request #3548 from matyasselmeci/pr/lenient-authfile.SOFTW…
Browse files Browse the repository at this point in the history 
…ARE-5787

Be lenient in Authfile generation (SOFTWARE-5787)
  • Loading branch information
@matyasselmeci
matyasselmeci authored Jan 2, 2024
2 parents 289318c + c8a95fb commit e9a6332
Showing 1 changed file with 43 additions and 34 deletions.
77 changes: 43 additions & 34 deletions src/stashcache.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,12 @@ def __init__(self):
self.id_to_paths = defaultdict(set)
self.id_to_str = {}
self.grid_mapfile_lines = set()
self.warnings = []
self.warnings_auth = []
self.warnings_public = []

@classmethod
def for_cache(cls, global_data: GlobalData, topology: Topology, vos_data: VOsData, legacy: bool,
cache_resource: Optional[Resource], public_cache: bool) -> "_IdNamespaceData":
def for_cache(cls, global_data: GlobalData, vos_data: VOsData, legacy: bool,
cache_resource: Optional[Resource]) -> "_IdNamespaceData":
self = cls()

ligo_authz_list: List[AuthMethod] = []
Expand All @@ -160,16 +161,14 @@ def fetch_ligo_authz_list_if_needed():
if namespace.is_public():
self.public_paths.add(path)
continue
if public_cache:
continue

# Extend authz list with LIGO DNs if applicable
extended_authz_list = namespace.authz_list
if vo_name.lower() == "ligo":
if legacy:
extended_authz_list += fetch_ligo_authz_list_if_needed()
else:
self.warnings.append("# LIGO DNs unavailable\n")
self.warnings_auth.append("# LIGO DNs unavailable\n")

for authz in extended_authz_list:
if authz.used_in_authfile:
Expand All @@ -181,8 +180,8 @@ def fetch_ligo_authz_list_if_needed():
return self

@classmethod
def for_origin(cls, topology: Topology, vos_data: VOsData, origin_resource: Optional[Resource],
public_origin: bool) -> "_IdNamespaceData":
def for_origin(cls, topology: Topology, vos_data: VOsData,
origin_resource: Optional[Resource]) -> "_IdNamespaceData":
self = cls()
for vo_name, stashcache_obj in vos_data.stashcache_by_vo_name.items():
for path, namespace in stashcache_obj.namespaces.items():
Expand All @@ -193,8 +192,6 @@ def for_origin(cls, topology: Topology, vos_data: VOsData, origin_resource: Opti
if namespace.is_public():
self.public_paths.add(path)
continue
if public_origin:
continue

# The Authfile for origins should contain only caches and the origin itself, via SSL (i.e. DNs).
# Ignore FQANs and DNs listed in the namespace's authz list.
Expand All @@ -207,14 +204,15 @@ def for_origin(cls, topology: Topology, vos_data: VOsData, origin_resource: Opti
allowed_resources.extend(allowed_caches)
else:
# TODO This situation should be caught by the CI
self.warnings.append(f"# WARNING: No working cache / namespace combinations found for {path}")
self.warnings_auth.append(f"# WARNING: No working cache / namespace combinations found for {path};"
" this path may not be cached")

for resource in allowed_resources:
dn = resource.data.get("DN")
if dn:
authz_list.append(DNAuth(dn))
else:
self.warnings.append(
self.warnings_auth.append(
f"# WARNING: Resource {resource.name} was skipped for VO {vo_name}, namespace {path}"
f" because the resource does not provide a DN."
)
Expand Down Expand Up @@ -247,19 +245,20 @@ def generate_cache_authfile(global_data: GlobalData,

idns = _IdNamespaceData.for_cache(
global_data=global_data,
topology=topology,
vos_data=vos_data,
legacy=legacy,
cache_resource=resource,
public_cache=False,
)

# TODO: improve message and turn this into a warning
if not idns.id_to_paths:
raise DataError("Cache does not support any protected namespaces")
if not idns.public_paths:
raise DataError("Cache does not support any namespaces") # TODO Catch this in the CI
else:
return ("# This cache does not support any protected/authenticated namespaces, only public namespaces.\n"
"# You must use the 'stash-cache' xrootd instance instead.\n")

authfile_lines = []
authfile_lines.extend(idns.warnings)
authfile_lines.extend(idns.warnings_auth)
for authfile_id in idns.id_to_paths:
paths_acl = " ".join(f"{p} rl" for p in sorted(idns.id_to_paths[authfile_id]))
authfile_lines.append(f"# {idns.id_to_str[authfile_id]}")
Expand All @@ -283,19 +282,20 @@ def generate_public_cache_authfile(global_data: GlobalData, fqdn=None, legacy=Tr

idns = _IdNamespaceData.for_cache(
global_data=global_data,
topology=topology,
vos_data=vos_data,
legacy=legacy,
cache_resource=resource,
public_cache=True,
)

# TODO: improve message and turn this into a warning
if not idns.public_paths:
raise DataError("Cache does not support any public namespaces")
if not idns.id_to_paths:
raise DataError("Cache does not support any namespaces") # TODO Catch this in the CI
else:
return ("# This cache does not support any public namespaces, only protected/authenticated namespaces.\n"
"# You must use the 'stash-cache-auth' xrootd instance instead.\n")

authfile_lines = []
authfile_lines.extend(idns.warnings)
authfile_lines.extend(idns.warnings_public)
authfile_lines.append("u * /user/ligo -rl \\")

for dirname in sorted(idns.public_paths):
Expand Down Expand Up @@ -325,15 +325,13 @@ def generate_cache_grid_mapfile(global_data: GlobalData,

idns = _IdNamespaceData.for_cache(
global_data=global_data,
topology=topology,
vos_data=vos_data,
legacy=legacy,
cache_resource=resource,
public_cache=False,
)

grid_mapfile_lines = []
grid_mapfile_lines.extend(idns.warnings)
grid_mapfile_lines.extend(idns.warnings_auth)
grid_mapfile_lines.extend(sorted(idns.grid_mapfile_lines))

return "\n".join(grid_mapfile_lines) + "\n"
Expand Down Expand Up @@ -410,21 +408,32 @@ def generate_origin_authfile(global_data: GlobalData, fqdn: str, suppress_errors
if not origin_resource:
return ""

idns = _IdNamespaceData.for_origin(topology, vos_data, origin_resource, public_origin)
idns = _IdNamespaceData.for_origin(topology, vos_data, origin_resource)

if not idns.id_to_paths and not idns.public_paths:
raise DataError("Origin does not support any namespaces")
raise DataError("Origin does not support any namespaces") # TODO Catch this in the CI
elif public_origin and not idns.public_paths:
return ("# This origin does not support any public namespaces, only protected/authenticated namespaces.\n"
"# You must use the 'stash-origin-auth' xrootd instance instead.\n")
elif not public_origin and not idns.id_to_paths:
return ("# This origin does not support any protected/authenticated namespaces, only public namespaces.\n"
"# You must use the 'stash-origin' xrootd instance instead.\n")

authfile_lines = []
authfile_lines.extend(idns.warnings)
for authfile_id in idns.id_to_paths:
paths_acl = " ".join(f"{p} lr" for p in sorted(idns.id_to_paths[authfile_id]))
authfile_lines.append(f"# {idns.id_to_str[authfile_id]}")
authfile_lines.append(f"{authfile_id} {paths_acl}")

# Only auth origins should serve paths requiring authentication
if not public_origin:
authfile_lines.extend(idns.warnings_auth)
for authfile_id in idns.id_to_paths:
paths_acl = " ".join(f"{p} lr" for p in sorted(idns.id_to_paths[authfile_id]))
authfile_lines.append(f"# {idns.id_to_str[authfile_id]}")
authfile_lines.append(f"{authfile_id} {paths_acl}")

# Public paths must be at the end
# XXX Should auth origins _also_ serve public paths?
if public_origin and idns.public_paths:
authfile_lines.append("")
authfile_lines.extend(idns.warnings_public)
paths_acl = " ".join(f"{p} lr" for p in sorted(idns.public_paths))
authfile_lines.append(f"u * {paths_acl}")

Expand All @@ -444,10 +453,10 @@ def generate_origin_grid_mapfile(global_data: GlobalData, fqdn: str, suppress_er
if not origin_resource:
return ""

idns = _IdNamespaceData.for_origin(topology, vos_data, origin_resource, public_origin=False)
idns = _IdNamespaceData.for_origin(topology, vos_data, origin_resource)

grid_mapfile_lines = []
grid_mapfile_lines.extend(idns.warnings)
grid_mapfile_lines.extend(idns.warnings_auth)
grid_mapfile_lines.extend(sorted(idns.grid_mapfile_lines))

return "\n".join(grid_mapfile_lines) + "\n"
Expand Down

0 comments on commit e9a6332

Please sign in to comment.