diff --git a/src/README.md b/src/README.md index 01c627baa..fa9e6e282 100644 --- a/src/README.md +++ b/src/README.md @@ -513,9 +513,12 @@ base_path = /ospool/PROTECTED ### Namespaces JSON generation The JSON file containing cache and namespace information for stashcp/OSDF is served at `/osdf/namespaces`. -The endpoint takes two optional parameters, `include_downed=1`, and `include_inactive=1`; -if they are set, caches in downtime or that are not marked as active, respectively, are also included in the result. -Otherwise, they are not included. +The endpoint takes some optional parameters for filtering: +- `include_downed=1` includes caches that are in downtime in the result; otherwise they are omitted +- `include_inactive=1` includes caches that are not marked as active in the result; otherwise they are omitted +- `production=1` includes resources in "production" (as opposed to ITB) in the result +- `itb=1` includes resources in "itb" in the result + if neither `production` nor `itb` are specified then both production and itb resources are included The JSON contains an attribute `caches` that is a list of caches. Each cache in the list contains the following attributes: diff --git a/src/app.py b/src/app.py index 68fbfbd8d..cf0c92cf5 100755 --- a/src/app.py +++ b/src/app.py @@ -18,12 +18,11 @@ from webapp import default_config from webapp.common import readfile, to_xml_bytes, to_json_bytes, Filters, support_cors, simplify_attr_list, is_null, \ - escape, cache_control_private, PreJSON, is_true + escape, cache_control_private, PreJSON, is_true, GRIDTYPE_1, GRIDTYPE_2, NamespacesFilters from webapp.flask_common import create_accepted_response from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService from webapp.forms import GenerateDowntimeForm, GenerateResourceGroupDowntimeForm, GenerateProjectForm from webapp.models import GlobalData -from webapp.topology import GRIDTYPE_1, GRIDTYPE_2 from webapp.oasis_managers import get_oasis_manager_endpoint_info from webapp.github import create_file_pr, update_file_pr, GithubUser, GitHubAuth, GitHubRepoAPI, GithubRequestException, GithubReferenceExistsException, GithubNotFoundException @@ -514,10 +513,20 @@ def scitokens(): def stashcache_namespaces_json(): if not stashcache: return Response("Can't get scitokens config: stashcache module unavailable", status=503) - include_downed = is_true(request.args.get("include_downed", False)) - include_inactive = is_true(request.args.get("include_inactive", False)) + args = request.args + filters = NamespacesFilters() + filters.include_downed = is_true(args.get("include_downed", False)) + filters.include_inactive = is_true(args.get("include_inactive", False)) + if "production" not in args and "itb" not in args: + # default: include both production and itb + filters.production = True + filters.itb = True + else: + filters.production = is_true(request.args.get("production", False)) + filters.itb = is_true(request.args.get("itb", False)) + try: - return Response(to_json_bytes(stashcache.get_namespaces_info(global_data, include_downed, include_inactive)), + return Response(to_json_bytes(stashcache.get_namespaces_info(global_data, filters=filters)), mimetype='application/json') except ResourceNotRegistered as e: return Response("# {}\n" diff --git a/src/stashcache.py b/src/stashcache.py index 47eb6986d..f2db0de99 100644 --- a/src/stashcache.py +++ b/src/stashcache.py @@ -1,7 +1,7 @@ from collections import defaultdict from typing import Dict, List, Optional -from webapp.common import is_null, PreJSON, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER +from webapp.common import is_null, PreJSON, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER, NamespacesFilters from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService from webapp.models import GlobalData from webapp.topology import Resource, ResourceGroup, Topology @@ -538,7 +538,7 @@ def get_scitokens_list_for_namespace(ns: Namespace) -> List[Dict]: ) -def get_namespaces_info(global_data: GlobalData, include_downed=False, include_inactive=False) -> PreJSON: +def get_namespaces_info(global_data: GlobalData, filters: Optional[NamespacesFilters] = None) -> PreJSON: """Return data for the /stashcache/namespaces JSON endpoint. This includes a list of caches and origins, with some data about their endpoints, @@ -547,6 +547,9 @@ def get_namespaces_info(global_data: GlobalData, include_downed=False, include_i If `include_downed` is True, caches/origins in downtime are also included. If `include_inactive` is True, caches/origins that are not marked as active are also included. """ + if filters is None: + filters = NamespacesFilters() + # Helper functions def _service_resource_dict( @@ -645,10 +648,14 @@ def _resource_has_downed_origin(r: Resource, t: Topology): cache_resource_dicts = {} # type: Dict[str, Dict] for group in resource_groups: + if group.production and not filters.production: + continue + if group.itb and not filters.itb: + continue for resource in group.resources: if (_resource_has_cache(resource) - and (include_inactive or resource.is_active) - and (include_downed or not _resource_has_downed_cache(resource, topology)) + and (filters.include_inactive or resource.is_active) + and (filters.include_downed or not _resource_has_downed_cache(resource, topology)) ): cache_resource_objs[resource.name] = resource cache_resource_dicts[resource.name] = _cache_resource_dict(resource) @@ -659,10 +666,14 @@ def _resource_has_downed_origin(r: Resource, t: Topology): origin_resource_dicts = {} # type: Dict[str, Dict] for group in resource_groups: + if group.production and not filters.production: + continue + if group.itb and not filters.itb: + continue for resource in group.resources: if (_resource_has_origin(resource) - and (include_inactive or resource.is_active) - and (include_downed or not _resource_has_downed_origin(resource, topology)) + and (filters.include_inactive or resource.is_active) + and (filters.include_downed or not _resource_has_downed_origin(resource, topology)) ): origin_resource_objs[resource.name] = resource origin_resource_dicts[resource.name] = _origin_resource_dict(resource) diff --git a/src/tests/test_stashcache.py b/src/tests/test_stashcache.py index ac927ed1f..53c0d1b72 100644 --- a/src/tests/test_stashcache.py +++ b/src/tests/test_stashcache.py @@ -20,7 +20,7 @@ from app import app, global_data from webapp import models, topology, vos_data -from webapp.common import load_yaml_file +from webapp.common import load_yaml_file, NamespacesFilters from webapp.data_federation import CredentialGeneration, StashCache import stashcache @@ -265,13 +265,35 @@ def caches(self, namespaces_json) -> List[Dict]: @pytest.fixture def caches_include_inactive(self, test_global_data) -> List[Dict]: - namespaces_json = stashcache.get_namespaces_info(test_global_data, include_inactive=True) + filters = NamespacesFilters() + filters.include_inactive = True + namespaces_json = stashcache.get_namespaces_info(test_global_data, filters) assert "caches" in namespaces_json return namespaces_json["caches"] @pytest.fixture def caches_include_downed(self, test_global_data) -> List[Dict]: - namespaces_json = stashcache.get_namespaces_info(test_global_data, include_downed=True) + filters = NamespacesFilters() + filters.include_downed = True + namespaces_json = stashcache.get_namespaces_info(test_global_data, filters) + assert "caches" in namespaces_json + return namespaces_json["caches"] + + @pytest.fixture + def caches_production(self, test_global_data) -> List[Dict]: + filters = NamespacesFilters() + filters.production = True + filters.itb = False + namespaces_json = stashcache.get_namespaces_info(test_global_data, filters) + assert "caches" in namespaces_json + return namespaces_json["caches"] + + @pytest.fixture + def caches_itb(self, test_global_data) -> List[Dict]: + filters = NamespacesFilters() + filters.production = False + filters.itb = True + namespaces_json = stashcache.get_namespaces_info(test_global_data, filters) assert "caches" in namespaces_json return namespaces_json["caches"] @@ -401,6 +423,14 @@ def test_caches_include_downed_param(self, caches, caches_include_downed): x["resource"] for x in caches_include_downed ), "Downed cache missing from namespaces JSON with ?include_downed=1" + def test_caches_production(self, caches_production, caches_itb): + assert "TEST_TIGER_CACHE" in ( + x["resource"] for x in caches_production + ), "Production cache not present in namespaces JSON with production filter" + assert "TEST_TIGER_CACHE" not in ( + x["resource"] for x in caches_itb + ), "Production cache wrongly present in namespaces JSON with itb filter" + if __name__ == '__main__': pytest.main() diff --git a/src/webapp/common.py b/src/webapp/common.py index da238cfae..d11c8e232 100644 --- a/src/webapp/common.py +++ b/src/webapp/common.py @@ -56,6 +56,17 @@ def populate_voown_name(self, vo_id_to_name: Dict): self.voown_name = [vo_id_to_name.get(i, "") for i in self.voown_id] +class NamespacesFilters: + """ + Filters for namespaces json + """ + def __init__(self): + self.include_inactive = False + self.include_downed = False + self.production = True + self.itb = True + + def to_csv(data: list) -> str: csv_string = StringIO() writer = csv.writer(csv_string) @@ -385,3 +396,5 @@ def wrapped(): XROOTD_CACHE_SERVER = "XRootD cache server" XROOTD_ORIGIN_SERVER = "XRootD origin server" +GRIDTYPE_1 = "OSG Production Resource" +GRIDTYPE_2 = "OSG Integration Test Bed Resource" diff --git a/src/webapp/topology.py b/src/webapp/topology.py index bcb9b5671..d09ae511c 100644 --- a/src/webapp/topology.py +++ b/src/webapp/topology.py @@ -7,14 +7,12 @@ import icalendar -from .common import RGDOWNTIME_SCHEMA_URL, RGSUMMARY_SCHEMA_URL, Filters, ParsedYaml,\ - is_null, expand_attr_list_single, expand_attr_list, ensure_list, XROOTD_ORIGIN_SERVER, XROOTD_CACHE_SERVER, gen_id_from_yaml +from .common import RGDOWNTIME_SCHEMA_URL, RGSUMMARY_SCHEMA_URL, Filters, ParsedYaml, \ + is_null, expand_attr_list_single, expand_attr_list, ensure_list, XROOTD_ORIGIN_SERVER, XROOTD_CACHE_SERVER, \ + gen_id_from_yaml, GRIDTYPE_1, GRIDTYPE_2, is_true from .contacts_reader import ContactsData, User from .exceptions import DataError -GRIDTYPE_1 = "OSG Production Resource" -GRIDTYPE_2 = "OSG Integration Test Bed Resource" - log = getLogger(__name__) @@ -372,7 +370,7 @@ def __init__(self, name: str, yaml_data: ParsedYaml, site: Site, common_data: Co self.site = site self.service_types = common_data.service_types self.common_data = common_data - self.production = yaml_data.get("Production", "") + self.production = is_true(yaml_data.get("Production", "")) scname = yaml_data["SupportCenter"] scid = int(common_data.support_centers[scname]["ID"]) @@ -395,6 +393,10 @@ def __init__(self, name: str, yaml_data: ParsedYaml, site: Site, common_data: Co def resources(self): return [self.resources_by_name[k] for k in sorted(self.resources_by_name)] + @property + def itb(self): + return not self.production + def get_tree(self, authorized=False, filters: Filters = None) -> Optional[OrderedDict]: if filters is None: filters = Filters() @@ -404,7 +406,7 @@ def get_tree(self, authorized=False, filters: Filters = None) -> Optional[Ordere (filters.rg_id, self.id)]: if filter_list and attribute not in filter_list: return - data_gridtype = GRIDTYPE_1 if self.data.get("Production", None) else GRIDTYPE_2 + data_gridtype = GRIDTYPE_1 if self.production else GRIDTYPE_2 if filters.grid_type is not None and data_gridtype != filters.grid_type: return @@ -455,8 +457,7 @@ def _expand_rg(self) -> OrderedDict: new_rg["GroupName"] = self.name new_rg["SupportCenter"] = self.support_center new_rg["IsCCStar"] = self.is_ccstar - production = new_rg.get("Production") - if production: + if self.production: new_rg["GridType"] = GRIDTYPE_1 else: new_rg["GridType"] = GRIDTYPE_2