Merge pull request #3901 from matyasselmeci/pr/production-in-namespac…

…es-json.SOFTWARE-5862

Add filters for production and itb in namespaces json (SOFTWARE-5862)

src/README.md

@@ -513,9 +513,12 @@ base_path = /ospool/PROTECTED
 ### Namespaces JSON generation
 
 The JSON file containing cache and namespace information for stashcp/OSDF is served at `/osdf/namespaces`.
-The endpoint takes two optional parameters, `include_downed=1`, and `include_inactive=1`;
-if they are set, caches in downtime or that are not marked as active, respectively, are also included in the result.
-Otherwise, they are not included.
+The endpoint takes some optional parameters for filtering:
+- `include_downed=1` includes caches that are in downtime in the result; otherwise they are omitted
+- `include_inactive=1` includes caches that are not marked as active in the result; otherwise they are omitted
+- `production=1` includes resources in "production" (as opposed to ITB) in the result
+- `itb=1` includes resources in "itb" in the result
+  if neither `production` nor `itb` are specified then both production and itb resources are included
 
 The JSON contains an attribute `caches` that is a list of caches.
 Each cache in the list contains the following attributes:

src/app.py

@@ -18,12 +18,11 @@
 
 from webapp import default_config
 from webapp.common import readfile, to_xml_bytes, to_json_bytes, Filters, support_cors, simplify_attr_list, is_null, \
-    escape, cache_control_private, PreJSON, is_true
+    escape, cache_control_private, PreJSON, is_true, GRIDTYPE_1, GRIDTYPE_2, NamespacesFilters
 from webapp.flask_common import create_accepted_response
 from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService
 from webapp.forms import GenerateDowntimeForm, GenerateResourceGroupDowntimeForm, GenerateProjectForm
 from webapp.models import GlobalData
-from webapp.topology import GRIDTYPE_1, GRIDTYPE_2
 from webapp.oasis_managers import get_oasis_manager_endpoint_info
 from webapp.github import create_file_pr, update_file_pr, GithubUser, GitHubAuth, GitHubRepoAPI, GithubRequestException, GithubReferenceExistsException, GithubNotFoundException
 
@@ -514,10 +513,20 @@ def scitokens():
 def stashcache_namespaces_json():
     if not stashcache:
         return Response("Can't get scitokens config: stashcache module unavailable", status=503)
-    include_downed = is_true(request.args.get("include_downed", False))
-    include_inactive = is_true(request.args.get("include_inactive", False))
+    args = request.args
+    filters = NamespacesFilters()
+    filters.include_downed = is_true(args.get("include_downed", False))
+    filters.include_inactive = is_true(args.get("include_inactive", False))
+    if "production" not in args and "itb" not in args:
+        # default: include both production and itb
+        filters.production = True
+        filters.itb = True
+    else:
+        filters.production = is_true(request.args.get("production", False))
+        filters.itb = is_true(request.args.get("itb", False))
+
     try:
-        return Response(to_json_bytes(stashcache.get_namespaces_info(global_data, include_downed, include_inactive)),
+        return Response(to_json_bytes(stashcache.get_namespaces_info(global_data, filters=filters)),
                         mimetype='application/json')
     except ResourceNotRegistered as e:
         return Response("# {}\n"

src/stashcache.py

@@ -1,7 +1,7 @@
 from collections import defaultdict
 from typing import Dict, List, Optional
 
-from webapp.common import is_null, PreJSON, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER
+from webapp.common import is_null, PreJSON, XROOTD_CACHE_SERVER, XROOTD_ORIGIN_SERVER, NamespacesFilters
 from webapp.exceptions import DataError, ResourceNotRegistered, ResourceMissingService
 from webapp.models import GlobalData
 from webapp.topology import Resource, ResourceGroup, Topology
@@ -538,7 +538,7 @@ def get_scitokens_list_for_namespace(ns: Namespace) -> List[Dict]:
     )
 
 
-def get_namespaces_info(global_data: GlobalData, include_downed=False, include_inactive=False) -> PreJSON:
+def get_namespaces_info(global_data: GlobalData, filters: Optional[NamespacesFilters] = None) -> PreJSON:
     """Return data for the /stashcache/namespaces JSON endpoint.
 
     This includes a list of caches and origins, with some data about their endpoints,
@@ -547,6 +547,9 @@ def get_namespaces_info(global_data: GlobalData, include_downed=False, include_i
     If `include_downed` is True, caches/origins in downtime are also included.
     If `include_inactive` is True, caches/origins that are not marked as active are also included.
     """
+    if filters is None:
+        filters = NamespacesFilters()
+
     # Helper functions
 
     def _service_resource_dict(
@@ -645,10 +648,14 @@ def _resource_has_downed_origin(r: Resource, t: Topology):
     cache_resource_dicts = {}  # type: Dict[str, Dict]
 
     for group in resource_groups:
+        if group.production and not filters.production:
+            continue
+        if group.itb and not filters.itb:
+            continue
         for resource in group.resources:
             if (_resource_has_cache(resource)
-                    and (include_inactive or resource.is_active)
-                    and (include_downed or not _resource_has_downed_cache(resource, topology))
+                    and (filters.include_inactive or resource.is_active)
+                    and (filters.include_downed or not _resource_has_downed_cache(resource, topology))
             ):
                 cache_resource_objs[resource.name] = resource
                 cache_resource_dicts[resource.name] = _cache_resource_dict(resource)
@@ -659,10 +666,14 @@ def _resource_has_downed_origin(r: Resource, t: Topology):
     origin_resource_dicts = {}  # type: Dict[str, Dict]
 
     for group in resource_groups:
+        if group.production and not filters.production:
+            continue
+        if group.itb and not filters.itb:
+            continue
         for resource in group.resources:
             if (_resource_has_origin(resource)
-                    and (include_inactive or resource.is_active)
-                    and (include_downed or not _resource_has_downed_origin(resource, topology))
+                    and (filters.include_inactive or resource.is_active)
+                    and (filters.include_downed or not _resource_has_downed_origin(resource, topology))
             ):
                 origin_resource_objs[resource.name] = resource
                 origin_resource_dicts[resource.name] = _origin_resource_dict(resource)

src/tests/test_stashcache.py

@@ -20,7 +20,7 @@
 
 from app import app, global_data
 from webapp import models, topology, vos_data
-from webapp.common import load_yaml_file
+from webapp.common import load_yaml_file, NamespacesFilters
 from webapp.data_federation import CredentialGeneration, StashCache
 import stashcache
 
@@ -265,13 +265,35 @@ def caches(self, namespaces_json) -> List[Dict]:
 
     @pytest.fixture
     def caches_include_inactive(self, test_global_data) -> List[Dict]:
-        namespaces_json = stashcache.get_namespaces_info(test_global_data, include_inactive=True)
+        filters = NamespacesFilters()
+        filters.include_inactive = True
+        namespaces_json = stashcache.get_namespaces_info(test_global_data, filters)
         assert "caches" in namespaces_json
         return namespaces_json["caches"]
 
     @pytest.fixture
     def caches_include_downed(self, test_global_data) -> List[Dict]:
-        namespaces_json = stashcache.get_namespaces_info(test_global_data, include_downed=True)
+        filters = NamespacesFilters()
+        filters.include_downed = True
+        namespaces_json = stashcache.get_namespaces_info(test_global_data, filters)
+        assert "caches" in namespaces_json
+        return namespaces_json["caches"]
+
+    @pytest.fixture
+    def caches_production(self, test_global_data) -> List[Dict]:
+        filters = NamespacesFilters()
+        filters.production = True
+        filters.itb = False
+        namespaces_json = stashcache.get_namespaces_info(test_global_data, filters)
+        assert "caches" in namespaces_json
+        return namespaces_json["caches"]
+
+    @pytest.fixture
+    def caches_itb(self, test_global_data) -> List[Dict]:
+        filters = NamespacesFilters()
+        filters.production = False
+        filters.itb = True
+        namespaces_json = stashcache.get_namespaces_info(test_global_data, filters)
         assert "caches" in namespaces_json
         return namespaces_json["caches"]
 
@@ -401,6 +423,14 @@ def test_caches_include_downed_param(self, caches, caches_include_downed):
             x["resource"] for x in caches_include_downed
         ), "Downed cache missing from namespaces JSON with ?include_downed=1"
 
+    def test_caches_production(self, caches_production, caches_itb):
+        assert "TEST_TIGER_CACHE" in (
+            x["resource"] for x in caches_production
+        ), "Production cache not present in namespaces JSON with production filter"
+        assert "TEST_TIGER_CACHE" not in (
+            x["resource"] for x in caches_itb
+        ), "Production cache wrongly present in namespaces JSON with itb filter"
+
 
 if __name__ == '__main__':
     pytest.main()

src/webapp/common.py

@@ -56,6 +56,17 @@ def populate_voown_name(self, vo_id_to_name: Dict):
         self.voown_name = [vo_id_to_name.get(i, "") for i in self.voown_id]
 
 
+class NamespacesFilters:
+    """
+    Filters for namespaces json
+    """
+    def __init__(self):
+        self.include_inactive = False
+        self.include_downed = False
+        self.production = True
+        self.itb = True
+
+
 def to_csv(data: list) -> str:
     csv_string = StringIO()
     writer = csv.writer(csv_string)
@@ -385,3 +396,5 @@ def wrapped():
 
 XROOTD_CACHE_SERVER = "XRootD cache server"
 XROOTD_ORIGIN_SERVER = "XRootD origin server"
+GRIDTYPE_1 = "OSG Production Resource"
+GRIDTYPE_2 = "OSG Integration Test Bed Resource"

src/webapp/topology.py

@@ -7,14 +7,12 @@
 
 import icalendar
 
-from .common import RGDOWNTIME_SCHEMA_URL, RGSUMMARY_SCHEMA_URL, Filters, ParsedYaml,\
-    is_null, expand_attr_list_single, expand_attr_list, ensure_list, XROOTD_ORIGIN_SERVER, XROOTD_CACHE_SERVER, gen_id_from_yaml
+from .common import RGDOWNTIME_SCHEMA_URL, RGSUMMARY_SCHEMA_URL, Filters, ParsedYaml, \
+    is_null, expand_attr_list_single, expand_attr_list, ensure_list, XROOTD_ORIGIN_SERVER, XROOTD_CACHE_SERVER, \
+    gen_id_from_yaml, GRIDTYPE_1, GRIDTYPE_2, is_true
 from .contacts_reader import ContactsData, User
 from .exceptions import DataError
 
-GRIDTYPE_1 = "OSG Production Resource"
-GRIDTYPE_2 = "OSG Integration Test Bed Resource"
-
 log = getLogger(__name__)
 
 
@@ -372,7 +370,7 @@ def __init__(self, name: str, yaml_data: ParsedYaml, site: Site, common_data: Co
         self.site = site
         self.service_types = common_data.service_types
         self.common_data = common_data
-        self.production = yaml_data.get("Production", "")
+        self.production = is_true(yaml_data.get("Production", ""))
 
         scname = yaml_data["SupportCenter"]
         scid = int(common_data.support_centers[scname]["ID"])
@@ -395,6 +393,10 @@ def __init__(self, name: str, yaml_data: ParsedYaml, site: Site, common_data: Co
     def resources(self):
         return [self.resources_by_name[k] for k in sorted(self.resources_by_name)]
 
+    @property
+    def itb(self):
+        return not self.production
+
     def get_tree(self, authorized=False, filters: Filters = None) -> Optional[OrderedDict]:
         if filters is None:
             filters = Filters()
@@ -404,7 +406,7 @@ def get_tree(self, authorized=False, filters: Filters = None) -> Optional[Ordere
                                        (filters.rg_id, self.id)]:
             if filter_list and attribute not in filter_list:
                 return
-        data_gridtype = GRIDTYPE_1 if self.data.get("Production", None) else GRIDTYPE_2
+        data_gridtype = GRIDTYPE_1 if self.production else GRIDTYPE_2
         if filters.grid_type is not None and data_gridtype != filters.grid_type:
             return
 
@@ -455,8 +457,7 @@ def _expand_rg(self) -> OrderedDict:
         new_rg["GroupName"] = self.name
         new_rg["SupportCenter"] = self.support_center
         new_rg["IsCCStar"] = self.is_ccstar
-        production = new_rg.get("Production")
-        if production:
+        if self.production:
             new_rg["GridType"] = GRIDTYPE_1
         else:
             new_rg["GridType"] = GRIDTYPE_2