Merge branch 'master' into itb

* master: (63 commits)
  Update FZU_downtime.yaml
  adding sdsc-xenon-origin
  adding FDP_OSDF_ORIGIN namespace
  node removed from NRP
  Bump urllib3 in /.github/scripts/check_project_fos_precision
  Activate HPC4L resources
  Add downtime for NEBRASKA_NRP_OSDF_ORIGIN due to OS testing
  Update University of California Irvine.yaml
  Update University of California Irvine.yaml
  Use COManage ID for Nathan Crawford
  New project: Illinois_Goyal
  Unregister the UChicago .well-known directories
  Revert "Disable directory listings that are causing redirect failures due to …"
  Add downtime for FNAL_OSDF_ORIGIN due to Testing the pelican sw
  Disable directory listings that are causing redirect failures due to 7.8.4 bug
  Fixup service type
  Fix Armen's ID
  Add SWT2 k8s cluster
  Add downtime for AMSTERDAM_ESNET_OSDF_CACHE due to toPelican
  Fixing inst name
  Add downtime for HOUSTON2_INTERNET2_OSDF_CACHE due to HW issues
  Update Lehigh - Hawk_downtime.yaml
  remove variable for institution ID since relative imports don't work based on the script's invocation method
  update new_org_rejected status message
  Use variable for institutions api url
  check against prod list of institution ids when adding new institution
  Update automerge check to error on invalid institution ID
  Fix Project form
  new dns
  Update UTA_Goplerud.yaml with Institution ID
  Update University of California Irvine.yaml
  adding endpoints
  removing old fdp cache
  fixing site position
  Fix multiline description in projects/UTA_Goplerud.yaml
  Correct base path for AP40 issuer
  Update the IssuerName for AP40 to match its local credmon setting
  Create UTA_Goplerud.yaml
  port fix
  change of startlight node
  Added Squid service for T2_LV_HPNET resource group
  adding end and auth point
  Update CHTC-OSDF_downtime.yaml
  Add downtime for SDSC_NRP_OSDF_ORIGIN due to <REASON>
  Adding a new compute entrypoint to the T2_LV_HPCNET
  Revert "Setting Esnet caches to Any VO"
  Added ORU contacts; FD #76429
  Update SWT2 CPU normalization factor per Armen
  Register CHTC-ap2001 and CHTC-ap2002
  Set "Active" to false for CHTC-submit2
  ...

.github/scripts/check_project_fos_precision/requirements.txt

@@ -9,4 +9,4 @@ PyYAML==6.0.1
 requests==2.32.0
 six==1.16.0
 tzdata==2024.1
-urllib3==2.2.1
+urllib3==2.2.2

projects/GATech_Ramprasad.yaml

@@ -0,0 +1,8 @@
+Department: Material Science
+Description: 'Develop and apply computational and machine learning tools to accelerate
+  materials discovery. More information can be found here: https://ramprasad.mse.gatech.edu/'
+FieldOfScience: Materials Science
+FieldOfScienceID: 14.1801b
+InstitutionID: https://osg-htc.org/iid/uvf22j6xjbtv
+Organization: Georgia Institute of Technology
+PIName: Rampi Ramprasad

projects/Illinois_Goyal.yaml

@@ -0,0 +1,8 @@
+Department: Computer Science
+Description: Diverse set of projects focusing on vision models/LLMs primarily falling
+  under the umbrella terms of "machine unlearning" and "human-AI alignment"
+FieldOfScience: Computer Science
+FieldOfScienceID: '11.0701'
+InstitutionID: https://osg-htc.org/iid/10izzs5e7v1r
+Organization: University of Illinois Urbana-Champaign
+PIName: Agam Goyal

projects/UTA_Goplerud.yaml

@@ -0,0 +1,13 @@
+Department: Department of Government
+Description: >
+  I create new methods to facilitate political science research by leveraging
+  the intersection of Bayesian methods and machine learning. My papers create new
+  methods to tackle a variety of common problems (heterogeneous effects, hierarchical
+  models, ideal point estimation) where existing methods have limitations that constrain
+  substantive researchers. My existing papers can be found on my website https://mgoplerud.com/
+  as well as on pre-print servers such as arxiv.
+FieldOfScience: Statistics
+FieldOfScienceID: 27.0599b
+InstitutionID: https://osg-htc.org/iid/6z0d22dz19io 
+Organization: University of Texas at Austin
+PIName: Max Goplerud

src/app.py

@@ -867,10 +867,9 @@ def generate_project_yaml():
     institution_short_names = {x[1]: x[0] for x in global_data.get_mappings().project_institution.items()}
     institutions = []
     for institution in institution_api_data:
-        institutions.append((institution['id'], institution['name'], institution_short_names.get(institution['name'], "")))
+        institutions.append((institution_short_names.get(institution['name'], ""), institution['name']))
 
     def render_form(**kwargs):
-        institutions = list(global_data.get_mappings().project_institution.items())
         session.pop("form_data", None)
 
         return render_template(

src/templates/generate_project_yaml.html.j2

@@ -152,7 +152,7 @@ Project YAML Generator
         function updateProjectName(){
             project_name_node.readOnly = false
 
-            if(!(pi_institution_node.value in institutions) || !pi_last_name_node.value){ return }
+            if(!(pi_institution_node.value in institutions) || !institutions[pi_institution_node.value] || !pi_last_name_node.value){ return }
 
             let pi_institution = institutions[pi_institution_node.value]
             let pi_last_name = pi_last_name_node.value

src/webapp/automerge_check.py

@@ -2,6 +2,7 @@
 
 import collections
 import subprocess
+import requests
 import stat
 import yaml
 import sys
@@ -20,6 +21,7 @@
 
 import xml.etree.ElementTree as et
 
+INSTITUTIONS_API = "https://topology-institutions.osg-htc.org"
 
 # NOTE: throughout this program, git shas are of type str, while paths and
 # filenames are of type bytes.  The motivation behind this is to handle
@@ -122,19 +124,27 @@ def main(args):
             errors += check_resource_contacts(BASE_SHA, rg_fname,
                                               resources_affected, contacts)
 
-    if any( re.match(br'^projects/.*\.yaml', fname) for fname in modified ):
+    updated_projects = [fname for fname in modified if re.match(br'^projects/.*\.yaml', fname)]
+    if updated_projects:
         orgs_base  = get_organizations_at_version(base)
         orgs_new   = get_organizations_at_version(head)
         orgs_added = orgs_new - orgs_base
         for org in sorted(orgs_added):
             errors += ["New Organization '%s' requires OSG approval" % org]
+        invalid_institutions = get_invalid_institution_ids(head, updated_projects)
+        if invalid_institutions:
+            errors += [
+                f"Unrecognized InstitutionID in project(s) {', '.join(invalid_institutions)}. "
+                f"See {INSTITUTIONS_API} for known ID list."
+            ]
     else:
         orgs_added = None
+        invalid_institutions = None
 
     print_errors(errors)
     return ( RC.ALL_CHECKS_PASS   if len(errors) == 0
         else RC.OUT_OF_DATE_ONLY  if len(errors) == 1 and not up_to_date
-        else RC.ORGS_ADDED        if orgs_added
+        else RC.ORGS_ADDED        if orgs_added or invalid_institutions
         else RC.DT_MOD_ERRORS     if len(DTs) > 0
         else RC.CONTACT_ERROR     if not contacts
         else RC.NON_DT_ERRORS )
@@ -212,6 +222,13 @@ def get_organizations_at_version(sha):
                  if re.search(br'.\.yaml$', fname) ]
     return set( p.get("Organization") for p in projects )
 
+def get_invalid_institution_ids(sha, fnames):
+    valid_institutions = requests.get(f'{INSTITUTIONS_API}/api/institution_ids').json()
+    institution_ids = [i['id'] for i in valid_institutions]
+    projects = { fname : parse_yaml_at_version(sha, fname, {}) for fname in fnames }
+    return [fname.decode() for fname, yaml in projects.items() if not yaml.get("InstitutionID", "") in institution_ids]
+
+
 def commit_is_merged(sha_a, sha_b):
     args = ['git', 'merge-base', '--is-ancestor', sha_a, sha_b]
     ret, out = runcmd(args, stderr=_devnull)

src/webapp/forms.py

@@ -200,7 +200,7 @@ def get_yaml(self, resources, service_names_by_resource) -> str:
         yaml = ""
         for index, resource in enumerate(resources):
             yaml += models.get_downtime_yaml(
-                id=dtid+index,
+                id_=dtid + index,
                 start_datetime=self.get_start_datetime(),
                 end_datetime=self.get_end_datetime(),
                 created_datetime=created_datetime,
@@ -293,7 +293,7 @@ def get_yaml(self) -> str:
         dtid = models._dtid(created_datetime)
 
         return models.get_downtime_yaml(
-            id=dtid,
+            id_=dtid,
             start_datetime=self.get_start_datetime(),
             end_datetime=self.get_end_datetime(),
             created_datetime=created_datetime,

src/webapp/models.py

@@ -203,10 +203,10 @@ def get_contact_db_data(self) -> Optional[ContactsData]:
                 if ok:
                     try:
                         self.contacts_data.update(contacts_reader.get_contacts_data(self.contacts_file))
-                    except Exception:
+                    except Exception as err:
                         if self.strict:
                             raise
-                        log.exception("Failed to update contacts data")
+                        log.exception("Failed to update contacts data (%s)", err)
                         self.contacts_data.try_again()
                 else:
                     self.contacts_data.try_again()
@@ -230,10 +230,10 @@ def get_comanage_data(self) -> Optional[ContactsData]:
                     idmap = self.get_cilogon_ldap_id_map()
                     data = ldap_data.cilogon_id_map_to_yaml_data(idmap)
                     self.comanage_data.update(ContactsData(data))
-                except Exception:
+                except Exception as err:
                     if self.strict:
                         raise
-                    log.exception("Failed to update comanage data")
+                    log.exception("Failed to update comanage data (%s)", err)
                     self.comanage_data.try_again()
 
         return self.comanage_data.data
@@ -255,10 +255,10 @@ def get_contacts_data(self) -> Optional[ContactsData]:
                 yd2 = self.get_contact_db_data().yaml_data
                 yd_merged = ldap_data.merge_yaml_data(yd1, yd2)
                 self.merged_contacts_data.update(ContactsData(yd_merged))
-            except Exception:
+            except Exception as err:
                 if self.strict:
                     raise
-                log.exception("Failed to update merged contacts data")
+                log.exception("Failed to update merged contacts data (%s)", err)
                 self.merged_contacts_data.try_again()
 
         return self.merged_contacts_data.data
@@ -279,10 +279,10 @@ def get_ligo_dn_list(self) -> Optional[List[str]]:
                     ligo_ldap_pass = readfile(self.ligo_ldap_passfile, log)
                     new_dn_list = ldap_data.get_ligo_ldap_dn_list(self.ligo_ldap_url, self.ligo_ldap_user, ligo_ldap_pass)
                     self.ligo_dn_list.update(new_dn_list)
-                except Exception:
+                except Exception as err:
                     if self.strict:
                         raise
-                    log.exception("Failed to update LIGO data")
+                    log.exception("Failed to update LIGO data (%s)", err)
                     self.ligo_dn_list.try_again()
 
         return self.ligo_dn_list.data
@@ -296,10 +296,10 @@ def get_dns(self) -> Optional[Set]:
             contacts_data = self.get_contacts_data()
             try:
                 self.dn_set.update(set(contacts_data.get_dns()))
-            except Exception:
+            except Exception as err:
                 if self.strict:
                     raise
-                log.exception("Failed to update DNs")
+                log.exception("Failed to update DNs (%s)", err)
                 self.contacts_data.try_again()
         return self.dn_set.data
 
@@ -324,10 +324,10 @@ def update_topology(self) -> None:
                 log.debug("Updating topology RG data")
                 self.topology.update(rg_reader.get_topology(self.topology_dir, self.get_contacts_data(), strict=self.strict))
                 log.debug("Updated topology RG data successfully")
-            except Exception:
+            except Exception as err:
                 if self.strict:
                     raise
-                log.exception("Failed to update topology RG data")
+                log.exception("Failed to update topology RG data (%s)", err)
                 self.topology.try_again()
         else:
             self.topology.try_again()
@@ -345,10 +345,10 @@ def get_vos_data(self) -> Optional[VOsData]:
                         log.debug("Updating VOs")
                         self.vos_data.update(vo_reader.get_vos_data(self.vos_dir, self.get_contacts_data(), strict=self.strict))
                         log.debug("Updated VOs successfully")
-                    except Exception:
+                    except Exception as err:
                         if self.strict:
                             raise
-                        log.exception("Failed to update VOs")
+                        log.exception("Failed to update VOs (%s)", err)
                         self.vos_data.try_again()
                 else:
                     self.vos_data.try_again()
@@ -368,10 +368,10 @@ def get_projects(self) -> Optional[Dict]:
                         log.debug("Updating projects")
                         self.projects.update(project_reader.get_projects(self.projects_dir, strict=self.strict))
                         log.debug("Updated projects successfully")
-                    except Exception:
+                    except Exception as err:
                         if self.strict:
                             raise
-                        log.exception("Failed to update projects")
+                        log.exception("Failed to update projects (%s)", err)
                         self.projects.try_again()
                 else:
                     self.projects.try_again()
@@ -393,10 +393,10 @@ def get_mappings(self, strict=None) -> Optional[mappings.Mappings]:
                         log.debug("Updating mappings")
                         self.mappings.update(mappings.get_mappings(indir=self.mappings_dir, strict=strict))
                         log.debug("Updated mappings successfully")
-                    except Exception:
+                    except Exception as err:
                         if self.strict:
                             raise
-                        log.exception("Failed to update mappings")
+                        log.exception("Failed to update mappings (%s)", err)
                         self.mappings.try_again()
                 else:
                     self.mappings.try_again()
@@ -412,7 +412,7 @@ def _dtid(created_datetime: datetime.datetime):
     return int((timestamp - dtid_offset) * multiplier)
 
 
-def get_downtime_yaml(id: int,
+def get_downtime_yaml(id_: int,
                       start_datetime: datetime.datetime,
                       end_datetime: datetime.datetime,
                       created_datetime: datetime.datetime,
@@ -428,8 +428,8 @@ def get_downtime_yaml(id: int,
 
     """
 
-    def render(key, value):
-        return yaml.safe_dump({key: value}, default_flow_style=False).strip()
+    def render(key_, value_):
+        return yaml.safe_dump({key_: value_}, default_flow_style=False).strip()
 
     def indent(in_str, amount):
         spaces = ' ' * amount
@@ -441,7 +441,7 @@ def indent(in_str, amount):
 
     result = "- " + render("Class", class_)
     for key, value in [
-        ("ID", id),
+        ("ID", id_),
         ("Description", description),
         ("Severity", severity),
         ("StartTime", start_time_str),

src/webapp/webhook_status_messages.py

@@ -76,7 +76,7 @@
     "Thank you for your pull request.\n"
     "\n"
     "Your [requested changes]({base_sha}...{head_sha}) add a new organization"
-    " to a project file; this requires manual review from OSG Staff.\n"
+    " or project ID to a project file; this requires manual review from OSG Staff.\n"
     "\n"
     "-- OSG-BOT  :oncoming_police_car:\n"
     "\n"

topology/American University of Beirut/HPC For Lebanon/HPC4L.yaml

@@ -2,7 +2,7 @@ GroupDescription: "HPC4L initiative (https://hpc4l-webpage.web.cern.ch/)"
 Production: true
 Resources:
   HPC4L-CE:
-    Active: false
+    Active: true
     ContactLists:
       Administrative Contact:
         Primary:
@@ -33,7 +33,7 @@ Resources:
       StorageCapacityMax: 0
       StorageCapacityMin: 0
   HPC4L-SE:
-    Active: false
+    Active: true
     ContactLists:
       Administrative Contact:
         Primary:
@@ -62,7 +62,7 @@ Resources:
       StorageCapacityMax: 8.2
       StorageCapacityMin: 1
   HPC4L-SE-2:
-    Active: false
+    Active: true
     ContactLists:
       Administrative Contact:
         Primary:
@@ -91,7 +91,7 @@ Resources:
       StorageCapacityMax: 8.2
       StorageCapacityMin: 1
   HPC4L-Squid:
-    Active: false
+    Active: true
     ContactLists:
       Administrative Contact:
         Primary:

topology/Compute Canada/ComputeCanada - Cedar/SITE.yaml

@@ -1,18 +1,10 @@
-# LongName is the expanded name of the site
 LongName: Compute Canada Cedar Cluster
-# Description is a brief description of your site
 Description: Cedar is a heterogeneous cluster suitable for a variety of workloads; it is located at Simon Fraser University.
-
-# If you have an up-to-date local git clone, fill ID with the output from `bin/next_site_id`
-# Otherwise, leave it blank and we will fill in the appropriate value for you.
 ID: 10281
-AddressLine1: 2152 Hillside Road
-City: Storrs
-
-Country: United States
-State: CT
-Zipcode: "06269-3046"
-
-Latitude: 41.8084
-Longitude: -72.2495
-
+AddressLine1: 8888 University Drive
+City: Burnaby
+Country: Canada
+State: BC
+Zipcode: "V5A 1S66"
+Latitude: 49.278639393188136
+Longitude: -122.91726450229353