Merge pull request #3907 from matyasselmeci/pr/cache-git

Cache and measure the update of the Topology Git repo separately

src/webapp/models.py

@@ -33,6 +33,7 @@ def time(self):
 log = logging.getLogger(__name__)
 
 topology_update_summary = Summary('topology_update_seconds', 'Time spent updating the topology repo data')
+topology_git_update_summary = Summary('topology_git_update_seconds', 'Time spent pulling/cloning the topology git repo')
 contact_update_summary = Summary('contact_update_seconds', 'Time spent updating the contact repo data')
 comanage_update_summary = Summary('comanage_update_seconds', 'Time spent updating the comanage LDAP data')
 ligo_update_summary = Summary('ligo_update_seconds', 'Time spent updating the LIGO LDAP data')
@@ -52,16 +53,16 @@ def should_update(self):
         """Return True if we should update, either because we're past the next update time
         or because force_update is True.
         """
-        return self.force_update or not self.data or time.time() > self.next_update
+        return self.force_update or not self.data or time.monotonic() > self.next_update
 
     def try_again(self):
         """Set the next update time to now + the retry delay."""
-        self.next_update = time.time() + self.retry_delay
+        self.next_update = time.monotonic() + self.retry_delay
 
     def update(self, data):
         """Cache new data and set the next update time to now + the cache lifetime."""
         self.data = data
-        self.timestamp = time.time()
+        self.timestamp = time.monotonic()
         self.next_update = self.timestamp + self.cache_lifetime
         self.force_update = False
 
@@ -90,6 +91,7 @@ def __init__(self, config=None, strict=False):
         self.topology = CachedData(cache_lifetime=topology_cache_lifetime)
         self.vos_data = CachedData(cache_lifetime=topology_cache_lifetime)
         self.mappings = CachedData(cache_lifetime=topology_cache_lifetime)
+        self.topology_repo_stamp = CachedData(cache_lifetime=topology_cache_lifetime)
         self.topology_data_dir = config["TOPOLOGY_DATA_DIR"]
         self.topology_data_repo = config.get("TOPOLOGY_DATA_REPO", "")
         self.topology_data_branch = config.get("TOPOLOGY_DATA_BRANCH", "")
@@ -153,6 +155,21 @@ def _update_topology_repo(self):
                 return False
         return True
 
+    def maybe_update_topology_repo(self) -> bool:
+        """Update the local git clone of the topology github repo if it hasn't
+        been updated recently (based on the cache time for self.topology_repo_stamp).
+        """
+        if self.topology_repo_stamp.should_update():
+            with topology_git_update_summary.time():
+                ok = self._update_topology_repo()
+            if ok:
+                self.topology_repo_stamp.update(time.monotonic())
+                return True
+            else:
+                self.topology_repo_stamp.try_again()
+                return False
+        return bool(self.topology_repo_stamp.data)
+
     def _update_contacts_repo(self):
         if not self.config["NO_GIT"]:
             parent = os.path.dirname(self.config["CONTACT_DATA_DIR"])
@@ -297,18 +314,20 @@ def get_topology(self) -> Optional[Topology]:
 
         return self.topology.data
 
-    def update_topology(self):
+    def update_topology(self) -> None:
         """
-        Update topology data
+        Update topology facility/site/ResourceGroup data
         """
-        ok = self._update_topology_repo()
+        ok = self.maybe_update_topology_repo()
         if ok:
             try:
+                log.debug("Updating topology RG data")
                 self.topology.update(rg_reader.get_topology(self.topology_dir, self.get_contacts_data(), strict=self.strict))
+                log.debug("Updated topology RG data successfully")
             except Exception:
                 if self.strict:
                     raise
-                log.exception("Failed to update topology")
+                log.exception("Failed to update topology RG data")
                 self.topology.try_again()
         else:
             self.topology.try_again()
@@ -320,10 +339,12 @@ def get_vos_data(self) -> Optional[VOsData]:
         """
         if self.vos_data.should_update():
             with topology_update_summary.time():
-                ok = self._update_topology_repo()
+                ok = self.maybe_update_topology_repo()
                 if ok:
                     try:
+                        log.debug("Updating VOs")
                         self.vos_data.update(vo_reader.get_vos_data(self.vos_dir, self.get_contacts_data(), strict=self.strict))
+                        log.debug("Updated VOs successfully")
                     except Exception:
                         if self.strict:
                             raise
@@ -341,10 +362,12 @@ def get_projects(self) -> Optional[Dict]:
         """
         if self.projects.should_update():
             with topology_update_summary.time():
-                ok = self._update_topology_repo()
+                ok = self.maybe_update_topology_repo()
                 if ok:
                     try:
+                        log.debug("Updating projects")
                         self.projects.update(project_reader.get_projects(self.projects_dir, strict=self.strict))
+                        log.debug("Updated projects successfully")
                     except Exception:
                         if self.strict:
                             raise
@@ -364,10 +387,12 @@ def get_mappings(self, strict=None) -> Optional[mappings.Mappings]:
             strict = self.strict
         if self.mappings.should_update():
             with topology_update_summary.time():
-                ok = self._update_topology_repo()
+                ok = self.maybe_update_topology_repo()
                 if ok:
                     try:
+                        log.debug("Updating mappings")
                         self.mappings.update(mappings.get_mappings(indir=self.mappings_dir, strict=strict))
+                        log.debug("Updated mappings successfully")
                     except Exception:
                         if self.strict:
                             raise