Merge pull request #1487 from perfsonar/issue-1463

Added timeout option to HTTP archiver. #1463
perfsonar · Nov 11, 2024 · 44c049b · 44c049b
2 parents 8c44ba7 + 6425d1e
commit 44c049b
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 6 deletions.
diff --git a/pscheduler-archiver-http/http/archive b/pscheduler-archiver-http/http/archive
@@ -12,7 +12,7 @@ import pycurl
 import http.client
 from urllib.parse import urlparse
 
-MAX_SCHEMA = 3
+MAX_SCHEMA = 4
 
 log = pscheduler.Log(prefix="archiver-http", quiet=True)
 
@@ -73,14 +73,16 @@ class PyCURLHandlePool(object):
                 self.pool[key].close()
                 del self.pool[key]
 
-    def __call__(self, key, bind=None):
+    def __call__(self, key, bind=None, timeout=None):
         """Create a new handle or return one that exists."""
         self.skim()
 
         if key not in self.pool:
             handle = PycURLHandle()
             if bind is not None:
                 handle().setopt(pycurl.INTERFACE, str(bind))
+            if timeout is not None:
+                handle().setopt(pycurl.TIMEOUT, timeout)
             self.pool[key] = handle
 
         return self.pool[key]()
@@ -109,6 +111,9 @@ def archive(json):
     log.debug("%s to %s", op, url)
     bind = json['data'].get('bind')
     verify_ssl = json['data'].get('verify-ssl', False)
+    timeout = int(pscheduler.timedelta_as_seconds(
+        pscheduler.iso8601_as_timedelta(json['data'].get('timeout', 'PT30S'))
+    ))
 
     # This must default to an empty hash so any previous headers are
     # cleared out.
@@ -129,17 +134,18 @@ def archive(json):
 
     parsed_url = urlparse(url)
 
-    key = "%s|%s|%s|%s|%s" % (
+    key = "%s|%s|%s|%s|%s|%s" % (
         parsed_url.scheme,
         parsed_url.hostname,
         '' if parsed_url.port is None else parsed_url.port,
         '' if bind is None else bind,
-        '' if not verify_ssl else 'verify-ssl'
+        '' if not verify_ssl else 'verify-ssl',
+        timeout
     )
 
     log.debug("Key is %s", key)
 
-    curl = handle_pool(key, bind)
+    curl = handle_pool(key, bind, timeout)
 
     # URL and headers may be different for each request.
 

diff --git a/pscheduler-archiver-http/http/data-is-valid b/pscheduler-archiver-http/http/data-is-valid
@@ -13,7 +13,7 @@ except ValueError as ex:
         "error": str(ex)
         })
 
-MAX_SCHEMA = 3
+MAX_SCHEMA = 4
 
 SPEC_SCHEMA = {
 
@@ -89,7 +89,35 @@ SPEC_SCHEMA = {
             },
             "required": [ "schema", "_url" ],
             "additionalProperties": False
+         },
+        "v4": {
+            "type": "object",
+            "properties": {
+                "schema": {" type": "integer", "enum": [ 4 ] },
+                "_url": { "$ref": "#/pScheduler/URL" },
+                "op": {
+                    "type": "string",
+                    "enum": [
+                        "put",
+                        "post",
+                    ]
+                },
+                "verify-ssl": { "$ref": "#/pScheduler/Boolean" },
+                "_headers": {
+                    "type": "object",
+                    "patternProperties": {
+                        "^.*$": { "$ref": "#/pScheduler/String" }
+                    },
+                    "additionalProperties": False
+                },
+                "timeout": { "$ref": "#/pScheduler/Duration" },
+                "bind": { "$ref": "#/pScheduler/Host" },
+                "retry-policy": { "$ref": "#/pScheduler/RetryPolicy" }
+            },
+            "required": [ "schema", "_url" ],
+            "additionalProperties": False
         }
+
     }
 }