ENH: Make regtap service aware

CHANGES.rst

@@ -45,6 +45,10 @@
 - Adding support for the VODataService 1.2 nrows attribute on table
   elements [#503]
 
+- registry.search now introspects the TAP service's capabilities and
+  only offers extended functionality or optimisations if the required
+  features are present [#386]
+
 
 1.4.3 (unreleased)
 ==================

docs/registry/index.rst

@@ -130,7 +130,7 @@ you would say:
   ...                             registry.Freetext("supernova"))
 
 After that, ``resources`` is an instance of
-:py:class:`pyvo.registry.RegistryResults`, which you can iterate over.  In
+:py:class:`pyvo.registry.regtap.RegistryResults`, which you can iterate over.  In
 interactive data discovery, however, it is usually preferable to use the
 ``to_table`` method for an overview of the resources available:
 
@@ -155,9 +155,9 @@ And to look for tap resources *in* a specific cone, you would do
   ...                 registry.Spatial((SkyCoord("23d +3d"), 3), intersect="enclosed"),
   ...                 includeaux=True) # doctest: +IGNORE_OUTPUT
   <DALResultsTable length=1>
-              ivoid                   res_type       short_name                   res_title                  ...  intf_types  intf_roles          alt_identifier         
-                                                                                                             ...                                                         
-              object                   object          object                       object                   ...    object      object                object             
+              ivoid                   res_type       short_name                   res_title                  ...  intf_types  intf_roles          alt_identifier
+                                                                                                             ...
+              object                   object          object                       object                   ...    object      object                object
   ------------------------------ ----------------- ------------- ------------------------------------------- ... ------------ ---------- --------------------------------
   ivo://cds.vizier/j/apj/835/123 vs:catalogservice J/ApJ/835/123 Globular clusters in NGC 474 from CFHT obs. ... vs:paramhttp        std doi:10.26093/cds/vizier.18350123
 
@@ -183,7 +183,7 @@ are not), but it is rather clunky, and in the real VO short name
 collisions should be very rare.
 
 Use the ``get_service`` method of
-:py:class:`pyvo.registry.RegistryResource` to obtain a DAL service
+:py:class:`pyvo.registry.regtap.RegistryResource` to obtain a DAL service
 object for a particular sort of interface.
 To query the fourth match using simple cone search, you would
 thus say:
@@ -513,6 +513,44 @@ and then you can run:
   {'flashheros.data': <VODataServiceTable name="flashheros.data">... 29 columns ...</VODataServiceTable>, 'ivoa.obscore': <VODataServiceTable name="ivoa.obscore">... 0 columns ...</VODataServiceTable>}
 
 
+Alternative Registries
+======================
+
+There are several RegTAP services in the VO.  PyVO by default uses the
+one at the TAP access URL http://reg.g-vo.org/tap.  You can use
+alternative ones, for instance, because they are nearer to you or
+because the default endpoint is down.
+
+You can pre-select the URL by setting the ``IVOA_REGISTRY`` environment
+variable to the TAP access URL of the service you would like to use.  In
+a bash-like shell, you would say::
+
+  export IVOA_REGISTRY="http://vao.stsci.edu/RegTAP/TapService.aspx"
+
+before starting python (or the notebook processor).
+
+Within a Python session, you can use the
+`pyvo.registry.choose_RegTAP_service` function, which also takes the
+TAP access URL.
+
+As long as you have on working registry endpoint, you can find the other
+RegTAP services using:
+
+.. We probably shouldn't test the result of the next code block; this
+   will change every time someone registers a new RegTAP service...
+
+.. doctest-remote-data::
+
+  >>> res = registry.search(datamodel="regtap")
+  >>> print("\n".join(sorted(r.get_interface("tap").access_url
+  ...   for r in res)))
+  http://dc.zah.uni-heidelberg.de/tap
+  http://gavo.aip.de/tap
+  http://voparis-rr.obspm.fr/tap
+  https://vao.stsci.edu/RegTAP/TapService.aspx
+
+
+
 Reference/API
 =============
 

pyvo/dal/tap.py

@@ -127,6 +127,20 @@ def __init__(self, baseurl, session=None):
         if hasattr(self._session, 'update_from_capabilities'):
             self._session.update_from_capabilities(self.capabilities)
 
+    def get_tap_capability(self):
+        """
+        returns the (first) TAP capability of this service.
+
+        Returns
+        -------
+        A `~pyvo.io.vosi.tapregext.TableAccess` instance.
+        """
+        for capa in self.capabilities:
+            if isinstance(capa, tr.TableAccess):
+                return capa
+        raise DALServiceError("Invalid TAP service: Does not"
+            " expose a tr:TableAccess capability")
+
     @property
     def tables(self):
         """
@@ -203,9 +217,7 @@ def maxrec(self):
             if the property is not exposed by the service
         """
         try:
-            for capa in self.capabilities:
-                if isinstance(capa, tr.TableAccess):
-                    return capa.outputlimit.default.content
+            return self.get_tap_capability().outputlimit.default.content
         except AttributeError:
             pass
         raise DALServiceError("Default limit not exposed by the service")
@@ -221,9 +233,7 @@ def hardlimit(self):
             if the property is not exposed by the service
         """
         try:
-            for capa in self.capabilities:
-                if isinstance(capa, tr.TableAccess):
-                    return capa.outputlimit.hard.content
+            return self.get_tap_capability().outputlimit.hard.content
         except AttributeError:
             pass
         raise DALServiceError("Hard limit not exposed by the service")
@@ -234,11 +244,7 @@ def upload_methods(self):
         a list of upload methods in form of
         :py:class:`~pyvo.io.vosi.tapregext.UploadMethod` objects
         """
-        upload_methods = []
-        for capa in self.capabilities:
-            if isinstance(capa, tr.TableAccess):
-                upload_methods += capa.uploadmethods
-        return upload_methods
+        return self.get_tap_capability().uploadmethods
 
     def run_sync(
             self, query, language="ADQL", maxrec=None, uploads=None,

pyvo/dal/tests/test_tap.py

@@ -14,10 +14,11 @@
 import requests_mock
 
 from pyvo.dal.tap import escape, search, AsyncTAPJob, TAPService
-from pyvo.dal import DALQueryError
+from pyvo.dal import DALQueryError, DALServiceError
 
 from pyvo.io.uws import JobFile
 from pyvo.io.uws.tree import Parameter, Result, ErrorSummary, Message
+from pyvo.io.vosi.exceptions import VOSIError
 from pyvo.utils import prototype
 
 from astropy.time import Time, TimeDelta
@@ -411,6 +412,17 @@ def callback(request, context):
         yield matcher
 
 
+@pytest.fixture()
+def tapservice(capabilities):
+    """
+    preferably use this fixture when you need a generic TAP service;
+    it saves a bit of parsing overhead.
+
+    (but of course make sure you don't modify it).
+    """
+    return TAPService('http://example.com/tap')
+
+
 def test_escape():
     query = 'SELECT * FROM ivoa.obscore WHERE dataproduct_type = {}'
     query = query.format(escape("'image'"))
@@ -448,6 +460,9 @@ def test_tables(self):
 
         assert list(vositables.keys()) == ['test.table1', 'test.table2']
 
+        assert "test.table1" in vositables
+        assert "any.random.stuff" not in vositables
+
         table1, table2 = list(vositables)
         self._test_tables(table1, table2)
 
@@ -717,3 +732,57 @@ def match_request_text(request):
                                              unique=True)
         finally:
             prototype.deactivate_features('cadc-tb-upload')
+
+
+@pytest.mark.usefixtures("tapservice")
+class TestTAPCapabilities:
+    def test_no_tap_cap(self):
+        svc = TAPService('http://example.com/tap')
+        svc.capabilities = []
+        with pytest.raises(DALServiceError) as excinfo:
+            svc.get_tap_capability()
+        assert str(excinfo.value) == ("Invalid TAP service:"
+            " Does not expose a tr:TableAccess capability")
+
+    def test_no_adql(self):
+        svc = TAPService('http://example.com/tap')
+        svc.get_tap_capability()._languages = []
+        with pytest.raises(VOSIError) as excinfo:
+            svc.get_tap_capability().get_adql()
+        assert str(excinfo.value) == ("Invalid TAP service:"
+            " Does not declare an ADQL language")
+
+    def test_get_adql(self, tapservice):
+        assert tapservice.get_tap_capability().get_adql().name == "ADQL"
+
+    def test_missing_featurelist(self, tapservice):
+        assert (
+            tapservice.get_tap_capability().get_adql().get_feature_list("fump")
+            == [])
+
+    def test_get_featurelist(self, tapservice):
+        features = tapservice.get_tap_capability().get_adql().get_feature_list(
+            "ivo://ivoa.net/std/TAPRegExt#features-adqlgeo")
+        assert set(f.form for f in features) == {
+            'CENTROID', 'CONTAINS', 'COORD1', 'POLYGON',
+            'INTERSECTS', 'COORD2', 'BOX', 'AREA', 'DISTANCE',
+            'REGION', 'CIRCLE', 'POINT'}
+
+    def test_get_missing_feature(self, tapservice):
+        assert tapservice.get_tap_capability().get_adql().get_feature(
+            "ivo://ivoa.net/std/TAPRegExt#features-adqlgeo",
+            "Garage") is None
+
+    def test_get_feature(self, tapservice):
+        feature = tapservice.get_tap_capability().get_adql().get_feature(
+            "ivo://ivoa.net/std/TAPRegExt#features-adqlgeo",
+            "AREA")
+        assert feature.form == "AREA"
+        assert feature.description is None
+
+    def test_missing_udf(self, tapservice):
+        assert tapservice.get_tap_capability().get_adql().get_udf("duff function") is None
+
+    def test_get_udf(self, tapservice):
+        func = tapservice.get_tap_capability().get_adql().get_udf("IVO_hasword")  # case insensitive!
+        assert func.form == "ivo_hasword(haystack TEXT, needle TEXT) -> INTEGER"

pyvo/dal/vosi.py

@@ -161,6 +161,9 @@ def __iter__(self):
         for tablename in self.keys():
             yield self._get_table(tablename)
 
+    def __contains__(self, tablename):
+        return tablename in self.keys()
+
     def _get_table(self, name):
         if name in self._cache:
             return self._cache[name]

pyvo/io/vosi/exceptions.py

@@ -463,3 +463,10 @@ class E10(VOSIWarning, XMLWarning, ValueError):
     Raised when then file doesn't appear to be valid capabilities xml
     """
     message_template = "File does not appear to be a VOSICapabilities file"
+
+
+class VOSIError(Exception):
+    """
+    Raised for non-XML VOSI errors
+    """
+    pass

pyvo/io/vosi/tapregext.py

@@ -9,7 +9,7 @@
 from . import voresource as vr
 from .exceptions import (
     W05, W06, W19, W20, W21, W22, W23, W24, W25, W26, W27, W28, W29, W30, W31,
-    E06, E08, E09)
+    E06, E08, E09, VOSIError)
 
 __all__ = [
     "TAPCapRestriction", "TableAccess", "DataModelType", "Language", "Version",
@@ -250,6 +250,82 @@ def describe(self):
 
             print()
 
+    def get_feature_list(self, ivoid):
+        """
+        returns a list of features groupd with the features id ivoid.
+
+        Parameters
+        ----------
+        ivoid : the ivoid of a TAPRegExt feature list.  It is compared
+            case-insensitively against the service's ivoids.
+
+        Returns
+        -------
+        A (possibly empty) list of `~pyvo.io.vosi.tapregext.LanguageFeature` elements
+        """
+        ivoid = ivoid.lower()
+        for features in self.languagefeaturelists:
+            if features.type.lower() == ivoid:
+                return features
+        return []
+
+    def get_feature(self, ivoid, form):
+        """
+        returns the `~pyvo.io.vosi.tapregext.LanguageFeature` with ivoid and form if present.
+
+        We return None rather than raising an error because we expect
+        the normal pattern of usage here will be "if feature is present",
+        and with None-s this is simpler to write than with exceptions.
+
+        Since it's hard to predict the form of UDFs, for those rather
+        use the get_udf method.
+
+        ivoid (regrettably) has to be compared case-insensitively;
+        form is compared case-sensitively.
+
+        Parameters
+        ----------
+        ivoid : str
+            The IVOA identifier of the feature group the form is in
+        form : str
+            The form of the feature requested
+
+        Returns
+        -------
+        A `~pyvo.io.vosi.tapregext.LanguageFeature` or None.
+        """
+        for feature in self.get_feature_list(ivoid):
+            if feature.form == form:
+                return feature
+
+        return None
+
+    def get_udf(self, function_name):
+        """
+        returns a `~pyvo.io.vosi.tapregext.LanguageFeature` corresponding to an ADQL user defined
+        function on the server, on None if the UDF is not available.
+
+        This is a bit heuristic in that it tries to parse the form, which
+        is specified only so-so.
+
+        Parameters
+        ----------
+        function_name : str
+            A function name.  This is matched against the server's function
+            names case-insensitively, as guided by ADQL's case insensitivity.
+
+        Returns:
+            A `~pyvo.io.vosi.tapregext.LanguageFeature` instance or None.
+        """
+        function_name = function_name.lower()
+        for udf in self.get_feature_list(
+                "ivo://ivoa.net/std/TAPRegExt#features-udf"):
+            this_name = udf.form.split("(")[0].strip()
+            if this_name.lower() == function_name:
+                return udf
+
+        return None
+
     @xmlelement(plain=True, multiple_exc=W05)
     def name(self):
         return self._name
@@ -421,6 +497,19 @@ def describe(self):
                 self.uploadlimit.hard.content, self.uploadlimit.hard.unit)))
             print()
 
+    def get_adql(self):
+        """
+        returns the (first) ADQL language element on this service.
+
+        ADQL support is mandatory for IVOA TAP, so in general you can
+        rely on this being present.
+        """
+        for lang in self.languages:
+            if lang.name == "ADQL":
+                return lang
+        raise VOSIError(
+            "Invalid TAP service: Does not declare an ADQL language")
+
     @xmlelement(name='dataModel', cls=DataModelType)
     def datamodels(self):
         """Identifier of IVOA-approved data model supported by the service."""

pyvo/registry/__init__.py

@@ -4,12 +4,15 @@
 
 The regtap module supports access to the IVOA Registries
 """
-from .regtap import search, ivoid2service, get_RegTAP_query
+
+from .regtap import (search, ivoid2service, get_RegTAP_query,
+    choose_RegTAP_service)
 
 from .rtcons import (Constraint,
     Freetext, Author, Servicetype, Waveband, Datamodel, Ivoid,
     UCD, Spatial, Spectral, Temporal)
 
 __all__ = ["search", "get_RegTAP_query", "Freetext", "Author",
     "Servicetype", "Waveband", "Datamodel", "Ivoid", "UCD",
-    "Spatial", "Spectral", "Temporal"]
+    "Spatial", "Spectral", "Temporal",
+    "choose_RegTAP_service"]