Adding support for DALI example continuation properties.

While this is a DALI feature, for now we only support examples in TAP, so that's where where this happens. Supporting continuations is desirable now that TOPCAT shows them, too.
astropy · Aug 28, 2023 · 89e04c4 · 89e04c4
1 parent 3ec378e
commit 89e04c4
Show file tree

Hide file tree

Showing 5 changed files with 89 additions and 22 deletions.
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -23,6 +23,7 @@
 
 - Adding python version to User-Agent. [#452]
 
+- TAP examples now support the continuation property [#483]
 
 1.4.3 (unreleased)
 ==================

diff --git a/pyvo/dal/tap.py b/pyvo/dal/tap.py
@@ -149,6 +149,38 @@ def tables(self):
                 vosi.parse_tables(response.raw.read), tables_url)
         return self._tables
 
+    def _parse_examples(self, examples_uri, /, depth=0):
+        """returns the TAP queries from a DALI examples URI.
+        """
+        if depth > 5:
+            raise Exception("Suspecting endless recursion when"
+                " parsing TAP examples")
+
+        response = self._session.get(examples_uri, stream=True)
+        if response.status_code == 404:
+            return []
+
+        try:
+            response.raise_for_status()
+        except requests.RequestException as ex:
+            raise DALServiceError.from_except(ex, examples_uri)
+
+        try:
+            root = xml.etree.ElementTree.parse(
+                io.BytesIO(response.content)).getroot()
+            exampleElements = root.findall('.//*[@property="query"]')
+        except Exception as ex:
+            raise DALServiceError.from_except(ex, examples_uri)
+
+        examples = [TAPQuery(self.baseurl, example.text)
+            for example in exampleElements]
+
+        for continuation in root.findall('.//*[@property="continuation"]'):
+            examples.extend(
+                self._parse_examples(continuation.get("href"), depth + 1))
+
+        return examples
+
     @property
     def examples(self):
         """
@@ -157,23 +189,7 @@ def examples(self):
         if self._examples is None:
             examples_url = '{}/examples'.format(self.baseurl)
 
-            response = self._session.get(examples_url, stream=True)
-            if response.status_code == 404:
-                return []
-
-            try:
-                response.raise_for_status()
-            except requests.RequestException as ex:
-                raise DALServiceError.from_except(ex, examples_url)
-
-            try:
-                root = xml.etree.ElementTree.parse(io.BytesIO(response.content)).getroot()
-                exampleElements = root.findall('.//*[@property="query"]')
-            except Exception as ex:
-                raise DALServiceError.from_except(ex, examples_url)
-
-            self._examples = [TAPQuery(self.baseurl, example.text) for example in exampleElements]
-
+            self._examples = self._parse_examples(examples_url)
         return self._examples
 
     @property

diff --git a/pyvo/dal/tests/data/tap/examples.htm b/pyvo/dal/tests/data/tap/examples.htm
@@ -1 +1,3 @@
-<!DOCTYPE html  PUBLIC "-//W3C//DTD XHTML+RDFa 1.1//EN"  "http://www.w3.org/MarkUp/DTD/xhtml-rdfa-2.dtd"><!-- used by taprender.TAPExamples --><html version="XHTML+RDFa 1.1" xmlns="http://www.w3.org/1999/xhtml">  <head>    <title>Examples queries for HEASARC's TAP service</title>  </head>  <body class="container" vocab="ivo://ivoa.net/std/DALI-examples#">    <div id="body">      <a name="body"></a>      <h1>Examples for the TAP service at HEASARC</h1>      <div id="exampleslist">         <div typeof="example" id="QueryContainsPointCircle" resource="#QueryContainsPointCircle">          <h2 property="name">Simple geometric query on rosmaster with circle and point</h2>          <p>            The Table Access Protocol Service at HEASARC allow for simple          geometric queries.  For example, this query searches for observations in the rostmaster          catalog within a circle of radius 1 degree of the          coordinates (ra,dec)=(50,-85) -- i.e., basically a cone          search -- and an exposure longer than 10000 seconds:            </p>          <pre property="query">            SELECT * FROM rosmaster                      WHERE exposure > 10000 and                            1=CONTAINS(POINT('ICRS', ra, dec),CIRCLE('ICRS', 50, -85, 1))          </pre>        </div>         <div typeof="example" id="CrossMatch" resource="#CrossMatch">          <h2 property="name">Simple geometric query on rosmaster with circle and point</h2>          <p>            The Table Access Protocol Service at HEASARC allow for simple          geometric and cross-match queries.  For example, this query searches for observations common to the rostmaster          catalog AND the chanmaster catalog with a minimum exposure of 10ks.          </p>          <pre property="query">	   SELECT * FROM rosmaster as ros	            INNER JOIN chanmaster as chan	                ON ros.name = chan.name	            WHERE ros.exposure > 10000 and chan.exposure > 10000	            ORDER by ros.exposure	             </pre>        </div>	<div typeof="example" id="QueryCirclesIntersect" resource="#QueryCirclesIntersect">          <h2 property="name">Simple geometric query on rosmaster with intersects, circle, point</h2>          <p>            The Table Access Protocol Service at HEASARC allow for simple          geometric queries.  For example, this (slow) query searches for observations in the rostmaster          catalog where a circle of radius 1 degree of the          coordinates (ra,dec)=(50,-85) intersects with a circle of 1          degree radius around the center of the pointing:            </p>          <pre property="query">            SELECT * FROM rosmaster                      WHERE 1=INTERSECTS(CIRCLE('ICRS', ra, dec,1),CIRCLE('ICRS', 50, -85, 1))          </pre>        </div>	        <div typeof="example" id="QueryPolygon" resource="#QueryPolygon">          <h2 property="name">Simple geometric query on rosmaster with polygon</h2>          <p>            The Table Access Protocol Service at HEASARC allow for simple          geometric queries.  For example, this query searches for observations in the rostmaster          catalog where a pointing lies with a user-defined polygon, in this case a triangle with	  vertices (-5,-5), (5,-5), and (0,5)):            </p>          <pre property="query">            SELECT * FROM rosmaster                      WHERE  exposure > 10000 AND                             1=CONTAINS(POINT('ICRS', ra, dec),POLYGON('ICRS', -5, -5, 5, -5, 0, 5))          </pre>        </div>		        <div typeof="example" id="QueryDistance" resource="#QueryDistance">          <h2 property="name">Simple geometric query on rosmaster, chanmaster with join on distance</h2>          <p>            The Table Access Protocol Service at HEASARC allow for simple          geometric queries.  For example, this query computes the distance between rostmaster observations	  and a given point, selecting those observations near it, and ordering the result by that distance:          </p>          <pre property="query">          SELECT DISTANCE(	      POINT('ICRS', ra, dec),              POINT('ICRS', 266.41683, -29.00781)) AS dist, *	  FROM rosmaster	  WHERE 1=CONTAINS(	      POINT('ICRS', ra, dec),	      CIRCLE('ICRS', 266.41683, -29.00781, 1))	  ORDER BY dist ASC          </pre>        </div>	      </div>    </div>  </body></html>
+<!DOCTYPE html  PUBLIC "-//W3C//DTD XHTML+RDFa 1.1//EN"  "http://www.w3.org/MarkUp/DTD/xhtml-rdfa-2.dtd"><!-- used by taprender.TAPExamples --><html version="XHTML+RDFa 1.1" xmlns="http://www.w3.org/1999/xhtml">  <head>    <title>Examples queries for HEASARC's TAP service</title>  </head>  <body class="container" vocab="ivo://ivoa.net/std/DALI-examples#">    <div id="body">      <a name="body"></a>      <h1>Examples for the TAP service at HEASARC</h1>      <div id="exampleslist">         <div typeof="example" id="QueryContainsPointCircle" resource="#QueryContainsPointCircle">          <h2 property="name">Simple geometric query on rosmaster with circle and point</h2>          <p>            The Table Access Protocol Service at HEASARC allow for simple          geometric queries.  For example, this query searches for observations in the rostmaster          catalog within a circle of radius 1 degree of the          coordinates (ra,dec)=(50,-85) -- i.e., basically a cone          search -- and an exposure longer than 10000 seconds:            </p>          <pre property="query">            SELECT * FROM rosmaster                      WHERE exposure > 10000 and                            1=CONTAINS(POINT('ICRS', ra, dec),CIRCLE('ICRS', 50, -85, 1))          </pre>        </div>         <div typeof="example" id="CrossMatch" resource="#CrossMatch">          <h2 property="name">Simple geometric query on rosmaster with circle and point</h2>          <p>            The Table Access Protocol Service at HEASARC allow for simple          geometric and cross-match queries.  For example, this query searches for observations common to the rostmaster          catalog AND the chanmaster catalog with a minimum exposure of 10ks.          </p>          <pre property="query">	   SELECT * FROM rosmaster as ros	            INNER JOIN chanmaster as chan	                ON ros.name = chan.name	            WHERE ros.exposure > 10000 and chan.exposure > 10000	            ORDER by ros.exposure	             </pre>        </div>	<div typeof="example" id="QueryCirclesIntersect" resource="#QueryCirclesIntersect">          <h2 property="name">Simple geometric query on rosmaster with intersects, circle, point</h2>          <p>            The Table Access Protocol Service at HEASARC allow for simple          geometric queries.  For example, this (slow) query searches for observations in the rostmaster          catalog where a circle of radius 1 degree of the          coordinates (ra,dec)=(50,-85) intersects with a circle of 1          degree radius around the center of the pointing:            </p>          <pre property="query">            SELECT * FROM rosmaster                      WHERE 1=INTERSECTS(CIRCLE('ICRS', ra, dec,1),CIRCLE('ICRS', 50, -85, 1))          </pre>        </div>	        <div typeof="example" id="QueryPolygon" resource="#QueryPolygon">          <h2 property="name">Simple geometric query on rosmaster with polygon</h2>          <p>            The Table Access Protocol Service at HEASARC allow for simple          geometric queries.  For example, this query searches for observations in the rostmaster          catalog where a pointing lies with a user-defined polygon, in this case a triangle with	  vertices (-5,-5), (5,-5), and (0,5)):            </p>          <pre property="query">            SELECT * FROM rosmaster                      WHERE  exposure > 10000 AND                             1=CONTAINS(POINT('ICRS', ra, dec),POLYGON('ICRS', -5, -5, 5, -5, 0, 5))          </pre>        </div>		        <div typeof="example" id="QueryDistance" resource="#QueryDistance">          <h2 property="name">Simple geometric query on rosmaster, chanmaster with join on distance</h2>          <p>            The Table Access Protocol Service at HEASARC allow for simple          geometric queries.  For example, this query computes the distance between rostmaster observations	  and a given point, selecting those observations near it, and ordering the result by that distance:          </p>          <pre property="query">          SELECT DISTANCE(	      POINT('ICRS', ra, dec),              POINT('ICRS', 266.41683, -29.00781)) AS dist, *	  FROM rosmaster	  WHERE 1=CONTAINS(	      POINT('ICRS', ra, dec),	      CIRCLE('ICRS', 266.41683, -29.00781, 1))	  ORDER BY dist ASC          </pre>        </div>	
+<a property="continuation" href="http://example.org/obscore-examples.html">ObsCore queries</a>
+</div>    </div>  </body></html>
diff --git a/pyvo/dal/tests/data/tap/obscore-examples.html b/pyvo/dal/tests/data/tap/obscore-examples.html
@@ -0,0 +1,38 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" version="XHTML+RDFa 1.1">
+  <head>
+    <meta http-equiv="Content-type" content="text/html;charset=UTF-8" />
+    <link rel="stylesheet" href="/static/css/gavo_dc.css" type="text/css" />
+  </head>
+  <body vocab="ivo://ivoa.net/std/DALI-examples#" class="container">
+    <h1>ObsCore Examples from Heidelberg</h1>
+    <p>These are examples for ADQL you can run in TAP services carrying
+      an <code>ivoa.obscore</code> table.  See
+      <a href="http://ivoa.net/documents/ObsCore/">ObsCore</a> for the underlying
+      data model.</p>
+   <div typeof="example" id="Findingplatesbytimeandplace" resource="#Findingplatesbytimeandplace">
+    <h2 property="name">Finding images by time and place</h2>
+    <p>Suppose you read in an old amateur observer's log there was an unexpected
+object on the night sky in the cold winter nights of the week between January
+12th and 18th, 1903 &#x2013; and now you would like to see whether there could
+be an observation of such a thing.</p>
+         <pre class="literal-block">
+SELECT s_ra, s_dec, t_min FROM ivoa.obscore
+  WHERE t_min BETWEEN gavo_to_mjd('1903-01-12')
+      AND gavo_to_mjd('1903-01-19')
+</pre>
+     <p>There is also a shortcut via user defined functions.  As an extension
+to regular ADQL, DaCHS lets you write gavo_simbadpoint('object') and replaces
+the result with a position obtained from simbad, like this:</p>
+        <pre class="dachs-ex-tapquery literal-block" property="query">
+SELECT access_url, t_exptime, t_min FROM ivoa.obscore
+  WHERE
+    t_min BETWEEN gavo_to_mjd('J2416128.5')
+      AND gavo_to_mjd('J2416133.5') AND
+    1=CONTAINS(gavo_simbadpoint('Aldebaran'),
+      CIRCLE('ICRS', s_ra, s_dec, 15))
+</pre>
+		</div>
+	</body>
+</html>
+
diff --git a/pyvo/dal/tests/test_tap.py b/pyvo/dal/tests/test_tap.py
@@ -386,10 +386,16 @@ def callback_table2(request, context):
 @pytest.fixture()
 def examples(mocker):
     def callback_examplesXHTML(request, context):
-        return get_pkg_data_contents('data/tap/examples.htm')
+        uri = f"://{request.netloc}{request.path}"
+        if uri == '://example.com/tap/examples':
+            return get_pkg_data_contents('data/tap/examples.htm')
+        elif uri == '://example.org/obscore-examples.html':
+            return get_pkg_data_contents('data/tap/obscore-examples.html')
+        else:
+            assert False, f"Unexpected examples URI: {uri}"
 
     with mocker.register_uri(
-        'GET', 'http://example.com/tap/examples', content=callback_examplesXHTML
+        'GET', requests_mock.ANY, content=callback_examplesXHTML
     ) as matcher:
         yield matcher
 
@@ -445,8 +451,12 @@ def test_tables(self):
         table1, table2 = list(vositables)
         self._test_tables(table1, table2)
 
-    def _test_examples(self, exampleXHTML):
-        assert "SELECT * FROM rosmaster" in exampleXHTML[0]['QUERY']
+    def _test_examples(self, parsed_examples):
+        assert len(parsed_examples) == 6
+        assert "SELECT * FROM rosmaster" in parsed_examples[0]['QUERY']
+        # the last query is from the continuation
+        assert parsed_examples[-1]['QUERY'].startswith(
+            "\nSELECT access_url, t_exptime, t_min FROM ivoa.obscore")
 
     @pytest.mark.usefixtures('examples')
     def test_examples(self):