From 464f21ed388f42b3c1246996ebf37c551bd0af00 Mon Sep 17 00:00:00 2001 From: Stefan Becker Date: Sun, 16 Oct 2016 17:45:45 +0200 Subject: [PATCH 1/9] Adding upload_methods to TAPService --- docs/pyvo/data_access.rst | 5 +++++ docs/pyvo/pyvo.dal.tap.TAPService.rst | 2 ++ pyvo/dal/tap.py | 11 +++++++++++ 3 files changed, 18 insertions(+) diff --git a/docs/pyvo/data_access.rst b/docs/pyvo/data_access.rst index 8e9c66adf..7744b8d3d 100644 --- a/docs/pyvo/data_access.rst +++ b/docs/pyvo/data_access.rst @@ -1270,6 +1270,7 @@ information through properties on service instead: ~pyvo.dal.tap.TAPService.maxrec ~pyvo.dal.tap.TAPService.hardlimit + ~pyvo.dal.tap.TAPService.upload_methods ------- Uploads @@ -1296,6 +1297,10 @@ Possible type/source combinations are: >>> service.run_sync(query, uploads = {'t1': ('uri', 'http://example.org/votable.xml')}) >>> service.run_sync(query, uploads = {'t1': ('uri', result)}) +.. note:: + To check if the service supports the desired URI scheme, evaluate the value of + :py:class:`~pyvo.dal.tap.TAPService.upload_methods` + >>> service.run_sync(query, uploads = {'t1': ('inline', open('/path/to/votable.xml'))}) >>> service.run_sync(query, uploads = {'t1': ('inline', result.votable.to_table())}) >>> service.run_sync(query, uploads = {'t1': ('inline', '/path/to/votable.xml')}) diff --git a/docs/pyvo/pyvo.dal.tap.TAPService.rst b/docs/pyvo/pyvo.dal.tap.TAPService.rst index 315782fa8..965d5b58e 100644 --- a/docs/pyvo/pyvo.dal.tap.TAPService.rst +++ b/docs/pyvo/pyvo.dal.tap.TAPService.rst @@ -26,6 +26,7 @@ TAPService ~TAPService.maxrec ~TAPService.tables ~TAPService.up_since + ~TAPService.upload_methods @@ -58,6 +59,7 @@ TAPService .. autoattribute:: maxrec .. autoattribute:: tables .. autoattribute:: up_since + .. autoattribute:: upload_methods diff --git a/pyvo/dal/tap.py b/pyvo/dal/tap.py index 72bb376ea..d1e4d0fde 100644 --- a/pyvo/dal/tap.py +++ b/pyvo/dal/tap.py @@ -179,6 +179,17 @@ def hardlimit(self): pass raise DALServiceError("Hard limit not exposed by the service") + @property + def upload_methods(self): + """ + a list of upload methods in form of IVOA identifiers + """ + _upload_methods = [] + for capa in self.capabilities: + if "uploadMethods" in capa: + _upload_methods += capa["uploadMethods"] + return _upload_methods + def run_sync(self, query, language="ADQL", maxrec=None, uploads=None): """ runs sync query and returns its result From 53086bef614e46df45dd1c805ee59bc3ff7d6438 Mon Sep 17 00:00:00 2001 From: Stefan Becker Date: Mon, 17 Oct 2016 12:02:01 +0200 Subject: [PATCH 2/9] result_uris now raises an exception when the query is in a error state --- pyvo/dal/tap.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyvo/dal/tap.py b/pyvo/dal/tap.py index d1e4d0fde..428a04099 100644 --- a/pyvo/dal/tap.py +++ b/pyvo/dal/tap.py @@ -566,6 +566,7 @@ def result_uris(self): """ a list of the last result uri's """ + self.raise_if_error() return self._job["results"] @property From ecc31b1ae99782e545f9ac92c26281fcd78ae10c Mon Sep 17 00:00:00 2001 From: Stefan Becker Date: Mon, 17 Oct 2016 12:05:15 +0200 Subject: [PATCH 3/9] adding docstrings for availability information --- pyvo/dal/tap.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyvo/dal/tap.py b/pyvo/dal/tap.py index 428a04099..9a947bef1 100644 --- a/pyvo/dal/tap.py +++ b/pyvo/dal/tap.py @@ -107,10 +107,16 @@ def availability(self): @property def available(self): + """ + True if the service is available, False otherwise + """ return self.availability[0] @property def up_since(self): + """ + datetime the service was started + """ return self.availability[1] @property From 3340419b13a123a8ddb1ed3d9d52d20f64f44da7 Mon Sep 17 00:00:00 2001 From: Stefan Becker Date: Mon, 17 Oct 2016 12:28:34 +0200 Subject: [PATCH 4/9] add table property to result object to directl return the astropy table --- pyvo/dal/query.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyvo/dal/query.py b/pyvo/dal/query.py index fc29b9c6b..0a4d62b22 100644 --- a/pyvo/dal/query.py +++ b/pyvo/dal/query.py @@ -591,6 +591,13 @@ def nrecs(self): """ return len(self.votable.to_table()) + @property + def table(self): + """ + the astropy table object + """ + return self.votable.to_table() + def __len__(self): """ return the value of the nrecs property From 9b48bba6a65db868ea2cfd31847ef3673e537f68 Mon Sep 17 00:00:00 2001 From: Stefan Becker Date: Mon, 17 Oct 2016 12:35:06 +0200 Subject: [PATCH 5/9] reflect .table attribute in documentation --- docs/pyvo/data_access.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pyvo/data_access.rst b/docs/pyvo/data_access.rst index 7744b8d3d..0b530a858 100644 --- a/docs/pyvo/data_access.rst +++ b/docs/pyvo/data_access.rst @@ -1302,7 +1302,7 @@ Possible type/source combinations are: :py:class:`~pyvo.dal.tap.TAPService.upload_methods` >>> service.run_sync(query, uploads = {'t1': ('inline', open('/path/to/votable.xml'))}) ->>> service.run_sync(query, uploads = {'t1': ('inline', result.votable.to_table())}) +>>> service.run_sync(query, uploads = {'t1': ('inline', result.table)}) >>> service.run_sync(query, uploads = {'t1': ('inline', '/path/to/votable.xml')}) Your upload can be referenced using 'TAP_UPLOAD.t1' as table name. From a4d655001af889067d6071a640de914d3f7b89ee Mon Sep 17 00:00:00 2001 From: Stefan Becker Date: Mon, 17 Oct 2016 14:40:56 +0200 Subject: [PATCH 6/9] set version number to 0.3.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eddca77f9..286963105 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ builtins._ASTROPY_PACKAGE_NAME_ = PACKAGENAME # VERSION should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386) -VERSION = '0.3.1' +VERSION = '0.3.2' # Indicates if this version is a release version RELEASE = 'dev' not in VERSION From 2c5024f2fbf4396072440d89d376d64cd02947ae Mon Sep 17 00:00:00 2001 From: Stefan Becker Date: Thu, 20 Oct 2016 13:14:04 +0200 Subject: [PATCH 7/9] Use astropy tables for table metadata (#71) Table metadata is now written into astropy.table infrastructure --- docs/pyvo/data_access.rst | 8 ++- pyvo/tools/vosi.py | 130 +++++++++++++------------------------- 2 files changed, 48 insertions(+), 90 deletions(-) diff --git a/docs/pyvo/data_access.rst b/docs/pyvo/data_access.rst index 0b530a858..1abf46e3a 100644 --- a/docs/pyvo/data_access.rst +++ b/docs/pyvo/data_access.rst @@ -1250,10 +1250,12 @@ datetime.datetime(2000, 0, 0, 0, 0, 0) >>> print(service.tables.keys()) The keys within tables are the fully qualified table names as they can -be used in queries. To inspect the column metadata for a table, see its -name's value in the tables dictionary. +be used in queries. To inspect the column metadata for a table, see the column +property of a give table. ->>> print(str(service.tables["rave.main"])) +>>> service.tables["rave.main"].columns + +See also http://docs.astropy.org/en/stable/table/index.html. .. note:: Some TAP services have tables metadata of several megabytes. diff --git a/pyvo/tools/vosi.py b/pyvo/tools/vosi.py index 94177e574..b00f992da 100644 --- a/pyvo/tools/vosi.py +++ b/pyvo/tools/vosi.py @@ -3,7 +3,9 @@ from . import plainxml import datetime from collections import OrderedDict -import StringIO +from astropy.table import Table, Column +from astropy.io.votable.converters import get_converter +import numpy as np class _CapabilitiesParser(plainxml.StartEndHandler): # VOSI; each capability is a dict with at least a key interfaces. @@ -196,7 +198,7 @@ def parse_capabilities(stream): class _TablesParser(plainxml.StartEndHandler): def __init__(self): plainxml.StartEndHandler.__init__(self) - self.tables = _TableDict() + self.tables = OrderedDict() def _start_schema(self, name, attrs): self.curSchema = "" @@ -205,128 +207,82 @@ def _end_schema(self, name, attrs, content): self.curSchema = None def _start_table(self, name, attrs): - self.curTable = _Table() + self.curTable = Table() def _end_table(self, name, attrs, content): - # table names should be exposed the same way they are accessed in ADQL - fqtn = ".".join((self.curSchema, self.curTable.name)) - self.curTable._fqdn = fqtn - self.curTable._schema = self.curSchema - self.tables[fqtn] = self.curTable + table_name = self.curTable.meta["name"] + self.tables[table_name] = self.curTable self.curTable = None def _start_column(self, name, attrs): - self.curColumn = _Column() + self.inColumn = True def _end_column(self, name, attrs, content): - self.curTable[self.curColumn.name] = self.curColumn - self.curColumn = None + column = Column( + name = self.curColumn, dtype = self.curDtype, + description = self.curDescription, + unit = getattr(self, "curUnit", None)) + column.meta["ucd"] = self.curUcd + column.meta["datatype"] = self.curDatatype + column.meta["arraysize"] = self.curArraysize + self.curTable[self.curColumn] = column + self.inColumn = False def _end_name(self, name, attrs, content): content = content.strip() - if getattr(self, "curColumn", None) is not None: - self.curColumn._name = content + if getattr(self, "inColumn", False): + self.curColumn = content elif getattr(self, "curTable", None) is not None: - # return the last tuple from dot notation - self.curTable._name = content.split(".")[-1] - elif getattr(self, "curSchema", None) is not None: - self.curSchema = content + self.curTable.meta["name"] = content def _end_description(self, name, attrs, content): content = content.strip() - if getattr(self, "curColumn", None) is not None: - self.curColumn._description = content + if getattr(self, "inColumn", False): + self.curDescription = content def _end_ucd(self, name, attrs, content): content = content.strip() - if getattr(self, "curColumn", None) is not None: - self.curColumn._ucd = content + if getattr(self, "inColumn", False): + self.curUcd = content def _end_dataType(self, name, attrs, content): content = content.strip() - if getattr(self, "curColumn", None) is not None: - self.curColumn._data_type = content + class _values(object): + null = None + + class _field(object): + datatype = content + arraysize = attrs.get("arraysize") if content in ( + "char", "unicodeChar") else None + precision = None + width = None + values = _values() + + + converter = get_converter(_field()) + self.curDtype = np.dtype(converter.format) + self.curDatatype = content + self.curArraysize = attrs.get("arraysize") def _end_unit(self, name, attrs, content): content = content.strip() - if getattr(self, "curColumn", None) is not None: - self.curColumn._unit = content + if getattr(self, "inColumn", False): + self.curUnit = content def getResult(self): return self.tables + def parse_tables(stream): parser = _TablesParser() parser.parse(stream) return parser.getResult() -class _TableDict(OrderedDict): - def __str__(self): - return "\n".join(unicode(t) for t in self.values()) - -class _Table(OrderedDict): - def __init__(self): - super(_Table, self).__init__() - self._schema = None - self._name = None - - def __str__(self): - io = StringIO.StringIO() - io.write("{0}.{1}\n".format(self.schema, self.name)) - - for k, v in self.items(): - s = "\n".join( - map(lambda x: "\t{0}".format(x), - unicode(v).split("\n")[:-1])) - io.write(unicode("{0}\n".format(s))) - - return io.getvalue() - - @property - def name(self): - return self._name - - @property - def schema(self): - return self._schema - -class _Column(object): - def __init__(self): - self._name = "" - self._ucd = "" - self._data_type = "" - self._unit = "" - self._description = "" - - def __str__(self): - ret = "{0} {1} {2} {3}\n\t{4}\n".format( - self.name, self.ucd, self.data_type, self.unit, self.description) - return ret - - @property - def name(self): - return self._name - - @property - def ucd(self): - return self._ucd - - @property - def data_type(self): - return self._data_type - - @property - def unit(self): - return self._unit - - @property - def description(self): - return self._description class _AvailabiliyParser(plainxml.StartEndHandler): def __init__(self): From 81521c3d1fe713e173143b2a395a7b283cd63018 Mon Sep 17 00:00:00 2001 From: Stefan Becker Date: Thu, 20 Oct 2016 14:44:14 +0200 Subject: [PATCH 8/9] fix another http encoding error (#72) --- pyvo/dal/tap.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pyvo/dal/tap.py b/pyvo/dal/tap.py index 9a947bef1..95bc9ab01 100644 --- a/pyvo/dal/tap.py +++ b/pyvo/dal/tap.py @@ -102,6 +102,10 @@ def availability(self): if self._availability == (None, None): r = requests.get( '{0}/availability'.format(self.baseurl), stream = True) + + # requests doesn't decode the content by default + r.raw.read = functools.partial(r.raw.read, decode_content=True) + self._availability = vosi.parse_availability(r.raw) return self._availability @@ -136,6 +140,10 @@ def capabilities(self): if self._capabilities is None: r = requests.get( '{0}/capabilities'.format(self.baseurl), stream = True) + + # requests doesn't decode the content by default + r.raw.read = functools.partial(r.raw.read, decode_content=True) + self._capabilities = vosi.parse_capabilities(r.raw) return self._capabilities @@ -146,6 +154,10 @@ def tables(self): """ if self._tables is None: r = requests.get('{0}/tables'.format(self.baseurl), stream = True) + + # requests doesn't decode the content by default + r.raw.read = functools.partial(r.raw.read, decode_content=True) + self._tables = vosi.parse_tables(r.raw) return self._tables @@ -403,6 +415,7 @@ def submit(self): r = requests.post(url, data = self._param, stream = True, files = files) + # requests doesn't decode the content by default r.raw.read = functools.partial(r.raw.read, decode_content=True) return r @@ -457,8 +470,11 @@ def _update(self): r.raise_for_status() except requests.exceptions.RequestException as ex: raise DALServiceError.from_except(ex, self.url, "TAP", "1.0") + + # requests doesn't decode the content by default + r.raw.read = functools.partial(r.raw.read, decode_content=True) + self._job.update(uws.parse_job(r.raw)) - pass @property def job(self): From ad3c5f9d9151b8e1bddbc8268d93d3333515df82 Mon Sep 17 00:00:00 2001 From: Stefan Becker Date: Thu, 20 Oct 2016 14:45:07 +0200 Subject: [PATCH 9/9] Release 0.4 --- CHANGES.rst | 22 ++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index c168906f2..ba139acef 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,25 @@ +0.4 (unreleased) +---------------- +* Use astropy tables for table metadata + +* fix another content encoding error + +0.3.2 (unreleased) +------------------ +* Adding table property to DALResults. This is a shortcut to access the astropy table + +* Improved Error Handling + +0.3.1 (unreleased) +------------------ +* fix an error where the content wasn't decoded properly + +* fix a bug where POST parameters are submitted as GET parameters + +0.3 (unreleased) +---------------- +Adding TAP API + 0.1 (unreleased) ---------------- diff --git a/setup.py b/setup.py index 286963105..b7da4b75b 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ builtins._ASTROPY_PACKAGE_NAME_ = PACKAGENAME # VERSION should be PEP386 compatible (http://www.python.org/dev/peps/pep-0386) -VERSION = '0.3.2' +VERSION = '0.4' # Indicates if this version is a release version RELEASE = 'dev' not in VERSION