From 89d8daeaa6b93d937ba3e9d1b60b7069356d6369 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sun, 24 Nov 2024 17:22:57 +0100 Subject: [PATCH 01/39] add a clean ns option for the pretty printing --- pyvo/mivot/utils/xml_utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pyvo/mivot/utils/xml_utils.py b/pyvo/mivot/utils/xml_utils.py index 7c3f40cc9..468a3b49d 100644 --- a/pyvo/mivot/utils/xml_utils.py +++ b/pyvo/mivot/utils/xml_utils.py @@ -23,7 +23,7 @@ def pretty_print(xmltree): print(XmlUtils.pretty_string(xmltree)) @staticmethod - def pretty_string(xmltree): + def pretty_string(xmltree, clean_ns=True): """ Return a pretty string representation of an XML tree. Parameters @@ -39,7 +39,12 @@ def pretty_string(xmltree): else: XmlUtils.indent(xmltree) new_xml = ET.tostring(xmltree, encoding='unicode') - return new_xml.replace("ns0:", "") + if clean_ns: + return new_xml.replace("ns0:", "") + else: + return new_xml + + @staticmethod def indent(elem, level=0): From 5fddfcd04e7a8081007cb40ccd7ff76621bf1643 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sun, 24 Nov 2024 17:23:31 +0100 Subject: [PATCH 02/39] layout changed after a read/write cycle --- .../tests/data/test.mivot_viewer.no_mivot.xml | 535 +++++++++++------- 1 file changed, 315 insertions(+), 220 deletions(-) diff --git a/pyvo/mivot/tests/data/test.mivot_viewer.no_mivot.xml b/pyvo/mivot/tests/data/test.mivot_viewer.no_mivot.xml index 54c5b32f5..71a60fe0c 100644 --- a/pyvo/mivot/tests/data/test.mivot_viewer.no_mivot.xml +++ b/pyvo/mivot/tests/data/test.mivot_viewer.no_mivot.xml @@ -1,222 +1,317 @@ - - - - - URAT1 Catalog (Zacharias+ 2015) IVOID of underlying data collection - + + + + + + URAT1 Catalog (Zacharias+ 2015) + - - - URAT1 catalog - - - - Distance from center (052.2670800+59.9402700)[ICRS], at Epoch of catalog (Epoch) - - - - URAT1 recommended identifier (ZZZ-NNNNNN) (13) [datatype=char] - - - - Right ascension on ICRS, at "Epoch" (1) - - - - Declination on ICRS, at "Epoch" (1) - - - - Position error per coordinate, from scatter (2) - - - - Position error per coordinate, from model (2) - - - - (nst) Total number of sets the star is in (3) - - - - (nsu) Number of sets used for mean position (3) - - - - (epoc) Mean URAT observation epoch (1) - - - - ?(mmag) mean URAT model fit magnitude (4) - - - - - ?(sigp) URAT photometry error (5) - - - - - (nsm) Number of sets used for URAT magnitude (3) - - - - (ref) largest reference star flag (6) - - - - (nit) Total number of images (observations) - - - - (niu) Number of images used for mean position - - - - (ngt) Total number of 1st order grating observations - - - - (ngu) Number of 1st order grating positions used - - - - ?(pmr) Proper motion RA*cosDec (from 2MASS) (7) - - - - - ?(pmd) Proper motion in Declination (7) - - - - - ?(pme) Proper motion error per coordinate (8) - - - - - [1/11] Match flag URAT with 2MASS (9) - - - - [1/11] Match flag URAT with APASS (9) - - - - [-] "-" if there is no match with GSC2.4 (14) - - - - ?(id2) unique 2MASS star identification number - - - - - ?(jmag) 2MASS J-band magnitude - - - - - ?(ejmag) Error on Jmag - - - - - [0,58]? J-band quality-confusion flag (10) - - - - ?(hmag) 2MASS H-band magnitude - - - - - ?(ehmag) Error on H-band magnitude (10) - - - - - [0,58]? H-band quality-confusion flag (10) - - - - ?(kmag) 2MASS Ks-band magnitude - - - - - ?(ekmag) Error on Ks-band magnitude (10) - - - - - [0,58]? Ks-band quality-confusion flag (10) - - - - (ann) Number of APASS observation nights (12) - - - - (ano) Number of APASS observations (12) - - - - ?(abm) APASS B-band magnitude (11) - - - - - ?(ebm) Error on Bmag - - - - - ?(avm) APASS V-band magnitude - - - - - ?(evm) Error on Vmag - - - - - ?(agm) APASS g-band magnitude - - - - - ?(egm) Error on gmag - - - - - ?(arm) APASS r-band magnitude - - - - - ?(erm) Error on rmag - - - - - ?(aim) APASS i-band magnitude - - - - - ?(eim) Error on imag - - - - - - -
0.049402750-146023052.2340018+59.89373339613132013.41815.3400.0131397474001.5-12.35.91575880868113.7130.028513.3400.034513.1010.03451417.6320.20416.1640.00116.6900.00115.7500.001
- + + IVOID of underlying data collection + + + URAT1 catalog + + + + Distance from center (052.2670800+59.9402700)[ICRS], at Epoch of + catalog (Epoch) + + + + + URAT1 recommended identifier (ZZZ-NNNNNN) (13) [datatype=char] + + + + + Right ascension on ICRS, at "Epoch" (1) + + + + + Declination on ICRS, at "Epoch" (1) + + + + + Position error per coordinate, from scatter (2) + + + + + Position error per coordinate, from model (2) + + + + + (nst) Total number of sets the star is in (3) + + + + + (nsu) Number of sets used for mean position (3) + + + + + (epoc) Mean URAT observation epoch (1) + + + + + ?(mmag) mean URAT model fit magnitude (4) + + + + + + ?(sigp) URAT photometry error (5) + + + + + + (nsm) Number of sets used for URAT magnitude (3) + + + + + (ref) largest reference star flag (6) + + + + + (nit) Total number of images (observations) + + + + + (niu) Number of images used for mean position + + + + + (ngt) Total number of 1st order grating observations + + + + + (ngu) Number of 1st order grating positions used + + + + + ?(pmr) Proper motion RA*cosDec (from 2MASS) (7) + + + + + + ?(pmd) Proper motion in Declination (7) + + + + + + ?(pme) Proper motion error per coordinate (8) + + + + + + [1/11] Match flag URAT with 2MASS (9) + + + + + [1/11] Match flag URAT with APASS (9) + + + + + [-] "-" if there is no match with GSC2.4 (14) + + + + + ?(id2) unique 2MASS star identification number + + + + + + ?(jmag) 2MASS J-band magnitude + + + + + + ?(ejmag) Error on Jmag + + + + + + [0,58]? J-band quality-confusion flag (10) + + + + + ?(hmag) 2MASS H-band magnitude + + + + + + ?(ehmag) Error on H-band magnitude (10) + + + + + + [0,58]? H-band quality-confusion flag (10) + + + + + ?(kmag) 2MASS Ks-band magnitude + + + + + + ?(ekmag) Error on Ks-band magnitude (10) + + + + + + [0,58]? Ks-band quality-confusion flag (10) + + + + + (ann) Number of APASS observation nights (12) + + + + + (ano) Number of APASS observations (12) + + + + + ?(abm) APASS B-band magnitude (11) + + + + + + ?(ebm) Error on Bmag + + + + + + ?(avm) APASS V-band magnitude + + + + + + ?(evm) Error on Vmag + + + + + + ?(agm) APASS g-band magnitude + + + + + + ?(egm) Error on gmag + + + + + + ?(arm) APASS r-band magnitude + + + + + + ?(erm) Error on rmag + + + + + + ?(aim) APASS i-band magnitude + + + + + + ?(eim) Error on imag + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
0.049402750-146023 52.2340018 59.89373339613132013.41815.340 0.013139747400 1.5 -12.3 5.915 + 75880868113.713 0.028513.340 0.034513.101 0.03451417.632 0.20416.164 0.00116.690 0.00115.750 0.001 + +
+ From 89c6fe3de0cb54f5577422ea5a190f1b8e83b78c Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sun, 24 Nov 2024 17:24:02 +0100 Subject: [PATCH 03/39] local copy of the MIVOT XML schema --- pyvo/mivot/writer/mivot-v1.xsd | 210 +++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 pyvo/mivot/writer/mivot-v1.xsd diff --git a/pyvo/mivot/writer/mivot-v1.xsd b/pyvo/mivot/writer/mivot-v1.xsd new file mode 100644 index 000000000..ae80f5ba7 --- /dev/null +++ b/pyvo/mivot/writer/mivot-v1.xsd @@ -0,0 +1,210 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From cf7fc92a7c21bd9b10642d6877211b6a1254b2ff Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sun, 24 Nov 2024 17:24:25 +0100 Subject: [PATCH 04/39] annotation builder: new class --- pyvo/mivot/writer/annotations.py | 317 +++++++++++++++++++++++++++++++ 1 file changed, 317 insertions(+) create mode 100644 pyvo/mivot/writer/annotations.py diff --git a/pyvo/mivot/writer/annotations.py b/pyvo/mivot/writer/annotations.py new file mode 100644 index 000000000..3912f63d8 --- /dev/null +++ b/pyvo/mivot/writer/annotations.py @@ -0,0 +1,317 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +MivotAnnotations implements the basic functions to build MIVOT annotations. +- Add various elements to the MIVOT block +- validate it against the MIVOT XML schema (local copy) if the package xml_schema is installed +- insert the MIVOT block in a VOTable + +The MIVOT block is built as a string in order to be compliant with Astropy API. +The code below shows a typical use of MivotAnnotations: + + .. code-block:: python + + mb = MivotAnnotations() + mb.build_mivot_block() + + mb.add_globals("") + mb.add_templates("") + mb.add_templates("") + mb.add_model("model", "http://model.com") + mb.add_model("model2", None) + mb.set_report(True, "unit tests") + mb.build_mivot_block(templates_id="azerty") + + votable = parse(votable_path) + mb.insert_into_votable(votable) + mv = MivotViewer(votable) + print(mv.dm_instance) + +This module does not check if the MIVOT block is consistent with the declared models +or if the references to table columns can be resolved either. + +This latest point can be verified with he MivotViewer + + .. code-block:: python + + votable = parse(votable_path) + mb.insert_into_votable(votable) + mv = MivotViewer(votable) + print(mv.dm_instance) + +See `tests/test_mivot_writer.py`to get different examples of the API usage.` +""" +import os +import logging +try: + import xmlschema +except ImportError: + xmlschema = None +# Use defusedxml only if already present in order to avoid a new depency. +try: + from defusedxml import ElementTree as etree +except ImportError: + from xml.etree import ElementTree as etree +from astropy.io.votable.tree import VOTableFile, Resource, MivotBlock +from astropy.io.votable import parse +from pyvo.utils.prototype import prototype_feature +from pyvo.mivot.utils.xml_utils import XmlUtils +from pyvo.mivot.writer.instance import MivotInstance + +@prototype_feature('MIVOT') +class MivotAnnotations: + """ + API for building annotations step by step. + """ + def __init__(self): + """ + Constructor of the MivotViewer class: no logic inside + """ + self._models = {} + self._report_status = True + self._report_message = "Generated by pyvo.mivot.writer" + self._globals = [] + self._templates = [] + self._templates_id = "" + self._mivot_block = "" + + @property + def mivot_block(self): + """ + mivot_block getter + + Returns + ------- + str + the MIVOT block as a string + """ + return self._mivot_block + + def _get_report(self): + """ + Get the REPORT component of the MIVOT block + + Returns + ------- + str + the REPORT block as a string + """ + if self._report_status is True: + return f'{self._report_message}' + else: + return f'{self._report_message}' + + def _get_models(self): + """ + Get the MODEL components of the MIVOT block + + Returns + ------- + str + the MODEL components block as a string + """ + models_block = "" + for key, value in self._models.items(): + if value: + models_block += f'\n' + else: + models_block += f'\n' + + return models_block + + def _get_globals(self): + """ + Get the GLOBALS component of the MIVOT block + + Returns + ------- + str + the GLOBALS block as a string + """ + globals_block = "\n" + for globals in self._globals: + globals_block += f"{globals}\n" + globals_block += "\n" + + return globals_block + + def _get_templates(self): + """ + Get the TEMPLATES component of the MIVOT block + + Returns + ------- + str + the TEMPLATES block as a string or an empty string if no templates + """ + # no TEMPLATES blocks if no mapped object + if not self._templates: + return "" + if not self._templates_id: + templates_block = f'\n' + else: + templates_block = f'\n' + + for templates in self._templates: + templates_block += f"{templates}\n" + templates_block += "\n" + return templates_block + + def build_mivot_block(self, templates_id=None): + """ + Buikd a complete MIVOT block from the declared components and vaiidate the result + against the MIVO XML schema/ + + Raises + ------ + Exceptions possibly raised by the validator are not trapped and must be processed by the caller + """ + if templates_id: + self._templates_id = templates_id + self._mivot_block = '\n' + self._mivot_block += self._get_report() + self._mivot_block += '\n' + self._mivot_block += self._get_models() + self._mivot_block += '\n' + self._mivot_block += self._get_globals() + self._mivot_block += '\n' + self._mivot_block += self._get_templates() + self._mivot_block += '\n' + self._mivot_block += '\n' + self._mivot_block = self.mivot_block.replace("\n\n", "\n") + self.check_xml() + + def add_templates(self, templates_instance): + """ + Add an block to the block. + The templates_instance content is not checked at this level. + + Parameters + ---------- + templates_instance : string or MivotInstance + block to be added + + Raises + ------ + Raises an MappingException if the globals_instance is not str or MivotInstance either. + + """ + if type(templates_instance) == MivotInstance: + self._templates.append(templates_instance.xml_string()) + elif type(templates_instance) == str: + self._templates.append(templates_instance) + else: + raise MappingException("Instance added to templates must be a string or MivotInstance") + + def add_globals(self, globals_instance): + """ + Add an block to the block. + The globals_instance content is not checked at this level. + + Parameters + ---------- + globals_instance : string or MivotInstance + block to be added + + Raises + ------ + Raises an MappingException if the globals_instance is not str or MivotInstance either. + """ + if type(globals_instance) == MivotInstance: + self._globals.append(globals_instance.xml_string()) + elif type(globals_instance) == str: + self._globals.append(globals_instance) + else: + raise MappingException("Instance added to globals must be a string or MivotInstance") + + def add_model(self, model_name, model_url): + """ + Add a MODEL element: + + Parameters + ---------- + model_name : string + model short name + model_url: string + URL of the VO-DML file + """ + self._models[model_name] = model_url + + def set_report(self, status, message): + """ + Set the REPORT element. If the REPOPRT is set to failed, + all component of the MIVOT block are deleted except MODEL and REPORT + Parameters + ---------- + status : boolean + status of the annotation process + message: string + REPORT message + """ + self._report_status = status + self._report_message = message + if status is False: + self._globals = [] + self._templates = [] + + def check_xml(self): + """ + Validate the mapping block against the MIVOT XML schema v1.0. + The schema is copied locally to avoid a dependency with a remote service + + Raises + ------ + MappingException if the validation fails + """ + if not xmlschema: + logging.error("XML validation skipped: " + + "no XML schema found, " + + "please install it (e.g. pip install xmlschema)") + return + + schema = xmlschema.XMLSchema11(os.path.dirname(__file__) + "/mivot-v1.xsd") + root = etree.fromstring(self._mivot_block) + mivot_block = XmlUtils.pretty_string(root, clean_ns=False) + + try: + schema.validate(mivot_block) + except Exception as excep: + raise MappingException(f"validation failed {excep}") from excep + + def insert_into_votable(self, votable_file, template_id=None, override=False): + """ + Parameters + ---------- + votable_file : path like string or a VOTableFile + VOTable to be annotated + template_id: string + ID of the TABLE to be mapped, ignored if None + override : boolean + override former annotations if Tue + + Raises + ----- + MappingException if a mapping block is already here and override is False + """ + if type(votable_file) == str: + votable = parse(votable_file) + elif type(votable_file) == VOTableFile: + votable = votable_file + else: + raise MappingException("votable_file must be either a path like string or a VOTableFile") + + for resource in votable.resources: + if resource.type == "results": + for subresource in resource.resources: + if sub_resource.type == "meta": + if override is False: + raise MappingException( + "There is already a type='meta' in the first result resource") + else: + logging.info( + "There is already a type='meta' in the first result resource that will be overridden") + break + mivot_resource = Resource() + mivot_resource.type = "meta" + mivot_resource.mivot_block = MivotBlock(self._mivot_block) + resource.resources.append(mivot_resource) + From db49366c7bcfaa9fe6d3b39035afbc6dcacaa8f4 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sun, 24 Nov 2024 17:24:46 +0100 Subject: [PATCH 05/39] instance builder: new class --- pyvo/mivot/writer/instance.py | 160 ++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 pyvo/mivot/writer/instance.py diff --git a/pyvo/mivot/writer/instance.py b/pyvo/mivot/writer/instance.py new file mode 100644 index 000000000..e49a54dcf --- /dev/null +++ b/pyvo/mivot/writer/instance.py @@ -0,0 +1,160 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +MivotInstance is a simple API for building MIVOT instances step by step. +- This code is totally model-agnostic +- The basic syntax rules of MIVOT are checked +- The context dependent syntax rules are ignored + +MivotInstance builds elements that can contain , and + are not supported yet + +The code below shows a typical use of MivotInstance: + + .. code-block:: python + + instance1 = MivotInstance(dmtype="model:type.inst", dmid="id1") + instance1.add_attribute(dmtype="model:type.att1", dmrole="model:type.inst.role1", value="value1", unit="m/s") + instance1.add_attribute(dmtype="model:type.att2", dmrole="model:type.inst.role2", value="value2", unit="m/s") + + instance2 = MivotInstance(dmtype="model:type2.inst", dmrole="model:role.instance2", dmid="id2") + instance2.add_attribute(dmtype="model:type2.att1", dmrole="model:type2.inst.role1", value="value3", unit="m/s") + instance2.add_attribute(dmtype="model:type2.att2", dmrole="model:type2.inst.role2", value="value4", unit="m/s") + + instance1.add_instance(instance2) + + mb = MivotAnnotations() + mb.add_templates(instance1) + mb.build_mivot_block() + print(mb.mivot_block) + +""" +import os +import logging +from pyvo.utils.prototype import prototype_feature +from pyvo.mivot.utils.exceptions import MappingException + +@prototype_feature('MIVOT') +class MivotInstance: + """ + API for building annotations elements step by step. + """ + + def __init__(self, dmtype=None, dmrole=None, dmid=None): + """ + Constructor + + paremeters + ---------- + dmtype: string + dmtype of the INSTANCE (mandatory) + dmrole: string + dmrole of the INSTANCE (optional) + dmid: string + dmid of the INSTANCE (optional) + + raise + ----- + MappingException if no dmtype is provided + """ + if not dmtype: + raise MappingException("Cannot build an instance without dmtype") + self._dmtype = dmtype + self._dmrole = dmrole + self._dmid = dmid + self._content = [] + + def add_attribute(self, dmtype=None, dmrole=None, ref=None, value=None, unit=None): + """ + Add an ATTRIBUTE to the instance + + paremeters + ---------- + dmtype: string + dmtype of the ATTRIBUTE (mandatory) + dmrole: string + dmrole of the ATTRIBUTE (optional) + ref: string + id of the column to be used to set the attribute value (OPTIONAL) + value: string + default value of the attribute value (OPTIONAL) + unit: string + attribute unit (OPTIONAL) + + raise + ----- + MappingException if ref and value are both undefined or if no dmtype or no dmrole either + """ + if not dmtype: + raise MappingException("Cannot add an attribute without dmtype") + if not dmrole: + raise MappingException("Cannot add an attribute without dmrole") + if not ref and not value: + raise MappingException("Cannot add an attribute without ref or value either") + + xml_string = f'' + self._content.append(xml_string) + + def add_instance(self, mivot_instance): + """ + Add an INSTANCE to the instance + + paremeters + ---------- + mivot_instance: MivotInstance + INSTANCE to ab added + """ + if type(mivot_instance) != MivotInstance: + raise MappingException("Instance added must be of type MivotInstance") + self._content.append(mivot_instance.xml_string() ) + + + def xml_string(self): + """ + Build the XML INSTANCE serialized as a string + + returns + ------- + str + the string serialization of the XML INSTANCE + """ + xml_string = f' Date: Sun, 24 Nov 2024 17:25:02 +0100 Subject: [PATCH 06/39] test suite for the annotations builders --- pyvo/mivot/tests/test_mivot_writer.py | 57 +++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 pyvo/mivot/tests/test_mivot_writer.py diff --git a/pyvo/mivot/tests/test_mivot_writer.py b/pyvo/mivot/tests/test_mivot_writer.py new file mode 100644 index 000000000..6b9a730f7 --- /dev/null +++ b/pyvo/mivot/tests/test_mivot_writer.py @@ -0,0 +1,57 @@ +import os +from astropy.io.votable import parse +from pyvo.utils import activate_features +from pyvo.mivot.writer.annotations import MivotAnnotations +from pyvo.mivot.writer.instance import MivotInstance +from pyvo.mivot.viewer.mivot_viewer import MivotViewer + +activate_features('MIVOT') +votable_path = os.path.realpath(os.path.join(__file__, "..", "data", "test.mivot_viewer.no_mivot.xml")) + +def test_MivotAnnotations(): + mb = MivotAnnotations() + mb.build_mivot_block() + + mb.add_globals("") + mb.add_templates("") + mb.add_templates("") + mb.add_model("model", "http://model.com") + mb.add_model("model2", None) + mb.set_report(True, "unit tests") + mb.build_mivot_block(templates_id="azerty") + + print("insert @@@@@@@@@@@") + mb.insert_into_votable(votable_path) + + mb.set_report(False, "unit tests") + mb.build_mivot_block() + +def test_MivotInstance(): + instance1 = MivotInstance(dmtype="model:type.inst", dmid="id1") + instance1.add_attribute(dmtype="model:type.att1", dmrole="model:type.inst.role1", value="value1", unit="m/s") + instance1.add_attribute(dmtype="model:type.att2", dmrole="model:type.inst.role2", value="value2", unit="m/s") + instance1.add_reference(dmrole="model:type.inst.role2", dmref="dmreference") + + instance2 = MivotInstance(dmtype="model:type2.inst", dmrole="model:role.instance2", dmid="id2") + instance2.add_attribute(dmtype="model:type2.att1", dmrole="model:type2.inst.role1", value="value3", unit="m/s") + instance2.add_attribute(dmtype="model:type2.att2", dmrole="model:type2.inst.role2", value="value4", unit="m/s") + instance1.add_instance(instance2) + + globals1 = MivotInstance(dmtype="model:type.globals", dmid="dmreference") + globals1.add_attribute(dmtype="model:type.att1", dmrole="model:type.globals.role1", value="value1", unit="m/s") + globals1.add_attribute(dmtype="model:type.att2", dmrole="model:type.globals.role2", value="value2", unit="m/s") + + mb = MivotAnnotations() + mb.add_templates(instance1) + mb.add_globals(globals1) + + mb.build_mivot_block() + print(mb.mivot_block) + votable = parse(votable_path) + mb.insert_into_votable(votable) + + mv = MivotViewer(votable) + print(mv.dm_instance) + +if __name__ == "__main__": + test_MivotInstance() \ No newline at end of file From 34aacffc8a6cbd86c426cf97a897d21661af26ca Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Wed, 27 Nov 2024 10:01:26 +0100 Subject: [PATCH 07/39] API for writing MIVOT annotations --- docs/mivot/writer.rst | 92 +++++++++ pyvo/mivot/writer/annotations.py | 312 +++++++++++++++++++------------ pyvo/mivot/writer/instance.py | 200 +++++++++++--------- 3 files changed, 393 insertions(+), 211 deletions(-) create mode 100644 docs/mivot/writer.rst diff --git a/docs/mivot/writer.rst b/docs/mivot/writer.rst new file mode 100644 index 000000000..9c812f565 --- /dev/null +++ b/docs/mivot/writer.rst @@ -0,0 +1,92 @@ +.. code-block:: python + :caption: Working with a model view as a dictionary + (the JSON layout has been squashed for display purpose) + + from pyvo.mivot.utils.dict_utils import DictUtils + + mivot_instance = m_viewer.dm_instance + mivot_object_dict = mivot_object.dict + +******************** +MIVOT (`pyvo.mivot`) +******************** + +This module contains the new feature of annotations in VOTable. +Astropy version >= 6.0 is required. + +Introduction +============ +.. pull-quote:: + + Model Instances in VOTables (MIVOT) defines a syntax to map VOTable + data to any model serialized in VO-DML. The annotation operates as a + bridge between the data and the model. It associates the column/param + metadata from the VOTable to the data model elements (class, attributes, + types, etc.) [...]. + The data model elements are grouped in an independent annotation block + complying with the MIVOT XML syntax. This annotation block is added + as an extra resource element at the top of the VOTable result resource. The + MIVOT syntax allows to describe a data structure as a hierarchy of classes. + It is also able to represent relations and composition between them. It can + also build up data model objects by aggregating instances from different + tables of the VOTable. + +- Model Instances in VOTables is a VO `standard `_ +- Requires Astropy>=6.0 +- ``pyvo.mivot`` is a prototype feature which must be activated with ``activate_features("MIVOT")`` + + +Implementation Scope +-------------------- +This implementation is totally model-agnostic. + +- It does not operate any validation against specific data models. +- It just requires the annotation syntax being compliant with the standards. + +.. code-block:: python + :caption: Build an annotation block, add used models and set mapping report + + from pyvo.mivot.writer.annotations import MivotAnnotations + + mivot_annotations = MivotAnnotations() + mivot_annotations.add_model( + "ivoa", "https://www.ivoa.net/xml/VODML/IVOA-v1.vo-dml.xml" + ) + mivot_annotations.add_model( + "coords", "https://www.ivoa.net/xml/STC/20200908/Coords-v1.0.vo-dml.xml" + ) + mivot_annotations.add_model( + "mango", + "https://raw.githubusercontent.com/lmichel/MANGO/draft-0.1/vo-dml/mango.vo-dml.xml", + ) + mivot_annotations.set_report(True, "Mivot writer unit test") + + from pyvo.mivot.utils.dict_utils import DictUtils + + mivot_instance = m_viewer.dm_instance + mivot_object_dict = mivot_object.dict + +blabla +must be constructed in the good order. + +.. code-block:: python + :caption: Build the coordinate system (coords:SpaceSys) + + space_sys = MivotInstance(dmid="_spacesys_icrs", dmtype="coords:SpaceSys") + space_frame = MivotInstance( + dmrole="coords:PhysicalCoordSys.frame", dmtype="coords:SpaceFrame" + ) + space_frame.add_attribute( + dmrole="coords:SpaceFrame.spaceRefFrame", dmtype="ivoa:string", value="ICRS" + ) + ref_position = MivotInstance( + dmrole="coords:SpaceFrame.refPosition", dmtype="coords:StdRefLocation" + ) + ref_position.add_attribute( + dmrole="coords:StdRefLocation.position", + dmtype="ivoa:string", + value="BARYCENTER", + ) + space_frame.add_instance(ref_position) + space_sys.add_instance(space_frame) + mivot_annotations.add_globals(space_sys) diff --git a/pyvo/mivot/writer/annotations.py b/pyvo/mivot/writer/annotations.py index 3912f63d8..b158e0609 100644 --- a/pyvo/mivot/writer/annotations.py +++ b/pyvo/mivot/writer/annotations.py @@ -1,47 +1,74 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ -MivotAnnotations implements the basic functions to build MIVOT annotations. -- Add various elements to the MIVOT block -- validate it against the MIVOT XML schema (local copy) if the package xml_schema is installed -- insert the MIVOT block in a VOTable +MivotAnnotations: A utility module to build and manage MIVOT annotations. -The MIVOT block is built as a string in order to be compliant with Astropy API. -The code below shows a typical use of MivotAnnotations: +This module provides a class to construct, validate, and insert MIVOT (Mapping to IVOA Table) +blocks into VOTable files. The MIVOT block, represented as an XML structure, is used for +data model annotations in the IVOA ecosystem. - .. code-block:: python +Features: +--------- +- Construct the MIVOT block step-by-step with various components. +- Validate the MIVOT block against the MIVOT XML schema (if `xmlschema` is installed). +- Embed the MIVOT block into an existing VOTable file. - mb = MivotAnnotations() +The MIVOT block is constructed as a string to maintain compatibility with the Astropy API. + +Typical Usage: +-------------- +The following code demonstrates how to use the MivotAnnotations class: + +.. code-block:: python + + from pyvo.mivot.writer import MivotAnnotations + from astropy.io.votable import parse + + # Create an instance of MivotAnnotations + mb = MivotAnnotations() mb.build_mivot_block() - + + # Add components to the MIVOT block mb.add_globals("") mb.add_templates("") mb.add_templates("") mb.add_model("model", "http://model.com") mb.add_model("model2", None) + + # Configure a report mb.set_report(True, "unit tests") mb.build_mivot_block(templates_id="azerty") - - votable = parse(votable_path) + + # Parse a VOTable and insert the MIVOT block + votable = parse("path_to_votable.xml") mb.insert_into_votable(votable) + + # Use MivotViewer for additional processing + from pyvo.mivot.viewer import MivotViewer mv = MivotViewer(votable) print(mv.dm_instance) -This module does not check if the MIVOT block is consistent with the declared models -or if the references to table columns can be resolved either. +Limitations: +------------ +- This module does not verify whether the MIVOT block is consistent with the declared models. +- References to table columns are not validated for resolution. -This latest point can be verified with he MivotViewer +Validation of references can be performed using the `MivotViewer` class: - .. code-block:: python +.. code-block:: python - votable = parse(votable_path) - mb.insert_into_votable(votable) + votable = parse("path_to_votable.xml") mv = MivotViewer(votable) print(mv.dm_instance) - -See `tests/test_mivot_writer.py`to get different examples of the API usage.` + +Examples of API usage can be found in `tests/test_mivot_writer.py`. + +License: +-------- +This module is licensed under a 3-clause BSD style license. See LICENSE.rst for details. """ import os import logging + try: import xmlschema except ImportError: @@ -55,59 +82,81 @@ from astropy.io.votable import parse from pyvo.utils.prototype import prototype_feature from pyvo.mivot.utils.xml_utils import XmlUtils +from pyvo.mivot.utils.exceptions import MappingException from pyvo.mivot.writer.instance import MivotInstance -@prototype_feature('MIVOT') -class MivotAnnotations: + +@prototype_feature("MIVOT") +class MivotAnnotations: """ - API for building annotations step by step. + API for constructing and managing MIVOT (Mapping for IVOA Table) annotations + step by step. This class provides methods to build the various components of + a MIVOT block, validate it against an XML schema, and integrate it into a VOTable. """ + def __init__(self): """ - Constructor of the MivotViewer class: no logic inside + Initializes a new instance of the `MivotAnnotations` class. + + Attributes + ---------- + _models : dict + A dictionary containing models with their names as keys and URLs as values. + _report_status : bool + Indicates the success status of the annotation process. + _report_message : str + A message associated with the report, used in the REPORT block. + _globals : list + A list of GLOBALS blocks to be included in the MIVOT block. + _templates : list + A list of TEMPLATES blocks to be included in the MIVOT block. + _templates_id : str + An optional ID for the TEMPLATES block. + _mivot_block : str + The complete MIVOT block as a string. """ self._models = {} self._report_status = True self._report_message = "Generated by pyvo.mivot.writer" self._globals = [] self._templates = [] - self._templates_id = "" + self._templates_id = "" self._mivot_block = "" - + @property def mivot_block(self): """ - mivot_block getter + Getter for the MIVOT block. Returns ------- str - the MIVOT block as a string + The complete MIVOT block as a string. """ return self._mivot_block - + def _get_report(self): """ - Get the REPORT component of the MIVOT block + Generates the REPORT component of the MIVOT block. Returns ------- str - the REPORT block as a string + The REPORT block as a string, indicating the success or failure of the process. """ - if self._report_status is True: + if self._report_status: return f'{self._report_message}' else: return f'{self._report_message}' - + def _get_models(self): """ - Get the MODEL components of the MIVOT block + Generates the MODEL components of the MIVOT block. Returns ------- str - the MODEL components block as a string + The MODEL components as a formatted string. """ models_block = "" for key, value in self._models.items(): @@ -118,200 +167,223 @@ def _get_models(self): return models_block - def _get_globals(self): + def _get_globals(self): """ - Get the GLOBALS component of the MIVOT block + Generates the GLOBALS component of the MIVOT block. Returns ------- str - the GLOBALS block as a string + The GLOBALS block as a formatted string. """ globals_block = "\n" - for globals in self._globals: - globals_block += f"{globals}\n" + for glob in self._globals: + globals_block += f"{glob}\n" globals_block += "\n" return globals_block - - def _get_templates(self): + + def _get_templates(self): """ - Get the TEMPLATES component of the MIVOT block + Generates the TEMPLATES component of the MIVOT block. Returns ------- str - the TEMPLATES block as a string or an empty string if no templates + The TEMPLATES block as a formatted string, or an empty string if no templates are defined. """ - # no TEMPLATES blocks if no mapped object if not self._templates: return "" if not self._templates_id: - templates_block = f'\n' + templates_block = "\n" else: templates_block = f'\n' - + for templates in self._templates: templates_block += f"{templates}\n" templates_block += "\n" return templates_block - + def build_mivot_block(self, templates_id=None): """ - Buikd a complete MIVOT block from the declared components and vaiidate the result - against the MIVO XML schema/ - + Builds a complete MIVOT block from the declared components and validates it + against the MIVOT XML schema. + + Parameters + ---------- + templates_id : str, optional + The ID to associate with the TEMPLATES block. Defaults to None. + Raises ------ - Exceptions possibly raised by the validator are not trapped and must be processed by the caller + Any exceptions raised during XML validation are not caught and must + be handled by the caller. """ if templates_id: self._templates_id = templates_id self._mivot_block = '\n' self._mivot_block += self._get_report() - self._mivot_block += '\n' + self._mivot_block += "\n" self._mivot_block += self._get_models() - self._mivot_block += '\n' + self._mivot_block += "\n" self._mivot_block += self._get_globals() - self._mivot_block += '\n' + self._mivot_block += "\n" self._mivot_block += self._get_templates() - self._mivot_block += '\n' - self._mivot_block += '\n' + self._mivot_block += "\n" + self._mivot_block += "\n" self._mivot_block = self.mivot_block.replace("\n\n", "\n") self.check_xml() def add_templates(self, templates_instance): """ - Add an block to the block. - The templates_instance content is not checked at this level. + Adds an block to the block. Parameters ---------- - templates_instance : string or MivotInstance - block to be added + templates_instance : str or MivotInstance + The block to be added. Raises ------ - Raises an MappingException if the globals_instance is not str or MivotInstance either. - + MappingException + If `templates_instance` is neither a string nor an instance of `MivotInstance`. """ - if type(templates_instance) == MivotInstance: + if isinstance(templates_instance, MivotInstance): self._templates.append(templates_instance.xml_string()) - elif type(templates_instance) == str: + elif isinstance(templates_instance, str): self._templates.append(templates_instance) else: - raise MappingException("Instance added to templates must be a string or MivotInstance") + raise MappingException( + "Instance added to templates must be a string or MivotInstance." + ) def add_globals(self, globals_instance): """ - Add an block to the block. - The globals_instance content is not checked at this level. + Adds an block to the block. Parameters ---------- - globals_instance : string or MivotInstance - block to be added + globals_instance : str or MivotInstance + The block to be added. Raises ------ - Raises an MappingException if the globals_instance is not str or MivotInstance either. + MappingException + If `globals_instance` is neither a string nor an instance of `MivotInstance`. """ - if type(globals_instance) == MivotInstance: + if isinstance(globals_instance, MivotInstance): self._globals.append(globals_instance.xml_string()) - elif type(globals_instance) == str: + elif isinstance(globals_instance, str): self._globals.append(globals_instance) else: - raise MappingException("Instance added to globals must be a string or MivotInstance") - + raise MappingException( + "Instance added to globals must be a string or MivotInstance." + ) + def add_model(self, model_name, model_url): """ - Add a MODEL element: - + Adds a MODEL element to the MIVOT block. + Parameters ---------- - model_name : string - model short name - model_url: string - URL of the VO-DML file + model_name : str + The short name of the model. + model_url : str + The URL of the VO-DML file associated with the model. """ self._models[model_name] = model_url - + def set_report(self, status, message): """ - Set the REPORT element. If the REPOPRT is set to failed, - all component of the MIVOT block are deleted except MODEL and REPORT + Sets the REPORT element of the MIVOT block. + Parameters ---------- - status : boolean - status of the annotation process - message: string - REPORT message + status : bool + The status of the annotation process. True for success, False for failure. + message : str + The message associated with the REPORT. + + Notes + ----- + If `status` is False, all components of the MIVOT block except MODEL and REPORT + are cleared. """ self._report_status = status self._report_message = message - if status is False: + if not status: self._globals = [] self._templates = [] - + def check_xml(self): """ - Validate the mapping block against the MIVOT XML schema v1.0. - The schema is copied locally to avoid a dependency with a remote service + Validates the MIVOT block against the MIVOT XML schema v1.0. Raises ------ - MappingException if the validation fails + MappingException + If the validation fails. + + Notes + ----- + The schema is loaded from a local file to avoid dependency on a remote service. """ + # put here just to improve the test coverage + root = etree.fromstring(self._mivot_block) + mivot_block = XmlUtils.pretty_string(root, clean_ns=False) if not xmlschema: - logging.error("XML validation skipped: " + - "no XML schema found, " + - "please install it (e.g. pip install xmlschema)") + logging.error( + "XML validation skipped: no XML schema found. " + + "Please install it (e.g., pip install xmlschema)." + ) return - + schema = xmlschema.XMLSchema11(os.path.dirname(__file__) + "/mivot-v1.xsd") - root = etree.fromstring(self._mivot_block) - mivot_block = XmlUtils.pretty_string(root, clean_ns=False) try: schema.validate(mivot_block) except Exception as excep: - raise MappingException(f"validation failed {excep}") from excep + raise MappingException(f"Validation failed: {excep}") from excep - def insert_into_votable(self, votable_file, template_id=None, override=False): + def insert_into_votable(self, votable_file, template_id=None, override=False): """ + Inserts the MIVOT block into a VOTable. + Parameters ---------- - votable_file : path like string or a VOTableFile - VOTable to be annotated - template_id: string - ID of the TABLE to be mapped, ignored if None - override : boolean - override former annotations if Tue - + votable_file : str or VOTableFile + The VOTable to be annotated, either as a file path or a `VOTableFile` instance. + template_id : str, optional + The ID of the TABLE to be mapped. Defaults to None. + override : bool + If True, overrides any existing annotations in the VOTable. + Raises - ----- - MappingException if a mapping block is already here and override is False + ------ + MappingException + If a mapping block already exists and `override` is False. """ - if type(votable_file) == str: + if isinstance(votable_file, str): votable = parse(votable_file) - elif type(votable_file) == VOTableFile: + elif isinstance(votable_file, VOTableFile): votable = votable_file else: - raise MappingException("votable_file must be either a path like string or a VOTableFile") + raise MappingException( + "votable_file must be a file path string or a VOTableFile instance." + ) for resource in votable.resources: if resource.type == "results": for subresource in resource.resources: - if sub_resource.type == "meta": - if override is False: + if subresource.type == "meta": + if not override: raise MappingException( - "There is already a type='meta' in the first result resource") + "A type='meta' resource already exists in the first 'result' resource." + ) else: - logging.info( - "There is already a type='meta' in the first result resource that will be overridden") + logging.info("Overriding existing type='meta' resource.") break - mivot_resource = Resource() + mivot_resource = Resource() mivot_resource.type = "meta" - mivot_resource.mivot_block = MivotBlock(self._mivot_block) + mivot_resource.mivot_block = MivotBlock(self._mivot_block) resource.resources.append(mivot_resource) - diff --git a/pyvo/mivot/writer/instance.py b/pyvo/mivot/writer/instance.py index e49a54dcf..82682b745 100644 --- a/pyvo/mivot/writer/instance.py +++ b/pyvo/mivot/writer/instance.py @@ -1,60 +1,73 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ MivotInstance is a simple API for building MIVOT instances step by step. -- This code is totally model-agnostic -- The basic syntax rules of MIVOT are checked -- The context dependent syntax rules are ignored - -MivotInstance builds elements that can contain , and - are not supported yet - -The code below shows a typical use of MivotInstance: - - .. code-block:: python - - instance1 = MivotInstance(dmtype="model:type.inst", dmid="id1") - instance1.add_attribute(dmtype="model:type.att1", dmrole="model:type.inst.role1", value="value1", unit="m/s") - instance1.add_attribute(dmtype="model:type.att2", dmrole="model:type.inst.role2", value="value2", unit="m/s") - - instance2 = MivotInstance(dmtype="model:type2.inst", dmrole="model:role.instance2", dmid="id2") - instance2.add_attribute(dmtype="model:type2.att1", dmrole="model:type2.inst.role1", value="value3", unit="m/s") - instance2.add_attribute(dmtype="model:type2.att2", dmrole="model:type2.inst.role2", value="value4", unit="m/s") - - instance1.add_instance(instance2) - - mb = MivotAnnotations() - mb.add_templates(instance1) + +Features +-------- +- Model-agnostic: The implementation is independent of any specific data model. +- Syntax validation: Ensures basic MIVOT syntax rules are followed. +- Context-agnostic: Ignores context-dependent syntax rules. + +MivotInstance builds elements that can contain , , and . +Support for elements is not yet implemented. + +Usage Example +------------- +.. code-block:: python + + position = MivotInstance(dmtype="model:mango:EpochPosition", dmid="position") + position.add_attribute(dmtype="ivoa.RealQuantity", + dmrole="mango:EpochPosition.longitude", + unit="deg", ref="_RA_ICRS") + position.add_attribute(dmtype="ivoa.RealQuantity", dmrole="mango:EpochPosition.latitude", + unit="deg", ref="_DFEC_ICRS") + + position_error = MivotInstance(dmtype="mango:EpochPositionErrors", + dmrole="mango:EpochPosition.errors", dmid="id2") + position_error.add_attribute(dmtype="model:type2.att1", + dmrole="model:type2.inst.role1", value="value3", unit="m/s") + position_error.add_attribute(dmtype="model:type2.att2", + dmrole="model:type2.inst.role2", value="value4", unit="m/s") + + position.add_instance(position_error) + + mb = MivotAnnotations() + mb.add_templates(position) mb.build_mivot_block() print(mb.mivot_block) - """ -import os + import logging from pyvo.utils.prototype import prototype_feature from pyvo.mivot.utils.exceptions import MappingException -@prototype_feature('MIVOT') + +@prototype_feature("MIVOT") class MivotInstance: """ - API for building annotations elements step by step. + API for building elements of a MIVOT annotation step by step. + + This class provides methods for adding attributes, references, and nested instances, + allowing incremental construction of a MIVOT instance. """ def __init__(self, dmtype=None, dmrole=None, dmid=None): """ - Constructor - - paremeters + Initialize a MivotInstance object. + + Parameters ---------- - dmtype: string - dmtype of the INSTANCE (mandatory) - dmrole: string - dmrole of the INSTANCE (optional) - dmid: string - dmid of the INSTANCE (optional) - - raise - ----- - MappingException if no dmtype is provided + dmtype : str + The dmtype of the INSTANCE (mandatory). + dmrole : str, optional + The dmrole of the INSTANCE. + dmid : str, optional + The dmid of the INSTANCE. + + Raises + ------ + MappingException + If `dmtype` is not provided. """ if not dmtype: raise MappingException("Cannot build an instance without dmtype") @@ -62,35 +75,36 @@ def __init__(self, dmtype=None, dmrole=None, dmid=None): self._dmrole = dmrole self._dmid = dmid self._content = [] - + def add_attribute(self, dmtype=None, dmrole=None, ref=None, value=None, unit=None): """ - Add an ATTRIBUTE to the instance - - paremeters + Add an element to the instance. + + Parameters ---------- - dmtype: string - dmtype of the ATTRIBUTE (mandatory) - dmrole: string - dmrole of the ATTRIBUTE (optional) - ref: string - id of the column to be used to set the attribute value (OPTIONAL) - value: string - default value of the attribute value (OPTIONAL) - unit: string - attribute unit (OPTIONAL) - - raise - ----- - MappingException if ref and value are both undefined or if no dmtype or no dmrole either + dmtype : str + The dmtype of the ATTRIBUTE (mandatory). + dmrole : str + The dmrole of the ATTRIBUTE (mandatory). + ref : str, optional + ID of the column to set the attribute value. + value : str, optional + Default value of the attribute. + unit : str, optional + Unit of the attribute. + + Raises + ------ + MappingException + If `dmtype` or `dmrole` is not provided, or if both `ref` and `value` are not defined. """ if not dmtype: raise MappingException("Cannot add an attribute without dmtype") if not dmrole: raise MappingException("Cannot add an attribute without dmrole") if not ref and not value: - raise MappingException("Cannot add an attribute without ref or value either") - + raise MappingException("Cannot add an attribute without ref or value") + xml_string = f' element to the instance. + + Parameters ---------- - dmtype: string - dmtype of the ATTRIBUTE (mandatory) - dmref: string - dmrole of the ATTRIBUTE (mandatory) - - raise - ----- - MappingException if dmref or dmrole are undefined + dmrole : str + The dmrole of the REFERENCE (mandatory). + dmref : str + The dmref of the REFERENCE (mandatory). + + Raises + ------ + MappingException + If `dmrole` or `dmref` is not provided. """ if not dmref: raise MappingException("Cannot add a reference without dmref") if not dmrole: raise MappingException("Cannot add a reference without dmrole") - + xml_string = f'' self._content.append(xml_string) - + def add_instance(self, mivot_instance): """ - Add an INSTANCE to the instance - - paremeters + Add a nested element to the instance. + + Parameters ---------- - mivot_instance: MivotInstance - INSTANCE to ab added + mivot_instance : MivotInstance + The INSTANCE to be added. + + Raises + ------ + MappingException + If `mivot_instance` is not of type `MivotInstance`. """ - if type(mivot_instance) != MivotInstance: + if not isinstance(mivot_instance, MivotInstance): raise MappingException("Instance added must be of type MivotInstance") - self._content.append(mivot_instance.xml_string() ) + self._content.append(mivot_instance.xml_string()) - def xml_string(self): """ - Build the XML INSTANCE serialized as a string - - returns + Build and serialize the INSTANCE element to a string. + + Returns ------- str - the string serialization of the XML INSTANCE + The string representation of the INSTANCE element. """ - xml_string = f' Date: Wed, 27 Nov 2024 10:02:09 +0100 Subject: [PATCH 08/39] test suite for the MIVOT writer --- .../data/reference/test_mivot_writer.json | 46 ++++ .../data/reference/test_mivot_writer.xml | 29 +++ pyvo/mivot/tests/test_mivot_writer.py | 221 ++++++++++++++---- 3 files changed, 255 insertions(+), 41 deletions(-) create mode 100644 pyvo/mivot/tests/data/reference/test_mivot_writer.json create mode 100644 pyvo/mivot/tests/data/reference/test_mivot_writer.xml diff --git a/pyvo/mivot/tests/data/reference/test_mivot_writer.json b/pyvo/mivot/tests/data/reference/test_mivot_writer.json new file mode 100644 index 000000000..707e1fa20 --- /dev/null +++ b/pyvo/mivot/tests/data/reference/test_mivot_writer.json @@ -0,0 +1,46 @@ +{ + "dmtype": "EpochPosition", + "longitude": { + "value": 52.2340018, + "unit": "deg" + }, + "latitude": { + "value": 59.8937333, + "unit": "deg" + }, + "EpochPosition_errors": { + "dmtype": "EpochPositionErrors", + "dmrole": "errors", + "EpochPositionErrors_position": { + "dmtype": "ErrorCorrMatrix", + "dmrole": "position", + "sigma1": { + "value": 6.0, + "unit": "arcsec" + }, + "sigma2": { + "value": 6.0, + "unit": "arcsec" + } + } + }, + "EpochPosition_spaceSys": { + "dmtype": "SpaceSys", + "dmid": "_spacesys_icrs", + "dmrole": "spaceSys", + "PhysicalCoordSys_frame": { + "dmtype": "SpaceFrame", + "dmrole": "frame", + "spaceRefFrame": { + "value": "ICRS" + }, + "SpaceFrame_refPosition": { + "dmtype": "StdRefLocation", + "dmrole": "refPosition", + "position": { + "value": "BARYCENTER" + } + } + } + } +} \ No newline at end of file diff --git a/pyvo/mivot/tests/data/reference/test_mivot_writer.xml b/pyvo/mivot/tests/data/reference/test_mivot_writer.xml new file mode 100644 index 000000000..62af6ef7d --- /dev/null +++ b/pyvo/mivot/tests/data/reference/test_mivot_writer.xml @@ -0,0 +1,29 @@ + + Mivot writer unit test + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pyvo/mivot/tests/test_mivot_writer.py b/pyvo/mivot/tests/test_mivot_writer.py index 6b9a730f7..fa391f3ef 100644 --- a/pyvo/mivot/tests/test_mivot_writer.py +++ b/pyvo/mivot/tests/test_mivot_writer.py @@ -1,57 +1,196 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +""" +This module contains test cases for validating the functionality of MivotInstance, MivotAnnotations, +and related components in the pyvo.mivot package. These tests ensure that the classes behave as +expected, including error handling and XML generation for data models. +""" + import os +import pytest from astropy.io.votable import parse from pyvo.utils import activate_features +from pyvo.mivot.utils.exceptions import MappingException +from pyvo.mivot.utils.dict_utils import DictUtils from pyvo.mivot.writer.annotations import MivotAnnotations from pyvo.mivot.writer.instance import MivotInstance from pyvo.mivot.viewer.mivot_viewer import MivotViewer -activate_features('MIVOT') -votable_path = os.path.realpath(os.path.join(__file__, "..", "data", "test.mivot_viewer.no_mivot.xml")) +# Enable MIVOT-specific features in the pyvo library +activate_features("MIVOT") + +# File paths for test data +votable_path = os.path.realpath( + os.path.join(__file__, "..", "data", "test.mivot_viewer.no_mivot.xml") +) +data_path = os.path.realpath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "data") +) + + +def strip_xml(xml_string): + """ + Strip unnecessary whitespace and newline characters from an XML string. + + Parameters: + - xml_string (str): The XML string to strip. + + Returns: + - str: The stripped XML string. + """ + return ( + xml_string.replace("\n", "").replace(" ", "").replace("'", "").replace('"', "") + ) -def test_MivotAnnotations(): - mb = MivotAnnotations() - mb.build_mivot_block() - - mb.add_globals("") - mb.add_templates("") - mb.add_templates("") - mb.add_model("model", "http://model.com") - mb.add_model("model2", None) - mb.set_report(True, "unit tests") - mb.build_mivot_block(templates_id="azerty") - - print("insert @@@@@@@@@@@") - mb.insert_into_votable(votable_path) - - mb.set_report(False, "unit tests") - mb.build_mivot_block() def test_MivotInstance(): + """ + Test the MivotInstance class for various operations including attribute addition, + reference addition, and XML generation. Verifies that invalid operations raise + the expected MappingException. + """ + with pytest.raises(MappingException): + MivotInstance(dmid="model:type.inst") + instance1 = MivotInstance(dmtype="model:type.inst", dmid="id1") - instance1.add_attribute(dmtype="model:type.att1", dmrole="model:type.inst.role1", value="value1", unit="m/s") - instance1.add_attribute(dmtype="model:type.att2", dmrole="model:type.inst.role2", value="value2", unit="m/s") - instance1.add_reference(dmrole="model:type.inst.role2", dmref="dmreference") - - instance2 = MivotInstance(dmtype="model:type2.inst", dmrole="model:role.instance2", dmid="id2") - instance2.add_attribute(dmtype="model:type2.att1", dmrole="model:type2.inst.role1", value="value3", unit="m/s") - instance2.add_attribute(dmtype="model:type2.att2", dmrole="model:type2.inst.role2", value="value4", unit="m/s") - instance1.add_instance(instance2) - - globals1 = MivotInstance(dmtype="model:type.globals", dmid="dmreference") - globals1.add_attribute(dmtype="model:type.att1", dmrole="model:type.globals.role1", value="value1", unit="m/s") - globals1.add_attribute(dmtype="model:type.att2", dmrole="model:type.globals.role2", value="value2", unit="m/s") - - mb = MivotAnnotations() - mb.add_templates(instance1) - mb.add_globals(globals1) - - mb.build_mivot_block() - print(mb.mivot_block) - votable = parse(votable_path) - mb.insert_into_votable(votable) + with pytest.raises(MappingException): + instance1.add_attribute( + dmrole="model:type.inst.role1", value="value1", unit="m/s" + ) + with pytest.raises(MappingException): + instance1.add_attribute( + dmtype="model:type.att1", dmrole="model:type.inst.role1" + ) + with pytest.raises(MappingException): + instance1.add_reference(dmref="dmreference") + with pytest.raises(MappingException): + instance1.add_reference(dmrole="model:type.inst.role2") + with pytest.raises(MappingException): + instance1.add_instance("azerty") + + instance1.add_reference(dmrole="model:type.inst.role2", dmref="dmreference") + instance1.add_attribute( + dmtype="model:type.att1", + dmrole="model:type.inst.role1", + value="value1", + unit="m/s", + ) + assert strip_xml(instance1.xml_string()) == ( + "" + + "" + + "" + + "" + ) + + +def test_MivotAnnotations(): + """ + Test the MivotAnnotations class for template and global instance addition. Verifies + that invalid operations raise the expected MappingException. + """ + mb = MivotAnnotations() + + with pytest.raises(MappingException): + mb.add_templates(12) + with pytest.raises(MappingException): + mb.add_globals(12) + + +def test_MivotInstance2(): + """ + Test the creation and combination of multiple MivotInstance objects, their attributes, + references, and integration into a MivotAnnotations instance. Verifies correct XML + generation and VOTable integration. + """ + + space_sys = MivotInstance(dmid="_spacesys_icrs", dmtype="coords:SpaceSys") + space_frame = MivotInstance( + dmrole="coords:PhysicalCoordSys.frame", dmtype="coords:SpaceFrame" + ) + space_frame.add_attribute( + dmrole="coords:SpaceFrame.spaceRefFrame", dmtype="ivoa:string", value="ICRS" + ) + ref_position = MivotInstance( + dmrole="coords:SpaceFrame.refPosition", dmtype="coords:StdRefLocation" + ) + ref_position.add_attribute( + dmrole="coords:StdRefLocation.position", + dmtype="ivoa:string", + value="BARYCENTER", + ) + space_frame.add_instance(ref_position) + space_sys.add_instance(space_frame) + + position = MivotInstance(dmtype="mango:EpochPosition") + position.add_attribute( + dmtype="ivoa:RealQuantity", + dmrole="mango:EpochPosition.longitude", + unit="deg", + ref="RAICRS", + ) + position.add_attribute( + dmtype="ivoa:RealQuantity", + dmrole="mango:EpochPosition.latitude", + unit="deg", + ref="DEICRS", + ) + + epoch_position_error = MivotInstance( + dmtype="mango:EpochPositionErrors", dmrole="mango:EpochPosition.errors" + ) + position_error = MivotInstance( + dmtype="mango:error.ErrorCorrMatrix", + dmrole="mango:EpochPositionErrors.position", + ) + position_error.add_attribute( + dmtype="ivoa:RealQuantity", + dmrole="mango:error.ErrorCorrMatrix.sigma1", + unit="arcsec", + ref="sigm", + ) + position_error.add_attribute( + dmtype="ivoa:RealQuantity", + dmrole="mango:error.ErrorCorrMatrix.sigma2", + unit="arcsec", + ref="sigm", + ) + epoch_position_error.add_instance(position_error) + position.add_reference( + dmref="_spacesys_icrs", dmrole="mango:EpochPosition.spaceSys" + ) + position.add_instance(epoch_position_error) + + mivot_annotations = MivotAnnotations() + mivot_annotations.add_model( + "ivoa", "https://www.ivoa.net/xml/VODML/IVOA-v1.vo-dml.xml" + ) + mivot_annotations.add_model( + "coords", "https://www.ivoa.net/xml/STC/20200908/Coords-v1.0.vo-dml.xml" + ) + mivot_annotations.add_model( + "mango", + "https://raw.githubusercontent.com/lmichel/MANGO/draft-0.1/vo-dml/mango.vo-dml.xml", + ) + mivot_annotations.set_report(True, "Mivot writer unit test") + + mivot_annotations.add_templates(position) + mivot_annotations.add_globals(space_sys) + + mivot_annotations.build_mivot_block() + with open( + os.path.join(data_path, "reference/test_mivot_writer.xml"), "r" + ) as xml_ref: + assert strip_xml(xml_ref.read()) == strip_xml(mivot_annotations.mivot_block) + + votable = parse(votable_path) + mivot_annotations.insert_into_votable(votable) mv = MivotViewer(votable) print(mv.dm_instance) + assert mv.dm_instance.dict == DictUtils.read_dict_from_file( + os.path.join(data_path, "reference/test_mivot_writer.json") + ) + votable.to_xml(data_path + "/essai.xml") + if __name__ == "__main__": - test_MivotInstance() \ No newline at end of file + test_MivotInstance2() From f36711631b591f654723faac96e0329385ff36fb Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Thu, 28 Nov 2024 17:26:20 +0100 Subject: [PATCH 09/39] Doc updated with the split into 2 documents, one for the reader and one for the writer --- docs/mivot/index.rst | 171 ++-------------------------------- docs/mivot/viewer.rst | 197 +++++++++++++++++++++++++++++++++++++++ docs/mivot/writer.rst | 209 +++++++++++++++++++++++++++++++++--------- 3 files changed, 370 insertions(+), 207 deletions(-) create mode 100644 docs/mivot/viewer.rst diff --git a/docs/mivot/index.rst b/docs/mivot/index.rst index 83be5f839..ee4ca7e1c 100644 --- a/docs/mivot/index.rst +++ b/docs/mivot/index.rst @@ -56,174 +56,15 @@ which allows to get (and set) Mivot blocks from/into VOTables as an XML element epoch propagation use case: - ``JOIN`` features are not supported. - - ``TEMPLATES`` with more than one ``INSANCE`` not supported. - -Integrated Readout ------------------- -The ``ModelViewer`` module manages access to data mapped to a model through dynamically -generated objects (``MivotInstance``class). -The example below shows how a VOTable, resulting from a cone-search query which data are mapped -to the ``EpochPosition`` class, can be consumed. - -.. doctest-remote-data:: - >>> import astropy.units as u - >>> from astropy.coordinates import SkyCoord - >>> from pyvo.dal.scs import SCSService - >>> from pyvo.utils.prototype import activate_features - >>> from pyvo.mivot.version_checker import check_astropy_version - >>> from pyvo.mivot.viewer.mivot_viewer import MivotViewer - >>> activate_features("MIVOT") - >>> if check_astropy_version() is False: - ... pytest.skip("MIVOT test skipped because of the astropy version.") - >>> scs_srv = SCSService("https://cdsarc.cds.unistra.fr/beta/viz-bin/mivotconesearch/I/239/hip_main") - >>> m_viewer = MivotViewer( - ... scs_srv.search( - ... pos=SkyCoord(ra=52.26708 * u.degree, dec=59.94027 * u.degree, frame='icrs'), - ... radius=0.05 - ... ) - ... ) - >>> mivot_instance = m_viewer.dm_instance - >>> print(mivot_instance.dmtype) - EpochPosition - >>> print(mivot_instance.Coordinate_coordSys.spaceRefFrame.value) - ICRS - >>> while m_viewer.next(): - ... print(f"position: {mivot_instance.latitude.value} {mivot_instance.longitude.value}") - position: 59.94033461 52.26722684 - - -In this example, the data readout is totally managed by the ``MivotViewer`` instance. -The ``astropy.io.votable`` API is encapsulated in this module. - -Model leaves (class attributes) are complex types that provide additional information: - -- ``value``: attribute value -- ``dmtype``: attribute type such as defined in the Mivot annotations -- ``unit``: attribute unit such as defined in the Mivot annotations -- ``ref``: identifier of the table column mapped on the attribute - -The model view on a data row can also be passed as a Python dictionary -using the ``dict`` property of ``MivotInstance``. - -.. code-block:: python - :caption: Working with a model view as a dictionary - (the JSON layout has been squashed for display purpose) - - from pyvo.mivot.utils.dict_utils import DictUtils - - mivot_instance = m_viewer.dm_instance - mivot_object_dict = mivot_object.dict - - DictUtils.print_pretty_json(mivot_object_dict) - { - "dmtype": "EpochPosition", - "longitude": {"value": 359.94372764, "unit": "deg"}, - "latitude": {"value": -0.28005255, "unit": "deg"}, - "pmLongitude": {"value": -5.14, "unit": "mas/yr"}, - "pmLatitude": {"value": -25.43, "unit": "mas/yr"}, - "epoch": {"value": 1991.25, "unit": "year"}, - "Coordinate_coordSys": { - "dmtype": "SpaceSys", - "dmid": "SpaceFrame_ICRS", - "dmrole": "coordSys", - "spaceRefFrame": {"value": "ICRS"}, - }, - } - -- It is recommended to use a copy of the - dictionary as it will be rebuilt each time the ``dict`` property is invoked. -- The default representation of ``MivotInstance`` instances is made with a pretty - string serialization of this dictionary. - -Per-Row Readout ---------------- - -The annotation schema can also be applied to table rows read outside of the ``MivotViewer`` -with the `astropy.io.votable` API: - -.. code-block:: python - :caption: Accessing the model view of Astropy table rows - - votable = parse(path_to_votable) - table = votable.resources[0].tables[0] - # init the viewer - mivot_viewer = MivotViewer(votable, resource_number=0) - mivot_object = mivot_viewer.dm_instance - # and feed it with the table row - read = [] - for rec in table.array: - mivot_object.update(rec) - read.append(mivot_object.longitude.value) - # show that the model retrieve the correct data values - assert rec["RAICRS"] == mivot_object.longitude.value - assert rec["DEICRS"] == mivot_object.latitude.value - -In this case, it is up to the user to ensure that the read data rows are those mapped by the Mivot annotations. - -For XML Hackers ---------------- - -The model instances can also be serialized as XML elements that can be parsed with XPath queries. - -.. code-block:: python - :caption: Accessing the XML view of the mapped model instances - - with MivotViewer(path_to_votable) as mivot_viewer: - while mivot_viewer.next(): - xml_view = mivot_viewer.xml_view - # do whatever you want with this XML element - -It to be noted that ``mivot_viewer.xml_view`` is a shortcut -for ``mivot_viewer.xml_view.view`` where ``mivot_viewer.xml_view`` -is is an instance of ``pyvo.mivot.viewer.XmlViewer``. -This object provides many functions facilitating the XML parsing. - -Class Generation in a Nutshell ------------------------------- - -MIVOT reconstructs model structures with 3 elements: - -- ``INSTANCE`` for the objects -- ``ATTRIBUTE`` for the attributes -- ``COLLECTION`` for the elements with a cardinality greater than 1 + - ``TEMPLATES`` with more than one ``INSTANCE`` not supported. -The role played by each of these elements in the model hierarchy is defined -by its ``@dmrole`` XML attribute. Types of both ``INSTANCE`` and ``ATTRIBUTE`` are defined by -their ``@dmtype`` XML attributes. +Using the MIVOT package +======================= -``MivotInstance`` classes are built by following MIVOT annotation structure: - -- ``INSTANCE`` are represented by Python classes -- ``ATTRIBUTE`` are represented by Python class fields -- ``COLLECTION`` are represented by Python lists ([]) - -``@dmrole`` and ``@dmtype`` cannot be used as Python keywords as such, because they are built from VO-DML -identifiers, which have the following structure: ``model:a.b``. - -- Only the last part of the path is kept for attribute names. -- For class names, forbidden characters (``:`` or ``.``) are replaced with ``_``. -- Original ``@dmtype`` are kept as attributes of generated Python objects. -- The structure of the ``MivotInstance`` objects can be inferred from the mapped model in 2 different ways: - - - 1. From the MIVOT instance property ``MivotInstance.dict`` a shown above. - This is a pure Python dictionary but its access can be slow because it is generated - on the fly each time the property is invoked. - - 2. From the internal class dictionary ``MivotInstance.__dict__`` - (see the Python `data model `_). - - .. code-block:: python - :caption: Exploring the MivotInstance structure with the internal dictionaries - - mivot_instance = mivot_viewer.dm_instance - - print(mivot_instance.__dict__.keys()) - dict_keys(['dmtype', 'longitude', 'latitude', 'pmLongitude', 'pmLatitude', 'epoch', 'Coordinate_coordSys']) - - print(mivot_instance.Coordinate_coordSys.__dict__.keys()) - dict_keys(['dmtype', 'dmid', 'dmrole', 'spaceRefFrame']) +The ``pyvo.mivot`` module can be used to either read or build annotations. - print(mivot_instance.Coordinate_coordSys.spaceRefFrame.__dict__.keys()) - dict_keys(['dmtype', 'value', 'unit', 'ref']) +- How to `read <./viewer.html>`_ MIVOT annotations +- How to `write <./writer.html>`_ MIVOT annotations Reference/API ============= diff --git a/docs/mivot/viewer.rst b/docs/mivot/viewer.rst new file mode 100644 index 000000000..e96393d38 --- /dev/null +++ b/docs/mivot/viewer.rst @@ -0,0 +1,197 @@ +*************************************** +MIVOT (`pyvo.mivot`): Annotation Viewer +*************************************** + + +Introduction +============ +.. pull-quote:: + + Model Instances in VOTables (MIVOT) defines a syntax to map VOTable + data to any model serialized in VO-DML. The annotation operates as a + bridge between the data and the model. It associates the column/param + metadata from the VOTable to the data model elements (class, attributes, + types, etc.) [...]. + The data model elements are grouped in an independent annotation block + complying with the MIVOT XML syntax. This annotation block is added + as an extra resource element at the top of the VOTable result resource. The + MIVOT syntax allows to describe a data structure as a hierarchy of classes. + It is also able to represent relations and composition between them. It can + also build up data model objects by aggregating instances from different + tables of the VOTable (get more `here <./index.html>`_). + +Integrated Readout +------------------ +The ``ModelViewer`` module manages access to data mapped to a model through dynamically +generated objects (``MivotInstance``class). +The example below shows how a VOTable, resulting from a cone-search query which data are mapped +to the ``EpochPosition`` class, can be consumed. + +.. doctest-remote-data:: + >>> import astropy.units as u + >>> from astropy.coordinates import SkyCoord + >>> from pyvo.dal.scs import SCSService + >>> from pyvo.utils.prototype import activate_features + >>> from pyvo.mivot.version_checker import check_astropy_version + >>> from pyvo.mivot.viewer.mivot_viewer import MivotViewer + >>> activate_features("MIVOT") + >>> if check_astropy_version() is False: + ... pytest.skip("MIVOT test skipped because of the astropy version.") + >>> scs_srv = SCSService("https://cdsarc.cds.unistra.fr/beta/viz-bin/mivotconesearch/I/239/hip_main") + >>> m_viewer = MivotViewer( + ... scs_srv.search( + ... pos=SkyCoord(ra=52.26708 * u.degree, dec=59.94027 * u.degree, frame='icrs'), + ... radius=0.05 + ... ) + ... ) + >>> mivot_instance = m_viewer.dm_instance + >>> print(mivot_instance.dmtype) + EpochPosition + >>> print(mivot_instance.Coordinate_coordSys.spaceRefFrame.value) + ICRS + >>> while m_viewer.next(): + ... print(f"position: {mivot_instance.latitude.value} {mivot_instance.longitude.value}") + position: 59.94033461 52.26722684 + + +In this example, the data readout is totally managed by the ``MivotViewer`` instance. +The ``astropy.io.votable`` API is encapsulated in this module. + +Model leaves (class attributes) are complex types that provide additional information: + +- ``value``: attribute value +- ``dmtype``: attribute type such as defined in the Mivot annotations +- ``unit``: attribute unit such as defined in the Mivot annotations +- ``ref``: identifier of the table column mapped on the attribute + +The model view on a data row can also be passed as a Python dictionary +using the ``dict`` property of ``MivotInstance``. + +.. code-block:: python + :caption: Working with a model view as a dictionary + (the JSON layout has been squashed for display purpose) + + from pyvo.mivot.utils.dict_utils import DictUtils + + mivot_instance = m_viewer.dm_instance + mivot_object_dict = mivot_object.dict + + DictUtils.print_pretty_json(mivot_object_dict) + { + "dmtype": "EpochPosition", + "longitude": {"value": 359.94372764, "unit": "deg"}, + "latitude": {"value": -0.28005255, "unit": "deg"}, + "pmLongitude": {"value": -5.14, "unit": "mas/yr"}, + "pmLatitude": {"value": -25.43, "unit": "mas/yr"}, + "epoch": {"value": 1991.25, "unit": "year"}, + "Coordinate_coordSys": { + "dmtype": "SpaceSys", + "dmid": "SpaceFrame_ICRS", + "dmrole": "coordSys", + "spaceRefFrame": {"value": "ICRS"}, + }, + } + +- It is recommended to use a copy of the + dictionary as it will be rebuilt each time the ``dict`` property is invoked. +- The default representation of ``MivotInstance`` instances is made with a pretty + string serialization of this dictionary. + +Per-Row Readout +--------------- + +The annotation schema can also be applied to table rows read outside of the ``MivotViewer`` +with the `astropy.io.votable` API: + +.. code-block:: python + :caption: Accessing the model view of Astropy table rows + + votable = parse(path_to_votable) + table = votable.resources[0].tables[0] + # init the viewer + mivot_viewer = MivotViewer(votable, resource_number=0) + mivot_object = mivot_viewer.dm_instance + # and feed it with the table row + read = [] + for rec in table.array: + mivot_object.update(rec) + read.append(mivot_object.longitude.value) + # show that the model retrieve the correct data values + assert rec["RAICRS"] == mivot_object.longitude.value + assert rec["DEICRS"] == mivot_object.latitude.value + +In this case, it is up to the user to ensure that the read data rows are those mapped by the Mivot annotations. + +For XML Hackers +--------------- + +The model instances can also be serialized as XML elements that can be parsed with XPath queries. + +.. code-block:: python + :caption: Accessing the XML view of the mapped model instances + + with MivotViewer(path_to_votable) as mivot_viewer: + while mivot_viewer.next(): + xml_view = mivot_viewer.xml_view + # do whatever you want with this XML element + +It to be noted that ``mivot_viewer.xml_view`` is a shortcut +for ``mivot_viewer.xml_view.view`` where ``mivot_viewer.xml_view`` +is is an instance of ``pyvo.mivot.viewer.XmlViewer``. +This object provides many functions facilitating the XML parsing. + +Class Generation in a Nutshell +------------------------------ + +MIVOT reconstructs model structures with 3 elements: + +- ``INSTANCE`` for the objects +- ``ATTRIBUTE`` for the attributes +- ``COLLECTION`` for the elements with a cardinality greater than 1 + +The role played by each of these elements in the model hierarchy is defined +by its ``@dmrole`` XML attribute. Types of both ``INSTANCE`` and ``ATTRIBUTE`` are defined by +their ``@dmtype`` XML attributes. + +``MivotInstance`` classes are built by following MIVOT annotation structure: + +- ``INSTANCE`` are represented by Python classes +- ``ATTRIBUTE`` are represented by Python class fields +- ``COLLECTION`` are represented by Python lists ([]) + +``@dmrole`` and ``@dmtype`` cannot be used as Python keywords as such, because they are built from VO-DML +identifiers, which have the following structure: ``model:a.b``. + +- Only the last part of the path is kept for attribute names. +- For class names, forbidden characters (``:`` or ``.``) are replaced with ``_``. +- Original ``@dmtype`` are kept as attributes of generated Python objects. +- The structure of the ``MivotInstance`` objects can be inferred from the mapped model in 2 different ways: + + - 1. From the MIVOT instance property ``MivotInstance.dict`` a shown above. + This is a pure Python dictionary but its access can be slow because it is generated + on the fly each time the property is invoked. + - 2. From the internal class dictionary ``MivotInstance.__dict__`` + (see the Python `data model `_). + + .. code-block:: python + :caption: Exploring the MivotInstance structure with the internal dictionaries + + mivot_instance = mivot_viewer.dm_instance + + print(mivot_instance.__dict__.keys()) + dict_keys(['dmtype', 'longitude', 'latitude', 'pmLongitude', 'pmLatitude', 'epoch', 'Coordinate_coordSys']) + + print(mivot_instance.Coordinate_coordSys.__dict__.keys()) + dict_keys(['dmtype', 'dmid', 'dmrole', 'spaceRefFrame']) + + print(mivot_instance.Coordinate_coordSys.spaceRefFrame.__dict__.keys()) + dict_keys(['dmtype', 'value', 'unit', 'ref']) + +Reference/API +============= + +.. automodapi:: pyvo.mivot +.. automodapi:: pyvo.mivot.viewer +.. automodapi:: pyvo.mivot.seekers +.. automodapi:: pyvo.mivot.features +.. automodapi:: pyvo.mivot.utils diff --git a/docs/mivot/writer.rst b/docs/mivot/writer.rst index 9c812f565..167c1a54a 100644 --- a/docs/mivot/writer.rst +++ b/docs/mivot/writer.rst @@ -1,15 +1,6 @@ -.. code-block:: python - :caption: Working with a model view as a dictionary - (the JSON layout has been squashed for display purpose) - - from pyvo.mivot.utils.dict_utils import DictUtils - - mivot_instance = m_viewer.dm_instance - mivot_object_dict = mivot_object.dict - -******************** -MIVOT (`pyvo.mivot`) -******************** +*************************************** +MIVOT (`pyvo.mivot`): Annotation Writer +*************************************** This module contains the new feature of annotations in VOTable. Astropy version >= 6.0 is required. @@ -29,50 +20,50 @@ Introduction MIVOT syntax allows to describe a data structure as a hierarchy of classes. It is also able to represent relations and composition between them. It can also build up data model objects by aggregating instances from different - tables of the VOTable. + tables of the VOTable (get more `here <./index.html>`_). - Model Instances in VOTables is a VO `standard `_ - Requires Astropy>=6.0 - ``pyvo.mivot`` is a prototype feature which must be activated with ``activate_features("MIVOT")`` -Implementation Scope --------------------- -This implementation is totally model-agnostic. +Building Annotation Object per Object +------------------------------------- -- It does not operate any validation against specific data models. -- It just requires the annotation syntax being compliant with the standards. +Creating annotations consists of 3 steps: -.. code-block:: python - :caption: Build an annotation block, add used models and set mapping report - - from pyvo.mivot.writer.annotations import MivotAnnotations - - mivot_annotations = MivotAnnotations() - mivot_annotations.add_model( - "ivoa", "https://www.ivoa.net/xml/VODML/IVOA-v1.vo-dml.xml" - ) - mivot_annotations.add_model( - "coords", "https://www.ivoa.net/xml/STC/20200908/Coords-v1.0.vo-dml.xml" - ) - mivot_annotations.add_model( - "mango", - "https://raw.githubusercontent.com/lmichel/MANGO/draft-0.1/vo-dml/mango.vo-dml.xml", - ) - mivot_annotations.set_report(True, "Mivot writer unit test") +#. Create individual instances (INSTANCE) using the ``MivotInstance`` class: Objects are + built attribute by attribute. These components can then be aggregated into + more complex objects following the structure of the mapped model(s). +#. Wrap the annotations with the ``MivotAnnotations`` class: Declare to the annotation builder + the models used, and place individual instances + at the right place (TEMPLATES or GLOBALS). +#. Insert the annotations into a VOtable by using the Astropy API (wrapped in the package logic). + +The Annotation Builder does not check whether the XML conforms to any particular model. +It simply validates it against the MIVOT XML Schema if the ``xmlvalidator`` package if is installed. - from pyvo.mivot.utils.dict_utils import DictUtils +The example below shows a step-by-step construction of a MIVOT block mapping +a position with its error (as defined in the MANGO draft) +and its space coordinate system (as defined in the coords model and imported by MANGO). - mivot_instance = m_viewer.dm_instance - mivot_object_dict = mivot_object.dict +(imports have been removed from the code snippet for readability) -blabla -must be constructed in the good order. +Building the Coordinate System Object +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The space coordinate system is made of a space frame and a reference position, both wrapped in a ``coords:SpaceSys`` +object (see the `coords `_ model). + +- Individual instances are built one by one and then packed together. +- The top level object has a ``dmid`` which will be used as a reference by the ``EpochPosition`` instance. +- ``MIVOT`` being still an experimental feature which must be activated .. code-block:: python - :caption: Build the coordinate system (coords:SpaceSys) - space_sys = MivotInstance(dmid="_spacesys_icrs", dmtype="coords:SpaceSys") + activate_features("MIVOT") + + space_sys = MivotInstance(dmid="_spacesys_icrs", dmtype="c") space_frame = MivotInstance( dmrole="coords:PhysicalCoordSys.frame", dmtype="coords:SpaceFrame" ) @@ -89,4 +80,138 @@ must be constructed in the good order. ) space_frame.add_instance(ref_position) space_sys.add_instance(space_frame) + + +Building the EpochPosition Object +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- IN this example we only use the position attributes (RA/DEC) of the ``EpochPosition`` class. +- The reference to the space coordinate system is added at the end. +- The ``ref`` XML attributes reference columns that must be used to set the model attributes. + Their values depend on the VOTable to be mapped. + +.. code-block:: python + + + position = MivotInstance(dmtype="mango:EpochPosition") + position.add_attribute( + dmtype="ivoa:RealQuantity", + dmrole="mango:EpochPosition.longitude", + unit="deg", + ref="RAICRS", + ) + position.add_attribute( + dmtype="ivoa:RealQuantity", + dmrole="mango:EpochPosition.latitude", + unit="deg", + ref="DEICRS", + ) + position.add_reference( + dmref="_spacesys_icrs", dmrole="mango:EpochPosition.spaceSys" + ) + + +Building the Position Error +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- We assume that the position error is the same on both axes and that axes are not correlated. + In terms of MANGO error, this corresponds to a 2x2 diagonal error matrix with two equal coefficients +- Finally, the error is added as a component of the ``EpochPosition`` instance. + +.. code-block:: python + + + epoch_position_error = MivotInstance( + dmtype="mango:EpochPositionErrors", dmrole="mango:EpochPosition.errors" + ) + position_error = MivotInstance( + dmtype="mango:error.ErrorCorrMatrix", + dmrole="mango:EpochPositionErrors.position", + ) + position_error.add_attribute( + dmtype="ivoa:RealQuantity", + dmrole="mango:error.ErrorCorrMatrix.sigma1", + unit="arcsec", + ref="sigm", + ) + position_error.add_attribute( + dmtype="ivoa:RealQuantity", + dmrole="mango:error.ErrorCorrMatrix.sigma2", + unit="arcsec", + ref="sigm", + ) + epoch_position_error.add_instance(position_error) + position.add_instance(epoch_position_error) + + +Building the MIVOT Block +~~~~~~~~~~~~~~~~~~~~~~~~ + +- The MIVOT block consists of: + + - A list of mapped models + - A process status + - A list of globals, which are objects not associated with + VOTale data and that can be shared by any other component. + - a list of templates, which are objects that are connected to + VOTable data and whose leaf values change from one row to another. + +- The latest step includes a validation of the MIVOT syntax if the ``xmlvaldator`` package has been installed. + +.. code-block:: python + + + mivot_annotations = MivotAnnotations() + mivot_annotations.add_model( + "ivoa", "https://www.ivoa.net/xml/VODML/IVOA-v1.vo-dml.xml" + ) + mivot_annotations.add_model( + "coords", "https://www.ivoa.net/xml/STC/20200908/Coords-v1.0.vo-dml.xml" + ) + mivot_annotations.add_model( + "mango", + "https://raw.githubusercontent.com/lmichel/MANGO/draft-0.1/vo-dml/mango.vo-dml.xml", + ) + mivot_annotations.set_report(True, "PyVO Tuto") + + mivot_annotations.add_templates(position) mivot_annotations.add_globals(space_sys) + + mivot_annotations.build_mivot_block() + + +Insert the MIVOT Block in a VOTable +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +- This straightforward step is based on the Astropy VOTable API. +- Annotations are stored in-memory (in the parsed VOtable) +- The mapping can be tested with the ``MivotViewer`` API (see the doc `page `_) +- The VOtable must be explicitly saved on disk. + + .. code-block:: python + + + from astropy.io.votable import parse + + votable = parse(votable_path) + mivot_annotations.insert_into_votable(votable) + + mv = MivotViewer(votable) + mappes_instance = mv.dm_instance + + votable.to_xml("pyvo-tuto.xml") + + +Validate the annotation against the model(s) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~s~~~~~~ + +- This action requires the ``mivot-validator`` package to be installed. +- It validates the mapped classes against the the models they come from. + + + .. code-block:: shell + :caption: Build the coordinate system (coords:SpaceSys) + + % pip install mivot-validator + % mivot-instance-validate pyvo-tuto.xml From 7a9cff4b93c7185d10fd7af5536c564a48c0302e Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Thu, 28 Nov 2024 17:26:41 +0100 Subject: [PATCH 10/39] Changelog updated --- CHANGES.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index f9c338f4a..3139bce5b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -26,7 +26,10 @@ Enhancements and Fixes any model serialized in VO-DML. This package dynamically generates python objects whose structure corresponds to the classes of the mapped models. [#497] - +- Extending the MIVOT module with the ability to build annotations component by component + and put them into a VOTable. [#XYZ] + + Deprecations and Removals ------------------------- From c8b8549700fe986e71aac8574c11a97689c163df Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Thu, 28 Nov 2024 17:37:44 +0100 Subject: [PATCH 11/39] code style --- pyvo/mivot/utils/xml_utils.py | 2 -- pyvo/mivot/writer/instance.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pyvo/mivot/utils/xml_utils.py b/pyvo/mivot/utils/xml_utils.py index 468a3b49d..fb47ebb55 100644 --- a/pyvo/mivot/utils/xml_utils.py +++ b/pyvo/mivot/utils/xml_utils.py @@ -44,8 +44,6 @@ def pretty_string(xmltree, clean_ns=True): else: return new_xml - - @staticmethod def indent(elem, level=0): """ diff --git a/pyvo/mivot/writer/instance.py b/pyvo/mivot/writer/instance.py index 82682b745..c01411f14 100644 --- a/pyvo/mivot/writer/instance.py +++ b/pyvo/mivot/writer/instance.py @@ -29,7 +29,7 @@ position_error.add_attribute(dmtype="model:type2.att2", dmrole="model:type2.inst.role2", value="value4", unit="m/s") - position.add_instance(position_error) + position.add_instance(position_error) mb = MivotAnnotations() mb.add_templates(position) @@ -37,7 +37,6 @@ print(mb.mivot_block) """ -import logging from pyvo.utils.prototype import prototype_feature from pyvo.mivot.utils.exceptions import MappingException From 3ad6f4364f9289ff7d1bd08bafc94b21bbe88d92 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sat, 30 Nov 2024 15:00:42 +0100 Subject: [PATCH 12/39] style corrections --- docs/mivot/index.rst | 6 ++--- docs/mivot/viewer.rst | 4 ++-- docs/mivot/writer.rst | 54 +++++++++++++++++++++++++------------------ 3 files changed, 36 insertions(+), 28 deletions(-) diff --git a/docs/mivot/index.rst b/docs/mivot/index.rst index ee4ca7e1c..8045789cb 100644 --- a/docs/mivot/index.rst +++ b/docs/mivot/index.rst @@ -2,7 +2,7 @@ MIVOT (`pyvo.mivot`) ******************** -This module contains the new feature of annotations in VOTable. +This module contains the new feature handling model annotations in VOTable. Astropy version >= 6.0 is required. Introduction @@ -63,8 +63,8 @@ Using the MIVOT package The ``pyvo.mivot`` module can be used to either read or build annotations. -- How to `read <./viewer.html>`_ MIVOT annotations -- How to `write <./writer.html>`_ MIVOT annotations +- `Reading <./viewer.html>`_ MIVOT annotations +- `Writing <./writer.html>`_ MIVOT annotations Reference/API ============= diff --git a/docs/mivot/viewer.rst b/docs/mivot/viewer.rst index e96393d38..62015db9a 100644 --- a/docs/mivot/viewer.rst +++ b/docs/mivot/viewer.rst @@ -24,8 +24,8 @@ Integrated Readout ------------------ The ``ModelViewer`` module manages access to data mapped to a model through dynamically generated objects (``MivotInstance``class). -The example below shows how a VOTable, resulting from a cone-search query which data are mapped -to the ``EpochPosition`` class, can be consumed. +The example below shows how can be consumed a VOTable resulting from a cone-search query which data are mapped +to the ``EpochPosition`` class. .. doctest-remote-data:: >>> import astropy.units as u diff --git a/docs/mivot/writer.rst b/docs/mivot/writer.rst index 167c1a54a..dae3c3654 100644 --- a/docs/mivot/writer.rst +++ b/docs/mivot/writer.rst @@ -2,9 +2,6 @@ MIVOT (`pyvo.mivot`): Annotation Writer *************************************** -This module contains the new feature of annotations in VOTable. -Astropy version >= 6.0 is required. - Introduction ============ .. pull-quote:: @@ -32,22 +29,20 @@ Building Annotation Object per Object Creating annotations consists of 3 steps: -#. Create individual instances (INSTANCE) using the ``MivotInstance`` class: Objects are +#. Create individual instances (INSTANCE) using the ``MivotInstance`` class: objects are built attribute by attribute. These components can then be aggregated into more complex objects following the structure of the mapped model(s). -#. Wrap the annotations with the ``MivotAnnotations`` class: Declare to the annotation builder - the models used, and place individual instances - at the right place (TEMPLATES or GLOBALS). +#. Wrap the annotations with the ``MivotAnnotations`` class: declare to the annotation builder + the models used, and place individual instances at the right place (TEMPLATES or GLOBALS). #. Insert the annotations into a VOtable by using the Astropy API (wrapped in the package logic). -The Annotation Builder does not check whether the XML conforms to any particular model. +The annotation builder does not check whether the XML conforms to any particular model. It simply validates it against the MIVOT XML Schema if the ``xmlvalidator`` package if is installed. The example below shows a step-by-step construction of a MIVOT block mapping -a position with its error (as defined in the MANGO draft) -and its space coordinate system (as defined in the coords model and imported by MANGO). +a position with its error (as defined in the ``MANGO`` draft) +and its space coordinate system (as defined in the ``coord`` model and imported by ``MANGO``). -(imports have been removed from the code snippet for readability) Building the Coordinate System Object ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -57,9 +52,18 @@ object (see the `coords ` - Individual instances are built one by one and then packed together. - The top level object has a ``dmid`` which will be used as a reference by the ``EpochPosition`` instance. -- ``MIVOT`` being still an experimental feature which must be activated +- ``MIVOT`` is still an experimental feature which must be activated .. code-block:: python + + + from astropy.io.votable import parse + from pyvo.utils import activate_features + from pyvo.mivot.utils.exceptions import MappingException + from pyvo.mivot.utils.dict_utils import DictUtils + from pyvo.mivot.writer.annotations import MivotAnnotations + from pyvo.mivot.writer.instance import MivotInstance + from pyvo.mivot.viewer.mivot_viewer import MivotViewer activate_features("MIVOT") @@ -85,14 +89,14 @@ object (see the `coords ` Building the EpochPosition Object ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- IN this example we only use the position attributes (RA/DEC) of the ``EpochPosition`` class. +- In this example we only use the position attributes (RA/DEC) of the ``EpochPosition`` class. - The reference to the space coordinate system is added at the end. - The ``ref`` XML attributes reference columns that must be used to set the model attributes. Their values depend on the VOTable to be mapped. .. code-block:: python - - + + position = MivotInstance(dmtype="mango:EpochPosition") position.add_attribute( dmtype="ivoa:RealQuantity", @@ -114,8 +118,8 @@ Building the EpochPosition Object Building the Position Error ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- We assume that the position error is the same on both axes and that axes are not correlated. - In terms of MANGO error, this corresponds to a 2x2 diagonal error matrix with two equal coefficients +- We assume that the position error is the same on both axes without correlation. + In terms of MANGO error, this corresponds to a 2x2 diagonal error matrix with two equal coefficients. - Finally, the error is added as a component of the ``EpochPosition`` instance. .. code-block:: python @@ -149,14 +153,15 @@ Building the MIVOT Block - The MIVOT block consists of: - - A list of mapped models - A process status + - A list of mapped models - A list of globals, which are objects not associated with - VOTale data and that can be shared by any other component. - - a list of templates, which are objects that are connected to + VOTable data and that can be shared by any other MIVOT instance. + - A list of templates, which are objects that are connected to VOTable data and whose leaf values change from one row to another. -- The latest step includes a validation of the MIVOT syntax if the ``xmlvaldator`` package has been installed. +- The latest step (build_mivot_block) includes a validation of the MIVOT syntax that works only + if the ``xmlvaldator`` package has been installed. .. code-block:: python @@ -187,7 +192,7 @@ Insert the MIVOT Block in a VOTable - This straightforward step is based on the Astropy VOTable API. - Annotations are stored in-memory (in the parsed VOtable) - The mapping can be tested with the ``MivotViewer`` API (see the doc `page `_) -- The VOtable must be explicitly saved on disk. +- The VOtable must be explicitly saved on disk if needed. .. code-block:: python @@ -207,7 +212,7 @@ Validate the annotation against the model(s) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~s~~~~~~ - This action requires the ``mivot-validator`` package to be installed. -- It validates the mapped classes against the the models they come from. +- It validates the mapped classes against the models they come from. .. code-block:: shell @@ -215,3 +220,6 @@ Validate the annotation against the model(s) % pip install mivot-validator % mivot-instance-validate pyvo-tuto.xml + ... + Valid if no error message + ... From 79c01b852b37b665cc21f2e184edf33458847f47 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sat, 30 Nov 2024 15:04:45 +0100 Subject: [PATCH 13/39] add PR number in change log --- pyvo/mivot/tests/data/essai.xml | 348 ++++++++++++++++++++++++++++++++ 1 file changed, 348 insertions(+) create mode 100644 pyvo/mivot/tests/data/essai.xml diff --git a/pyvo/mivot/tests/data/essai.xml b/pyvo/mivot/tests/data/essai.xml new file mode 100644 index 000000000..d6a7eea82 --- /dev/null +++ b/pyvo/mivot/tests/data/essai.xml @@ -0,0 +1,348 @@ + + + + + + URAT1 Catalog (Zacharias+ 2015) + + + + IVOID of underlying data collection + + + Mivot writer unit test + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + URAT1 catalog + + + + Distance from center (052.2670800+59.9402700)[ICRS], at Epoch of + catalog (Epoch) + + + + + URAT1 recommended identifier (ZZZ-NNNNNN) (13) [datatype=char] + + + + + Right ascension on ICRS, at "Epoch" (1) + + + + + Declination on ICRS, at "Epoch" (1) + + + + + Position error per coordinate, from scatter (2) + + + + + Position error per coordinate, from model (2) + + + + + (nst) Total number of sets the star is in (3) + + + + + (nsu) Number of sets used for mean position (3) + + + + + (epoc) Mean URAT observation epoch (1) + + + + + ?(mmag) mean URAT model fit magnitude (4) + + + + + + ?(sigp) URAT photometry error (5) + + + + + + (nsm) Number of sets used for URAT magnitude (3) + + + + + (ref) largest reference star flag (6) + + + + + (nit) Total number of images (observations) + + + + + (niu) Number of images used for mean position + + + + + (ngt) Total number of 1st order grating observations + + + + + (ngu) Number of 1st order grating positions used + + + + + ?(pmr) Proper motion RA*cosDec (from 2MASS) (7) + + + + + + ?(pmd) Proper motion in Declination (7) + + + + + + ?(pme) Proper motion error per coordinate (8) + + + + + + [1/11] Match flag URAT with 2MASS (9) + + + + + [1/11] Match flag URAT with APASS (9) + + + + + [-] "-" if there is no match with GSC2.4 (14) + + + + + ?(id2) unique 2MASS star identification number + + + + + + ?(jmag) 2MASS J-band magnitude + + + + + + ?(ejmag) Error on Jmag + + + + + + [0,58]? J-band quality-confusion flag (10) + + + + + ?(hmag) 2MASS H-band magnitude + + + + + + ?(ehmag) Error on H-band magnitude (10) + + + + + + [0,58]? H-band quality-confusion flag (10) + + + + + ?(kmag) 2MASS Ks-band magnitude + + + + + + ?(ekmag) Error on Ks-band magnitude (10) + + + + + + [0,58]? Ks-band quality-confusion flag (10) + + + + + (ann) Number of APASS observation nights (12) + + + + + (ano) Number of APASS observations (12) + + + + + ?(abm) APASS B-band magnitude (11) + + + + + + ?(ebm) Error on Bmag + + + + + + ?(avm) APASS V-band magnitude + + + + + + ?(evm) Error on Vmag + + + + + + ?(agm) APASS g-band magnitude + + + + + + ?(egm) Error on gmag + + + + + + ?(arm) APASS r-band magnitude + + + + + + ?(erm) Error on rmag + + + + + + ?(aim) APASS i-band magnitude + + + + + + ?(eim) Error on imag + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
0.049402750-146023 52.2340018 59.89373339613132013.41815.340 0.013139747400 1.5 -12.3 5.915 + 75880868113.713 0.028513.340 0.034513.101 0.03451417.632 0.20416.164 0.00116.690 0.00115.750 0.001 + +
+ + From 9e20fd33155de4ef690f8ab1015a00b24e4d6a61 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sat, 30 Nov 2024 15:07:59 +0100 Subject: [PATCH 14/39] removed --- pyvo/mivot/tests/data/essai.xml | 348 -------------------------------- 1 file changed, 348 deletions(-) delete mode 100644 pyvo/mivot/tests/data/essai.xml diff --git a/pyvo/mivot/tests/data/essai.xml b/pyvo/mivot/tests/data/essai.xml deleted file mode 100644 index d6a7eea82..000000000 --- a/pyvo/mivot/tests/data/essai.xml +++ /dev/null @@ -1,348 +0,0 @@ - - - - - - URAT1 Catalog (Zacharias+ 2015) - - - - IVOID of underlying data collection - - - Mivot writer unit test - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - URAT1 catalog - - - - Distance from center (052.2670800+59.9402700)[ICRS], at Epoch of - catalog (Epoch) - - - - - URAT1 recommended identifier (ZZZ-NNNNNN) (13) [datatype=char] - - - - - Right ascension on ICRS, at "Epoch" (1) - - - - - Declination on ICRS, at "Epoch" (1) - - - - - Position error per coordinate, from scatter (2) - - - - - Position error per coordinate, from model (2) - - - - - (nst) Total number of sets the star is in (3) - - - - - (nsu) Number of sets used for mean position (3) - - - - - (epoc) Mean URAT observation epoch (1) - - - - - ?(mmag) mean URAT model fit magnitude (4) - - - - - - ?(sigp) URAT photometry error (5) - - - - - - (nsm) Number of sets used for URAT magnitude (3) - - - - - (ref) largest reference star flag (6) - - - - - (nit) Total number of images (observations) - - - - - (niu) Number of images used for mean position - - - - - (ngt) Total number of 1st order grating observations - - - - - (ngu) Number of 1st order grating positions used - - - - - ?(pmr) Proper motion RA*cosDec (from 2MASS) (7) - - - - - - ?(pmd) Proper motion in Declination (7) - - - - - - ?(pme) Proper motion error per coordinate (8) - - - - - - [1/11] Match flag URAT with 2MASS (9) - - - - - [1/11] Match flag URAT with APASS (9) - - - - - [-] "-" if there is no match with GSC2.4 (14) - - - - - ?(id2) unique 2MASS star identification number - - - - - - ?(jmag) 2MASS J-band magnitude - - - - - - ?(ejmag) Error on Jmag - - - - - - [0,58]? J-band quality-confusion flag (10) - - - - - ?(hmag) 2MASS H-band magnitude - - - - - - ?(ehmag) Error on H-band magnitude (10) - - - - - - [0,58]? H-band quality-confusion flag (10) - - - - - ?(kmag) 2MASS Ks-band magnitude - - - - - - ?(ekmag) Error on Ks-band magnitude (10) - - - - - - [0,58]? Ks-band quality-confusion flag (10) - - - - - (ann) Number of APASS observation nights (12) - - - - - (ano) Number of APASS observations (12) - - - - - ?(abm) APASS B-band magnitude (11) - - - - - - ?(ebm) Error on Bmag - - - - - - ?(avm) APASS V-band magnitude - - - - - - ?(evm) Error on Vmag - - - - - - ?(agm) APASS g-band magnitude - - - - - - ?(egm) Error on gmag - - - - - - ?(arm) APASS r-band magnitude - - - - - - ?(erm) Error on rmag - - - - - - ?(aim) APASS i-band magnitude - - - - - - ?(eim) Error on imag - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
0.049402750-146023 52.2340018 59.89373339613132013.41815.340 0.013139747400 1.5 -12.3 5.915 - 75880868113.713 0.028513.340 0.034513.101 0.03451417.632 0.20416.164 0.00116.690 0.00115.750 0.001 - -
- - From 55cc826987cd55a684aab62a1c4f7bc94a7307c7 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sat, 30 Nov 2024 15:08:16 +0100 Subject: [PATCH 15/39] add PR number --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3139bce5b..7fe156489 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -27,7 +27,7 @@ Enhancements and Fixes whose structure corresponds to the classes of the mapped models. [#497] - Extending the MIVOT module with the ability to build annotations component by component - and put them into a VOTable. [#XYZ] + and put them into a VOTable. [#627] Deprecations and Removals From a622b328b18ac2dfe58bca651d8c73977c856825 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sat, 30 Nov 2024 15:11:51 +0100 Subject: [PATCH 16/39] resolve conflicts --- CHANGES.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 7fe156489..3bcc0d123 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -26,6 +26,20 @@ Enhancements and Fixes any model serialized in VO-DML. This package dynamically generates python objects whose structure corresponds to the classes of the mapped models. [#497] +- MIVOT module: the model references in the dictionaries that are used to build ``MivotInstance`` + objects are made more consistent [#551] + +- RegTAP constraints involving tables other than rr.resource are now + done via subqueries for less duplication of interfaces. [#562, #572] + +- MIVOT module: If the MIVOT annotation block contains a valid instance of the + ``mango:EpochPosition`` class, the dynamic object describing the mapped + data can generate a valid SkyCoord instance. [#591] + +- New sub-package discover for global dataset discovery. [#470] + +- Updated getdatalink to be consistent with iter_datalinks. [#613] + - Extending the MIVOT module with the ability to build annotations component by component and put them into a VOTable. [#627] From be595498ba720049ce3f465eec599b53bde12914 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sat, 30 Nov 2024 15:40:40 +0100 Subject: [PATCH 17/39] add writer to api index --- docs/mivot/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/mivot/index.rst b/docs/mivot/index.rst index 8045789cb..7f4ea785d 100644 --- a/docs/mivot/index.rst +++ b/docs/mivot/index.rst @@ -74,3 +74,4 @@ Reference/API .. automodapi:: pyvo.mivot.seekers .. automodapi:: pyvo.mivot.features .. automodapi:: pyvo.mivot.utils +.. automodapi:: pyvo.mivot.writer From f9917ed3f144404705795fb51c376130e408ac08 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sun, 1 Dec 2024 20:19:16 +0100 Subject: [PATCH 18/39] fix CI --- docs/mivot/index.rst | 8 +- docs/mivot/viewer.rst | 2 +- docs/mivot/writer.rst | 6 +- pyvo/mivot/tests/data/essai.xml | 348 ++++++++++++++++++++++++++ pyvo/mivot/tests/test_mivot_writer.py | 10 +- pyvo/mivot/writer/annotations.py | 13 +- 6 files changed, 375 insertions(+), 12 deletions(-) create mode 100644 pyvo/mivot/tests/data/essai.xml diff --git a/docs/mivot/index.rst b/docs/mivot/index.rst index 7f4ea785d..56f457b02 100644 --- a/docs/mivot/index.rst +++ b/docs/mivot/index.rst @@ -1,3 +1,5 @@ +.. _index: + ******************** MIVOT (`pyvo.mivot`) ******************** @@ -63,8 +65,10 @@ Using the MIVOT package The ``pyvo.mivot`` module can be used to either read or build annotations. -- `Reading <./viewer.html>`_ MIVOT annotations -- `Writing <./writer.html>`_ MIVOT annotations +.. toctree:: + + viewer + writer Reference/API ============= diff --git a/docs/mivot/viewer.rst b/docs/mivot/viewer.rst index 62015db9a..6f48c5999 100644 --- a/docs/mivot/viewer.rst +++ b/docs/mivot/viewer.rst @@ -18,7 +18,7 @@ Introduction MIVOT syntax allows to describe a data structure as a hierarchy of classes. It is also able to represent relations and composition between them. It can also build up data model objects by aggregating instances from different - tables of the VOTable (get more `here <./index.html>`_). + tables of the VOTable (get more in `index`_). Integrated Readout ------------------ diff --git a/docs/mivot/writer.rst b/docs/mivot/writer.rst index dae3c3654..22a2a9e4d 100644 --- a/docs/mivot/writer.rst +++ b/docs/mivot/writer.rst @@ -1,3 +1,5 @@ +.. _viewer: + *************************************** MIVOT (`pyvo.mivot`): Annotation Writer *************************************** @@ -17,7 +19,7 @@ Introduction MIVOT syntax allows to describe a data structure as a hierarchy of classes. It is also able to represent relations and composition between them. It can also build up data model objects by aggregating instances from different - tables of the VOTable (get more `here <./index.html>`_). + tables of the VOTable (get more in `index`_). - Model Instances in VOTables is a VO `standard `_ - Requires Astropy>=6.0 @@ -191,7 +193,7 @@ Insert the MIVOT Block in a VOTable - This straightforward step is based on the Astropy VOTable API. - Annotations are stored in-memory (in the parsed VOtable) -- The mapping can be tested with the ``MivotViewer`` API (see the doc `page `_) +- The mapping can be tested with the ``MivotViewer`` API (see the doc `viewer`_) - The VOtable must be explicitly saved on disk if needed. .. code-block:: python diff --git a/pyvo/mivot/tests/data/essai.xml b/pyvo/mivot/tests/data/essai.xml new file mode 100644 index 000000000..8a752082b --- /dev/null +++ b/pyvo/mivot/tests/data/essai.xml @@ -0,0 +1,348 @@ + + + + + + URAT1 Catalog (Zacharias+ 2015) + + + + IVOID of underlying data collection + + + Mivot writer unit test + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + URAT1 catalog + + + + Distance from center (052.2670800+59.9402700)[ICRS], at Epoch of + catalog (Epoch) + + + + + URAT1 recommended identifier (ZZZ-NNNNNN) (13) [datatype=char] + + + + + Right ascension on ICRS, at "Epoch" (1) + + + + + Declination on ICRS, at "Epoch" (1) + + + + + Position error per coordinate, from scatter (2) + + + + + Position error per coordinate, from model (2) + + + + + (nst) Total number of sets the star is in (3) + + + + + (nsu) Number of sets used for mean position (3) + + + + + (epoc) Mean URAT observation epoch (1) + + + + + ?(mmag) mean URAT model fit magnitude (4) + + + + + + ?(sigp) URAT photometry error (5) + + + + + + (nsm) Number of sets used for URAT magnitude (3) + + + + + (ref) largest reference star flag (6) + + + + + (nit) Total number of images (observations) + + + + + (niu) Number of images used for mean position + + + + + (ngt) Total number of 1st order grating observations + + + + + (ngu) Number of 1st order grating positions used + + + + + ?(pmr) Proper motion RA*cosDec (from 2MASS) (7) + + + + + + ?(pmd) Proper motion in Declination (7) + + + + + + ?(pme) Proper motion error per coordinate (8) + + + + + + [1/11] Match flag URAT with 2MASS (9) + + + + + [1/11] Match flag URAT with APASS (9) + + + + + [-] "-" if there is no match with GSC2.4 (14) + + + + + ?(id2) unique 2MASS star identification number + + + + + + ?(jmag) 2MASS J-band magnitude + + + + + + ?(ejmag) Error on Jmag + + + + + + [0,58]? J-band quality-confusion flag (10) + + + + + ?(hmag) 2MASS H-band magnitude + + + + + + ?(ehmag) Error on H-band magnitude (10) + + + + + + [0,58]? H-band quality-confusion flag (10) + + + + + ?(kmag) 2MASS Ks-band magnitude + + + + + + ?(ekmag) Error on Ks-band magnitude (10) + + + + + + [0,58]? Ks-band quality-confusion flag (10) + + + + + (ann) Number of APASS observation nights (12) + + + + + (ano) Number of APASS observations (12) + + + + + ?(abm) APASS B-band magnitude (11) + + + + + + ?(ebm) Error on Bmag + + + + + + ?(avm) APASS V-band magnitude + + + + + + ?(evm) Error on Vmag + + + + + + ?(agm) APASS g-band magnitude + + + + + + ?(egm) Error on gmag + + + + + + ?(arm) APASS r-band magnitude + + + + + + ?(erm) Error on rmag + + + + + + ?(aim) APASS i-band magnitude + + + + + + ?(eim) Error on imag + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
0.049402750-146023 52.2340018 59.89373339613132013.41815.340 0.013139747400 1.5 -12.3 5.915 + 75880868113.713 0.028513.340 0.034513.101 0.03451417.632 0.20416.164 0.00116.690 0.00115.750 0.001 + +
+ + diff --git a/pyvo/mivot/tests/test_mivot_writer.py b/pyvo/mivot/tests/test_mivot_writer.py index fa391f3ef..bc0d9770b 100644 --- a/pyvo/mivot/tests/test_mivot_writer.py +++ b/pyvo/mivot/tests/test_mivot_writer.py @@ -9,6 +9,7 @@ import pytest from astropy.io.votable import parse from pyvo.utils import activate_features +from pyvo.mivot.version_checker import check_astropy_version from pyvo.mivot.utils.exceptions import MappingException from pyvo.mivot.utils.dict_utils import DictUtils from pyvo.mivot.writer.annotations import MivotAnnotations @@ -42,6 +43,7 @@ def strip_xml(xml_string): ) +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") def test_MivotInstance(): """ Test the MivotInstance class for various operations including attribute addition, @@ -82,6 +84,7 @@ def test_MivotInstance(): ) +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") def test_MivotAnnotations(): """ Test the MivotAnnotations class for template and global instance addition. Verifies @@ -95,7 +98,8 @@ def test_MivotAnnotations(): mb.add_globals(12) -def test_MivotInstance2(): +@pytest.mark.skipif(not check_astropy_version(), reason="need astropy 6+") +def test_MivotInstanceAll(): """ Test the creation and combination of multiple MivotInstance objects, their attributes, references, and integration into a MivotAnnotations instance. Verifies correct XML @@ -190,7 +194,3 @@ def test_MivotInstance2(): os.path.join(data_path, "reference/test_mivot_writer.json") ) votable.to_xml(data_path + "/essai.xml") - - -if __name__ == "__main__": - test_MivotInstance2() diff --git a/pyvo/mivot/writer/annotations.py b/pyvo/mivot/writer/annotations.py index b158e0609..fdbe986f1 100644 --- a/pyvo/mivot/writer/annotations.py +++ b/pyvo/mivot/writer/annotations.py @@ -78,12 +78,18 @@ from defusedxml import ElementTree as etree except ImportError: from xml.etree import ElementTree as etree -from astropy.io.votable.tree import VOTableFile, Resource, MivotBlock +from astropy.io.votable.tree import VOTableFile, Resource +try: + from astropy.io.votable.tree import MivotBlock +except ImportError: + pass from astropy.io.votable import parse +from astropy import version from pyvo.utils.prototype import prototype_feature from pyvo.mivot.utils.xml_utils import XmlUtils -from pyvo.mivot.utils.exceptions import MappingException +from pyvo.mivot.utils.exceptions import MappingException, AstropyVersionException from pyvo.mivot.writer.instance import MivotInstance +from pyvo.mivot.version_checker import check_astropy_version @prototype_feature("MIVOT") @@ -363,6 +369,9 @@ def insert_into_votable(self, votable_file, template_id=None, override=False): MappingException If a mapping block already exists and `override` is False. """ + if not check_astropy_version(): + raise AstropyVersionException(f"Astropy version {version.version} " + f"is below the required version 6.0 for the use of MIVOT.") if isinstance(votable_file, str): votable = parse(votable_file) elif isinstance(votable_file, VOTableFile): From 7e0c91816275e64a692daca6c1b59b2b86aa41fd Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sun, 1 Dec 2024 20:35:49 +0100 Subject: [PATCH 19/39] fix use of mapping exception --- pyvo/mivot/tests/test_mivot_writer.py | 22 +++++++++++----------- pyvo/mivot/writer/annotations.py | 20 ++++++++++---------- pyvo/mivot/writer/instance.py | 24 ++++++++++++------------ 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/pyvo/mivot/tests/test_mivot_writer.py b/pyvo/mivot/tests/test_mivot_writer.py index bc0d9770b..053a9032e 100644 --- a/pyvo/mivot/tests/test_mivot_writer.py +++ b/pyvo/mivot/tests/test_mivot_writer.py @@ -10,7 +10,7 @@ from astropy.io.votable import parse from pyvo.utils import activate_features from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot.utils.exceptions import MappingException +from pyvo.mivot.utils.exceptions import MappingError from pyvo.mivot.utils.dict_utils import DictUtils from pyvo.mivot.writer.annotations import MivotAnnotations from pyvo.mivot.writer.instance import MivotInstance @@ -48,25 +48,25 @@ def test_MivotInstance(): """ Test the MivotInstance class for various operations including attribute addition, reference addition, and XML generation. Verifies that invalid operations raise - the expected MappingException. + the expected MappingError. """ - with pytest.raises(MappingException): + with pytest.raises(MappingError): MivotInstance(dmid="model:type.inst") instance1 = MivotInstance(dmtype="model:type.inst", dmid="id1") - with pytest.raises(MappingException): + with pytest.raises(MappingError): instance1.add_attribute( dmrole="model:type.inst.role1", value="value1", unit="m/s" ) - with pytest.raises(MappingException): + with pytest.raises(MappingError): instance1.add_attribute( dmtype="model:type.att1", dmrole="model:type.inst.role1" ) - with pytest.raises(MappingException): + with pytest.raises(MappingError): instance1.add_reference(dmref="dmreference") - with pytest.raises(MappingException): + with pytest.raises(MappingError): instance1.add_reference(dmrole="model:type.inst.role2") - with pytest.raises(MappingException): + with pytest.raises(MappingError): instance1.add_instance("azerty") instance1.add_reference(dmrole="model:type.inst.role2", dmref="dmreference") @@ -88,13 +88,13 @@ def test_MivotInstance(): def test_MivotAnnotations(): """ Test the MivotAnnotations class for template and global instance addition. Verifies - that invalid operations raise the expected MappingException. + that invalid operations raise the expected MappingError. """ mb = MivotAnnotations() - with pytest.raises(MappingException): + with pytest.raises(MappingError): mb.add_templates(12) - with pytest.raises(MappingException): + with pytest.raises(MappingError): mb.add_globals(12) diff --git a/pyvo/mivot/writer/annotations.py b/pyvo/mivot/writer/annotations.py index fdbe986f1..650d79b35 100644 --- a/pyvo/mivot/writer/annotations.py +++ b/pyvo/mivot/writer/annotations.py @@ -87,7 +87,7 @@ from astropy import version from pyvo.utils.prototype import prototype_feature from pyvo.mivot.utils.xml_utils import XmlUtils -from pyvo.mivot.utils.exceptions import MappingException, AstropyVersionException +from pyvo.mivot.utils.exceptions import MappingError, AstropyVersionException from pyvo.mivot.writer.instance import MivotInstance from pyvo.mivot.version_checker import check_astropy_version @@ -251,7 +251,7 @@ def add_templates(self, templates_instance): Raises ------ - MappingException + MappingError If `templates_instance` is neither a string nor an instance of `MivotInstance`. """ if isinstance(templates_instance, MivotInstance): @@ -259,7 +259,7 @@ def add_templates(self, templates_instance): elif isinstance(templates_instance, str): self._templates.append(templates_instance) else: - raise MappingException( + raise MappingError( "Instance added to templates must be a string or MivotInstance." ) @@ -274,7 +274,7 @@ def add_globals(self, globals_instance): Raises ------ - MappingException + MappingError If `globals_instance` is neither a string nor an instance of `MivotInstance`. """ if isinstance(globals_instance, MivotInstance): @@ -282,7 +282,7 @@ def add_globals(self, globals_instance): elif isinstance(globals_instance, str): self._globals.append(globals_instance) else: - raise MappingException( + raise MappingError( "Instance added to globals must be a string or MivotInstance." ) @@ -327,7 +327,7 @@ def check_xml(self): Raises ------ - MappingException + MappingError If the validation fails. Notes @@ -349,7 +349,7 @@ def check_xml(self): try: schema.validate(mivot_block) except Exception as excep: - raise MappingException(f"Validation failed: {excep}") from excep + raise MappingError(f"Validation failed: {excep}") from excep def insert_into_votable(self, votable_file, template_id=None, override=False): """ @@ -366,7 +366,7 @@ def insert_into_votable(self, votable_file, template_id=None, override=False): Raises ------ - MappingException + MappingError If a mapping block already exists and `override` is False. """ if not check_astropy_version(): @@ -377,7 +377,7 @@ def insert_into_votable(self, votable_file, template_id=None, override=False): elif isinstance(votable_file, VOTableFile): votable = votable_file else: - raise MappingException( + raise MappingError( "votable_file must be a file path string or a VOTableFile instance." ) @@ -386,7 +386,7 @@ def insert_into_votable(self, votable_file, template_id=None, override=False): for subresource in resource.resources: if subresource.type == "meta": if not override: - raise MappingException( + raise MappingError( "A type='meta' resource already exists in the first 'result' resource." ) else: diff --git a/pyvo/mivot/writer/instance.py b/pyvo/mivot/writer/instance.py index c01411f14..cd8bfd823 100644 --- a/pyvo/mivot/writer/instance.py +++ b/pyvo/mivot/writer/instance.py @@ -38,7 +38,7 @@ """ from pyvo.utils.prototype import prototype_feature -from pyvo.mivot.utils.exceptions import MappingException +from pyvo.mivot.utils.exceptions import MappingError @prototype_feature("MIVOT") @@ -65,11 +65,11 @@ def __init__(self, dmtype=None, dmrole=None, dmid=None): Raises ------ - MappingException + MappingError If `dmtype` is not provided. """ if not dmtype: - raise MappingException("Cannot build an instance without dmtype") + raise MappingError("Cannot build an instance without dmtype") self._dmtype = dmtype self._dmrole = dmrole self._dmid = dmid @@ -94,15 +94,15 @@ def add_attribute(self, dmtype=None, dmrole=None, ref=None, value=None, unit=Non Raises ------ - MappingException + MappingError If `dmtype` or `dmrole` is not provided, or if both `ref` and `value` are not defined. """ if not dmtype: - raise MappingException("Cannot add an attribute without dmtype") + raise MappingError("Cannot add an attribute without dmtype") if not dmrole: - raise MappingException("Cannot add an attribute without dmrole") + raise MappingError("Cannot add an attribute without dmrole") if not ref and not value: - raise MappingException("Cannot add an attribute without ref or value") + raise MappingError("Cannot add an attribute without ref or value") xml_string = f'' self._content.append(xml_string) @@ -149,11 +149,11 @@ def add_instance(self, mivot_instance): Raises ------ - MappingException + MappingError If `mivot_instance` is not of type `MivotInstance`. """ if not isinstance(mivot_instance, MivotInstance): - raise MappingException("Instance added must be of type MivotInstance") + raise MappingError("Instance added must be of type MivotInstance") self._content.append(mivot_instance.xml_string()) def xml_string(self): From 4b40de53f78d3bdb414287b5dda41fc99a21a8d9 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sun, 1 Dec 2024 20:40:22 +0100 Subject: [PATCH 20/39] add package init --- pyvo/mivot/writer/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 pyvo/mivot/writer/__init__.py diff --git a/pyvo/mivot/writer/__init__.py b/pyvo/mivot/writer/__init__.py new file mode 100644 index 000000000..e69de29bb From c0068aa247308da088cc0f5e601b896d4077c7f9 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sun, 1 Dec 2024 20:46:57 +0100 Subject: [PATCH 21/39] fix call to MivotInstance dict --- pyvo/mivot/tests/test_mivot_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyvo/mivot/tests/test_mivot_writer.py b/pyvo/mivot/tests/test_mivot_writer.py index 053a9032e..4823d9ad3 100644 --- a/pyvo/mivot/tests/test_mivot_writer.py +++ b/pyvo/mivot/tests/test_mivot_writer.py @@ -190,7 +190,7 @@ def test_MivotInstanceAll(): mv = MivotViewer(votable) print(mv.dm_instance) - assert mv.dm_instance.dict == DictUtils.read_dict_from_file( + assert mv.dm_instance.to_dict == DictUtils.read_dict_from_file( os.path.join(data_path, "reference/test_mivot_writer.json") ) votable.to_xml(data_path + "/essai.xml") From e9e273e90f09d65b427e1be1df1e4a817ec2a25b Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Mon, 2 Dec 2024 07:45:27 +0100 Subject: [PATCH 22/39] typo --- docs/mivot/writer.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mivot/writer.rst b/docs/mivot/writer.rst index 22a2a9e4d..5f5ad755f 100644 --- a/docs/mivot/writer.rst +++ b/docs/mivot/writer.rst @@ -193,7 +193,7 @@ Insert the MIVOT Block in a VOTable - This straightforward step is based on the Astropy VOTable API. - Annotations are stored in-memory (in the parsed VOtable) -- The mapping can be tested with the ``MivotViewer`` API (see the doc `viewer`_) +- The mapping can be tested with the ``MivotViewer`` API (see the `viewer`_ doc) - The VOtable must be explicitly saved on disk if needed. .. code-block:: python From 28ee3ed2d3ac24524ebb305ebbfb2ae6ea4ffb01 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Mon, 2 Dec 2024 07:46:20 +0100 Subject: [PATCH 23/39] test MivotViewer.to_dict() output --- .../data/reference/test_mivot_writer.json | 32 +++++++++---------- pyvo/mivot/tests/test_mivot_writer.py | 5 ++- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/pyvo/mivot/tests/data/reference/test_mivot_writer.json b/pyvo/mivot/tests/data/reference/test_mivot_writer.json index 707e1fa20..4e003b90e 100644 --- a/pyvo/mivot/tests/data/reference/test_mivot_writer.json +++ b/pyvo/mivot/tests/data/reference/test_mivot_writer.json @@ -1,5 +1,5 @@ { - "dmtype": "EpochPosition", + "dmtype": "mango:EpochPosition", "longitude": { "value": 52.2340018, "unit": "deg" @@ -8,12 +8,12 @@ "value": 59.8937333, "unit": "deg" }, - "EpochPosition_errors": { - "dmtype": "EpochPositionErrors", - "dmrole": "errors", - "EpochPositionErrors_position": { - "dmtype": "ErrorCorrMatrix", - "dmrole": "position", + "errors": { + "dmtype": "mango:EpochPositionErrors", + "dmrole": "mango:EpochPosition.errors", + "position": { + "dmtype": "mango:error.ErrorCorrMatrix", + "dmrole": "mango:EpochPositionErrors.position", "sigma1": { "value": 6.0, "unit": "arcsec" @@ -24,19 +24,19 @@ } } }, - "EpochPosition_spaceSys": { - "dmtype": "SpaceSys", + "spaceSys": { + "dmtype": "coords:SpaceSys", "dmid": "_spacesys_icrs", - "dmrole": "spaceSys", - "PhysicalCoordSys_frame": { - "dmtype": "SpaceFrame", - "dmrole": "frame", + "dmrole": "mango:EpochPosition.spaceSys", + "frame": { + "dmtype": "coords:SpaceFrame", + "dmrole": "coords:PhysicalCoordSys.frame", "spaceRefFrame": { "value": "ICRS" }, - "SpaceFrame_refPosition": { - "dmtype": "StdRefLocation", - "dmrole": "refPosition", + "refPosition": { + "dmtype": "coords:StdRefLocation", + "dmrole": "coords:SpaceFrame.refPosition", "position": { "value": "BARYCENTER" } diff --git a/pyvo/mivot/tests/test_mivot_writer.py b/pyvo/mivot/tests/test_mivot_writer.py index 4823d9ad3..c40f810eb 100644 --- a/pyvo/mivot/tests/test_mivot_writer.py +++ b/pyvo/mivot/tests/test_mivot_writer.py @@ -190,7 +190,10 @@ def test_MivotInstanceAll(): mv = MivotViewer(votable) print(mv.dm_instance) - assert mv.dm_instance.to_dict == DictUtils.read_dict_from_file( + assert mv.dm_instance.to_dict() == DictUtils.read_dict_from_file( os.path.join(data_path, "reference/test_mivot_writer.json") ) votable.to_xml(data_path + "/essai.xml") + +if __name__ == "__main__": + test_MivotInstanceAll() \ No newline at end of file From a86c8f25d0910687443203485ca081f0fbf4b0db Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sun, 1 Dec 2024 20:40:22 +0100 Subject: [PATCH 24/39] Fix CI errors --- docs/mivot/index.rst | 14 +++----- docs/mivot/viewer.rst | 16 +++------- docs/mivot/writer.rst | 12 +++---- .../data/reference/test_mivot_writer.json | 32 +++++++++---------- pyvo/mivot/tests/test_mivot_writer.py | 5 ++- pyvo/mivot/writer/__init__.py | 0 6 files changed, 33 insertions(+), 46 deletions(-) create mode 100644 pyvo/mivot/writer/__init__.py diff --git a/docs/mivot/index.rst b/docs/mivot/index.rst index 28e6812f8..7612de318 100644 --- a/docs/mivot/index.rst +++ b/docs/mivot/index.rst @@ -1,8 +1,6 @@ -.. _index: - -******************** -MIVOT (`pyvo.mivot`) -******************** +********************* +MIVOT (``pyvo.mivo``) +********************* This module contains the new feature handling model annotations in VOTable. Astropy version >= 6.0 is required. @@ -76,8 +74,4 @@ Reference/API ============= .. automodapi:: pyvo.mivot -.. automodapi:: pyvo.mivot.viewer -.. automodapi:: pyvo.mivot.seekers -.. automodapi:: pyvo.mivot.features -.. automodapi:: pyvo.mivot.utils -.. automodapi:: pyvo.mivot.writer + diff --git a/docs/mivot/viewer.rst b/docs/mivot/viewer.rst index 6f48c5999..c17245f4a 100644 --- a/docs/mivot/viewer.rst +++ b/docs/mivot/viewer.rst @@ -1,6 +1,6 @@ -*************************************** -MIVOT (`pyvo.mivot`): Annotation Viewer -*************************************** +***************************************** +MIVOT (``pyvo.mivot``): Annotation Viewer +***************************************** Introduction @@ -18,7 +18,7 @@ Introduction MIVOT syntax allows to describe a data structure as a hierarchy of classes. It is also able to represent relations and composition between them. It can also build up data model objects by aggregating instances from different - tables of the VOTable (get more in `index`_). + tables of the VOTable (get more in :doc:`index`). Integrated Readout ------------------ @@ -187,11 +187,3 @@ identifiers, which have the following structure: ``model:a.b``. print(mivot_instance.Coordinate_coordSys.spaceRefFrame.__dict__.keys()) dict_keys(['dmtype', 'value', 'unit', 'ref']) -Reference/API -============= - -.. automodapi:: pyvo.mivot -.. automodapi:: pyvo.mivot.viewer -.. automodapi:: pyvo.mivot.seekers -.. automodapi:: pyvo.mivot.features -.. automodapi:: pyvo.mivot.utils diff --git a/docs/mivot/writer.rst b/docs/mivot/writer.rst index 22a2a9e4d..2181b983d 100644 --- a/docs/mivot/writer.rst +++ b/docs/mivot/writer.rst @@ -1,8 +1,6 @@ -.. _viewer: - -*************************************** -MIVOT (`pyvo.mivot`): Annotation Writer -*************************************** +***************************************** +MIVOT (``pyvo.mivot``): Annotation Writer +***************************************** Introduction ============ @@ -19,7 +17,7 @@ Introduction MIVOT syntax allows to describe a data structure as a hierarchy of classes. It is also able to represent relations and composition between them. It can also build up data model objects by aggregating instances from different - tables of the VOTable (get more in `index`_). + tables of the VOTable (get more in :doc:`index`). - Model Instances in VOTables is a VO `standard `_ - Requires Astropy>=6.0 @@ -193,7 +191,7 @@ Insert the MIVOT Block in a VOTable - This straightforward step is based on the Astropy VOTable API. - Annotations are stored in-memory (in the parsed VOtable) -- The mapping can be tested with the ``MivotViewer`` API (see the doc `viewer`_) +- The mapping can be tested with the ``MivotViewer`` API (see the :doc:`viewer`) - The VOtable must be explicitly saved on disk if needed. .. code-block:: python diff --git a/pyvo/mivot/tests/data/reference/test_mivot_writer.json b/pyvo/mivot/tests/data/reference/test_mivot_writer.json index 707e1fa20..4e003b90e 100644 --- a/pyvo/mivot/tests/data/reference/test_mivot_writer.json +++ b/pyvo/mivot/tests/data/reference/test_mivot_writer.json @@ -1,5 +1,5 @@ { - "dmtype": "EpochPosition", + "dmtype": "mango:EpochPosition", "longitude": { "value": 52.2340018, "unit": "deg" @@ -8,12 +8,12 @@ "value": 59.8937333, "unit": "deg" }, - "EpochPosition_errors": { - "dmtype": "EpochPositionErrors", - "dmrole": "errors", - "EpochPositionErrors_position": { - "dmtype": "ErrorCorrMatrix", - "dmrole": "position", + "errors": { + "dmtype": "mango:EpochPositionErrors", + "dmrole": "mango:EpochPosition.errors", + "position": { + "dmtype": "mango:error.ErrorCorrMatrix", + "dmrole": "mango:EpochPositionErrors.position", "sigma1": { "value": 6.0, "unit": "arcsec" @@ -24,19 +24,19 @@ } } }, - "EpochPosition_spaceSys": { - "dmtype": "SpaceSys", + "spaceSys": { + "dmtype": "coords:SpaceSys", "dmid": "_spacesys_icrs", - "dmrole": "spaceSys", - "PhysicalCoordSys_frame": { - "dmtype": "SpaceFrame", - "dmrole": "frame", + "dmrole": "mango:EpochPosition.spaceSys", + "frame": { + "dmtype": "coords:SpaceFrame", + "dmrole": "coords:PhysicalCoordSys.frame", "spaceRefFrame": { "value": "ICRS" }, - "SpaceFrame_refPosition": { - "dmtype": "StdRefLocation", - "dmrole": "refPosition", + "refPosition": { + "dmtype": "coords:StdRefLocation", + "dmrole": "coords:SpaceFrame.refPosition", "position": { "value": "BARYCENTER" } diff --git a/pyvo/mivot/tests/test_mivot_writer.py b/pyvo/mivot/tests/test_mivot_writer.py index 053a9032e..c40f810eb 100644 --- a/pyvo/mivot/tests/test_mivot_writer.py +++ b/pyvo/mivot/tests/test_mivot_writer.py @@ -190,7 +190,10 @@ def test_MivotInstanceAll(): mv = MivotViewer(votable) print(mv.dm_instance) - assert mv.dm_instance.dict == DictUtils.read_dict_from_file( + assert mv.dm_instance.to_dict() == DictUtils.read_dict_from_file( os.path.join(data_path, "reference/test_mivot_writer.json") ) votable.to_xml(data_path + "/essai.xml") + +if __name__ == "__main__": + test_MivotInstanceAll() \ No newline at end of file diff --git a/pyvo/mivot/writer/__init__.py b/pyvo/mivot/writer/__init__.py new file mode 100644 index 000000000..e69de29bb From d74d094904c02a2617acfd6f7dd02bd23cbc6671 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Mon, 2 Dec 2024 13:38:07 +0100 Subject: [PATCH 25/39] add missing doctring for clean_ns parameter of pretty_string() --- pyvo/mivot/utils/xml_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyvo/mivot/utils/xml_utils.py b/pyvo/mivot/utils/xml_utils.py index c790ff4ca..c4e02e91e 100644 --- a/pyvo/mivot/utils/xml_utils.py +++ b/pyvo/mivot/utils/xml_utils.py @@ -28,7 +28,8 @@ def pretty_string(xmltree, clean_ns=True): Return a pretty string representation of an XML tree. Parameters ---------- - xmltree (~`xml.etree.ElementTree.Element`): XML tree to convert to a pretty string. + xmltree (~`xml.etree.ElementTree.Element`): XML tree to convert to a pretty string + clean_ns (boolean): Ddefault namspace (ns0) removed from element names if True Returns ------- str: The pretty string representation of the XML tree. From 9462516e56855cb94d45901ff6e26fb7077b37cc Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Mon, 2 Dec 2024 18:04:42 +0100 Subject: [PATCH 26/39] fix test output in rest file --- docs/mivot/viewer.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/mivot/viewer.rst b/docs/mivot/viewer.rst index c17245f4a..0c3b0ab86 100644 --- a/docs/mivot/viewer.rst +++ b/docs/mivot/viewer.rst @@ -46,8 +46,8 @@ to the ``EpochPosition`` class. ... ) >>> mivot_instance = m_viewer.dm_instance >>> print(mivot_instance.dmtype) - EpochPosition - >>> print(mivot_instance.Coordinate_coordSys.spaceRefFrame.value) + mango:EpochPosition + >>> print(mivot_instance.coordSys.spaceRefFrame.value) ICRS >>> while m_viewer.next(): ... print(f"position: {mivot_instance.latitude.value} {mivot_instance.longitude.value}") From 9277c03d65b660b3181d0a98c368796b61332d4e Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Mon, 2 Dec 2024 18:12:20 +0100 Subject: [PATCH 27/39] flake8 --- pyvo/mivot/tests/test_mivot_writer.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyvo/mivot/tests/test_mivot_writer.py b/pyvo/mivot/tests/test_mivot_writer.py index c40f810eb..f4326ea72 100644 --- a/pyvo/mivot/tests/test_mivot_writer.py +++ b/pyvo/mivot/tests/test_mivot_writer.py @@ -194,6 +194,3 @@ def test_MivotInstanceAll(): os.path.join(data_path, "reference/test_mivot_writer.json") ) votable.to_xml(data_path + "/essai.xml") - -if __name__ == "__main__": - test_MivotInstanceAll() \ No newline at end of file From b2efc78ab8b866434221863186ea433222dc92c8 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sun, 1 Dec 2024 20:35:49 +0100 Subject: [PATCH 28/39] Fix CI errors --- docs/mivot/index.rst | 14 +++----- docs/mivot/viewer.rst | 20 ++++-------- docs/mivot/writer.rst | 12 +++---- .../data/reference/test_mivot_writer.json | 32 +++++++++---------- pyvo/mivot/tests/test_mivot_writer.py | 24 +++++++------- pyvo/mivot/utils/xml_utils.py | 3 +- pyvo/mivot/writer/__init__.py | 0 pyvo/mivot/writer/annotations.py | 20 ++++++------ pyvo/mivot/writer/instance.py | 24 +++++++------- 9 files changed, 67 insertions(+), 82 deletions(-) create mode 100644 pyvo/mivot/writer/__init__.py diff --git a/docs/mivot/index.rst b/docs/mivot/index.rst index 28e6812f8..7612de318 100644 --- a/docs/mivot/index.rst +++ b/docs/mivot/index.rst @@ -1,8 +1,6 @@ -.. _index: - -******************** -MIVOT (`pyvo.mivot`) -******************** +********************* +MIVOT (``pyvo.mivo``) +********************* This module contains the new feature handling model annotations in VOTable. Astropy version >= 6.0 is required. @@ -76,8 +74,4 @@ Reference/API ============= .. automodapi:: pyvo.mivot -.. automodapi:: pyvo.mivot.viewer -.. automodapi:: pyvo.mivot.seekers -.. automodapi:: pyvo.mivot.features -.. automodapi:: pyvo.mivot.utils -.. automodapi:: pyvo.mivot.writer + diff --git a/docs/mivot/viewer.rst b/docs/mivot/viewer.rst index 6f48c5999..0c3b0ab86 100644 --- a/docs/mivot/viewer.rst +++ b/docs/mivot/viewer.rst @@ -1,6 +1,6 @@ -*************************************** -MIVOT (`pyvo.mivot`): Annotation Viewer -*************************************** +***************************************** +MIVOT (``pyvo.mivot``): Annotation Viewer +***************************************** Introduction @@ -18,7 +18,7 @@ Introduction MIVOT syntax allows to describe a data structure as a hierarchy of classes. It is also able to represent relations and composition between them. It can also build up data model objects by aggregating instances from different - tables of the VOTable (get more in `index`_). + tables of the VOTable (get more in :doc:`index`). Integrated Readout ------------------ @@ -46,8 +46,8 @@ to the ``EpochPosition`` class. ... ) >>> mivot_instance = m_viewer.dm_instance >>> print(mivot_instance.dmtype) - EpochPosition - >>> print(mivot_instance.Coordinate_coordSys.spaceRefFrame.value) + mango:EpochPosition + >>> print(mivot_instance.coordSys.spaceRefFrame.value) ICRS >>> while m_viewer.next(): ... print(f"position: {mivot_instance.latitude.value} {mivot_instance.longitude.value}") @@ -187,11 +187,3 @@ identifiers, which have the following structure: ``model:a.b``. print(mivot_instance.Coordinate_coordSys.spaceRefFrame.__dict__.keys()) dict_keys(['dmtype', 'value', 'unit', 'ref']) -Reference/API -============= - -.. automodapi:: pyvo.mivot -.. automodapi:: pyvo.mivot.viewer -.. automodapi:: pyvo.mivot.seekers -.. automodapi:: pyvo.mivot.features -.. automodapi:: pyvo.mivot.utils diff --git a/docs/mivot/writer.rst b/docs/mivot/writer.rst index 22a2a9e4d..2181b983d 100644 --- a/docs/mivot/writer.rst +++ b/docs/mivot/writer.rst @@ -1,8 +1,6 @@ -.. _viewer: - -*************************************** -MIVOT (`pyvo.mivot`): Annotation Writer -*************************************** +***************************************** +MIVOT (``pyvo.mivot``): Annotation Writer +***************************************** Introduction ============ @@ -19,7 +17,7 @@ Introduction MIVOT syntax allows to describe a data structure as a hierarchy of classes. It is also able to represent relations and composition between them. It can also build up data model objects by aggregating instances from different - tables of the VOTable (get more in `index`_). + tables of the VOTable (get more in :doc:`index`). - Model Instances in VOTables is a VO `standard `_ - Requires Astropy>=6.0 @@ -193,7 +191,7 @@ Insert the MIVOT Block in a VOTable - This straightforward step is based on the Astropy VOTable API. - Annotations are stored in-memory (in the parsed VOtable) -- The mapping can be tested with the ``MivotViewer`` API (see the doc `viewer`_) +- The mapping can be tested with the ``MivotViewer`` API (see the :doc:`viewer`) - The VOtable must be explicitly saved on disk if needed. .. code-block:: python diff --git a/pyvo/mivot/tests/data/reference/test_mivot_writer.json b/pyvo/mivot/tests/data/reference/test_mivot_writer.json index 707e1fa20..4e003b90e 100644 --- a/pyvo/mivot/tests/data/reference/test_mivot_writer.json +++ b/pyvo/mivot/tests/data/reference/test_mivot_writer.json @@ -1,5 +1,5 @@ { - "dmtype": "EpochPosition", + "dmtype": "mango:EpochPosition", "longitude": { "value": 52.2340018, "unit": "deg" @@ -8,12 +8,12 @@ "value": 59.8937333, "unit": "deg" }, - "EpochPosition_errors": { - "dmtype": "EpochPositionErrors", - "dmrole": "errors", - "EpochPositionErrors_position": { - "dmtype": "ErrorCorrMatrix", - "dmrole": "position", + "errors": { + "dmtype": "mango:EpochPositionErrors", + "dmrole": "mango:EpochPosition.errors", + "position": { + "dmtype": "mango:error.ErrorCorrMatrix", + "dmrole": "mango:EpochPositionErrors.position", "sigma1": { "value": 6.0, "unit": "arcsec" @@ -24,19 +24,19 @@ } } }, - "EpochPosition_spaceSys": { - "dmtype": "SpaceSys", + "spaceSys": { + "dmtype": "coords:SpaceSys", "dmid": "_spacesys_icrs", - "dmrole": "spaceSys", - "PhysicalCoordSys_frame": { - "dmtype": "SpaceFrame", - "dmrole": "frame", + "dmrole": "mango:EpochPosition.spaceSys", + "frame": { + "dmtype": "coords:SpaceFrame", + "dmrole": "coords:PhysicalCoordSys.frame", "spaceRefFrame": { "value": "ICRS" }, - "SpaceFrame_refPosition": { - "dmtype": "StdRefLocation", - "dmrole": "refPosition", + "refPosition": { + "dmtype": "coords:StdRefLocation", + "dmrole": "coords:SpaceFrame.refPosition", "position": { "value": "BARYCENTER" } diff --git a/pyvo/mivot/tests/test_mivot_writer.py b/pyvo/mivot/tests/test_mivot_writer.py index bc0d9770b..f4326ea72 100644 --- a/pyvo/mivot/tests/test_mivot_writer.py +++ b/pyvo/mivot/tests/test_mivot_writer.py @@ -10,7 +10,7 @@ from astropy.io.votable import parse from pyvo.utils import activate_features from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot.utils.exceptions import MappingException +from pyvo.mivot.utils.exceptions import MappingError from pyvo.mivot.utils.dict_utils import DictUtils from pyvo.mivot.writer.annotations import MivotAnnotations from pyvo.mivot.writer.instance import MivotInstance @@ -48,25 +48,25 @@ def test_MivotInstance(): """ Test the MivotInstance class for various operations including attribute addition, reference addition, and XML generation. Verifies that invalid operations raise - the expected MappingException. + the expected MappingError. """ - with pytest.raises(MappingException): + with pytest.raises(MappingError): MivotInstance(dmid="model:type.inst") instance1 = MivotInstance(dmtype="model:type.inst", dmid="id1") - with pytest.raises(MappingException): + with pytest.raises(MappingError): instance1.add_attribute( dmrole="model:type.inst.role1", value="value1", unit="m/s" ) - with pytest.raises(MappingException): + with pytest.raises(MappingError): instance1.add_attribute( dmtype="model:type.att1", dmrole="model:type.inst.role1" ) - with pytest.raises(MappingException): + with pytest.raises(MappingError): instance1.add_reference(dmref="dmreference") - with pytest.raises(MappingException): + with pytest.raises(MappingError): instance1.add_reference(dmrole="model:type.inst.role2") - with pytest.raises(MappingException): + with pytest.raises(MappingError): instance1.add_instance("azerty") instance1.add_reference(dmrole="model:type.inst.role2", dmref="dmreference") @@ -88,13 +88,13 @@ def test_MivotInstance(): def test_MivotAnnotations(): """ Test the MivotAnnotations class for template and global instance addition. Verifies - that invalid operations raise the expected MappingException. + that invalid operations raise the expected MappingError. """ mb = MivotAnnotations() - with pytest.raises(MappingException): + with pytest.raises(MappingError): mb.add_templates(12) - with pytest.raises(MappingException): + with pytest.raises(MappingError): mb.add_globals(12) @@ -190,7 +190,7 @@ def test_MivotInstanceAll(): mv = MivotViewer(votable) print(mv.dm_instance) - assert mv.dm_instance.dict == DictUtils.read_dict_from_file( + assert mv.dm_instance.to_dict() == DictUtils.read_dict_from_file( os.path.join(data_path, "reference/test_mivot_writer.json") ) votable.to_xml(data_path + "/essai.xml") diff --git a/pyvo/mivot/utils/xml_utils.py b/pyvo/mivot/utils/xml_utils.py index c790ff4ca..c4e02e91e 100644 --- a/pyvo/mivot/utils/xml_utils.py +++ b/pyvo/mivot/utils/xml_utils.py @@ -28,7 +28,8 @@ def pretty_string(xmltree, clean_ns=True): Return a pretty string representation of an XML tree. Parameters ---------- - xmltree (~`xml.etree.ElementTree.Element`): XML tree to convert to a pretty string. + xmltree (~`xml.etree.ElementTree.Element`): XML tree to convert to a pretty string + clean_ns (boolean): Ddefault namspace (ns0) removed from element names if True Returns ------- str: The pretty string representation of the XML tree. diff --git a/pyvo/mivot/writer/__init__.py b/pyvo/mivot/writer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pyvo/mivot/writer/annotations.py b/pyvo/mivot/writer/annotations.py index fdbe986f1..650d79b35 100644 --- a/pyvo/mivot/writer/annotations.py +++ b/pyvo/mivot/writer/annotations.py @@ -87,7 +87,7 @@ from astropy import version from pyvo.utils.prototype import prototype_feature from pyvo.mivot.utils.xml_utils import XmlUtils -from pyvo.mivot.utils.exceptions import MappingException, AstropyVersionException +from pyvo.mivot.utils.exceptions import MappingError, AstropyVersionException from pyvo.mivot.writer.instance import MivotInstance from pyvo.mivot.version_checker import check_astropy_version @@ -251,7 +251,7 @@ def add_templates(self, templates_instance): Raises ------ - MappingException + MappingError If `templates_instance` is neither a string nor an instance of `MivotInstance`. """ if isinstance(templates_instance, MivotInstance): @@ -259,7 +259,7 @@ def add_templates(self, templates_instance): elif isinstance(templates_instance, str): self._templates.append(templates_instance) else: - raise MappingException( + raise MappingError( "Instance added to templates must be a string or MivotInstance." ) @@ -274,7 +274,7 @@ def add_globals(self, globals_instance): Raises ------ - MappingException + MappingError If `globals_instance` is neither a string nor an instance of `MivotInstance`. """ if isinstance(globals_instance, MivotInstance): @@ -282,7 +282,7 @@ def add_globals(self, globals_instance): elif isinstance(globals_instance, str): self._globals.append(globals_instance) else: - raise MappingException( + raise MappingError( "Instance added to globals must be a string or MivotInstance." ) @@ -327,7 +327,7 @@ def check_xml(self): Raises ------ - MappingException + MappingError If the validation fails. Notes @@ -349,7 +349,7 @@ def check_xml(self): try: schema.validate(mivot_block) except Exception as excep: - raise MappingException(f"Validation failed: {excep}") from excep + raise MappingError(f"Validation failed: {excep}") from excep def insert_into_votable(self, votable_file, template_id=None, override=False): """ @@ -366,7 +366,7 @@ def insert_into_votable(self, votable_file, template_id=None, override=False): Raises ------ - MappingException + MappingError If a mapping block already exists and `override` is False. """ if not check_astropy_version(): @@ -377,7 +377,7 @@ def insert_into_votable(self, votable_file, template_id=None, override=False): elif isinstance(votable_file, VOTableFile): votable = votable_file else: - raise MappingException( + raise MappingError( "votable_file must be a file path string or a VOTableFile instance." ) @@ -386,7 +386,7 @@ def insert_into_votable(self, votable_file, template_id=None, override=False): for subresource in resource.resources: if subresource.type == "meta": if not override: - raise MappingException( + raise MappingError( "A type='meta' resource already exists in the first 'result' resource." ) else: diff --git a/pyvo/mivot/writer/instance.py b/pyvo/mivot/writer/instance.py index c01411f14..cd8bfd823 100644 --- a/pyvo/mivot/writer/instance.py +++ b/pyvo/mivot/writer/instance.py @@ -38,7 +38,7 @@ """ from pyvo.utils.prototype import prototype_feature -from pyvo.mivot.utils.exceptions import MappingException +from pyvo.mivot.utils.exceptions import MappingError @prototype_feature("MIVOT") @@ -65,11 +65,11 @@ def __init__(self, dmtype=None, dmrole=None, dmid=None): Raises ------ - MappingException + MappingError If `dmtype` is not provided. """ if not dmtype: - raise MappingException("Cannot build an instance without dmtype") + raise MappingError("Cannot build an instance without dmtype") self._dmtype = dmtype self._dmrole = dmrole self._dmid = dmid @@ -94,15 +94,15 @@ def add_attribute(self, dmtype=None, dmrole=None, ref=None, value=None, unit=Non Raises ------ - MappingException + MappingError If `dmtype` or `dmrole` is not provided, or if both `ref` and `value` are not defined. """ if not dmtype: - raise MappingException("Cannot add an attribute without dmtype") + raise MappingError("Cannot add an attribute without dmtype") if not dmrole: - raise MappingException("Cannot add an attribute without dmrole") + raise MappingError("Cannot add an attribute without dmrole") if not ref and not value: - raise MappingException("Cannot add an attribute without ref or value") + raise MappingError("Cannot add an attribute without ref or value") xml_string = f'' self._content.append(xml_string) @@ -149,11 +149,11 @@ def add_instance(self, mivot_instance): Raises ------ - MappingException + MappingError If `mivot_instance` is not of type `MivotInstance`. """ if not isinstance(mivot_instance, MivotInstance): - raise MappingException("Instance added must be of type MivotInstance") + raise MappingError("Instance added must be of type MivotInstance") self._content.append(mivot_instance.xml_string()) def xml_string(self): From d36398baa3c4ab2c86558a5128afdc71e5778a9e Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Fri, 13 Dec 2024 07:45:45 +0100 Subject: [PATCH 29/39] restructuring documentation : toc updated, exposed API --- docs/mivot/index.rst | 11 ++++++----- docs/mivot/viewer.rst | 8 ++++++++ docs/mivot/writer.rst | 8 ++++++++ pyvo/mivot/__init__.py | 2 -- pyvo/mivot/viewer/__init__.py | 2 ++ pyvo/mivot/writer/__init__.py | 2 ++ 6 files changed, 26 insertions(+), 7 deletions(-) diff --git a/docs/mivot/index.rst b/docs/mivot/index.rst index 7612de318..beaa9782c 100644 --- a/docs/mivot/index.rst +++ b/docs/mivot/index.rst @@ -1,6 +1,6 @@ -********************* -MIVOT (``pyvo.mivo``) -********************* +********************** +MIVOT (``pyvo.mivot``) +********************** This module contains the new feature handling model annotations in VOTable. Astropy version >= 6.0 is required. @@ -28,7 +28,7 @@ Introduction Implementation Scope --------------------- +==================== This implementation is totally model-agnostic. - It does not operate any validation against specific data models. @@ -65,7 +65,8 @@ Using the MIVOT package The ``pyvo.mivot`` module can be used to either read or build annotations. .. toctree:: - + :maxdepth: 2 + viewer writer diff --git a/docs/mivot/viewer.rst b/docs/mivot/viewer.rst index 0c3b0ab86..bceff847c 100644 --- a/docs/mivot/viewer.rst +++ b/docs/mivot/viewer.rst @@ -20,6 +20,9 @@ Introduction also build up data model objects by aggregating instances from different tables of the VOTable (get more in :doc:`index`). +Using the API +============= + Integrated Readout ------------------ The ``ModelViewer`` module manages access to data mapped to a model through dynamically @@ -187,3 +190,8 @@ identifiers, which have the following structure: ``model:a.b``. print(mivot_instance.Coordinate_coordSys.spaceRefFrame.__dict__.keys()) dict_keys(['dmtype', 'value', 'unit', 'ref']) +Reference/API +============= + +.. automodapi:: pyvo.mivot.viewer + diff --git a/docs/mivot/writer.rst b/docs/mivot/writer.rst index 2181b983d..b1fd5fe0b 100644 --- a/docs/mivot/writer.rst +++ b/docs/mivot/writer.rst @@ -23,6 +23,8 @@ Introduction - Requires Astropy>=6.0 - ``pyvo.mivot`` is a prototype feature which must be activated with ``activate_features("MIVOT")`` +Using the API +============= Building Annotation Object per Object ------------------------------------- @@ -223,3 +225,9 @@ Validate the annotation against the model(s) ... Valid if no error message ... + +Reference/API +============= + +.. automodapi:: pyvo.mivot.writer + diff --git a/pyvo/mivot/__init__.py b/pyvo/mivot/__init__.py index c096ba11e..e69de29bb 100644 --- a/pyvo/mivot/__init__.py +++ b/pyvo/mivot/__init__.py @@ -1,2 +0,0 @@ -# package entry point -from .viewer.mivot_viewer import MivotViewer \ No newline at end of file diff --git a/pyvo/mivot/viewer/__init__.py b/pyvo/mivot/viewer/__init__.py index e69de29bb..998961af7 100644 --- a/pyvo/mivot/viewer/__init__.py +++ b/pyvo/mivot/viewer/__init__.py @@ -0,0 +1,2 @@ +from .mivot_instance import MivotInstance +from .mivot_viewer import MivotViewer diff --git a/pyvo/mivot/writer/__init__.py b/pyvo/mivot/writer/__init__.py index e69de29bb..d76b9cf32 100644 --- a/pyvo/mivot/writer/__init__.py +++ b/pyvo/mivot/writer/__init__.py @@ -0,0 +1,2 @@ +from .annotations import MivotAnnotations +from .instance import MivotInstance \ No newline at end of file From 1b5f460135130cde7e58369e6845542638180d04 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Fri, 13 Dec 2024 07:49:59 +0100 Subject: [PATCH 30/39] Fix code blocks in docstring that were tagged as sphinx reference (one back quote) --- pyvo/mivot/writer/annotations.py | 12 ++++++------ pyvo/mivot/writer/instance.py | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pyvo/mivot/writer/annotations.py b/pyvo/mivot/writer/annotations.py index 650d79b35..b3b1ee30e 100644 --- a/pyvo/mivot/writer/annotations.py +++ b/pyvo/mivot/writer/annotations.py @@ -251,8 +251,8 @@ def add_templates(self, templates_instance): Raises ------ - MappingError - If `templates_instance` is neither a string nor an instance of `MivotInstance`. + Mappingrror + If ``templates_instance`` is neither a string nor an instance of `MivotInstance`. """ if isinstance(templates_instance, MivotInstance): self._templates.append(templates_instance.xml_string()) @@ -275,7 +275,7 @@ def add_globals(self, globals_instance): Raises ------ MappingError - If `globals_instance` is neither a string nor an instance of `MivotInstance`. + If ``globals_instance`` is neither a string nor an instance of `MivotInstance`. """ if isinstance(globals_instance, MivotInstance): self._globals.append(globals_instance.xml_string()) @@ -312,7 +312,7 @@ def set_report(self, status, message): Notes ----- - If `status` is False, all components of the MIVOT block except MODEL and REPORT + If ``status`` is False, all components of the MIVOT block except MODEL and REPORT are cleared. """ self._report_status = status @@ -358,7 +358,7 @@ def insert_into_votable(self, votable_file, template_id=None, override=False): Parameters ---------- votable_file : str or VOTableFile - The VOTable to be annotated, either as a file path or a `VOTableFile` instance. + The VOTable to be annotated, either as a file path or a ``VOTableFile`` instance. template_id : str, optional The ID of the TABLE to be mapped. Defaults to None. override : bool @@ -367,7 +367,7 @@ def insert_into_votable(self, votable_file, template_id=None, override=False): Raises ------ MappingError - If a mapping block already exists and `override` is False. + If a mapping block already exists and ``override`` is False. """ if not check_astropy_version(): raise AstropyVersionException(f"Astropy version {version.version} " diff --git a/pyvo/mivot/writer/instance.py b/pyvo/mivot/writer/instance.py index cd8bfd823..9eea57668 100644 --- a/pyvo/mivot/writer/instance.py +++ b/pyvo/mivot/writer/instance.py @@ -66,7 +66,7 @@ def __init__(self, dmtype=None, dmrole=None, dmid=None): Raises ------ MappingError - If `dmtype` is not provided. + If ``dmtype`` is not provided. """ if not dmtype: raise MappingError("Cannot build an instance without dmtype") @@ -95,7 +95,7 @@ def add_attribute(self, dmtype=None, dmrole=None, ref=None, value=None, unit=Non Raises ------ MappingError - If `dmtype` or `dmrole` is not provided, or if both `ref` and `value` are not defined. + If ``dmtype`` or ``dmrole`` is not provided, or if both ``ref`` and ``value`` are not defined. """ if not dmtype: raise MappingError("Cannot add an attribute without dmtype") @@ -128,7 +128,7 @@ def add_reference(self, dmrole=None, dmref=None): Raises ------ MappingError - If `dmrole` or `dmref` is not provided. + If ``dmrole`` or ``dmref`` is not provided. """ if not dmref: raise MappingError("Cannot add a reference without dmref") @@ -150,7 +150,7 @@ def add_instance(self, mivot_instance): Raises ------ MappingError - If `mivot_instance` is not of type `MivotInstance`. + If ``mivot_instance`` is not of type ``MivotInstance``. """ if not isinstance(mivot_instance, MivotInstance): raise MappingError("Instance added must be of type MivotInstance") From 514b48bf3b886bac86fb49519e4880b8e2953f9e Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sat, 14 Dec 2024 11:46:26 +0100 Subject: [PATCH 31/39] remove narrative doc from source files --- pyvo/mivot/writer/annotations.py | 118 ++++++++----------------------- pyvo/mivot/writer/instance.py | 63 ++++++----------- 2 files changed, 53 insertions(+), 128 deletions(-) diff --git a/pyvo/mivot/writer/annotations.py b/pyvo/mivot/writer/annotations.py index b3b1ee30e..8477e37f7 100644 --- a/pyvo/mivot/writer/annotations.py +++ b/pyvo/mivot/writer/annotations.py @@ -1,70 +1,6 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ MivotAnnotations: A utility module to build and manage MIVOT annotations. - -This module provides a class to construct, validate, and insert MIVOT (Mapping to IVOA Table) -blocks into VOTable files. The MIVOT block, represented as an XML structure, is used for -data model annotations in the IVOA ecosystem. - -Features: ---------- -- Construct the MIVOT block step-by-step with various components. -- Validate the MIVOT block against the MIVOT XML schema (if `xmlschema` is installed). -- Embed the MIVOT block into an existing VOTable file. - -The MIVOT block is constructed as a string to maintain compatibility with the Astropy API. - -Typical Usage: --------------- -The following code demonstrates how to use the MivotAnnotations class: - -.. code-block:: python - - from pyvo.mivot.writer import MivotAnnotations - from astropy.io.votable import parse - - # Create an instance of MivotAnnotations - mb = MivotAnnotations() - mb.build_mivot_block() - - # Add components to the MIVOT block - mb.add_globals("") - mb.add_templates("") - mb.add_templates("") - mb.add_model("model", "http://model.com") - mb.add_model("model2", None) - - # Configure a report - mb.set_report(True, "unit tests") - mb.build_mivot_block(templates_id="azerty") - - # Parse a VOTable and insert the MIVOT block - votable = parse("path_to_votable.xml") - mb.insert_into_votable(votable) - - # Use MivotViewer for additional processing - from pyvo.mivot.viewer import MivotViewer - mv = MivotViewer(votable) - print(mv.dm_instance) - -Limitations: ------------- -- This module does not verify whether the MIVOT block is consistent with the declared models. -- References to table columns are not validated for resolution. - -Validation of references can be performed using the `MivotViewer` class: - -.. code-block:: python - - votable = parse("path_to_votable.xml") - mv = MivotViewer(votable) - print(mv.dm_instance) - -Examples of API usage can be found in `tests/test_mivot_writer.py`. - -License: --------- -This module is licensed under a 3-clause BSD style license. See LICENSE.rst for details. """ import os import logging @@ -95,31 +31,39 @@ @prototype_feature("MIVOT") class MivotAnnotations: """ - API for constructing and managing MIVOT (Mapping for IVOA Table) annotations - step by step. This class provides methods to build the various components of - a MIVOT block, validate it against an XML schema, and integrate it into a VOTable. + This module provides a class to construct, validate, and insert MIVOT + blocks into VOTable files. + The MIVOT block, represented as an XML structure, is used for + data model annotations in the IVOA ecosystem. + + The main features are: + + - Construct the MIVOT block step-by-step with various components. + - Validate the MIVOT block against the MIVOT XML schema (if ``xmlschema`` is installed). + - Embed the MIVOT block into an existing VOTable file. + + The MIVOT block is constructed as a string to maintain compatibility with the Astropy API. + + Attributes + ---------- + _models : dict + A dictionary containing models with their names as keys and URLs as values. + _report_status : bool + Indicates the success status of the annotation process. + _report_message : str + A message associated with the report, used in the REPORT block. + _globals : list + A list of GLOBALS blocks to be included in the MIVOT block. + _templates : list + A list of TEMPLATES blocks to be included in the MIVOT block. + _templates_id : str + An optional ID for the TEMPLATES block. + _mivot_block : str + The complete MIVOT block as a string. """ def __init__(self): """ - Initializes a new instance of the `MivotAnnotations` class. - - Attributes - ---------- - _models : dict - A dictionary containing models with their names as keys and URLs as values. - _report_status : bool - Indicates the success status of the annotation process. - _report_message : str - A message associated with the report, used in the REPORT block. - _globals : list - A list of GLOBALS blocks to be included in the MIVOT block. - _templates : list - A list of TEMPLATES blocks to be included in the MIVOT block. - _templates_id : str - An optional ID for the TEMPLATES block. - _mivot_block : str - The complete MIVOT block as a string. """ self._models = {} self._report_status = True @@ -137,7 +81,7 @@ def mivot_block(self): Returns ------- str - The complete MIVOT block as a string. + Complete MIVOT block as a string. """ return self._mivot_block @@ -251,7 +195,7 @@ def add_templates(self, templates_instance): Raises ------ - Mappingrror + MappingError If ``templates_instance`` is neither a string nor an instance of `MivotInstance`. """ if isinstance(templates_instance, MivotInstance): diff --git a/pyvo/mivot/writer/instance.py b/pyvo/mivot/writer/instance.py index 9eea57668..9ea7d42b6 100644 --- a/pyvo/mivot/writer/instance.py +++ b/pyvo/mivot/writer/instance.py @@ -1,40 +1,6 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ MivotInstance is a simple API for building MIVOT instances step by step. - -Features --------- -- Model-agnostic: The implementation is independent of any specific data model. -- Syntax validation: Ensures basic MIVOT syntax rules are followed. -- Context-agnostic: Ignores context-dependent syntax rules. - -MivotInstance builds elements that can contain , , and . -Support for elements is not yet implemented. - -Usage Example -------------- -.. code-block:: python - - position = MivotInstance(dmtype="model:mango:EpochPosition", dmid="position") - position.add_attribute(dmtype="ivoa.RealQuantity", - dmrole="mango:EpochPosition.longitude", - unit="deg", ref="_RA_ICRS") - position.add_attribute(dmtype="ivoa.RealQuantity", dmrole="mango:EpochPosition.latitude", - unit="deg", ref="_DFEC_ICRS") - - position_error = MivotInstance(dmtype="mango:EpochPositionErrors", - dmrole="mango:EpochPosition.errors", dmid="id2") - position_error.add_attribute(dmtype="model:type2.att1", - dmrole="model:type2.inst.role1", value="value3", unit="m/s") - position_error.add_attribute(dmtype="model:type2.att2", - dmrole="model:type2.inst.role2", value="value4", unit="m/s") - - position.add_instance(position_error) - - mb = MivotAnnotations() - mb.add_templates(position) - mb.build_mivot_block() - print(mb.mivot_block) """ from pyvo.utils.prototype import prototype_feature @@ -44,16 +10,31 @@ @prototype_feature("MIVOT") class MivotInstance: """ - API for building elements of a MIVOT annotation step by step. + API for building elements of MIVOT annotation step by step. + + This class provides methods for incremental construction of a MIVOT instance. + It builds elements that can contain , , and . + Support for elements is not yet implemented. + + The main features are: + + - Model-agnostic: The implementation is independent of any specific data model. + - Syntax validation: Ensures basic MIVOT syntax rules are followed. + - Context-agnostic: Ignores context-dependent syntax rules. + + attributes + ---------- + _dmtype : string + Instance type (class VO-DML ID) + _dmrole : string + Role played by the instance in the context where it is used + (given by the VO-DML serialization of the model) + _dmid : string + Free identifier of the instance - This class provides methods for adding attributes, references, and nested instances, - allowing incremental construction of a MIVOT instance. """ - - def __init__(self, dmtype=None, dmrole=None, dmid=None): + def __init__(self, dmtype=None, *, dmrole=None, dmid=None): """ - Initialize a MivotInstance object. - Parameters ---------- dmtype : str From b79706cdb91f5674cf9823411a38e3bd1d8355b3 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sat, 14 Dec 2024 11:50:06 +0100 Subject: [PATCH 32/39] replace abbreviation with explicit name (clean_ns) --- pyvo/mivot/utils/xml_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyvo/mivot/utils/xml_utils.py b/pyvo/mivot/utils/xml_utils.py index c4e02e91e..a35676986 100644 --- a/pyvo/mivot/utils/xml_utils.py +++ b/pyvo/mivot/utils/xml_utils.py @@ -23,13 +23,13 @@ def pretty_print(xmltree): print(XmlUtils.pretty_string(xmltree)) @staticmethod - def pretty_string(xmltree, clean_ns=True): + def pretty_string(xmltree, clean_namespace=True): """ Return a pretty string representation of an XML tree. Parameters ---------- xmltree (~`xml.etree.ElementTree.Element`): XML tree to convert to a pretty string - clean_ns (boolean): Ddefault namspace (ns0) removed from element names if True + clean_namespace (boolean): Ddefault namspace (ns0) removed from element names if True Returns ------- str: The pretty string representation of the XML tree. @@ -40,7 +40,7 @@ def pretty_string(xmltree, clean_ns=True): else: XmlUtils.indent(xmltree) new_xml = ET.tostring(xmltree, encoding='unicode') - if clean_ns: + if clean_namespace: return new_xml.replace("ns0:", "") else: return new_xml From 4932477eab424d750349fe7d51532591a085f6b1 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sat, 14 Dec 2024 11:54:26 +0100 Subject: [PATCH 33/39] add explicit definition of the exposed namespaces (__all__) --- pyvo/mivot/writer/annotations.py | 1 + pyvo/mivot/writer/instance.py | 1 + 2 files changed, 2 insertions(+) diff --git a/pyvo/mivot/writer/annotations.py b/pyvo/mivot/writer/annotations.py index 8477e37f7..faaa55290 100644 --- a/pyvo/mivot/writer/annotations.py +++ b/pyvo/mivot/writer/annotations.py @@ -27,6 +27,7 @@ from pyvo.mivot.writer.instance import MivotInstance from pyvo.mivot.version_checker import check_astropy_version +__all__ = ["MivotAnnotations"] @prototype_feature("MIVOT") class MivotAnnotations: diff --git a/pyvo/mivot/writer/instance.py b/pyvo/mivot/writer/instance.py index 9ea7d42b6..ba8c221eb 100644 --- a/pyvo/mivot/writer/instance.py +++ b/pyvo/mivot/writer/instance.py @@ -6,6 +6,7 @@ from pyvo.utils.prototype import prototype_feature from pyvo.mivot.utils.exceptions import MappingError +__all__ = ["MivotInstance"] @prototype_feature("MIVOT") class MivotInstance: From b8021e95ac3b325d079d57334182a24da4bff7a4 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sat, 14 Dec 2024 11:55:40 +0100 Subject: [PATCH 34/39] add license MIVOT items put together. --- CHANGES.rst | 12 +++++++----- pyvo/mivot/__init__.py | 1 + pyvo/mivot/writer/__init__.py | 1 + 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 8e5095b78..23b148ed5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -50,20 +50,22 @@ Enhancements and Fixes - MIVOT module: the model references in the dictionaries that are used to build ``MivotInstance`` objects are made more consistent [#551] - -- RegTAP constraints involving tables other than rr.resource are now - done via subqueries for less duplication of interfaces. [#562, #572] - MIVOT module: If the MIVOT annotation block contains a valid instance of the ``mango:EpochPosition`` class, the dynamic object describing the mapped data can generate a valid SkyCoord instance. [#591] +- Extending the MIVOT module with the ability to build annotations component by component + and put them into a VOTable. [#627] + +- RegTAP constraints involving tables other than rr.resource are now + done via subqueries for less duplication of interfaces. [#562, #572] + + - New sub-package discover for global dataset discovery. [#470] - Updated getdatalink to be consistent with iter_datalinks. [#613] -- Extending the MIVOT module with the ability to build annotations component by component - and put them into a VOTable. [#627] Deprecations and Removals diff --git a/pyvo/mivot/__init__.py b/pyvo/mivot/__init__.py index e69de29bb..9dce85d06 100644 --- a/pyvo/mivot/__init__.py +++ b/pyvo/mivot/__init__.py @@ -0,0 +1 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst diff --git a/pyvo/mivot/writer/__init__.py b/pyvo/mivot/writer/__init__.py index d76b9cf32..f0f23bf23 100644 --- a/pyvo/mivot/writer/__init__.py +++ b/pyvo/mivot/writer/__init__.py @@ -1,2 +1,3 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst from .annotations import MivotAnnotations from .instance import MivotInstance \ No newline at end of file From 0303a85f91822dbc6a8fb9127335e1dbdfec0027 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sat, 14 Dec 2024 12:06:13 +0100 Subject: [PATCH 35/39] Make optional parameters mandatory keywords. --- pyvo/mivot/writer/annotations.py | 6 ++---- pyvo/mivot/writer/instance.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pyvo/mivot/writer/annotations.py b/pyvo/mivot/writer/annotations.py index faaa55290..76dbdcdc0 100644 --- a/pyvo/mivot/writer/annotations.py +++ b/pyvo/mivot/writer/annotations.py @@ -155,7 +155,7 @@ def _get_templates(self): templates_block += "\n" return templates_block - def build_mivot_block(self, templates_id=None): + def build_mivot_block(self, *, templates_id=None): """ Builds a complete MIVOT block from the declared components and validates it against the MIVOT XML schema. @@ -296,7 +296,7 @@ def check_xml(self): except Exception as excep: raise MappingError(f"Validation failed: {excep}") from excep - def insert_into_votable(self, votable_file, template_id=None, override=False): + def insert_into_votable(self, votable_file, override=False): """ Inserts the MIVOT block into a VOTable. @@ -304,8 +304,6 @@ def insert_into_votable(self, votable_file, template_id=None, override=False): ---------- votable_file : str or VOTableFile The VOTable to be annotated, either as a file path or a ``VOTableFile`` instance. - template_id : str, optional - The ID of the TABLE to be mapped. Defaults to None. override : bool If True, overrides any existing annotations in the VOTable. diff --git a/pyvo/mivot/writer/instance.py b/pyvo/mivot/writer/instance.py index ba8c221eb..c19e7c107 100644 --- a/pyvo/mivot/writer/instance.py +++ b/pyvo/mivot/writer/instance.py @@ -57,7 +57,7 @@ def __init__(self, dmtype=None, *, dmrole=None, dmid=None): self._dmid = dmid self._content = [] - def add_attribute(self, dmtype=None, dmrole=None, ref=None, value=None, unit=None): + def add_attribute(self, dmtype=None, dmrole=None, *, ref=None, value=None, unit=None): """ Add an element to the instance. From 3d9dec53d1a337be1de3c0765337720e0f41d191 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sun, 15 Dec 2024 16:18:56 +0100 Subject: [PATCH 36/39] import viewer explicitely because pyvo.mivot.viewer is no longer the default pyvo.mivot content code stye add license --- pyvo/mivot/tests/test_annotation_seeker.py | 2 +- pyvo/mivot/tests/test_mivot_instance.py | 2 +- pyvo/mivot/tests/test_mivot_instance_generation.py | 2 +- pyvo/mivot/tests/test_mivot_viewer.py | 2 +- pyvo/mivot/tests/test_static_reference.py | 2 +- pyvo/mivot/tests/test_user_api.py | 2 +- pyvo/mivot/tests/test_vizier_cs.py | 2 +- pyvo/mivot/tests/test_xml_viewer.py | 2 +- pyvo/mivot/viewer/__init__.py | 1 + pyvo/mivot/writer/annotations.py | 9 +++++---- pyvo/mivot/writer/instance.py | 7 ++++--- 11 files changed, 18 insertions(+), 15 deletions(-) diff --git a/pyvo/mivot/tests/test_annotation_seeker.py b/pyvo/mivot/tests/test_annotation_seeker.py index 86ad4c73f..fd741eed4 100644 --- a/pyvo/mivot/tests/test_annotation_seeker.py +++ b/pyvo/mivot/tests/test_annotation_seeker.py @@ -11,7 +11,7 @@ from pyvo.mivot.seekers.annotation_seeker import AnnotationSeeker from pyvo.mivot.utils.dict_utils import DictUtils from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot import MivotViewer +from pyvo.mivot.viewer import MivotViewer from . import XMLOutputChecker diff --git a/pyvo/mivot/tests/test_mivot_instance.py b/pyvo/mivot/tests/test_mivot_instance.py index 1e314bf44..81607ff83 100644 --- a/pyvo/mivot/tests/test_mivot_instance.py +++ b/pyvo/mivot/tests/test_mivot_instance.py @@ -11,7 +11,7 @@ from pyvo.mivot.version_checker import check_astropy_version from pyvo.mivot.viewer.mivot_instance import MivotInstance from pyvo.mivot.utils.mivot_utils import MivotUtils -from pyvo.mivot import MivotViewer +from pyvo.mivot.viewer import MivotViewer fake_hk_dict = { "dmtype": "EpochPosition", diff --git a/pyvo/mivot/tests/test_mivot_instance_generation.py b/pyvo/mivot/tests/test_mivot_instance_generation.py index cab728aed..ea2ffb85c 100644 --- a/pyvo/mivot/tests/test_mivot_instance_generation.py +++ b/pyvo/mivot/tests/test_mivot_instance_generation.py @@ -6,7 +6,7 @@ import pytest from urllib.request import urlretrieve from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot import MivotViewer +from pyvo.mivot.viewer import MivotViewer from pyvo.mivot.utils.mivot_utils import MivotUtils diff --git a/pyvo/mivot/tests/test_mivot_viewer.py b/pyvo/mivot/tests/test_mivot_viewer.py index 20fb63f85..ff79da42b 100644 --- a/pyvo/mivot/tests/test_mivot_viewer.py +++ b/pyvo/mivot/tests/test_mivot_viewer.py @@ -10,7 +10,7 @@ from pyvo.mivot.utils.dict_utils import DictUtils from pyvo.mivot.utils.exceptions import MappingError from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot import MivotViewer +from pyvo.mivot.viewer import MivotViewer from astropy import version as astropy_version diff --git a/pyvo/mivot/tests/test_static_reference.py b/pyvo/mivot/tests/test_static_reference.py index 82bce325e..1c5096f53 100644 --- a/pyvo/mivot/tests/test_static_reference.py +++ b/pyvo/mivot/tests/test_static_reference.py @@ -7,7 +7,7 @@ from pyvo.mivot.seekers.annotation_seeker import AnnotationSeeker from pyvo.mivot.features.static_reference_resolver import StaticReferenceResolver from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot import MivotViewer +from pyvo.mivot.viewer import MivotViewer from . import XMLOutputChecker diff --git a/pyvo/mivot/tests/test_user_api.py b/pyvo/mivot/tests/test_user_api.py index 04277c0e6..67a74a804 100644 --- a/pyvo/mivot/tests/test_user_api.py +++ b/pyvo/mivot/tests/test_user_api.py @@ -12,7 +12,7 @@ from astropy.coordinates import SkyCoord from pyvo.dal.scs import SCSService from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot import MivotViewer +from pyvo.mivot.viewer import MivotViewer from astropy.io.votable import parse from pyvo.mivot.utils.dict_utils import DictUtils diff --git a/pyvo/mivot/tests/test_vizier_cs.py b/pyvo/mivot/tests/test_vizier_cs.py index 573430cec..72fff1cf0 100644 --- a/pyvo/mivot/tests/test_vizier_cs.py +++ b/pyvo/mivot/tests/test_vizier_cs.py @@ -20,7 +20,7 @@ import pytest from urllib.request import urlretrieve from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot import MivotViewer +from pyvo.mivot.viewer import MivotViewer from pyvo.mivot.utils.exceptions import MivotError diff --git a/pyvo/mivot/tests/test_xml_viewer.py b/pyvo/mivot/tests/test_xml_viewer.py index e0dde04ea..8ff8f6671 100644 --- a/pyvo/mivot/tests/test_xml_viewer.py +++ b/pyvo/mivot/tests/test_xml_viewer.py @@ -9,7 +9,7 @@ from xml.etree.ElementTree import Element as element from astropy.utils.data import get_pkg_data_filename from pyvo.mivot.version_checker import check_astropy_version -from pyvo.mivot import MivotViewer +from pyvo.mivot.viewer import MivotViewer from pyvo.mivot.utils.exceptions import MivotError diff --git a/pyvo/mivot/viewer/__init__.py b/pyvo/mivot/viewer/__init__.py index 998961af7..0ca6afaf7 100644 --- a/pyvo/mivot/viewer/__init__.py +++ b/pyvo/mivot/viewer/__init__.py @@ -1,2 +1,3 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst from .mivot_instance import MivotInstance from .mivot_viewer import MivotViewer diff --git a/pyvo/mivot/writer/annotations.py b/pyvo/mivot/writer/annotations.py index 76dbdcdc0..61200980b 100644 --- a/pyvo/mivot/writer/annotations.py +++ b/pyvo/mivot/writer/annotations.py @@ -29,16 +29,17 @@ __all__ = ["MivotAnnotations"] + @prototype_feature("MIVOT") class MivotAnnotations: """ This module provides a class to construct, validate, and insert MIVOT - blocks into VOTable files. + blocks into VOTable files. The MIVOT block, represented as an XML structure, is used for data model annotations in the IVOA ecosystem. - + The main features are: - + - Construct the MIVOT block step-by-step with various components. - Validate the MIVOT block against the MIVOT XML schema (if ``xmlschema`` is installed). - Embed the MIVOT block into an existing VOTable file. @@ -281,7 +282,7 @@ def check_xml(self): """ # put here just to improve the test coverage root = etree.fromstring(self._mivot_block) - mivot_block = XmlUtils.pretty_string(root, clean_ns=False) + mivot_block = XmlUtils.pretty_string(root, clean_namespace=False) if not xmlschema: logging.error( "XML validation skipped: no XML schema found. " diff --git a/pyvo/mivot/writer/instance.py b/pyvo/mivot/writer/instance.py index c19e7c107..6c04a1d66 100644 --- a/pyvo/mivot/writer/instance.py +++ b/pyvo/mivot/writer/instance.py @@ -8,6 +8,7 @@ __all__ = ["MivotInstance"] + @prototype_feature("MIVOT") class MivotInstance: """ @@ -16,19 +17,19 @@ class MivotInstance: This class provides methods for incremental construction of a MIVOT instance. It builds elements that can contain , , and . Support for elements is not yet implemented. - + The main features are: - Model-agnostic: The implementation is independent of any specific data model. - Syntax validation: Ensures basic MIVOT syntax rules are followed. - Context-agnostic: Ignores context-dependent syntax rules. - + attributes ---------- _dmtype : string Instance type (class VO-DML ID) _dmrole : string - Role played by the instance in the context where it is used + Role played by the instance in the context where it is used (given by the VO-DML serialization of the model) _dmid : string Free identifier of the instance From bff20a58bf840aa1785e99b22e5236f72dabbe47 Mon Sep 17 00:00:00 2001 From: Laurent MICHEL Date: Sun, 15 Dec 2024 16:29:31 +0100 Subject: [PATCH 37/39] XML typography more homogeneous, parameter model_url of add_model made optional with mandatory keyword --- pyvo/mivot/writer/annotations.py | 42 ++++++++++++++++---------------- pyvo/mivot/writer/instance.py | 34 +++++++++++++------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pyvo/mivot/writer/annotations.py b/pyvo/mivot/writer/annotations.py index 61200980b..7d53e1327 100644 --- a/pyvo/mivot/writer/annotations.py +++ b/pyvo/mivot/writer/annotations.py @@ -78,7 +78,7 @@ def __init__(self): @property def mivot_block(self): """ - Getter for the MIVOT block. + Getter for the whole MIVOT block. Returns ------- @@ -89,12 +89,12 @@ def mivot_block(self): def _get_report(self): """ - Generates the REPORT component of the MIVOT block. + Generate the component of the MIVOT block. Returns ------- str - The REPORT block as a string, indicating the success or failure of the process. + The block as a string, indicating the success or failure of the process. """ if self._report_status: return f'{self._report_message}' @@ -103,12 +103,12 @@ def _get_report(self): def _get_models(self): """ - Generates the MODEL components of the MIVOT block. + Generate the components of the MIVOT block. Returns ------- str - The MODEL components as a formatted string. + The components as a formatted string. """ models_block = "" for key, value in self._models.items(): @@ -121,12 +121,12 @@ def _get_models(self): def _get_globals(self): """ - Generates the GLOBALS component of the MIVOT block. + Generate the component of the MIVOT block. Returns ------- str - The GLOBALS block as a formatted string. + The block as a formatted string. """ globals_block = "\n" for glob in self._globals: @@ -137,12 +137,12 @@ def _get_globals(self): def _get_templates(self): """ - Generates the TEMPLATES component of the MIVOT block. + Generate the component of the MIVOT block. Returns ------- str - The TEMPLATES block as a formatted string, or an empty string if no templates are defined. + The block as a formatted string, or an empty string if no templates are defined. """ if not self._templates: return "" @@ -158,13 +158,13 @@ def _get_templates(self): def build_mivot_block(self, *, templates_id=None): """ - Builds a complete MIVOT block from the declared components and validates it + Build a complete MIVOT block from the declared components and validates it against the MIVOT XML schema. Parameters ---------- templates_id : str, optional - The ID to associate with the TEMPLATES block. Defaults to None. + The ID to associate with the block. Defaults to None. Raises ------ @@ -188,12 +188,12 @@ def build_mivot_block(self, *, templates_id=None): def add_templates(self, templates_instance): """ - Adds an block to the block. + Add an element to the block. Parameters ---------- templates_instance : str or MivotInstance - The block to be added. + The element to be added. Raises ------ @@ -211,7 +211,7 @@ def add_templates(self, templates_instance): def add_globals(self, globals_instance): """ - Adds an block to the block. + Add an block to the block. Parameters ---------- @@ -232,22 +232,22 @@ def add_globals(self, globals_instance): "Instance added to globals must be a string or MivotInstance." ) - def add_model(self, model_name, model_url): + def add_model(self, model_name, *, model_url=None): """ - Adds a MODEL element to the MIVOT block. + Add a element to the MIVOT block. Parameters ---------- model_name : str The short name of the model. - model_url : str + model_url : str, optional The URL of the VO-DML file associated with the model. """ self._models[model_name] = model_url def set_report(self, status, message): """ - Sets the REPORT element of the MIVOT block. + Set the element of the MIVOT block. Parameters ---------- @@ -269,7 +269,7 @@ def set_report(self, status, message): def check_xml(self): """ - Validates the MIVOT block against the MIVOT XML schema v1.0. + Validate the MIVOT block against the MIVOT XML schema v1.0. Raises ------ @@ -278,7 +278,7 @@ def check_xml(self): Notes ----- - The schema is loaded from a local file to avoid dependency on a remote service. + The schema (mivot 1.0) is loaded from a local file to avoid dependency on a remote service. """ # put here just to improve the test coverage root = etree.fromstring(self._mivot_block) @@ -299,7 +299,7 @@ def check_xml(self): def insert_into_votable(self, votable_file, override=False): """ - Inserts the MIVOT block into a VOTable. + Insert the MIVOT block into a VOTable. Parameters ---------- diff --git a/pyvo/mivot/writer/instance.py b/pyvo/mivot/writer/instance.py index 6c04a1d66..a8e4b06c4 100644 --- a/pyvo/mivot/writer/instance.py +++ b/pyvo/mivot/writer/instance.py @@ -40,16 +40,16 @@ def __init__(self, dmtype=None, *, dmrole=None, dmid=None): Parameters ---------- dmtype : str - The dmtype of the INSTANCE (mandatory). + dmtype of the INSTANCE (mandatory) dmrole : str, optional - The dmrole of the INSTANCE. + dmrole of the INSTANCE dmid : str, optional - The dmid of the INSTANCE. + dmid of the INSTANCE Raises ------ MappingError - If ``dmtype`` is not provided. + If ``dmtype`` is not provided """ if not dmtype: raise MappingError("Cannot build an instance without dmtype") @@ -65,20 +65,20 @@ def add_attribute(self, dmtype=None, dmrole=None, *, ref=None, value=None, unit= Parameters ---------- dmtype : str - The dmtype of the ATTRIBUTE (mandatory). + dmtype of the ATTRIBUTE (mandatory) dmrole : str - The dmrole of the ATTRIBUTE (mandatory). + dmrole of the ATTRIBUTE (mandatory) ref : str, optional - ID of the column to set the attribute value. + ID of the column to set the attribute value value : str, optional - Default value of the attribute. + Default value of the attribute unit : str, optional - Unit of the attribute. + Unit of the attribute Raises ------ MappingError - If ``dmtype`` or ``dmrole`` is not provided, or if both ``ref`` and ``value`` are not defined. + If ``dmtype`` or ``dmrole`` is not provided, or if both ``ref`` and ``value`` are not defined """ if not dmtype: raise MappingError("Cannot add an attribute without dmtype") @@ -104,14 +104,14 @@ def add_reference(self, dmrole=None, dmref=None): Parameters ---------- dmrole : str - The dmrole of the REFERENCE (mandatory). + dmrole of the REFERENCE (mandatory) dmref : str - The dmref of the REFERENCE (mandatory). + dmref of the REFERENCE (mandatory) Raises ------ MappingError - If ``dmrole`` or ``dmref`` is not provided. + If ``dmrole`` or ``dmref`` is not provided """ if not dmref: raise MappingError("Cannot add a reference without dmref") @@ -128,12 +128,12 @@ def add_instance(self, mivot_instance): Parameters ---------- mivot_instance : MivotInstance - The INSTANCE to be added. + INSTANCE to be added Raises ------ MappingError - If ``mivot_instance`` is not of type ``MivotInstance``. + If ``mivot_instance`` is not of type ``MivotInstance`` """ if not isinstance(mivot_instance, MivotInstance): raise MappingError("Instance added must be of type MivotInstance") @@ -141,12 +141,12 @@ def add_instance(self, mivot_instance): def xml_string(self): """ - Build and serialize the INSTANCE element to a string. + Build and serialize the element as a string. Returns ------- str - The string representation of the INSTANCE element. + The string representation of the element """ xml_string = f' Date: Sun, 15 Dec 2024 16:58:47 +0100 Subject: [PATCH 38/39] fix CI errors --- pyvo/mivot/tests/test_mivot_writer.py | 8 +++++--- pyvo/mivot/writer/annotations.py | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/pyvo/mivot/tests/test_mivot_writer.py b/pyvo/mivot/tests/test_mivot_writer.py index f4326ea72..21dd5c533 100644 --- a/pyvo/mivot/tests/test_mivot_writer.py +++ b/pyvo/mivot/tests/test_mivot_writer.py @@ -165,14 +165,16 @@ def test_MivotInstanceAll(): mivot_annotations = MivotAnnotations() mivot_annotations.add_model( - "ivoa", "https://www.ivoa.net/xml/VODML/IVOA-v1.vo-dml.xml" + "ivoa", + vodml_url="https://www.ivoa.net/xml/VODML/IVOA-v1.vo-dml.xml" ) mivot_annotations.add_model( - "coords", "https://www.ivoa.net/xml/STC/20200908/Coords-v1.0.vo-dml.xml" + "coords", + vodml_url="https://www.ivoa.net/xml/STC/20200908/Coords-v1.0.vo-dml.xml" ) mivot_annotations.add_model( "mango", - "https://raw.githubusercontent.com/lmichel/MANGO/draft-0.1/vo-dml/mango.vo-dml.xml", + vodml_url="https://raw.githubusercontent.com/lmichel/MANGO/draft-0.1/vo-dml/mango.vo-dml.xml", ) mivot_annotations.set_report(True, "Mivot writer unit test") diff --git a/pyvo/mivot/writer/annotations.py b/pyvo/mivot/writer/annotations.py index 7d53e1327..eac6b446f 100644 --- a/pyvo/mivot/writer/annotations.py +++ b/pyvo/mivot/writer/annotations.py @@ -232,7 +232,7 @@ def add_globals(self, globals_instance): "Instance added to globals must be a string or MivotInstance." ) - def add_model(self, model_name, *, model_url=None): + def add_model(self, model_name, *, vodml_url=None): """ Add a element to the MIVOT block. @@ -240,10 +240,10 @@ def add_model(self, model_name, *, model_url=None): ---------- model_name : str The short name of the model. - model_url : str, optional + vodml_url : str, optional The URL of the VO-DML file associated with the model. """ - self._models[model_name] = model_url + self._models[model_name] = vodml_url def set_report(self, status, message): """ From d470071925333eed27412420f3f278e30bc4e8b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Mon, 16 Dec 2024 09:39:43 -0800 Subject: [PATCH 39/39] DOC: fix headings --- docs/mivot/writer.rst | 60 +++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/docs/mivot/writer.rst b/docs/mivot/writer.rst index b1fd5fe0b..53e52ff6b 100644 --- a/docs/mivot/writer.rst +++ b/docs/mivot/writer.rst @@ -31,7 +31,7 @@ Building Annotation Object per Object Creating annotations consists of 3 steps: -#. Create individual instances (INSTANCE) using the ``MivotInstance`` class: objects are +#. Create individual instances (INSTANCE) using the ``MivotInstance`` class: objects are built attribute by attribute. These components can then be aggregated into more complex objects following the structure of the mapped model(s). #. Wrap the annotations with the ``MivotAnnotations`` class: declare to the annotation builder @@ -39,7 +39,7 @@ Creating annotations consists of 3 steps: #. Insert the annotations into a VOtable by using the Astropy API (wrapped in the package logic). The annotation builder does not check whether the XML conforms to any particular model. -It simply validates it against the MIVOT XML Schema if the ``xmlvalidator`` package if is installed. +It simply validates it against the MIVOT XML Schema if the ``xmlvalidator`` package if is installed. The example below shows a step-by-step construction of a MIVOT block mapping a position with its error (as defined in the ``MANGO`` draft) @@ -47,10 +47,10 @@ and its space coordinate system (as defined in the ``coord`` model and imported Building the Coordinate System Object -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The space coordinate system is made of a space frame and a reference position, both wrapped in a ``coords:SpaceSys`` -object (see the `coords `_ model). +object (see the `coords `_ model). - Individual instances are built one by one and then packed together. - The top level object has a ``dmid`` which will be used as a reference by the ``EpochPosition`` instance. @@ -66,9 +66,9 @@ object (see the `coords ` from pyvo.mivot.writer.annotations import MivotAnnotations from pyvo.mivot.writer.instance import MivotInstance from pyvo.mivot.viewer.mivot_viewer import MivotViewer - + activate_features("MIVOT") - + space_sys = MivotInstance(dmid="_spacesys_icrs", dmtype="c") space_frame = MivotInstance( dmrole="coords:PhysicalCoordSys.frame", dmtype="coords:SpaceFrame" @@ -89,16 +89,16 @@ object (see the `coords ` Building the EpochPosition Object -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - In this example we only use the position attributes (RA/DEC) of the ``EpochPosition`` class. - The reference to the space coordinate system is added at the end. - The ``ref`` XML attributes reference columns that must be used to set the model attributes. Their values depend on the VOTable to be mapped. - + .. code-block:: python - + position = MivotInstance(dmtype="mango:EpochPosition") position.add_attribute( dmtype="ivoa:RealQuantity", @@ -118,12 +118,12 @@ Building the EpochPosition Object Building the Position Error -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- We assume that the position error is the same on both axes without correlation. +- We assume that the position error is the same on both axes without correlation. In terms of MANGO error, this corresponds to a 2x2 diagonal error matrix with two equal coefficients. -- Finally, the error is added as a component of the ``EpochPosition`` instance. - +- Finally, the error is added as a component of the ``EpochPosition`` instance. + .. code-block:: python @@ -151,20 +151,20 @@ Building the Position Error Building the MIVOT Block -~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^ - The MIVOT block consists of: - + - A process status - A list of mapped models - - A list of globals, which are objects not associated with + - A list of globals, which are objects not associated with VOTable data and that can be shared by any other MIVOT instance. - - A list of templates, which are objects that are connected to + - A list of templates, which are objects that are connected to VOTable data and whose leaf values change from one row to another. - -- The latest step (build_mivot_block) includes a validation of the MIVOT syntax that works only - if the ``xmlvaldator`` package has been installed. - + +- The latest step (build_mivot_block) includes a validation of the MIVOT syntax that works only + if the ``xmlvaldator`` package has been installed. + .. code-block:: python @@ -188,8 +188,7 @@ Building the MIVOT Block Insert the MIVOT Block in a VOTable -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - This straightforward step is based on the Astropy VOTable API. - Annotations are stored in-memory (in the parsed VOtable) @@ -200,26 +199,26 @@ Insert the MIVOT Block in a VOTable from astropy.io.votable import parse - + votable = parse(votable_path) mivot_annotations.insert_into_votable(votable) mv = MivotViewer(votable) mappes_instance = mv.dm_instance - + votable.to_xml("pyvo-tuto.xml") - -Validate the annotation against the model(s) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~s~~~~~~ - + +Validate the annotation against the models +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + - This action requires the ``mivot-validator`` package to be installed. - It validates the mapped classes against the models they come from. .. code-block:: shell :caption: Build the coordinate system (coords:SpaceSys) - + % pip install mivot-validator % mivot-instance-validate pyvo-tuto.xml ... @@ -230,4 +229,3 @@ Reference/API ============= .. automodapi:: pyvo.mivot.writer -