From 904494d4291bf8f414a411b3e911a91d771735b5 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Tue, 9 Oct 2018 14:31:10 -0400 Subject: [PATCH 01/55] Implement basic threshold for inlining array data --- asdf/block.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/asdf/block.py b/asdf/block.py index 45a5ac797..f87f5b0d5 100644 --- a/asdf/block.py +++ b/asdf/block.py @@ -25,6 +25,9 @@ from . import yamlutil +_DEFAULT_INLINE_THRESHOLD_SIZE = 100 + + class BlockManager(object): """ Manages the `Block`s associated with a ASDF file. @@ -45,6 +48,8 @@ def __init__(self, asdffile, copy_arrays=False, lazy_load=True, 'streamed': self._streamed_blocks } + self._inline_threshold_size = _DEFAULT_INLINE_THRESHOLD_SIZE + self._data_to_block_mapping = {} self._validate_checksums = False self._memmap = not copy_arrays @@ -730,8 +735,7 @@ def find_or_create_block_for_array(self, arr, ctx): block : Block """ from .tags.core import ndarray - if (isinstance(arr, ndarray.NDArrayType) and - arr.block is not None): + if (isinstance(arr, ndarray.NDArrayType) and arr.block is not None): if arr.block in self.blocks: return arr.block else: @@ -742,6 +746,10 @@ def find_or_create_block_for_array(self, arr, ctx): if block is not None: return block block = Block(base) + + if arr.size <= self._inline_threshold_size: + block._array_storage = 'inline' + self.add(block) self._handle_global_block_settings(ctx, block) return block From 6b0d3023c26fa631e2c09bc0bcf7b9a5036b9d5f Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Tue, 9 Oct 2018 14:52:19 -0400 Subject: [PATCH 02/55] Add top-level option for controlling inline threshold size --- asdf/asdf.py | 10 +++++++--- asdf/block.py | 7 +++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/asdf/asdf.py b/asdf/asdf.py index 5cd804e4e..1b62172dd 100644 --- a/asdf/asdf.py +++ b/asdf/asdf.py @@ -51,7 +51,8 @@ class AsdfFile(versioning.VersionedMixin): def __init__(self, tree=None, uri=None, extensions=None, version=None, ignore_version_mismatch=True, ignore_unrecognized_tag=False, ignore_implicit_conversion=False, copy_arrays=False, - lazy_load=True, custom_schema=None, _readonly=False): + lazy_load=True, custom_schema=None, _readonly=False, + inline_threshold=None): """ Parameters ---------- @@ -108,7 +109,10 @@ def __init__(self, tree=None, uri=None, extensions=None, version=None, files follow custom conventions beyond those enforced by the standard. - """ + inline_threshold : int, optional + Optional threshold size below which arrays will automatically be + stored inline. Defaults to {0}. + """.format(block._DEFAULT_INLINE_THRESHOLD_SIZE) if custom_schema is not None: self._custom_schema = schema.load_custom_schema(custom_schema) @@ -131,7 +135,7 @@ def __init__(self, tree=None, uri=None, extensions=None, version=None, self._external_asdf_by_uri = {} self._blocks = block.BlockManager( self, copy_arrays=copy_arrays, lazy_load=lazy_load, - readonly=_readonly) + readonly=_readonly, inline_threshold=inline_threshold) self._uri = None if tree is None: self.tree = {} diff --git a/asdf/block.py b/asdf/block.py index f87f5b0d5..527ebb404 100644 --- a/asdf/block.py +++ b/asdf/block.py @@ -33,7 +33,7 @@ class BlockManager(object): Manages the `Block`s associated with a ASDF file. """ def __init__(self, asdffile, copy_arrays=False, lazy_load=True, - readonly=False): + readonly=False, inline_threshold=None): self._asdffile = weakref.ref(asdffile) self._internal_blocks = [] @@ -48,7 +48,10 @@ def __init__(self, asdffile, copy_arrays=False, lazy_load=True, 'streamed': self._streamed_blocks } - self._inline_threshold_size = _DEFAULT_INLINE_THRESHOLD_SIZE + if inline_threshold is not None: + self._inline_threshold_size = inline_threshold + else: + self._inline_threshold_size = _DEFAULT_INLINE_THRESHOLD_SIZE self._data_to_block_mapping = {} self._validate_checksums = False From 5ccd9d775469489536508ead91f250d963c72ed9 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Tue, 9 Oct 2018 14:59:11 -0400 Subject: [PATCH 03/55] Set default inline threshold level to 0 for test helpers... Since many tests assume that arrays will be stored internally --- asdf/tests/helpers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/asdf/tests/helpers.py b/asdf/tests/helpers.py index 24ccb1038..e38c08bca 100644 --- a/asdf/tests/helpers.py +++ b/asdf/tests/helpers.py @@ -180,6 +180,9 @@ def _assert_roundtrip_tree(tree, tmpdir, *, asdf_check_func=None, fname = str(tmpdir.join('test.asdf')) + # Most tests assume that all blocks will be stored internally + init_options.setdefault('inline_threshold', 0) + # First, test writing/reading a BytesIO buffer buff = io.BytesIO() AsdfFile(tree, extensions=extensions, **init_options).write_to(buff, **write_options) From 0520d3606852264e3cab04ac667bf8ecc437abd5 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Tue, 9 Oct 2018 15:05:26 -0400 Subject: [PATCH 04/55] Do not automatically inline masked arrays --- asdf/block.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/asdf/block.py b/asdf/block.py index 527ebb404..50a3866b3 100644 --- a/asdf/block.py +++ b/asdf/block.py @@ -12,6 +12,7 @@ from urllib import parse as urlparse import numpy as np +from numpy.ma.core import masked_array import yaml @@ -751,7 +752,8 @@ def find_or_create_block_for_array(self, arr, ctx): block = Block(base) if arr.size <= self._inline_threshold_size: - block._array_storage = 'inline' + if not isinstance(arr, masked_array): + block._array_storage = 'inline' self.add(block) self._handle_global_block_settings(ctx, block) From 85d30f8ea04778e6b682913929356c8fbacad006 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Tue, 9 Oct 2018 15:21:23 -0400 Subject: [PATCH 05/55] Adjust inline threshold size down to 50... This is pretty arbitrary but it makes a lot more tests pass without modification. --- asdf/block.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asdf/block.py b/asdf/block.py index 50a3866b3..fe81354c2 100644 --- a/asdf/block.py +++ b/asdf/block.py @@ -26,7 +26,7 @@ from . import yamlutil -_DEFAULT_INLINE_THRESHOLD_SIZE = 100 +_DEFAULT_INLINE_THRESHOLD_SIZE = 50 class BlockManager(object): From 896a3f38363bf4ff2ab2e45cf3a3274dabcb9f72 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Tue, 9 Oct 2018 16:26:12 -0400 Subject: [PATCH 06/55] More sophisticated check for automatic inlining of arrays --- asdf/block.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/asdf/block.py b/asdf/block.py index fe81354c2..d9a5a0fd4 100644 --- a/asdf/block.py +++ b/asdf/block.py @@ -724,6 +724,20 @@ def get_source(self, block): raise ValueError("block not found.") + def _should_inline(self, array): + + if not np.issubdtype(array.dtype, np.number): + return False + + if isinstance(array, masked_array): + return False + + # Make sure none of the values are too large to store as literals + if (array > 2**52).any(): + return False + + return array.size <= self._inline_threshold_size + def find_or_create_block_for_array(self, arr, ctx): """ For a given array, looks for an existing block containing its @@ -751,9 +765,8 @@ def find_or_create_block_for_array(self, arr, ctx): return block block = Block(base) - if arr.size <= self._inline_threshold_size: - if not isinstance(arr, masked_array): - block._array_storage = 'inline' + if self._should_inline(arr): + block._array_storage = 'inline' self.add(block) self._handle_global_block_settings(ctx, block) From 7fa87df5623cc066b9fef196adeb6e505a3fe532 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Tue, 9 Oct 2018 16:26:57 -0400 Subject: [PATCH 07/55] Update tests to account for automatic inlining of arrays --- asdf/commands/tests/test_exploded.py | 5 ++++- asdf/tests/test_generic_io.py | 6 ++++++ asdf/tests/test_low_level.py | 20 ++++++++++++++++---- asdf/tests/test_reference.py | 7 +++++-- asdf/tests/test_stream.py | 7 +++++++ 5 files changed, 38 insertions(+), 7 deletions(-) diff --git a/asdf/commands/tests/test_exploded.py b/asdf/commands/tests/test_exploded.py index e3b2855a6..47629f939 100644 --- a/asdf/commands/tests/test_exploded.py +++ b/asdf/commands/tests/test_exploded.py @@ -23,7 +23,10 @@ def test_explode_then_implode(tmpdir): path = os.path.join(str(tmpdir), 'original.asdf') ff = AsdfFile(tree) - ff.write_to(path) + # Since we're testing with small arrays, force all arrays to be stored + # in internal blocks rather than letting some of them be automatically put + # inline. + ff.write_to(path, all_array_storage='internal') assert len(ff.blocks) == 2 result = main.main_from_args(['explode', path]) diff --git a/asdf/tests/test_generic_io.py b/asdf/tests/test_generic_io.py index 8237cc3a8..3e51f1b5c 100644 --- a/asdf/tests/test_generic_io.py +++ b/asdf/tests/test_generic_io.py @@ -26,6 +26,12 @@ def tree(request): def _roundtrip(tree, get_write_fd, get_read_fd, write_options={}, read_options={}): + + # Since we're testing with small arrays, force all arrays to be stored + # in internal blocks rather than letting some of them be automatically put + # inline. + write_options.setdefault('all_array_storage', 'internal') + with get_write_fd() as fd: asdf.AsdfFile(tree).write_to(fd, **write_options) # Work around the fact that generic_io's get_file doesn't have a way of diff --git a/asdf/tests/test_low_level.py b/asdf/tests/test_low_level.py index 51e054185..c8bf6e271 100644 --- a/asdf/tests/test_low_level.py +++ b/asdf/tests/test_low_level.py @@ -128,7 +128,10 @@ def test_invalid_source(small_tree): buff = io.BytesIO() ff = asdf.AsdfFile(small_tree) - ff.write_to(buff) + # Since we're testing with small arrays, force all arrays to be stored + # in internal blocks rather than letting some of them be automatically put + # inline. + ff.write_to(buff, all_array_storage='internal') buff.seek(0) with asdf.open(buff) as ff2: @@ -802,7 +805,10 @@ def test_deferred_block_loading(small_tree): buff = io.BytesIO() ff = asdf.AsdfFile(small_tree) - ff.write_to(buff, include_block_index=False) + # Since we're testing with small arrays, force all arrays to be stored + # in internal blocks rather than letting some of them be automatically put + # inline. + ff.write_to(buff, include_block_index=False, all_array_storage='internal') buff.seek(0) with asdf.open(buff) as ff2: @@ -869,7 +875,10 @@ def test_large_block_index(): } ff = asdf.AsdfFile(tree) - ff.write_to(buff) + # Since we're testing with small arrays, force all arrays to be stored + # in internal blocks rather than letting some of them be automatically put + # inline. + ff.write_to(buff, all_array_storage='internal') buff.seek(0) with asdf.open(buff) as ff2: @@ -927,7 +936,10 @@ def test_short_file_find_block_index(): buff = io.BytesIO() ff = asdf.AsdfFile({'arr': np.ndarray([1]), 'arr2': np.ndarray([2])}) - ff.write_to(buff, include_block_index=False) + # Since we're testing with small arrays, force all arrays to be stored + # in internal blocks rather than letting some of them be automatically put + # inline. + ff.write_to(buff, include_block_index=False, all_array_storage='internal') buff.write(b'#ASDF BLOCK INDEX\n') buff.write(b'0' * (io.DEFAULT_BUFFER_SIZE * 4)) diff --git a/asdf/tests/test_reference.py b/asdf/tests/test_reference.py index 993c22204..43b7d0f39 100644 --- a/asdf/tests/test_reference.py +++ b/asdf/tests/test_reference.py @@ -32,11 +32,14 @@ def test_external_reference(tmpdir): } external_path = os.path.join(str(tmpdir), 'external.asdf') ext = asdf.AsdfFile(exttree) - ext.write_to(external_path) + # Since we're testing with small arrays, force all arrays to be stored + # in internal blocks rather than letting some of them be automatically put + # inline. + ext.write_to(external_path, all_array_storage='internal') external_path = os.path.join(str(tmpdir), 'external2.asdf') ff = asdf.AsdfFile(exttree) - ff.write_to(external_path) + ff.write_to(external_path, all_array_storage='internal') tree = { # The special name "data" here must be an array. This is diff --git a/asdf/tests/test_stream.py b/asdf/tests/test_stream.py index 2c12135fe..2c0eabecb 100644 --- a/asdf/tests/test_stream.py +++ b/asdf/tests/test_stream.py @@ -87,6 +87,9 @@ def test_stream_with_nonstream(): } ff = asdf.AsdfFile(tree) + # Since we're testing with small arrays, force this array to be stored in + # an internal block rather than letting it be automatically put inline. + ff.set_array_storage(ff['nonstream'], 'internal') ff.write_to(buff) for i in range(100): buff.write(np.array([i] * 12, np.float64).tostring()) @@ -112,6 +115,10 @@ def test_stream_real_file(tmpdir): with open(path, 'wb') as fd: ff = asdf.AsdfFile(tree) + # Since we're testing with small arrays, force this array to be stored + # in an internal block rather than letting it be automatically put + # inline. + ff.set_array_storage(ff['nonstream'], 'internal') ff.write_to(fd) for i in range(100): fd.write(np.array([i] * 12, np.float64).tostring()) From 0371b95b578f4e14697fc6d4d70f9bbd45acb282 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Thu, 11 Oct 2018 16:46:19 -0400 Subject: [PATCH 08/55] Add test cases for inline array threshold size --- asdf/tests/test_low_level.py | 78 ++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/asdf/tests/test_low_level.py b/asdf/tests/test_low_level.py index c8bf6e271..0404e6874 100644 --- a/asdf/tests/test_low_level.py +++ b/asdf/tests/test_low_level.py @@ -1245,3 +1245,81 @@ def test_open_readonly(tmpdir): with pytest.raises(PermissionError): with asdf.open(tmpfile, mode='rw'): pass + +def test_inline_threshold(tmpdir): + + tree = { + 'small': np.ones(10), + 'large': np.ones(100) + } + + with asdf.AsdfFile(tree) as af: + assert len(list(af.blocks.inline_blocks)) == 1 + assert len(list(af.blocks.internal_blocks)) == 1 + + with asdf.AsdfFile(tree, inline_threshold=10) as af: + assert len(list(af.blocks.inline_blocks)) == 1 + assert len(list(af.blocks.internal_blocks)) == 1 + + with asdf.AsdfFile(tree, inline_threshold=5) as af: + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 2 + + with asdf.AsdfFile(tree, inline_threshold=100) as af: + assert len(list(af.blocks.inline_blocks)) == 2 + assert len(list(af.blocks.internal_blocks)) == 0 + + +def test_inline_threshold_masked(tmpdir): + + mask = np.random.randint(0, 1+1, 20) + masked_array = np.ma.masked_array(np.ones(20), mask=mask) + + tree = { + 'masked': masked_array + } + + # Make sure that masked arrays aren't automatically inlined, even if they + # are small enough + with asdf.AsdfFile(tree) as af: + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 2 + + tree = { + 'masked': masked_array, + 'normal': np.random.random(20) + } + + with asdf.AsdfFile(tree) as af: + assert len(list(af.blocks.inline_blocks)) == 1 + assert len(list(af.blocks.internal_blocks)) == 2 + + +def test_inline_threshold_override(tmpdir): + + tmpfile = str(tmpdir.join('inline.asdf')) + + tree = { + 'small': np.ones(10), + 'large': np.ones(100) + } + + with asdf.AsdfFile(tree) as af: + af.set_array_storage(tree['small'], 'internal') + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 2 + + with asdf.AsdfFile(tree) as af: + af.set_array_storage(tree['large'], 'inline') + assert len(list(af.blocks.inline_blocks)) == 2 + assert len(list(af.blocks.internal_blocks)) == 0 + + with asdf.AsdfFile(tree) as af: + af.write_to(tmpfile, all_array_storage='internal') + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 2 + + with asdf.AsdfFile(tree) as af: + af.write_to(tmpfile, all_array_storage='inline') + assert len(list(af.blocks.inline_blocks)) == 2 + assert len(list(af.blocks.internal_blocks)) == 0 From 1be49f60de22c1a04266cb04f60fcc1312292e64 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Fri, 12 Oct 2018 13:33:34 -0400 Subject: [PATCH 09/55] Update documentation --- docs/asdf/arrays.rst | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/docs/asdf/arrays.rst b/docs/asdf/arrays.rst index 5e588c120..371dc9163 100644 --- a/docs/asdf/arrays.rst +++ b/docs/asdf/arrays.rst @@ -56,11 +56,37 @@ data being saved. Saving inline arrays -------------------- -For small arrays, you may not care about the efficiency of a binary -representation and just want to save the array contents directly in the YAML -tree. The `~asdf.AsdfFile.set_array_storage` method can be used to set the -storage type of the associated data. The allowed values are ``internal``, -``external``, and ``inline``. +As of `asdf-2.2.0`, small numerical arrays are automatically stored inline. The +default threshold size for inline versus internal arrays can be found with the +following: + +.. code:: + + >>> from asdf.block import _DEFAULT_INLINE_THRESHOLD_SIZE + >>> print(_DEFAULT_INLINE_THRESHOLD_SIZE) + 50 + +The default threshold can be overridden passing the `inline_threshold` argument +to the `asdf.AsdfFile` constructor. Setting `inline_threshold=0` has the effect +of making all small arrays be stored in internal blocks: + +.. runcode:: + + from asdf import AsdfFile + import numpy as np + + # Ordinarilly an array this size would be automatically inlined + my_array = np.ones(10) + tree = {'my_array': my_array} + # Set the inline threshold to 0 to force internal storage + with AsdfFile(tree, inline_threshold=0) as ff: + ff.write_to("test.asdf") + +.. asdf:: test.asdf + +The `~asdf.AsdfFile.set_array_storage` method can be used to set or override +the default storage type of a particular data array. The allowed values are +``internal``, ``external``, and ``inline``. - ``internal``: The default. The array data will be stored in a binary block in the same ASDF file. From b1bdab255473ffd34536dd07996fedf4e0c4e921 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Wed, 17 Oct 2018 14:03:45 -0400 Subject: [PATCH 10/55] First-pass implementation of support for large integers --- asdf-standard | 2 +- asdf/__init__.py | 1 + asdf/tags/core/__init__.py | 1 + asdf/tags/core/integer.py | 60 ++++++++++++++++++++++++++++ asdf/tags/core/tests/test_integer.py | 32 +++++++++++++++ 5 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 asdf/tags/core/integer.py create mode 100644 asdf/tags/core/tests/test_integer.py diff --git a/asdf-standard b/asdf-standard index 0bc7aa7c6..e03385d64 160000 --- a/asdf-standard +++ b/asdf-standard @@ -1 +1 @@ -Subproject commit 0bc7aa7c6b7b2099174f6ac40e2590f366f8b092 +Subproject commit e03385d643a94cd80dd3254d7ace71c3cda96945 diff --git a/asdf/__init__.py b/asdf/__init__.py index 9b434e439..096ebe0d0 100644 --- a/asdf/__init__.py +++ b/asdf/__init__.py @@ -38,6 +38,7 @@ from .extension import AsdfExtension from .stream import Stream from . import commands +from .tags.core import IntegerType from .tags.core.external_reference import ExternalArrayReference from jsonschema import ValidationError diff --git a/asdf/tags/core/__init__.py b/asdf/tags/core/__init__.py index 5233f834e..7e0dadd02 100644 --- a/asdf/tags/core/__init__.py +++ b/asdf/tags/core/__init__.py @@ -44,4 +44,5 @@ def to_tree(cls, node, ctx): from .constant import ConstantType from .ndarray import NDArrayType from .complex import ComplexType +from .integer import IntegerType from .external_reference import ExternalArrayReference diff --git a/asdf/tags/core/integer.py b/asdf/tags/core/integer.py new file mode 100644 index 000000000..f05d62774 --- /dev/null +++ b/asdf/tags/core/integer.py @@ -0,0 +1,60 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +# -*- coding: utf-8 -*- + +from numbers import Integral + +import numpy as np + +from ...asdftypes import AsdfType +from ...yamlutil import custom_tree_to_tagged_tree + + +class IntegerType(AsdfType): + name = 'core/integer' + version = '1.0.0' + + def __init__(self, value): + self._value = value + self._sign = '-' if value < 0 else '+' + + @classmethod + def to_tree(cls, node, ctx): + # pack integer value into 32-bit words + words = [] + value = int(np.abs(node._value)) + while value > 0: + words.append(value & 0xffffffff) + value >>= 32 + + tree = dict() + array = np.array(words, dtype=np.uint32) + tree['words'] = custom_tree_to_tagged_tree(np.array(words), ctx) + tree['sign'] = node._sign + + return tree + + @classmethod + def from_tree(cls, tree, ctx): + + value = 0 + for x in tree['words'][::-1]: + value <<= 32 + value |= int(x) + + if tree['sign'] == '-': + value = -value + + return value + + def __eq__(self, other): + if isinstance(other, Integral): + return self._value == other + elif isinstance(other, IntegerType): + return self._value == other._value + else: + raise ValueError( + "Can't compare IntegralType to unknown type: {}".format( + type(other))) + + def __repr__(self): + return "IntegerType({})".format(self._value) diff --git a/asdf/tags/core/tests/test_integer.py b/asdf/tags/core/tests/test_integer.py new file mode 100644 index 000000000..fe534af2f --- /dev/null +++ b/asdf/tags/core/tests/test_integer.py @@ -0,0 +1,32 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +# -*- coding: utf-8 -*- + +import random + +import pytest + +from asdf import IntegerType +from asdf.tests import helpers + + +# Make sure tests are deterministic +random.seed(0) + + +@pytest.mark.parametrize('sign', ['+', '-']) +@pytest.mark.parametrize('value', [ + random.getrandbits(64), + random.getrandbits(65), + random.getrandbits(100), + random.getrandbits(128), + random.getrandbits(129), + random.getrandbits(200), +]) +def test_integer_value(tmpdir, value, sign): + + if sign == '-': + value = -value + + integer = IntegerType(value) + tree = dict(integer=integer) + helpers.assert_roundtrip_tree(tree, tmpdir) From 10bc1b58abfd4c689c35623ff3ee5f14d197105e Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Wed, 17 Oct 2018 14:54:14 -0400 Subject: [PATCH 11/55] Update documentation to include IntegerType --- asdf/__init__.py | 2 +- asdf/tags/core/integer.py | 31 +++++++++++++++++++++++++++++++ docs/asdf/features.rst | 13 +++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/asdf/__init__.py b/asdf/__init__.py index 096ebe0d0..f6daae3a1 100644 --- a/asdf/__init__.py +++ b/asdf/__init__.py @@ -15,7 +15,7 @@ __all__ = [ 'AsdfFile', 'CustomType', 'AsdfExtension', 'Stream', 'open', 'test', - 'commands', 'ExternalArrayReference' + 'commands', 'IntegerType', 'ExternalArrayReference' ] try: diff --git a/asdf/tags/core/integer.py b/asdf/tags/core/integer.py index f05d62774..e6ad1a2b9 100644 --- a/asdf/tags/core/integer.py +++ b/asdf/tags/core/integer.py @@ -10,6 +10,37 @@ class IntegerType(AsdfType): + """ + Enables the storage of arbitrarily large integer values + + The ASDF Standard mandates that integer literals in the tree can be no + larger than 52 bits. Use of this class enables the storage of arbitrarily + large integer values. + + When reading files that contain arbitrarily large integers, the values that + are restored in the tree will be raw Python `int` instances. + + Parameters + ---------- + + value: `numbers.Integral` + A Python integral value (e.g. `int` or `numpy.integer`) + + Examples + -------- + + >>> import asdf + >>> import random + >>> # Create a large integer value + >>> largeval = random.getrandombits(100) + >>> # Store the large integer value to the tree using asdf.IntegerType + >>> tree = dict(largeval=Asdf.IntegerType(largeval)) + >>> with asdf.AsdfFile(tree) as af: + ... af.write_to('largeval.asdf') + >>> with asdf.open('largeval.asdf') as aa: + ... assert aa['largeval'] == largeval + """ + name = 'core/integer' version = '1.0.0' diff --git a/docs/asdf/features.rst b/docs/asdf/features.rst index 3d19b565e..0f4cb42c4 100644 --- a/docs/asdf/features.rst +++ b/docs/asdf/features.rst @@ -18,6 +18,19 @@ respectively. The top-level tree object behaves like a Python dictionary and supports arbitrary nesting of data structures. For simple examples of creating and reading trees, see :ref:`overview`. +.. note:: + + The ASDF Standard imposes a maximum size of 52 bits for integer literals in + the tree (see `the docs `_ + for details and justification). Attempting to store a larger value will + result in a validation error. + + Integers and floats of up to 64 bits can be stored inside of :mod:`numpy` + arrays (see below). + + For arbitrary precision integer support, see `IntegerType`. + + One of the key features of ASDF is its ability to serialize :mod:`numpy` arrays. This is discussed in detail in :ref:`array-data`. From 84080ba735c5455ee11caaf69e2934f8ae768195 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Wed, 17 Oct 2018 15:39:50 -0400 Subject: [PATCH 12/55] Allow storage type of IntegerType array to be overridden --- asdf/tags/core/integer.py | 12 ++++++++++-- asdf/tags/core/tests/test_integer.py | 26 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/asdf/tags/core/integer.py b/asdf/tags/core/integer.py index e6ad1a2b9..288569fa0 100644 --- a/asdf/tags/core/integer.py +++ b/asdf/tags/core/integer.py @@ -26,6 +26,11 @@ class IntegerType(AsdfType): value: `numbers.Integral` A Python integral value (e.g. `int` or `numpy.integer`) + storage_type: `str`, optional + Optionally overrides the storage type of the array used to represent + the integer value. Valid values are "internal" (the default) and + "inline" + Examples -------- @@ -44,9 +49,11 @@ class IntegerType(AsdfType): name = 'core/integer' version = '1.0.0' - def __init__(self, value): + def __init__(self, value, storage_type='internal'): + assert storage_type in ['internal', 'inline'], "Invalid storage type given" self._value = value self._sign = '-' if value < 0 else '+' + self._storage = storage_type @classmethod def to_tree(cls, node, ctx): @@ -59,7 +66,8 @@ def to_tree(cls, node, ctx): tree = dict() array = np.array(words, dtype=np.uint32) - tree['words'] = custom_tree_to_tagged_tree(np.array(words), ctx) + ctx.set_array_storage(array, node._storage) + tree['words'] = custom_tree_to_tagged_tree(array, ctx) tree['sign'] = node._sign return tree diff --git a/asdf/tags/core/tests/test_integer.py b/asdf/tags/core/tests/test_integer.py index fe534af2f..1e5141959 100644 --- a/asdf/tags/core/tests/test_integer.py +++ b/asdf/tags/core/tests/test_integer.py @@ -5,6 +5,7 @@ import pytest +import asdf from asdf import IntegerType from asdf.tests import helpers @@ -30,3 +31,28 @@ def test_integer_value(tmpdir, value, sign): integer = IntegerType(value) tree = dict(integer=integer) helpers.assert_roundtrip_tree(tree, tmpdir) + + +@pytest.mark.parametrize('inline', [False, True]) +def test_integer_storage(tmpdir, inline): + + tmpfile = str(tmpdir.join('integer.asdf')) + + kwargs = dict() + if inline: + kwargs['storage_type'] = 'inline' + + random.seed(0) + value = random.getrandbits(1000) + tree = dict(integer=IntegerType(value, **kwargs)) + + with asdf.AsdfFile(tree) as af: + af.write_to(tmpfile) + + with asdf.open(tmpfile, _force_raw_types=True) as rf: + if inline: + assert 'source' not in rf.tree['integer']['words'] + assert 'data' in rf.tree['integer']['words'] + else: + assert 'source' in rf.tree['integer']['words'] + assert 'data' not in rf.tree['integer']['words'] From ed1862914ea97292e1332cdf073d5ded88ebdbf2 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Wed, 17 Oct 2018 15:56:55 -0400 Subject: [PATCH 13/55] Use the same underlying array for duplicate integer values --- asdf/tags/core/integer.py | 29 +++++++++++++++++++++------- asdf/tags/core/tests/test_integer.py | 21 ++++++++++++++++++++ 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/asdf/tags/core/integer.py b/asdf/tags/core/integer.py index 288569fa0..36ab27fa6 100644 --- a/asdf/tags/core/integer.py +++ b/asdf/tags/core/integer.py @@ -49,6 +49,8 @@ class IntegerType(AsdfType): name = 'core/integer' version = '1.0.0' + _value_cache = dict() + def __init__(self, value, storage_type='internal'): assert storage_type in ['internal', 'inline'], "Invalid storage type given" self._value = value @@ -57,15 +59,28 @@ def __init__(self, value, storage_type='internal'): @classmethod def to_tree(cls, node, ctx): - # pack integer value into 32-bit words - words = [] - value = int(np.abs(node._value)) - while value > 0: - words.append(value & 0xffffffff) - value >>= 32 + + if ctx not in cls._value_cache: + cls._value_cache[ctx] = dict() + + abs_value = int(np.abs(node._value)) + + # If the same value has already been stored, reuse the array + if abs_value in cls._value_cache[ctx]: + array = cls._value_cache[ctx][abs_value] + else: + # pack integer value into 32-bit words + words = [] + value = abs_value + while value > 0: + words.append(value & 0xffffffff) + value >>= 32 + + array = np.array(words, dtype=np.uint32) + if node._storage == 'internal': + cls._value_cache[ctx][abs_value] = array tree = dict() - array = np.array(words, dtype=np.uint32) ctx.set_array_storage(array, node._storage) tree['words'] = custom_tree_to_tagged_tree(array, ctx) tree['sign'] = node._sign diff --git a/asdf/tags/core/tests/test_integer.py b/asdf/tags/core/tests/test_integer.py index 1e5141959..7645d65b8 100644 --- a/asdf/tags/core/tests/test_integer.py +++ b/asdf/tags/core/tests/test_integer.py @@ -56,3 +56,24 @@ def test_integer_storage(tmpdir, inline): else: assert 'source' in rf.tree['integer']['words'] assert 'data' not in rf.tree['integer']['words'] + + +def test_integer_storage_duplication(tmpdir): + + tmpfile = str(tmpdir.join('integer.asdf')) + + random.seed(0) + value = random.getrandbits(1000) + tree = dict(integer1=IntegerType(value), integer2=IntegerType(value)) + + with asdf.AsdfFile(tree) as af: + af.write_to(tmpfile) + assert len(af.blocks) == 1 + + with asdf.open(tmpfile, _force_raw_types=True) as rf: + assert rf.tree['integer1']['words']['source'] == 0 + assert rf.tree['integer2']['words']['source'] == 0 + + with asdf.open(tmpfile) as aa: + assert aa.tree['integer1'] == value + assert aa.tree['integer2'] == value From 1f2812f3bea1c9b08582e65c91f4ce3db7e9cb78 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Thu, 18 Oct 2018 12:33:42 -0400 Subject: [PATCH 14/55] Add optional field to integer for string representation of value... This is intended solely to aid in human readability when looking at the ASDF file contents. --- asdf-standard | 2 +- asdf/tags/core/integer.py | 1 + asdf/tags/core/tests/test_integer.py | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/asdf-standard b/asdf-standard index e03385d64..9d17f2b78 160000 --- a/asdf-standard +++ b/asdf-standard @@ -1 +1 @@ -Subproject commit e03385d643a94cd80dd3254d7ace71c3cda96945 +Subproject commit 9d17f2b78c64388709117c2123d479ff427cb1b1 diff --git a/asdf/tags/core/integer.py b/asdf/tags/core/integer.py index 36ab27fa6..2bd69648e 100644 --- a/asdf/tags/core/integer.py +++ b/asdf/tags/core/integer.py @@ -84,6 +84,7 @@ def to_tree(cls, node, ctx): ctx.set_array_storage(array, node._storage) tree['words'] = custom_tree_to_tagged_tree(array, ctx) tree['sign'] = node._sign + tree['string'] = str(int(node._value)) return tree diff --git a/asdf/tags/core/tests/test_integer.py b/asdf/tags/core/tests/test_integer.py index 7645d65b8..d1a943d76 100644 --- a/asdf/tags/core/tests/test_integer.py +++ b/asdf/tags/core/tests/test_integer.py @@ -57,6 +57,9 @@ def test_integer_storage(tmpdir, inline): assert 'source' in rf.tree['integer']['words'] assert 'data' not in rf.tree['integer']['words'] + assert 'string' in rf.tree['integer'] + assert rf.tree['integer']['string'] == str(value) + def test_integer_storage_duplication(tmpdir): From 086c967c2c289d75b427f529e0d07f0cd4a5aed9 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Thu, 18 Oct 2018 14:19:17 -0400 Subject: [PATCH 15/55] Update standard to version 1.3.0 --- asdf-standard | 2 +- asdf/tags/core/integer.py | 4 ++-- asdf/versioning.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/asdf-standard b/asdf-standard index 9d17f2b78..dfada9046 160000 --- a/asdf-standard +++ b/asdf-standard @@ -1 +1 @@ -Subproject commit 9d17f2b78c64388709117c2123d479ff427cb1b1 +Subproject commit dfada904677a6b7f5e7473977fe9a14c4ecb8277 diff --git a/asdf/tags/core/integer.py b/asdf/tags/core/integer.py index 2bd69648e..d70d7a4ab 100644 --- a/asdf/tags/core/integer.py +++ b/asdf/tags/core/integer.py @@ -37,9 +37,9 @@ class IntegerType(AsdfType): >>> import asdf >>> import random >>> # Create a large integer value - >>> largeval = random.getrandombits(100) + >>> largeval = random.getrandbits(100) >>> # Store the large integer value to the tree using asdf.IntegerType - >>> tree = dict(largeval=Asdf.IntegerType(largeval)) + >>> tree = dict(largeval=asdf.IntegerType(largeval)) >>> with asdf.AsdfFile(tree) as af: ... af.write_to('largeval.asdf') >>> with asdf.open('largeval.asdf') as aa: diff --git a/asdf/versioning.py b/asdf/versioning.py index 312e6ac1b..3d2aff6e4 100644 --- a/asdf/versioning.py +++ b/asdf/versioning.py @@ -142,15 +142,15 @@ def __hash__(self): return super(AsdfSpec, self).__hash__() -default_version = AsdfVersion('1.2.0') - - supported_versions = [ AsdfVersion('1.0.0'), AsdfVersion('1.1.0'), - AsdfVersion('1.2.0') + AsdfVersion('1.2.0'), + AsdfVersion('1.3.0') ] +default_version = supported_versions[-1] + class VersionedMixin(object): _version = default_version From eb654c95922f3720455bd71bafd53aae599433c4 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Thu, 18 Oct 2018 14:57:34 -0400 Subject: [PATCH 16/55] Add workaround for reading files with large literal values... Even though this indicates an invalid ASDF file, we make this possible, albeit with a warning. --- asdf/asdf.py | 11 ++++++----- asdf/schema.py | 37 +++++++++++++++++++++++++++++-------- asdf/tests/test_schema.py | 18 ++++++++++++++++++ 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/asdf/asdf.py b/asdf/asdf.py index 1b62172dd..2f8b1feeb 100644 --- a/asdf/asdf.py +++ b/asdf/asdf.py @@ -415,13 +415,14 @@ def comments(self): """ return self._comments - def _validate(self, tree, custom=True): + def _validate(self, tree, custom=True, reading=False): tagged_tree = yamlutil.custom_tree_to_tagged_tree( tree, self) - schema.validate(tagged_tree, self) + schema.validate(tagged_tree, self, reading=reading) # Perform secondary validation pass if requested if custom and self._custom_schema: - schema.validate(tagged_tree, self, self._custom_schema) + schema.validate(tagged_tree, self, self._custom_schema, + reading=reading) def validate(self): """ @@ -654,10 +655,10 @@ def _open_asdf(cls, self, fd, uri=None, mode='r', tree = reference.find_references(tree, self) if not do_not_fill_defaults: - schema.fill_defaults(tree, self) + schema.fill_defaults(tree, self, reading=True) try: - self._validate(tree) + self._validate(tree, reading=True) except ValidationError: self.close() raise diff --git a/asdf/schema.py b/asdf/schema.py index 477b3a9b0..231cf43fd 100644 --- a/asdf/schema.py +++ b/asdf/schema.py @@ -458,21 +458,33 @@ def get_validator(schema={}, ctx=None, validators=None, url_mapping=None, return validator -def validate_large_literals(instance): +def validate_large_literals(instance, reading=False): """ Validate that the tree has no large numeric literals. """ # We can count on 52 bits of precision for instance in treeutil.iter_tree(instance): - if (isinstance(instance, (Integral)) and ( - instance > ((1 << 51) - 1) or - instance < -((1 << 51) - 2))): + + if not isinstance(instance, Integral): + continue + + if instance <= ((1 << 51) - 1) and instance >= -((1 << 51) - 2): + continue + + if not reading: raise ValidationError( "Integer value {0} is too large to safely represent as a " "literal in ASDF".format(instance)) + warnings.warn( + "Invalid integer literal value {0} detected while reading file. " + "The value has been read safely, but the file should be " + "fixed.".format(instance) + ) -def validate(instance, ctx=None, schema={}, validators=None, *args, **kwargs): + +def validate(instance, ctx=None, schema={}, validators=None, reading=False, + *args, **kwargs): """ Validate the given instance (which must be a tagged tree) against the appropriate schema. The schema itself is located using the @@ -495,6 +507,11 @@ def validate(instance, ctx=None, schema={}, validators=None, *args, **kwargs): validators : dict, optional A dictionary mapping properties to validators to use (instead of the built-in ones and ones provided by extension types). + + reading: bool, optional + Indicates whether validation is being performed when the file is being + read. This is useful to allow for different validation behavior when + reading vs writing files. """ if ctx is None: from .asdf import AsdfFile @@ -504,10 +521,10 @@ def validate(instance, ctx=None, schema={}, validators=None, *args, **kwargs): *args, **kwargs) validator.validate(instance, _schema=(schema or None)) - validate_large_literals(instance) + validate_large_literals(instance, reading=reading) -def fill_defaults(instance, ctx): +def fill_defaults(instance, ctx, reading=False): """ For any default values in the schema, add them to the tree if they don't exist. @@ -518,8 +535,12 @@ def fill_defaults(instance, ctx): ctx : AsdfFile context Used to resolve tags and urls + + reading: bool, optional + Indicates whether the ASDF file is being read (in contrast to being + written). """ - validate(instance, ctx, validators=FILL_DEFAULTS) + validate(instance, ctx, validators=FILL_DEFAULTS, reading=reading) def remove_defaults(instance, ctx): diff --git a/asdf/tests/test_schema.py b/asdf/tests/test_schema.py index 243370d7b..e1073eccd 100644 --- a/asdf/tests/test_schema.py +++ b/asdf/tests/test_schema.py @@ -500,6 +500,24 @@ def test_large_literals(use_numpy): print(buff.getvalue()) +def test_read_large_literal(): + + value = 1 << 64 + yaml = """integer: {}""".format(value) + + buff = helpers.yaml_to_asdf(yaml) + + with pytest.warns(UserWarning) as w: + with asdf.open(buff) as af: + assert af['integer'] == value + + # We get two warnings: one for validation time, and one when defaults + # are filled. It seems like we could improve this architecture, though... + assert len(w) == 2 + assert str(w[0].message).startswith('Invalid integer literal value') + assert str(w[1].message).startswith('Invalid integer literal value') + + def test_nested_array(): s = { 'type': 'object', From ca407a8d7c97833433c6c0d569ec81bbfa660d52 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Thu, 18 Oct 2018 16:30:12 -0400 Subject: [PATCH 17/55] Update change log --- CHANGES.rst | 6 ++++++ asdf-standard | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 099fd5b47..85a7f1e73 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -24,6 +24,12 @@ types. This warning is converted to an error when using ``assert_roundtrip_tree`` for tests. [#583] +- Storage of arbitrary precision integers is now provided by + ``asdf.IntegerType``. Reading a file with integer literals that are too + large now causes only a warning instead of a validation error. This is to + provide backwards compatibility for files that were created with a buggy + version of ASDF (see #553 below). [#566] + 2.1.2 (2018-11-13) ------------------ diff --git a/asdf-standard b/asdf-standard index dfada9046..0bc7aa7c6 160000 --- a/asdf-standard +++ b/asdf-standard @@ -1 +1 @@ -Subproject commit dfada904677a6b7f5e7473977fe9a14c4ecb8277 +Subproject commit 0bc7aa7c6b7b2099174f6ac40e2590f366f8b092 From 06e22923058b7c09b78dfabf78bb7ae141dabaa8 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Wed, 24 Oct 2018 13:25:29 -0400 Subject: [PATCH 18/55] IntegerType needs to roundtrip. Add conversion to int, float --- asdf/tags/core/integer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/asdf/tags/core/integer.py b/asdf/tags/core/integer.py index d70d7a4ab..0d1d62972 100644 --- a/asdf/tags/core/integer.py +++ b/asdf/tags/core/integer.py @@ -99,7 +99,13 @@ def from_tree(cls, tree, ctx): if tree['sign'] == '-': value = -value - return value + return IntegerType(value) + + def __int__(self): + return int(self._value) + + def __float__(self): + return float(self._value) def __eq__(self, other): if isinstance(other, Integral): From e2f931dcc537f39d8eb5826f1082ef68fd404271 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Wed, 24 Oct 2018 13:32:08 -0400 Subject: [PATCH 19/55] Add tests for IntegerType equality and conversion --- asdf/tags/core/tests/test_integer.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/asdf/tags/core/tests/test_integer.py b/asdf/tags/core/tests/test_integer.py index d1a943d76..5b171f966 100644 --- a/asdf/tags/core/tests/test_integer.py +++ b/asdf/tags/core/tests/test_integer.py @@ -80,3 +80,14 @@ def test_integer_storage_duplication(tmpdir): with asdf.open(tmpfile) as aa: assert aa.tree['integer1'] == value assert aa.tree['integer2'] == value + + +def test_integer_conversion(): + + random.seed(0) + value = random.getrandbits(1000) + + integer = asdf.IntegerType(value) + assert integer == value + assert int(integer) == int(value) + assert float(integer) == float(value) From 3ec2f8469d9931aa834264645b82a033b444a0d7 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Fri, 19 Oct 2018 16:20:24 -0400 Subject: [PATCH 20/55] Fix a regression introduced by #557 --- asdf/block.py | 2 +- asdf/tests/test_low_level.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/asdf/block.py b/asdf/block.py index d9a5a0fd4..f560a6a27 100644 --- a/asdf/block.py +++ b/asdf/block.py @@ -733,7 +733,7 @@ def _should_inline(self, array): return False # Make sure none of the values are too large to store as literals - if (array > 2**52).any(): + if (array[~np.isnan(array)] > 2**52).any(): return False return array.size <= self._inline_threshold_size diff --git a/asdf/tests/test_low_level.py b/asdf/tests/test_low_level.py index 0404e6874..4b20fe161 100644 --- a/asdf/tests/test_low_level.py +++ b/asdf/tests/test_low_level.py @@ -18,7 +18,8 @@ from asdf import versioning from asdf.exceptions import AsdfDeprecationWarning -from ..tests.helpers import assert_tree_match +from ..tests.helpers import (assert_tree_match, assert_roundtrip_tree, + display_warnings) def test_no_yaml_end_marker(tmpdir): @@ -1323,3 +1324,16 @@ def test_inline_threshold_override(tmpdir): af.write_to(tmpfile, all_array_storage='inline') assert len(list(af.blocks.inline_blocks)) == 2 assert len(list(af.blocks.internal_blocks)) == 0 + + +def test_no_warning_nan_array(tmpdir): + """ + Tests for a regression that was introduced by + https://github.com/spacetelescope/asdf/pull/557 + """ + + tree = dict(array=np.array([1, 2, np.nan])) + + with pytest.warns(None) as w: + assert_roundtrip_tree(tree, tmpdir) + assert len(w) == 0, display_warnings(w) From 9cc3315420edda25219f1cd5f99b5c0cd68ba864 Mon Sep 17 00:00:00 2001 From: Nadia Dencheva Date: Sun, 4 Nov 2018 14:14:47 -0500 Subject: [PATCH 21/55] remove WCS tags which are now available in gwcs --- asdf/tags/__init__.py | 1 - asdf/tags/wcs/__init__.py | 5 - asdf/tags/wcs/tests/__init__.py | 0 asdf/tags/wcs/tests/data/__init__.py | 1 - .../wcs/tests/data/test_frames-1.1.0.asdf | 97 ----- asdf/tags/wcs/tests/data/test_wcs-1.0.0.asdf | 46 --- asdf/tags/wcs/tests/data/test_wcs-1.1.0.asdf | 46 --- asdf/tags/wcs/tests/setup_package.py | 5 - asdf/tags/wcs/tests/test_wcs.py | 223 ---------- asdf/tags/wcs/wcs.py | 390 ------------------ 10 files changed, 814 deletions(-) delete mode 100644 asdf/tags/wcs/__init__.py delete mode 100644 asdf/tags/wcs/tests/__init__.py delete mode 100644 asdf/tags/wcs/tests/data/__init__.py delete mode 100644 asdf/tags/wcs/tests/data/test_frames-1.1.0.asdf delete mode 100644 asdf/tags/wcs/tests/data/test_wcs-1.0.0.asdf delete mode 100644 asdf/tags/wcs/tests/data/test_wcs-1.1.0.asdf delete mode 100644 asdf/tags/wcs/tests/setup_package.py delete mode 100644 asdf/tags/wcs/tests/test_wcs.py delete mode 100644 asdf/tags/wcs/wcs.py diff --git a/asdf/tags/__init__.py b/asdf/tags/__init__.py index deb4c841e..055bc2cf8 100644 --- a/asdf/tags/__init__.py +++ b/asdf/tags/__init__.py @@ -4,4 +4,3 @@ # TODO: Import entire tree automatically and make these work like "plugins"? from . import core -from . import wcs diff --git a/asdf/tags/wcs/__init__.py b/asdf/tags/wcs/__init__.py deleted file mode 100644 index 00bdb2ec1..000000000 --- a/asdf/tags/wcs/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -# -*- coding: utf-8 -*- - - -from .wcs import * diff --git a/asdf/tags/wcs/tests/__init__.py b/asdf/tags/wcs/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/asdf/tags/wcs/tests/data/__init__.py b/asdf/tags/wcs/tests/data/__init__.py deleted file mode 100644 index 9dce85d06..000000000 --- a/asdf/tags/wcs/tests/data/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst diff --git a/asdf/tags/wcs/tests/data/test_frames-1.1.0.asdf b/asdf/tags/wcs/tests/data/test_frames-1.1.0.asdf deleted file mode 100644 index fb5d066e7..000000000 --- a/asdf/tags/wcs/tests/data/test_frames-1.1.0.asdf +++ /dev/null @@ -1,97 +0,0 @@ -#ASDF 1.0.0 -#ASDF_STANDARD 1.1.0 -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- !core/asdf-1.0.0 -asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf', - name: asdf, version: 1.3.3} -frames: -- !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: {type: ICRS} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: {equinox: !time/time-1.1.0 '2018-01-01 00:00:00.000', type: FK5} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: {equinox: !time/time-1.1.0 '2018-01-01 00:00:00.000', obstime: !time/time-1.1.0 '2015-01-01 - 00:00:00.000', type: FK4} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: {equinox: !time/time-1.1.0 '2018-01-01 00:00:00.000', obstime: !time/time-1.1.0 '2015-01-01 - 00:00:00.000', type: FK4_noeterms} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: {type: galactic} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [x, y, z] - axes_order: [0, 1, 2] - name: CelestialFrame - reference_frame: - galcen_coord: !wcs/icrs_coord-1.1.0 - dec: {value: -28.936175} - ra: - value: 266.4051 - wrap_angle: !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 deg, value: 360.0} - galcen_distance: !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: 5.0} - galcen_v_sun: - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 km s-1, value: 11.1} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 km s-1, value: 232.24} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 km s-1, value: 7.25} - roll: !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 deg, value: 3.0} - type: galactocentric - z_sun: !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 pc, value: 3.0} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: - obsgeoloc: - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: 3.0856775814671916e+16} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: 9.257032744401574e+16} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: 6.1713551629343834e+19} - obsgeovel: - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: 2.0} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: 1.0} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: 8.0} - obstime: !time/time-1.1.0 2018-01-01 00:00:00.000 - type: GCRS - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: {obstime: !time/time-1.1.0 '2018-01-01 00:00:00.000', type: CIRS} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [x, y, z] - axes_order: [0, 1, 2] - name: CelestialFrame - reference_frame: {obstime: !time/time-1.1.0 '2018-01-03 00:00:00.000', type: ITRS} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: - equinox: !time/time-1.1.0 J2000.000 - obsgeoloc: - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: 3.0856775814671916e+16} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: 9.257032744401574e+16} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: 6.1713551629343834e+19} - obsgeovel: - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: 2.0} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: 1.0} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: 8.0} - obstime: !time/time-1.1.0 2018-01-01 00:00:00.000 - type: precessed_geocentric - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -... diff --git a/asdf/tags/wcs/tests/data/test_wcs-1.0.0.asdf b/asdf/tags/wcs/tests/data/test_wcs-1.0.0.asdf deleted file mode 100644 index 195251394..000000000 --- a/asdf/tags/wcs/tests/data/test_wcs-1.0.0.asdf +++ /dev/null @@ -1,46 +0,0 @@ -#ASDF 1.0.0 -#ASDF_STANDARD 1.0.0 -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- !core/asdf-1.0.0 -asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf', - name: asdf, version: 1.3.3} -gw1: !wcs/wcs-1.0.0 - name: '' - steps: - - !wcs/step-1.0.0 - frame: detector - transform: !transform/concatenate-1.1.0 - forward: - - !transform/shift-1.1.0 {offset: 12.4} - - !transform/shift-1.1.0 {offset: -2.0} - - !wcs/step-1.0.0 {frame: icrs} -gw2: !wcs/wcs-1.0.0 - name: '' - steps: - - !wcs/step-1.0.0 - frame: detector - transform: !transform/concatenate-1.1.0 - forward: - - !transform/shift-1.1.0 {offset: 12.4} - - !transform/shift-1.1.0 {offset: -2.0} - - !wcs/step-1.0.0 {frame: icrs} -gw3: !wcs/wcs-1.0.0 - name: '' - steps: - - !wcs/step-1.0.0 - frame: !wcs/frame-1.1.0 - axes_names: [x, y] - name: detector - unit: [!unit/unit-1.0.0 pixel, !unit/unit-1.0.0 pixel] - transform: !transform/concatenate-1.1.0 - forward: - - !transform/shift-1.1.0 {offset: 12.4} - - !transform/shift-1.1.0 {offset: -2.0} - - !wcs/step-1.0.0 - frame: !wcs/celestial_frame-1.0.0 - axes_names: [lon, lat] - name: icrs - reference_frame: {type: ICRS} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -... diff --git a/asdf/tags/wcs/tests/data/test_wcs-1.1.0.asdf b/asdf/tags/wcs/tests/data/test_wcs-1.1.0.asdf deleted file mode 100644 index 9e0281edd..000000000 --- a/asdf/tags/wcs/tests/data/test_wcs-1.1.0.asdf +++ /dev/null @@ -1,46 +0,0 @@ -#ASDF 1.0.0 -#ASDF_STANDARD 1.1.0 -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- !core/asdf-1.0.0 -asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf', - name: asdf, version: 1.3.3} -gw1: !wcs/wcs-1.0.0 - name: '' - steps: - - !wcs/step-1.0.0 - frame: detector - transform: !transform/concatenate-1.1.0 - forward: - - !transform/shift-1.1.0 {offset: 12.4} - - !transform/shift-1.1.0 {offset: -2.0} - - !wcs/step-1.0.0 {frame: icrs} -gw2: !wcs/wcs-1.0.0 - name: '' - steps: - - !wcs/step-1.0.0 - frame: detector - transform: !transform/concatenate-1.1.0 - forward: - - !transform/shift-1.1.0 {offset: 12.4} - - !transform/shift-1.1.0 {offset: -2.0} - - !wcs/step-1.0.0 {frame: icrs} -gw3: !wcs/wcs-1.0.0 - name: '' - steps: - - !wcs/step-1.0.0 - frame: !wcs/frame-1.1.0 - axes_names: [x, y] - name: detector - unit: [!unit/unit-1.0.0 pixel, !unit/unit-1.0.0 pixel] - transform: !transform/concatenate-1.1.0 - forward: - - !transform/shift-1.1.0 {offset: 12.4} - - !transform/shift-1.1.0 {offset: -2.0} - - !wcs/step-1.0.0 - frame: !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: icrs - reference_frame: {type: ICRS} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -... diff --git a/asdf/tags/wcs/tests/setup_package.py b/asdf/tags/wcs/tests/setup_package.py deleted file mode 100644 index 99c855276..000000000 --- a/asdf/tags/wcs/tests/setup_package.py +++ /dev/null @@ -1,5 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -# -*- coding: utf-8 -*- - -def get_package_data(): # pragma: no cover - return { str(_PACKAGE_NAME_ + '.tags.wcs.tests'): ['data/*.asdf'] } diff --git a/asdf/tags/wcs/tests/test_wcs.py b/asdf/tags/wcs/tests/test_wcs.py deleted file mode 100644 index d273993b0..000000000 --- a/asdf/tags/wcs/tests/test_wcs.py +++ /dev/null @@ -1,223 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -# -*- coding: utf-8 -*- - -import os -import pytest -import warnings -from functools import partial - -gwcs = pytest.importorskip('gwcs') -astropy = pytest.importorskip('astropy', minversion='3.0.0') - -_gwcs_version = gwcs.version.version -_astropy_version = astropy.version.version - -INCOMPATIBLE_VERSIONS = _gwcs_version == '0.9.0' and _astropy_version < '3.1.dev0' - - -from astropy.modeling import models -from astropy import coordinates as coord -from astropy import units as u -from astropy import time - -from gwcs import coordinate_frames as cf -from gwcs import wcs - -import asdf -from asdf import AsdfFile -from asdf.tests import helpers - -from . import data as test_data -get_test_data_path = partial(helpers.get_test_data_path, module=test_data) - - -@pytest.mark.parametrize('version', ['1.0.0', '1.1.0']) -def test_read_wcs(version): - """Simple test to make sure that we can read older versions of files - containing WCS objects. We do not test against versions of the ASDF format - more recent than 1.1.0 since the schemas and tags have moved to Astropy and - GWCS.""" - - filename = get_test_data_path("test_wcs-{}.asdf".format(version)) - with asdf.open(filename) as tree: - assert isinstance(tree['gw1'], wcs.WCS) - assert isinstance(tree['gw2'], wcs.WCS) - assert isinstance(tree['gw3'], wcs.WCS) - - -@pytest.mark.skipif(INCOMPATIBLE_VERSIONS, reason="Incompatible versions for GWCS and Astropy") -@pytest.mark.parametrize('version', ['1.0.0', '1.1.0', '1.2.0']) -def test_composite_frame(tmpdir, version): - icrs = coord.ICRS() - fk5 = coord.FK5() - cel1 = cf.CelestialFrame(reference_frame=icrs) - cel2 = cf.CelestialFrame(reference_frame=fk5) - - spec1 = cf.SpectralFrame(name='freq', unit=[u.Hz,], axes_order=(2,)) - spec2 = cf.SpectralFrame(name='wave', unit=[u.m,], axes_order=(2,)) - - comp1 = cf.CompositeFrame([cel1, spec1]) - comp2 = cf.CompositeFrame([cel2, spec2]) - comp = cf.CompositeFrame([comp1, cf.SpectralFrame(axes_order=(3,), unit=(u.m,))]) - - tree = { - 'comp1': comp1, - 'comp2': comp2, - 'comp': comp - } - - write_options = dict(version=version) - helpers.assert_roundtrip_tree(tree, tmpdir, write_options=write_options) - - -def test_frames(tmpdir): - """Simple check to make sure we can still read older files with frames. - Serialization of these frames was only introduced in v1.1.0, and we do not - test any subsequent ASDF versions since the schemas and tags for those - frames have moved to Astropy and gwcs.""" - - filename = get_test_data_path("test_frames-1.1.0.asdf") - with asdf.open(filename) as tree: - for frame in tree['frames']: - assert isinstance(frame, cf.CoordinateFrame) - - -def test_backwards_compat_galcen(): - # Hold these fields constant so that we can compare them - declination = 1.0208 # in degrees - right_ascension = 45.729 # in degrees - galcen_distance = 3.14 - roll = 4.0 - z_sun = 0.2084 - old_frame_yaml = """ -frames: - - !wcs/celestial_frame-1.0.0 - axes_names: [x, y, z] - axes_order: [0, 1, 2] - name: CelestialFrame - reference_frame: - type: galactocentric - galcen_dec: - - %f - - deg - galcen_ra: - - %f - - deg - galcen_distance: - - %f - - m - roll: - - %f - - deg - z_sun: - - %f - - pc - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -""" % (declination, right_ascension, galcen_distance, roll, z_sun) - - new_frame_yaml = """ -frames: - - !wcs/celestial_frame-1.1.0 - axes_names: [x, y, z] - axes_order: [0, 1, 2] - name: CelestialFrame - reference_frame: - type: galactocentric - galcen_coord: !wcs/icrs_coord-1.1.0 - dec: {value: %f} - ra: - value: %f - wrap_angle: - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 deg, value: 360.0} - galcen_distance: - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} - galcen_v_sun: - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 km s-1, value: 11.1} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 km s-1, value: 232.24} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 km s-1, value: 7.25} - roll: !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 deg, value: %f} - z_sun: !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 pc, value: %f} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -""" % (declination, right_ascension, galcen_distance, roll, z_sun) - - old_buff = helpers.yaml_to_asdf(old_frame_yaml) - old_asdf = asdf.open(old_buff) - old_frame = old_asdf.tree['frames'][0] - new_buff = helpers.yaml_to_asdf(new_frame_yaml) - new_asdf = asdf.open(new_buff) - new_frame = new_asdf.tree['frames'][0] - - # Poor man's frame comparison since it's not implemented by astropy - assert old_frame.axes_names == new_frame.axes_names - assert old_frame.axes_order == new_frame.axes_order - assert old_frame.unit == new_frame.unit - - old_refframe = old_frame.reference_frame - new_refframe = new_frame.reference_frame - - # v1.0.0 frames have no representation of galcen_v_center, so do not compare - assert old_refframe.galcen_distance == new_refframe.galcen_distance - assert old_refframe.galcen_coord.dec == new_refframe.galcen_coord.dec - assert old_refframe.galcen_coord.ra == new_refframe.galcen_coord.ra - - -def test_backwards_compat_gcrs(): - obsgeoloc = ( - 3.0856775814671916e+16, - 9.257032744401574e+16, - 6.1713551629343834e+19 - ) - obsgeovel = (2.0, 1.0, 8.0) - - old_frame_yaml = """ -frames: - - !wcs/celestial_frame-1.0.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: - type: GCRS - obsgeoloc: - - [%f, %f, %f] - - !unit/unit-1.0.0 m - obsgeovel: - - [%f, %f, %f] - - !unit/unit-1.0.0 m s-1 - obstime: !time/time-1.0.0 2010-01-01 00:00:00.000 - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -""" % (obsgeovel + obsgeoloc) - - new_frame_yaml = """ -frames: - - !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: - type: GCRS - obsgeoloc: - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} - obsgeovel: - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f} - obstime: !time/time-1.1.0 2010-01-01 00:00:00.000 - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -""" % (obsgeovel + obsgeoloc) - - old_buff = helpers.yaml_to_asdf(old_frame_yaml) - old_asdf = asdf.open(old_buff) - old_frame = old_asdf.tree['frames'][0] - old_loc = old_frame.reference_frame.obsgeoloc - old_vel = old_frame.reference_frame.obsgeovel - - new_buff = helpers.yaml_to_asdf(new_frame_yaml) - new_asdf = asdf.open(new_buff) - new_frame = new_asdf.tree['frames'][0] - new_loc = new_frame.reference_frame.obsgeoloc - new_vel = new_frame.reference_frame.obsgeovel - - assert (old_loc.x == new_loc.x and old_loc.y == new_loc.y and - old_loc.z == new_loc.z) - assert (old_vel.x == new_vel.x and old_vel.y == new_vel.y and - old_vel.z == new_vel.z) diff --git a/asdf/tags/wcs/wcs.py b/asdf/tags/wcs/wcs.py deleted file mode 100644 index 1a8c7de9e..000000000 --- a/asdf/tags/wcs/wcs.py +++ /dev/null @@ -1,390 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -# -*- coding: utf-8 -*- - -from ...asdftypes import AsdfType -from ... import yamlutil - - -_REQUIRES = ['gwcs', 'astropy'] - - -class WCSType(AsdfType): - name = "wcs/wcs" - requires = _REQUIRES - types = ['gwcs.WCS'] - version = '1.1.0' - - @classmethod - def from_tree(cls, node, ctx): - import gwcs - - steps = [(x['frame'], x.get('transform')) for x in node['steps']] - name = node['name'] - - return gwcs.WCS(steps, name=name) - - @classmethod - def to_tree(cls, gwcs, ctx): - def get_frame(frame_name): - frame = getattr(gwcs, frame_name) - if frame is None: - return frame_name - return frame - - frames = gwcs.available_frames - steps = [] - for i in range(len(frames) - 1): - frame_name = frames[i] - frame = get_frame(frame_name) - transform = gwcs.get_transform(frames[i], frames[i + 1]) - steps.append(StepType({'frame': frame, 'transform': transform})) - frame_name = frames[-1] - frame = get_frame(frame_name) - steps.append(StepType({'frame': frame})) - - return {'name': gwcs.name, - 'steps': yamlutil.custom_tree_to_tagged_tree(steps, ctx)} - - @classmethod - def assert_equal(cls, old, new): - from ...tests import helpers - - assert old.name == new.name - assert len(old.available_frames) == len(new.available_frames) - for (old_frame, old_transform), (new_frame, new_transform) in zip( - old.pipeline, new.pipeline): - helpers.assert_tree_match(old_frame, new_frame) - helpers.assert_tree_match(old_transform, new_transform) - - -class StepType(dict, AsdfType): - name = "wcs/step" - requires = _REQUIRES - version = '1.1.0' - - -class FrameType(AsdfType): - name = "wcs/frame" - requires = ['gwcs'] - types = ['gwcs.Frame2D'] - version = '1.1.0' - - @classmethod - def _get_reference_frame_mapping(cls): - if hasattr(cls, '_reference_frame_mapping'): - return cls._reference_frame_mapping - - from astropy.coordinates import builtin_frames - - cls._reference_frame_mapping = { - 'ICRS': builtin_frames.ICRS, - 'FK5': builtin_frames.FK5, - 'FK4': builtin_frames.FK4, - 'FK4_noeterms': builtin_frames.FK4NoETerms, - 'galactic': builtin_frames.Galactic, - 'galactocentric': builtin_frames.Galactocentric, - 'GCRS': builtin_frames.GCRS, - 'CIRS': builtin_frames.CIRS, - 'ITRS': builtin_frames.ITRS, - 'precessed_geocentric': builtin_frames.PrecessedGeocentric - } - - return cls._reference_frame_mapping - - @classmethod - def _get_inverse_reference_frame_mapping(cls): - if hasattr(cls, '_inverse_reference_frame_mapping'): - return cls._inverse_reference_frame_mapping - - reference_frame_mapping = cls._get_reference_frame_mapping() - - cls._inverse_reference_frame_mapping = {} - for key, val in reference_frame_mapping.items(): - cls._inverse_reference_frame_mapping[val] = key - - return cls._inverse_reference_frame_mapping - - @classmethod - def _reference_frame_from_tree(cls, node, ctx): - from astropy.units import Quantity - from astropy.io.misc.asdf.tags.unit.quantity import QuantityType - from astropy.coordinates import ICRS, CartesianRepresentation - - version = cls.version - reference_frame = node['reference_frame'] - reference_frame_name = reference_frame['type'] - - frame_cls = cls._get_reference_frame_mapping()[reference_frame_name] - - frame_kwargs = {} - for name in frame_cls.get_frame_attr_names().keys(): - val = reference_frame.get(name) - if val is not None: - # These are deprecated fields that must be handled as a special - # case for older versions of the schema - if name in ['galcen_ra', 'galcen_dec']: - continue - # There was no schema for quantities in v1.0.0 - if name in ['galcen_distance', 'roll', 'z_sun'] and version == '1.0.0': - val = Quantity(val[0], unit=val[1]) - # These fields are known to be CartesianRepresentations - if name in ['obsgeoloc', 'obsgeovel']: - if version == '1.0.0': - unit = val[1] - x = Quantity(val[0][0], unit=unit) - y = Quantity(val[0][1], unit=unit) - z = Quantity(val[0][2], unit=unit) - else: - x = QuantityType.from_tree(val[0], ctx) - y = QuantityType.from_tree(val[1], ctx) - z = QuantityType.from_tree(val[2], ctx) - val = CartesianRepresentation(x, y, z) - elif name == 'galcen_v_sun': - from astropy.coordinates import CartesianDifferential - # This field only exists since v1.1.0, and it only uses - # CartesianDifferential after v1.3.3 - d_x = QuantityType.from_tree(val[0], ctx) - d_y = QuantityType.from_tree(val[1], ctx) - d_z = QuantityType.from_tree(val[2], ctx) - val = CartesianDifferential(d_x, d_y, d_z) - else: - val = yamlutil.tagged_tree_to_custom_tree(val, ctx) - frame_kwargs[name] = val - has_ra_and_dec = reference_frame.get('galcen_dec') and \ - reference_frame.get('galcen_ra') - if version == '1.0.0' and has_ra_and_dec: - # Convert deprecated ra and dec fields into galcen_coord - galcen_dec = reference_frame['galcen_dec'] - galcen_ra = reference_frame['galcen_ra'] - dec = Quantity(galcen_dec[0], unit=galcen_dec[1]) - ra = Quantity(galcen_ra[0], unit=galcen_ra[1]) - frame_kwargs['galcen_coord'] = ICRS(dec=dec, ra=ra) - return frame_cls(**frame_kwargs) - - @classmethod - def _from_tree(cls, node, ctx): - kwargs = {'name': node['name']} - - if 'axes_names' in node: - kwargs['axes_names'] = node['axes_names'] - - if 'reference_frame' in node: - kwargs['reference_frame'] = \ - cls._reference_frame_from_tree(node, ctx) - - if 'axes_order' in node: - kwargs['axes_order'] = tuple(node['axes_order']) - - if 'unit' in node: - kwargs['unit'] = tuple( - yamlutil.tagged_tree_to_custom_tree(node['unit'], ctx)) - - return kwargs - - @classmethod - def _to_tree(cls, frame, ctx): - import numpy as np - from astropy.coordinates import CartesianRepresentation - from astropy.io.misc.asdf.tags.unit.quantity import QuantityType - from astropy.coordinates import CartesianDifferential - - node = {} - - node['name'] = frame.name - - if frame.axes_order != (0, 1): - node['axes_order'] = list(frame.axes_order) - - if frame.axes_names is not None: - node['axes_names'] = list(frame.axes_names) - - if frame.reference_frame is not None: - reference_frame = {} - reference_frame['type'] = cls._get_inverse_reference_frame_mapping()[ - type(frame.reference_frame)] - - for name in frame.reference_frame.get_frame_attr_names().keys(): - frameval = getattr(frame.reference_frame, name) - # CartesianRepresentation becomes a flat list of x,y,z - # coordinates with associated units - if isinstance(frameval, CartesianRepresentation): - value = [frameval.x, frameval.y, frameval.z] - frameval = value - elif isinstance(frameval, CartesianDifferential): - value = [frameval.d_x, frameval.d_y, frameval.d_z] - frameval = value - yamlval = yamlutil.custom_tree_to_tagged_tree(frameval, ctx) - reference_frame[name] = yamlval - - node['reference_frame'] = reference_frame - - if frame.unit is not None: - node['unit'] = yamlutil.custom_tree_to_tagged_tree( - list(frame.unit), ctx) - - return node - - @classmethod - def _assert_equal(cls, old, new): - from ...tests import helpers - - assert old.name == new.name - assert old.axes_order == new.axes_order - assert old.axes_names == new.axes_names - assert type(old.reference_frame) == type(new.reference_frame) - assert old.unit == new.unit - - if old.reference_frame is not None: - for name in old.reference_frame.get_frame_attr_names().keys(): - helpers.assert_tree_match( - getattr(old.reference_frame, name), - getattr(new.reference_frame, name)) - - @classmethod - def assert_equal(cls, old, new): - cls._assert_equal(old, new) - - @classmethod - def from_tree(cls, node, ctx): - import gwcs - - node = cls._from_tree(node, ctx) - - return gwcs.Frame2D(**node) - - @classmethod - def to_tree(cls, frame, ctx): - return cls._to_tree(frame, ctx) - -class CelestialFrameType(FrameType): - name = "wcs/celestial_frame" - types = ['gwcs.CelestialFrame'] - supported_versions = [(1,0,0), (1,1,0)] - - @classmethod - def from_tree(cls, node, ctx): - import gwcs - - node = cls._from_tree(node, ctx) - - return gwcs.CelestialFrame(**node) - - @classmethod - def to_tree(cls, frame, ctx): - return cls._to_tree(frame, ctx) - - @classmethod - def assert_equal(cls, old, new): - cls._assert_equal(old, new) - - assert old.reference_position == new.reference_position - - -class SpectralFrame(FrameType): - name = "wcs/spectral_frame" - types = ['gwcs.SpectralFrame'] - - @classmethod - def from_tree(cls, node, ctx): - import gwcs - - node = cls._from_tree(node, ctx) - - if 'reference_position' in node: - node['reference_position'] = node['reference_position'].upper() - - return gwcs.SpectralFrame(**node) - - @classmethod - def to_tree(cls, frame, ctx): - node = cls._to_tree(frame, ctx) - - if frame.reference_position is not None: - node['reference_position'] = frame.reference_position.lower() - - return node - - -class CompositeFrame(FrameType): - name = "wcs/composite_frame" - types = ['gwcs.CompositeFrame'] - version = '1.1.0' - - @classmethod - def from_tree(cls, node, ctx): - import gwcs - - if len(node) != 2: - raise ValueError("CompositeFrame has extra properties") - - name = node['name'] - frames = node['frames'] - - return gwcs.CompositeFrame(frames, name) - - @classmethod - def to_tree(cls, frame, ctx): - return { - 'name': frame.name, - 'frames': yamlutil.custom_tree_to_tagged_tree(frame.frames, ctx) - } - - @classmethod - def assert_equal(cls, old, new): - from ...tests import helpers - - assert old.name == new.name - for old_frame, new_frame in zip(old.frames, new.frames): - helpers.assert_tree_match(old_frame, new_frame) - -class ICRSCoord(AsdfType): - """The newest version of this tag and the associated schema have moved to - Astropy. This implementation is retained here for the purposes of backwards - compatibility with older files. - """ - name = "wcs/icrs_coord" - types = ['astropy.coordinates.ICRS'] - requires = ['astropy'] - version = "1.1.0" - - @classmethod - def from_tree(cls, node, ctx): - from astropy.io.misc.asdf.tags.unit.quantity import QuantityType - from astropy.coordinates import ICRS, Longitude, Latitude, Angle - - angle = QuantityType.from_tree(node['ra']['wrap_angle'], ctx) - wrap_angle = Angle(angle.value, unit=angle.unit) - ra = Longitude( - node['ra']['value'], - unit=node['ra']['unit'], - wrap_angle=wrap_angle) - dec = Latitude(node['dec']['value'], unit=node['dec']['unit']) - - return ICRS(ra=ra, dec=dec) - - @classmethod - def to_tree(cls, frame, ctx): # pragma: no cover - # We do not run coverage analysis since new ICRS objects will be - # serialized by the tag implementation in Astropy. Eventually if we - # have a better way to write older versions of tags, we can re-add - # tests for this code. - from astropy.units import Quantity - from astropy.coordinates import ICRS - from astropy.io.misc.asdf.tags.unit.quantity import QuantityType - - node = {} - - wrap_angle = Quantity( - frame.ra.wrap_angle.value, - unit=frame.ra.wrap_angle.unit) - node['ra'] = { - 'value': frame.ra.value, - 'unit': frame.ra.unit.to_string(), - 'wrap_angle': yamlutil.custom_tree_to_tagged_tree(wrap_angle, ctx) - } - node['dec'] = { - 'value': frame.dec.value, - 'unit': frame.dec.unit.to_string() - } - - return node From 450e62855460d22d417791dd6ffdd63d9eea73c8 Mon Sep 17 00:00:00 2001 From: Nadia Dencheva Date: Sun, 4 Nov 2018 14:29:54 -0500 Subject: [PATCH 22/55] clean tests --- asdf/tests/test_asdftypes.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/asdf/tests/test_asdftypes.py b/asdf/tests/test_asdftypes.py index d880426b2..40ca4bc1d 100644 --- a/asdf/tests/test_asdftypes.py +++ b/asdf/tests/test_asdftypes.py @@ -617,7 +617,6 @@ def test_extension_override(tmpdir): with open(tmpfile, 'rb') as ff: contents = str(ff.read()) assert gwcs.tags.WCSType.yaml_tag in contents - assert asdf.tags.wcs.WCSType.yaml_tag not in contents def test_extension_override_subclass(tmpdir): @@ -647,7 +646,6 @@ class SubclassWCS(gwcs.WCS): with open(tmpfile, 'rb') as ff: contents = str(ff.read()) assert gwcs.tags.WCSType.yaml_tag in contents - assert asdf.tags.wcs.WCSType.yaml_tag not in contents def test_tag_without_schema(tmpdir): From c632ba5754b692dd94279d9f79f482b1214a9ee9 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Mon, 5 Nov 2018 09:50:31 -0500 Subject: [PATCH 23/55] Add config option to schema tester to skip examples --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index f26212bbc..c1839ade6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,7 +17,7 @@ open_files_ignore = test.fits asdf.fits # Account for both the astropy test runner case and the native pytest case asdf_schema_root = asdf-standard/schemas asdf/schemas asdf_schema_skip_names = asdf-schema-1.0.0 draft-01 -asdf_schema_skip_examples = domain-1.0.0 +asdf_schema_skip_examples = domain-1.0.0 frame-1.0.0 frame-1.1.0 #addopts = --doctest-rst [ah_bootstrap] From 98a1f8fa70fec0112fce35f2594d89e6096619d4 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Tue, 6 Nov 2018 10:51:41 -0500 Subject: [PATCH 24/55] Update asdf-standard submodule with removal of WCS From 506defe44131fdaa210abd907f04f843ecbd9849 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Tue, 6 Nov 2018 11:34:36 -0500 Subject: [PATCH 25/55] Update change log --- CHANGES.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 85a7f1e73..4033c3ffb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -37,6 +37,9 @@ index before any others. This fixes a bug that was related to the way that subclass tags were overwritten by external extensions. [#598] +- Remove WCS tags. These are now provided by the `gwcs package + `_. [#593] + 2.1.1 (2018-11-01) ------------------ From d06b8be6d04411727989da11133dfeffb6d830f5 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Mon, 5 Nov 2018 09:50:31 -0500 Subject: [PATCH 26/55] Add config option to schema tester to skip examples From 79bd1267f011f92488778dc8f328ff699690eee6 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Mon, 5 Nov 2018 10:38:41 -0500 Subject: [PATCH 27/55] Update asdf-standard to remove domain from transform schema From 6d9c51f23ca8bd7fcbea25526816d88694b62ed1 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Thu, 15 Nov 2018 13:01:42 -0500 Subject: [PATCH 28/55] Update travis and appveyor to use pytest --- .travis.yml | 39 ++++++++++++++++++++------------------- appveyor.yml | 6 +++--- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index ed2f46dbe..aeeda9099 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,57 +29,55 @@ env: - CONDA_DEPENDENCIES='semantic_version jsonschema pyyaml six lz4 pytest-astropy' - GWCS_GIT='git+git://github.com/spacetelescope/gwcs.git#egg=gwcs' - GWCS_PIP='gwcs' - - MAIN_CMD='python setup.py' - - SETUP_CMD='test --remote-data' + - RUN_CMD='pytest' matrix: # Make sure that installation does not fail - - SETUP_CMD='install' + - RUN_CMD='python setup.py install' # Make sure README will display properly on pypi - PIP_DEPENDENCIES='twine' TWINE_CHECK=1 - - PYTHON_VERSION=3.5 SETUP_CMD='test' - - PYTHON_VERSION=3.6 SETUP_CMD='test' - - PYTHON_VERSION=3.7 PYTEST_VERSION=3.8 SETUP_CMD='test' + - PYTHON_VERSION=3.5 + - PYTHON_VERSION=3.6 + - PYTHON_VERSION=3.7 PYTEST_VERSION=3.8 matrix: fast_finish: true include: # Do a coverage test - - env: SETUP_CMD='test --coverage --open-files --remote-data' + - env: COVERAGE=1 # Check for sphinx doc build warnings - we do this first because it # may run for a long time - - env: SETUP_CMD='build_docs -w' + - env: RUN_CMD='python setup.py build_docs -w' # Do a code style check - - env: MAIN_CMD="flake8 asdf --count" SETUP_CMD='' + - env: RUN_CMD="flake8 asdf --count" # try older numpy versions - - env: PYTHON_VERSION=3.5 NUMPY_VERSION=1.11 SETUP_CMD='test' - - env: NUMPY_VERSION=1.12 SETUP_CMD='test' + - env: PYTHON_VERSION=3.5 NUMPY_VERSION=1.11 + - env: NUMPY_VERSION=1.12 - # run a test using native pytest # also test against development version of Astropy - - env: MAIN_CMD='pytest' SETUP_CMD='' ASTROPY_VERSION=development + - env: ASTROPY_VERSION=development GWCS_PIP="$GWCS_GIT" PYTEST_VERSION=3.8 # latest stable versions - - env: NUMPY_VERSION=stable SETUP_CMD='test' + - env: NUMPY_VERSION=stable # Test against development version of numpy (this job can fail) - - env: NUMPY_VERSION=development SETUP_CMD='test' + - env: NUMPY_VERSION=development # Try a run on OSX - os: osx - env: NUMPY_VERSION=stable SETUP_CMD='test' + env: NUMPY_VERSION=stable # Test against latest version of jsonschema - env: PIP_DEPENDENCIES='jsonschema pytest-faulthandler importlib_resources' allow_failures: - - env: NUMPY_VERSION=development SETUP_CMD='test' + - env: NUMPY_VERSION=development - env: PIP_DEPENDENCIES='jsonschema pytest-faulthandler importlib_resources' install: @@ -92,12 +90,15 @@ script: - if [[ $TWINE_CHECK ]]; then python setup.py build sdist; twine check dist/*; + elif [[ $COVERAGE ]]; then + coverage run --source=asdf -m pytest --remote-data --open-files; + coverage report -m; else - $MAIN_CMD $SETUP_CMD; + $RUN_CMD; fi after_success: # If coveralls.io is set up for this package, uncomment the line # below and replace "packagename" with the name of your package. # The coveragerc file may be customized as needed for your package. - - if [[ $SETUP_CMD == 'test --coverage --open-files --remote-data' ]]; then coveralls --rcfile='asdf/tests/coveragerc'; fi + - if [[ $COVERAGE ]]; then coveralls --rcfile='asdf/tests/coveragerc'; fi diff --git a/appveyor.yml b/appveyor.yml index 4da97f6d6..3b6829125 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -8,7 +8,7 @@ environment: PYTEST_VERSION: "3.7" MINICONDA_VERSION: "latest" CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci-helpers\\appveyor\\windows_sdk.cmd" - SETUP_CMD: "test" + RUN_CMD: "pytest" NUMPY_VERSION: "stable" ASTROPY_VERSION: "stable" GWCS_GIT: "git+git://github.com/spacetelescope/gwcs.git#egg=gwcs" @@ -20,7 +20,7 @@ environment: matrix: # Make sure that installation does not fail - PYTHON_VERSION: "3.6" - SETUP_CMD: "install" + RUN_CMD: "python setup.py install" platform: x64 - PYTHON_VERSION: "3.5" @@ -65,4 +65,4 @@ install: build: false test_script: - - "%CMD_IN_ENV% python setup.py %SETUP_CMD%" + - "%CMD_IN_ENV% %RUN_CMD%" From 3c7be7a13fded83190321efef8b47689ca246b7b Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Thu, 15 Nov 2018 15:32:05 -0500 Subject: [PATCH 29/55] Explicitly provide dependencies for flake8, build_docs, coverage --- .travis.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index aeeda9099..3a34345a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,8 +25,10 @@ env: - PYTEST_VERSION=3.7 - ASTROPY_VERSION=stable - NUMPY_VERSION=stable - - PIP_DEPENDENCIES='pytest-faulthandler importlib_resources' - - CONDA_DEPENDENCIES='semantic_version jsonschema pyyaml six lz4 pytest-astropy' + - ALL_PIP_DEPENDENCIES='pytest-faulthandler importlib_resources' + - PIP_DEPENDENCIES="$ALL_PIP_DEPENDENCIES" + - ALL_CONDA_DEPENDENCIES='semantic_version jsonschema pyyaml six lz4 pytest-astropy' + - CONDA_DEPENDENCIES="$ALL_CONDA_DEPENDENCIES" - GWCS_GIT='git+git://github.com/spacetelescope/gwcs.git#egg=gwcs' - GWCS_PIP='gwcs' - RUN_CMD='pytest' @@ -46,13 +48,16 @@ matrix: # Do a coverage test - env: COVERAGE=1 + PIP_DEPENDENCIES="$ALL_PIP_DEPENDENCIES coverage coveralls" # Check for sphinx doc build warnings - we do this first because it # may run for a long time - env: RUN_CMD='python setup.py build_docs -w' + CONDA_DEPENDENCIES="$ALL_CONDA_DEPENDENCIES sphinx-astropy" # Do a code style check - env: RUN_CMD="flake8 asdf --count" + PIP_DEPENDENCIES="$ALL_PIP_DEPENDENCIES flake8" # try older numpy versions - env: PYTHON_VERSION=3.5 NUMPY_VERSION=1.11 From 0a36350f8305a77c578513c7413ce82538e531d8 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Thu, 15 Nov 2018 16:00:14 -0500 Subject: [PATCH 30/55] Test against newer versions of pytest --- .travis.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3a34345a6..bd30ed9e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ env: # overidden underneath. They are defined here in order to save having # to repeat them for all configurations. - PYTHON_VERSION=3.6 - - PYTEST_VERSION=3.7 + - PYTEST_VERSION=3.10 - ASTROPY_VERSION=stable - NUMPY_VERSION=stable - ALL_PIP_DEPENDENCIES='pytest-faulthandler importlib_resources' @@ -39,8 +39,8 @@ env: # Make sure README will display properly on pypi - PIP_DEPENDENCIES='twine' TWINE_CHECK=1 - PYTHON_VERSION=3.5 - - PYTHON_VERSION=3.6 - - PYTHON_VERSION=3.7 PYTEST_VERSION=3.8 + - PYTHON_VERSION=3.6 PYTEST_VERSION=3.8 + - PYTHON_VERSION=3.7 PYTEST_VERSION=3.9 matrix: fast_finish: true @@ -64,9 +64,7 @@ matrix: - env: NUMPY_VERSION=1.12 # also test against development version of Astropy - - env: ASTROPY_VERSION=development - GWCS_PIP="$GWCS_GIT" - PYTEST_VERSION=3.8 + - env: ASTROPY_VERSION=development GWCS_PIP="$GWCS_GIT" # latest stable versions - env: NUMPY_VERSION=stable From 0a9b90ac01cd70ea4af33b79262fe6fba6a14606 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Thu, 15 Nov 2018 16:45:02 -0500 Subject: [PATCH 31/55] Update asdf-standard to prevent warning from pytest From c2aaf914f11a867808d695e3d064b395416fe325 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Thu, 15 Nov 2018 11:59:24 -0500 Subject: [PATCH 32/55] Revert "Implement basic threshold for inlining array data" This reverts commit 5ceed65e1a827868aa4861b31a4d4cceac758d8c. --- asdf/block.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/asdf/block.py b/asdf/block.py index f560a6a27..9e1c46610 100644 --- a/asdf/block.py +++ b/asdf/block.py @@ -26,9 +26,6 @@ from . import yamlutil -_DEFAULT_INLINE_THRESHOLD_SIZE = 50 - - class BlockManager(object): """ Manages the `Block`s associated with a ASDF file. @@ -49,11 +46,6 @@ def __init__(self, asdffile, copy_arrays=False, lazy_load=True, 'streamed': self._streamed_blocks } - if inline_threshold is not None: - self._inline_threshold_size = inline_threshold - else: - self._inline_threshold_size = _DEFAULT_INLINE_THRESHOLD_SIZE - self._data_to_block_mapping = {} self._validate_checksums = False self._memmap = not copy_arrays @@ -753,7 +745,8 @@ def find_or_create_block_for_array(self, arr, ctx): block : Block """ from .tags.core import ndarray - if (isinstance(arr, ndarray.NDArrayType) and arr.block is not None): + if (isinstance(arr, ndarray.NDArrayType) and + arr.block is not None): if arr.block in self.blocks: return arr.block else: @@ -764,10 +757,6 @@ def find_or_create_block_for_array(self, arr, ctx): if block is not None: return block block = Block(base) - - if self._should_inline(arr): - block._array_storage = 'inline' - self.add(block) self._handle_global_block_settings(ctx, block) return block From e29855ad8fa517d9ce1d6be20f1cc88e55b4e96b Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Thu, 15 Nov 2018 12:02:41 -0500 Subject: [PATCH 33/55] Revert "Add top-level option for controlling inline threshold size" This reverts commit b4b5fe2204ec0f7d951977f89d3bd5585883fb77. --- asdf/asdf.py | 10 +++------- asdf/block.py | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/asdf/asdf.py b/asdf/asdf.py index 2f8b1feeb..36a1b3001 100644 --- a/asdf/asdf.py +++ b/asdf/asdf.py @@ -51,8 +51,7 @@ class AsdfFile(versioning.VersionedMixin): def __init__(self, tree=None, uri=None, extensions=None, version=None, ignore_version_mismatch=True, ignore_unrecognized_tag=False, ignore_implicit_conversion=False, copy_arrays=False, - lazy_load=True, custom_schema=None, _readonly=False, - inline_threshold=None): + lazy_load=True, custom_schema=None, _readonly=False): """ Parameters ---------- @@ -109,10 +108,7 @@ def __init__(self, tree=None, uri=None, extensions=None, version=None, files follow custom conventions beyond those enforced by the standard. - inline_threshold : int, optional - Optional threshold size below which arrays will automatically be - stored inline. Defaults to {0}. - """.format(block._DEFAULT_INLINE_THRESHOLD_SIZE) + """ if custom_schema is not None: self._custom_schema = schema.load_custom_schema(custom_schema) @@ -135,7 +131,7 @@ def __init__(self, tree=None, uri=None, extensions=None, version=None, self._external_asdf_by_uri = {} self._blocks = block.BlockManager( self, copy_arrays=copy_arrays, lazy_load=lazy_load, - readonly=_readonly, inline_threshold=inline_threshold) + readonly=_readonly) self._uri = None if tree is None: self.tree = {} diff --git a/asdf/block.py b/asdf/block.py index 9e1c46610..6dd4ea885 100644 --- a/asdf/block.py +++ b/asdf/block.py @@ -31,7 +31,7 @@ class BlockManager(object): Manages the `Block`s associated with a ASDF file. """ def __init__(self, asdffile, copy_arrays=False, lazy_load=True, - readonly=False, inline_threshold=None): + readonly=False): self._asdffile = weakref.ref(asdffile) self._internal_blocks = [] From 981bc2bcf9a0e4d380b78384d3684b79c46e7eaa Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Thu, 15 Nov 2018 12:03:10 -0500 Subject: [PATCH 34/55] Revert "Set default inline threshold level to 0 for test helpers..." This reverts commit a25bb0b50d4ec5bffe8376044281a3b935a3e685. --- asdf/tests/helpers.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/asdf/tests/helpers.py b/asdf/tests/helpers.py index e38c08bca..24ccb1038 100644 --- a/asdf/tests/helpers.py +++ b/asdf/tests/helpers.py @@ -180,9 +180,6 @@ def _assert_roundtrip_tree(tree, tmpdir, *, asdf_check_func=None, fname = str(tmpdir.join('test.asdf')) - # Most tests assume that all blocks will be stored internally - init_options.setdefault('inline_threshold', 0) - # First, test writing/reading a BytesIO buffer buff = io.BytesIO() AsdfFile(tree, extensions=extensions, **init_options).write_to(buff, **write_options) From 28db7dbbcef3a1f91021237aa214265ba465a79c Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Fri, 16 Nov 2018 14:33:24 -0500 Subject: [PATCH 35/55] Skip inline_threshold tests for now... I don't want to eliminate the tests entirely since they will be useful for when we make the updates to auto_inline. --- asdf/tests/test_low_level.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/asdf/tests/test_low_level.py b/asdf/tests/test_low_level.py index 4b20fe161..52fecb6e2 100644 --- a/asdf/tests/test_low_level.py +++ b/asdf/tests/test_low_level.py @@ -1247,6 +1247,7 @@ def test_open_readonly(tmpdir): with asdf.open(tmpfile, mode='rw'): pass +@pytest.mark.skip(reason='Until inline_threshold is added as a write option') def test_inline_threshold(tmpdir): tree = { @@ -1271,6 +1272,7 @@ def test_inline_threshold(tmpdir): assert len(list(af.blocks.internal_blocks)) == 0 +@pytest.mark.skip(reason='Until inline_threshold is added as a write option') def test_inline_threshold_masked(tmpdir): mask = np.random.randint(0, 1+1, 20) @@ -1296,6 +1298,7 @@ def test_inline_threshold_masked(tmpdir): assert len(list(af.blocks.internal_blocks)) == 2 +@pytest.mark.skip(reason='Until inline_threshold is added as a write option') def test_inline_threshold_override(tmpdir): tmpfile = str(tmpdir.join('inline.asdf')) From 3873ae7183b00db975c9683b36c7bc5ee8a1fa16 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Fri, 16 Nov 2018 14:34:35 -0500 Subject: [PATCH 36/55] Revert "Update change log" This reverts commit deac0cd527fe639c6566f48a5c3b8b8620b24fbb. --- CHANGES.rst | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4033c3ffb..3a7a91224 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,15 @@ +2.3.0 (unreleased) +------------------ + +- Storage of arbitrary precision integers is now provided by + ``asdf.IntegerType``. Reading a file with integer literals that are too + large now causes only a warning instead of a validation error. This is to + provide backwards compatibility for files that were created with a buggy + version of ASDF (see #553 below). [#566] + +- Remove WCS tags. These are now provided by the `gwcs package + `_. [#593] + 2.2.1 (2018-11-15) ------------------ @@ -24,12 +36,6 @@ types. This warning is converted to an error when using ``assert_roundtrip_tree`` for tests. [#583] -- Storage of arbitrary precision integers is now provided by - ``asdf.IntegerType``. Reading a file with integer literals that are too - large now causes only a warning instead of a validation error. This is to - provide backwards compatibility for files that were created with a buggy - version of ASDF (see #553 below). [#566] - 2.1.2 (2018-11-13) ------------------ From 1b546916658e5caa3453808cf7760fb020076ac7 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Fri, 16 Nov 2018 14:34:46 -0500 Subject: [PATCH 37/55] Revert "Update documentation" This reverts commit 2d06d133303da315f8e417b773aa252fafd4ab00. --- docs/asdf/arrays.rst | 36 +++++------------------------------- 1 file changed, 5 insertions(+), 31 deletions(-) diff --git a/docs/asdf/arrays.rst b/docs/asdf/arrays.rst index 371dc9163..5e588c120 100644 --- a/docs/asdf/arrays.rst +++ b/docs/asdf/arrays.rst @@ -56,37 +56,11 @@ data being saved. Saving inline arrays -------------------- -As of `asdf-2.2.0`, small numerical arrays are automatically stored inline. The -default threshold size for inline versus internal arrays can be found with the -following: - -.. code:: - - >>> from asdf.block import _DEFAULT_INLINE_THRESHOLD_SIZE - >>> print(_DEFAULT_INLINE_THRESHOLD_SIZE) - 50 - -The default threshold can be overridden passing the `inline_threshold` argument -to the `asdf.AsdfFile` constructor. Setting `inline_threshold=0` has the effect -of making all small arrays be stored in internal blocks: - -.. runcode:: - - from asdf import AsdfFile - import numpy as np - - # Ordinarilly an array this size would be automatically inlined - my_array = np.ones(10) - tree = {'my_array': my_array} - # Set the inline threshold to 0 to force internal storage - with AsdfFile(tree, inline_threshold=0) as ff: - ff.write_to("test.asdf") - -.. asdf:: test.asdf - -The `~asdf.AsdfFile.set_array_storage` method can be used to set or override -the default storage type of a particular data array. The allowed values are -``internal``, ``external``, and ``inline``. +For small arrays, you may not care about the efficiency of a binary +representation and just want to save the array contents directly in the YAML +tree. The `~asdf.AsdfFile.set_array_storage` method can be used to set the +storage type of the associated data. The allowed values are ``internal``, +``external``, and ``inline``. - ``internal``: The default. The array data will be stored in a binary block in the same ASDF file. From c9297b5bddb93d20cb52b02f90251930f99176a2 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Mon, 19 Nov 2018 09:07:02 -0500 Subject: [PATCH 38/55] Refactor type index into a separate module --- asdf/asdftypes.py | 403 +----------------------------------- asdf/extension.py | 7 +- asdf/tests/schema_tester.py | 2 +- asdf/type_index.py | 393 +++++++++++++++++++++++++++++++++++ asdf/versioning.py | 18 +- asdf/yamlutil.py | 4 +- 6 files changed, 419 insertions(+), 408 deletions(-) create mode 100644 asdf/type_index.py diff --git a/asdf/asdftypes.py b/asdf/asdftypes.py index 2b9c51297..e40a6f850 100644 --- a/asdf/asdftypes.py +++ b/asdf/asdftypes.py @@ -3,25 +3,18 @@ import re -import bisect -import warnings import importlib -from collections import OrderedDict import six from copy import copy -from functools import lru_cache - from . import tagged from . import util -from .versioning import AsdfVersion, AsdfSpec, get_version_map, default_version - +from .versioning import AsdfVersion, AsdfSpec -__all__ = ['format_tag', 'CustomType', 'AsdfTypeIndex'] +__all__ = ['format_tag', 'CustomType'] -_BASIC_PYTHON_TYPES = [str, int, float, list, dict, tuple] # regex used to parse module name from optional version string MODULE_RE = re.compile(r'([a-zA-Z]+)(-(\d+\.\d+\.\d+))?') @@ -42,398 +35,6 @@ def format_tag(organization, standard, version, tag_name): return "{0}-{1}".format(tag, version) -def split_tag_version(tag): - """ - Split a tag into its base and version. - """ - name, version = tag.rsplit('-', 1) - version = AsdfVersion(version) - return name, version - - -def join_tag_version(name, version): - """ - Join the root and version of a tag back together. - """ - return '{0}-{1}'.format(name, version) - - -class _AsdfWriteTypeIndex(object): - """ - The _AsdfWriteTypeIndex is a helper class for AsdfTypeIndex that - manages an index of types for writing out ASDF files, i.e. from - converting from custom types to tagged_types. It is not always - the inverse of the mapping from tags to custom types, since there - are likely multiple versions present for a given tag. - - This uses the `version_map.yaml` file that ships with the ASDF - standard to figure out which schemas correspond to a particular - version of the ASDF standard. - - An AsdfTypeIndex manages multiple _AsdfWriteTypeIndex instances - for each version the user may want to write out, and they are - instantiated on-demand. - - If version is ``'latest'``, it will just use the highest-numbered - versions of each of the schemas. This is currently only used to - aid in testing. - - In the future, this may be renamed to _ExtensionWriteTypeIndex since it is - not specific to classes that inherit `AsdfType`. - """ - _version_map = None - - def __init__(self, version, index): - self._version = version - - self._type_by_cls = {} - self._type_by_name = {} - self._type_by_subclasses = {} - self._class_by_subclass = {} - self._types_with_dynamic_subclasses = {} - self._extension_by_cls = {} - self._extensions_used = set() - - try: - version_map = get_version_map(self._version) - core_version_map = version_map['core'] - standard_version_map = version_map['standard'] - except ValueError: - raise ValueError( - "Don't know how to write out ASDF version {0}".format( - self._version)) - - def should_overwrite(cls, new_type): - existing_type = self._type_by_cls[cls] - - # Types that are provided by extensions from other packages should - # only override the type index corresponding to the latest version - # of ASDF. - if existing_type.tag_base() != new_type.tag_base(): - return self._version == default_version - - return True - - def add_type_to_index(cls, typ): - if cls in self._type_by_cls and not should_overwrite(cls, typ): - return - - self._type_by_cls[cls] = typ - self._extension_by_cls[cls] = index._extension_by_type[typ] - - def add_subclasses(typ, asdftype): - for subclass in util.iter_subclasses(typ): - # Do not overwrite the tag type for an existing subclass if the - # new tag serializes a class that is higher in the type - # hierarchy than the existing subclass. - if subclass in self._class_by_subclass: - if issubclass(self._class_by_subclass[subclass], typ): - # Allow for cases where a subclass tag is being - # overridden by a tag from another extension. - if (self._extension_by_cls[subclass] == - index._extension_by_type[asdftype]): - continue - self._class_by_subclass[subclass] = typ - self._type_by_subclasses[subclass] = asdftype - self._extension_by_cls[subclass] = index._extension_by_type[asdftype] - - def add_all_types(asdftype): - add_type_to_index(asdftype, asdftype) - for typ in asdftype.types: - add_type_to_index(typ, asdftype) - add_subclasses(typ, asdftype) - - if asdftype.handle_dynamic_subclasses: - for typ in asdftype.types: - self._types_with_dynamic_subclasses[typ] = asdftype - - def add_by_tag(name, version): - tag = join_tag_version(name, version) - if tag in index._type_by_tag: - asdftype = index._type_by_tag[tag] - self._type_by_name[name] = asdftype - add_all_types(asdftype) - - # Process all types defined in the ASDF version map. It is important to - # make sure that tags that are associated with the core part of the - # standard are processed first in order to handle subclasses properly. - for name, _version in core_version_map.items(): - add_by_tag(name, AsdfVersion(_version)) - for name, _version in standard_version_map.items(): - add_by_tag(name, AsdfVersion(_version)) - - # Now add any extension types that aren't known to the ASDF standard. - # This expects that all types defined by ASDF will be encountered - # before any types that are defined by external packages. This - # allows external packages to override types that are also defined - # by ASDF. The ordering is guaranteed due to the use of OrderedDict - # for _versions_by_type_name, and due to the fact that the built-in - # extension will always be processed first. - for name, versions in index._versions_by_type_name.items(): - if name not in self._type_by_name: - add_by_tag(name, versions[-1]) - - for asdftype in index._unnamed_types: - add_all_types(asdftype) - - def _mark_used_extension(self, custom_type): - self._extensions_used.add(self._extension_by_cls[custom_type]) - - def _process_dynamic_subclass(self, custom_type): - for key, val in self._types_with_dynamic_subclasses.items(): - if issubclass(custom_type, key): - self._type_by_cls[custom_type] = val - self._mark_used_extension(key) - return val - - return None - - def from_custom_type(self, custom_type): - """ - Given a custom type, return the corresponding `ExtensionType` - definition. - """ - asdftype = None - - # Try to find an exact class match first... - try: - asdftype = self._type_by_cls[custom_type] - except KeyError: - # ...failing that, match any subclasses - try: - asdftype = self._type_by_subclasses[custom_type] - except KeyError: - # ...failing that, try any subclasses that we couldn't - # cache in _type_by_subclasses. This generally only - # includes classes that are created dynamically post - # Python-import, e.g. astropy.modeling._CompoundModel - # subclasses. - return self._process_dynamic_subclass(custom_type) - - if asdftype is not None: - extension = self._extension_by_cls.get(custom_type) - if extension is not None: - self._mark_used_extension(custom_type) - else: - # Handle the case where the dynamic subclass was identified as - # a proper subclass above, but it has not yet been registered - # as such. - self._process_dynamic_subclass(custom_type) - - return asdftype - - -class AsdfTypeIndex(object): - """ - An index of the known `ExtensionType` classes. - - In the future this class may be renamed to ExtensionTypeIndex, since it is - not specific to classes that inherit `AsdfType`. - """ - def __init__(self): - self._write_type_indices = {} - self._type_by_tag = {} - # Use OrderedDict here to preserve the order in which types are added - # to the type index. Since the ASDF built-in extension is always - # processed first, this ensures that types defined by external packages - # will always override corresponding types that are defined by ASDF - # itself. However, if two different external packages define tags for - # the same type, the result is currently undefined. - self._versions_by_type_name = OrderedDict() - self._best_matches = {} - self._real_tag = {} - self._unnamed_types = set() - self._hooks_by_type = {} - self._all_types = set() - self._has_warned = {} - self._extension_by_type = {} - - def add_type(self, asdftype, extension): - """ - Add a type to the index. - """ - self._all_types.add(asdftype) - self._extension_by_type[asdftype] = extension - - if asdftype.yaml_tag is None and asdftype.name is None: - return - - if isinstance(asdftype.name, list): - yaml_tags = [asdftype.make_yaml_tag(name) for name in asdftype.name] - elif isinstance(asdftype.name, str): - yaml_tags = [asdftype.yaml_tag] - elif asdftype.name is None: - yaml_tags = [] - else: - raise TypeError("name must be a string, list or None") - - for yaml_tag in yaml_tags: - self._type_by_tag[yaml_tag] = asdftype - name, version = split_tag_version(yaml_tag) - versions = self._versions_by_type_name.get(name) - if versions is None: - self._versions_by_type_name[name] = [version] - else: - idx = bisect.bisect_left(versions, version) - if idx == len(versions) or versions[idx] != version: - versions.insert(idx, version) - - if not len(yaml_tags): - self._unnamed_types.add(asdftype) - - def from_custom_type(self, custom_type, version=default_version): - """ - Given a custom type, return the corresponding `ExtensionType` - definition. - """ - # Basic Python types should not ever have an AsdfType associated with - # them. - if custom_type in _BASIC_PYTHON_TYPES: - return None - - write_type_index = self._write_type_indices.get(str(version)) - if write_type_index is None: - write_type_index = _AsdfWriteTypeIndex(version, self) - self._write_type_indices[version] = write_type_index - - return write_type_index.from_custom_type(custom_type) - - def _get_version_mismatch(self, name, version, latest_version): - warning_string = None - - if (latest_version.major, latest_version.minor) != \ - (version.major, version.minor): - warning_string = \ - "'{}' with version {} found in file{{}}, but latest " \ - "supported version is {}".format( - name, version, latest_version) - - return warning_string - - def _warn_version_mismatch(self, ctx, tag, warning_string, fname): - if warning_string is not None: - # Ensure that only a single warning occurs per tag per AsdfFile - # TODO: If it is useful to only have a single warning per file on - # disk, then use `fname` in the key instead of `ctx`. - if not (ctx, tag) in self._has_warned: - warnings.warn(warning_string.format(fname)) - self._has_warned[(ctx, tag)] = True - - def fix_yaml_tag(self, ctx, tag, ignore_version_mismatch=True): - """ - Given a YAML tag, adjust it to the best supported version. - - If there is no exact match, this finds the newest version - understood that is still less than the version in file. Or, - the earliest understood version if none are less than the - version in the file. - - If ``ignore_version_mismatch==False``, this function raises a warning - if it could not find a match where the major and minor numbers are the - same. - """ - warning_string = None - - name, version = split_tag_version(tag) - - fname = " '{}'".format(ctx._fname) if ctx._fname else '' - - if tag in self._type_by_tag: - asdftype = self._type_by_tag[tag] - # Issue warnings for the case where there exists a class for the - # given tag due to the 'supported_versions' attribute being - # defined, but this tag is not the latest version of the type. - # This prevents 'supported_versions' from affecting the behavior of - # warnings that are purely related to YAML validation. - if not ignore_version_mismatch and hasattr(asdftype, '_latest_version'): - warning_string = self._get_version_mismatch( - name, version, asdftype._latest_version) - self._warn_version_mismatch(ctx, tag, warning_string, fname) - return tag - - if tag in self._best_matches: - best_tag, warning_string = self._best_matches[tag] - - if not ignore_version_mismatch: - self._warn_version_mismatch(ctx, tag, warning_string, fname) - - return best_tag - - versions = self._versions_by_type_name.get(name) - if versions is None: - return tag - - # The versions list is kept sorted, so bisect can be used to - # quickly find the best option. - i = bisect.bisect_left(versions, version) - i = max(0, i - 1) - - if not ignore_version_mismatch: - warning_string = self._get_version_mismatch( - name, version, versions[-1]) - self._warn_version_mismatch(ctx, tag, warning_string, fname) - - best_version = versions[i] - best_tag = join_tag_version(name, best_version) - self._best_matches[tag] = best_tag, warning_string - if tag != best_tag: - self._real_tag[best_tag] = tag - return best_tag - - def get_real_tag(self, tag): - if tag in self._real_tag: - return self._real_tag[tag] - elif tag in self._type_by_tag: - return tag - return None - - def from_yaml_tag(self, ctx, tag): - """ - From a given YAML tag string, return the corresponding - AsdfType definition. - """ - tag = self.fix_yaml_tag(ctx, tag) - return self._type_by_tag.get(tag) - - @lru_cache(5) - def has_hook(self, hook_name): - """ - Returns `True` if the given hook name exists on any of the managed - types. - """ - for cls in self._all_types: - if hasattr(cls, hook_name): - return True - return False - - def get_hook_for_type(self, hookname, typ, version=default_version): - """ - Get the hook function for the given type, if it exists, - else return None. - """ - hooks = self._hooks_by_type.setdefault(hookname, {}) - hook = hooks.get(typ, None) - if hook is not None: - return hook - - tag = self.from_custom_type(typ, version) - if tag is not None: - hook = getattr(tag, hookname, None) - if hook is not None: - hooks[typ] = hook - return hook - - hooks[typ] = None - return None - - def get_extensions_used(self, version=default_version): - write_type_index = self._write_type_indices.get(str(version)) - if write_type_index is None: - return [] - - return list(write_type_index._extensions_used) - - _all_asdftypes = set() diff --git a/asdf/extension.py b/asdf/extension.py index c9a244868..7da26bad1 100644 --- a/asdf/extension.py +++ b/asdf/extension.py @@ -9,10 +9,11 @@ import six import importlib -from . import asdftypes from . import resolver -from .version import version as asdf_version +from . import asdftypes from .util import get_class_name +from .type_index import AsdfTypeIndex +from .version import version as asdf_version from .exceptions import AsdfDeprecationWarning @@ -120,7 +121,7 @@ def __init__(self, extensions): tag_mapping = [] url_mapping = [] validators = {} - self._type_index = asdftypes.AsdfTypeIndex() + self._type_index = AsdfTypeIndex() for extension in extensions: if not isinstance(extension, AsdfExtension): raise TypeError( diff --git a/asdf/tests/schema_tester.py b/asdf/tests/schema_tester.py index 85efa2c28..0f07a20dd 100644 --- a/asdf/tests/schema_tester.py +++ b/asdf/tests/schema_tester.py @@ -120,7 +120,7 @@ def should_skip(name, version): def parse_schema_filename(filename): components = filename[filename.find('schemas') + 1:].split(os.path.sep) tag = 'tag:{}:{}'.format(components[1], '/'.join(components[2:])) - name, version = asdftypes.split_tag_version(tag.replace('.yaml', '')) + name, version = versioning.split_tag_version(tag.replace('.yaml', '')) return name, version diff --git a/asdf/type_index.py b/asdf/type_index.py new file mode 100644 index 000000000..155119254 --- /dev/null +++ b/asdf/type_index.py @@ -0,0 +1,393 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +# -*- coding: utf-8 -*- + +import bisect +import warnings +from functools import lru_cache +from collections import OrderedDict + +from . import util +from .versioning import (AsdfVersion, get_version_map, default_version, + split_tag_version, join_tag_version) + + +__all__ = ['AsdfTypeIndex'] + + +_BASIC_PYTHON_TYPES = [str, int, float, list, dict, tuple] + + +class _AsdfWriteTypeIndex(object): + """ + The _AsdfWriteTypeIndex is a helper class for AsdfTypeIndex that + manages an index of types for writing out ASDF files, i.e. from + converting from custom types to tagged_types. It is not always + the inverse of the mapping from tags to custom types, since there + are likely multiple versions present for a given tag. + + This uses the `version_map.yaml` file that ships with the ASDF + standard to figure out which schemas correspond to a particular + version of the ASDF standard. + + An AsdfTypeIndex manages multiple _AsdfWriteTypeIndex instances + for each version the user may want to write out, and they are + instantiated on-demand. + + If version is ``'latest'``, it will just use the highest-numbered + versions of each of the schemas. This is currently only used to + aid in testing. + + In the future, this may be renamed to _ExtensionWriteTypeIndex since it is + not specific to classes that inherit `AsdfType`. + """ + _version_map = None + + def __init__(self, version, index): + self._version = version + + self._type_by_cls = {} + self._type_by_name = {} + self._type_by_subclasses = {} + self._class_by_subclass = {} + self._types_with_dynamic_subclasses = {} + self._extension_by_cls = {} + self._extensions_used = set() + + try: + version_map = get_version_map(self._version) + core_version_map = version_map['core'] + standard_version_map = version_map['standard'] + except ValueError: + raise ValueError( + "Don't know how to write out ASDF version {0}".format( + self._version)) + + def should_overwrite(cls, new_type): + existing_type = self._type_by_cls[cls] + + # Types that are provided by extensions from other packages should + # only override the type index corresponding to the latest version + # of ASDF. + if existing_type.tag_base() != new_type.tag_base(): + return self._version == default_version + + return True + + def add_type_to_index(cls, typ): + if cls in self._type_by_cls and not should_overwrite(cls, typ): + return + + self._type_by_cls[cls] = typ + self._extension_by_cls[cls] = index._extension_by_type[typ] + + def add_subclasses(typ, asdftype): + for subclass in util.iter_subclasses(typ): + # Do not overwrite the tag type for an existing subclass if the + # new tag serializes a class that is higher in the type + # hierarchy than the existing subclass. + if subclass in self._class_by_subclass: + if issubclass(self._class_by_subclass[subclass], typ): + # Allow for cases where a subclass tag is being + # overridden by a tag from another extension. + if (self._extension_by_cls[subclass] == + index._extension_by_type[asdftype]): + continue + self._class_by_subclass[subclass] = typ + self._type_by_subclasses[subclass] = asdftype + self._extension_by_cls[subclass] = index._extension_by_type[asdftype] + + def add_all_types(asdftype): + add_type_to_index(asdftype, asdftype) + for typ in asdftype.types: + add_type_to_index(typ, asdftype) + add_subclasses(typ, asdftype) + + if asdftype.handle_dynamic_subclasses: + for typ in asdftype.types: + self._types_with_dynamic_subclasses[typ] = asdftype + + def add_by_tag(name, version): + tag = join_tag_version(name, version) + if tag in index._type_by_tag: + asdftype = index._type_by_tag[tag] + self._type_by_name[name] = asdftype + add_all_types(asdftype) + + # Process all types defined in the ASDF version map. It is important to + # make sure that tags that are associated with the core part of the + # standard are processed first in order to handle subclasses properly. + for name, _version in core_version_map.items(): + add_by_tag(name, AsdfVersion(_version)) + for name, _version in standard_version_map.items(): + add_by_tag(name, AsdfVersion(_version)) + + # Now add any extension types that aren't known to the ASDF standard. + # This expects that all types defined by ASDF will be encountered + # before any types that are defined by external packages. This + # allows external packages to override types that are also defined + # by ASDF. The ordering is guaranteed due to the use of OrderedDict + # for _versions_by_type_name, and due to the fact that the built-in + # extension will always be processed first. + for name, versions in index._versions_by_type_name.items(): + if name not in self._type_by_name: + add_by_tag(name, versions[-1]) + + for asdftype in index._unnamed_types: + add_all_types(asdftype) + + def _mark_used_extension(self, custom_type): + self._extensions_used.add(self._extension_by_cls[custom_type]) + + def _process_dynamic_subclass(self, custom_type): + for key, val in self._types_with_dynamic_subclasses.items(): + if issubclass(custom_type, key): + self._type_by_cls[custom_type] = val + self._mark_used_extension(key) + return val + + return None + + def from_custom_type(self, custom_type): + """ + Given a custom type, return the corresponding `ExtensionType` + definition. + """ + asdftype = None + + # Try to find an exact class match first... + try: + asdftype = self._type_by_cls[custom_type] + except KeyError: + # ...failing that, match any subclasses + try: + asdftype = self._type_by_subclasses[custom_type] + except KeyError: + # ...failing that, try any subclasses that we couldn't + # cache in _type_by_subclasses. This generally only + # includes classes that are created dynamically post + # Python-import, e.g. astropy.modeling._CompoundModel + # subclasses. + return self._process_dynamic_subclass(custom_type) + + if asdftype is not None: + extension = self._extension_by_cls.get(custom_type) + if extension is not None: + self._mark_used_extension(custom_type) + else: + # Handle the case where the dynamic subclass was identified as + # a proper subclass above, but it has not yet been registered + # as such. + self._process_dynamic_subclass(custom_type) + + return asdftype + + +class AsdfTypeIndex(object): + """ + An index of the known `ExtensionType` classes. + + In the future this class may be renamed to ExtensionTypeIndex, since it is + not specific to classes that inherit `AsdfType`. + """ + def __init__(self): + self._write_type_indices = {} + self._type_by_tag = {} + # Use OrderedDict here to preserve the order in which types are added + # to the type index. Since the ASDF built-in extension is always + # processed first, this ensures that types defined by external packages + # will always override corresponding types that are defined by ASDF + # itself. However, if two different external packages define tags for + # the same type, the result is currently undefined. + self._versions_by_type_name = OrderedDict() + self._best_matches = {} + self._real_tag = {} + self._unnamed_types = set() + self._hooks_by_type = {} + self._all_types = set() + self._has_warned = {} + self._extension_by_type = {} + + def add_type(self, asdftype, extension): + """ + Add a type to the index. + """ + self._all_types.add(asdftype) + self._extension_by_type[asdftype] = extension + + if asdftype.yaml_tag is None and asdftype.name is None: + return + + if isinstance(asdftype.name, list): + yaml_tags = [asdftype.make_yaml_tag(name) for name in asdftype.name] + elif isinstance(asdftype.name, str): + yaml_tags = [asdftype.yaml_tag] + elif asdftype.name is None: + yaml_tags = [] + else: + raise TypeError("name must be a string, list or None") + + for yaml_tag in yaml_tags: + self._type_by_tag[yaml_tag] = asdftype + name, version = split_tag_version(yaml_tag) + versions = self._versions_by_type_name.get(name) + if versions is None: + self._versions_by_type_name[name] = [version] + else: + idx = bisect.bisect_left(versions, version) + if idx == len(versions) or versions[idx] != version: + versions.insert(idx, version) + + if not len(yaml_tags): + self._unnamed_types.add(asdftype) + + def from_custom_type(self, custom_type, version=default_version): + """ + Given a custom type, return the corresponding `ExtensionType` + definition. + """ + # Basic Python types should not ever have an AsdfType associated with + # them. + if custom_type in _BASIC_PYTHON_TYPES: + return None + + write_type_index = self._write_type_indices.get(str(version)) + if write_type_index is None: + write_type_index = _AsdfWriteTypeIndex(version, self) + self._write_type_indices[version] = write_type_index + + return write_type_index.from_custom_type(custom_type) + + def _get_version_mismatch(self, name, version, latest_version): + warning_string = None + + if (latest_version.major, latest_version.minor) != \ + (version.major, version.minor): + warning_string = \ + "'{}' with version {} found in file{{}}, but latest " \ + "supported version is {}".format( + name, version, latest_version) + + return warning_string + + def _warn_version_mismatch(self, ctx, tag, warning_string, fname): + if warning_string is not None: + # Ensure that only a single warning occurs per tag per AsdfFile + # TODO: If it is useful to only have a single warning per file on + # disk, then use `fname` in the key instead of `ctx`. + if not (ctx, tag) in self._has_warned: + warnings.warn(warning_string.format(fname)) + self._has_warned[(ctx, tag)] = True + + def fix_yaml_tag(self, ctx, tag, ignore_version_mismatch=True): + """ + Given a YAML tag, adjust it to the best supported version. + + If there is no exact match, this finds the newest version + understood that is still less than the version in file. Or, + the earliest understood version if none are less than the + version in the file. + + If ``ignore_version_mismatch==False``, this function raises a warning + if it could not find a match where the major and minor numbers are the + same. + """ + warning_string = None + + name, version = split_tag_version(tag) + + fname = " '{}'".format(ctx._fname) if ctx._fname else '' + + if tag in self._type_by_tag: + asdftype = self._type_by_tag[tag] + # Issue warnings for the case where there exists a class for the + # given tag due to the 'supported_versions' attribute being + # defined, but this tag is not the latest version of the type. + # This prevents 'supported_versions' from affecting the behavior of + # warnings that are purely related to YAML validation. + if not ignore_version_mismatch and hasattr(asdftype, '_latest_version'): + warning_string = self._get_version_mismatch( + name, version, asdftype._latest_version) + self._warn_version_mismatch(ctx, tag, warning_string, fname) + return tag + + if tag in self._best_matches: + best_tag, warning_string = self._best_matches[tag] + + if not ignore_version_mismatch: + self._warn_version_mismatch(ctx, tag, warning_string, fname) + + return best_tag + + versions = self._versions_by_type_name.get(name) + if versions is None: + return tag + + # The versions list is kept sorted, so bisect can be used to + # quickly find the best option. + i = bisect.bisect_left(versions, version) + i = max(0, i - 1) + + if not ignore_version_mismatch: + warning_string = self._get_version_mismatch( + name, version, versions[-1]) + self._warn_version_mismatch(ctx, tag, warning_string, fname) + + best_version = versions[i] + best_tag = join_tag_version(name, best_version) + self._best_matches[tag] = best_tag, warning_string + if tag != best_tag: + self._real_tag[best_tag] = tag + return best_tag + + def get_real_tag(self, tag): + if tag in self._real_tag: + return self._real_tag[tag] + elif tag in self._type_by_tag: + return tag + return None + + def from_yaml_tag(self, ctx, tag): + """ + From a given YAML tag string, return the corresponding + AsdfType definition. + """ + tag = self.fix_yaml_tag(ctx, tag) + return self._type_by_tag.get(tag) + + @lru_cache(5) + def has_hook(self, hook_name): + """ + Returns `True` if the given hook name exists on any of the managed + types. + """ + for cls in self._all_types: + if hasattr(cls, hook_name): + return True + return False + + def get_hook_for_type(self, hookname, typ, version=default_version): + """ + Get the hook function for the given type, if it exists, + else return None. + """ + hooks = self._hooks_by_type.setdefault(hookname, {}) + hook = hooks.get(typ, None) + if hook is not None: + return hook + + tag = self.from_custom_type(typ, version) + if tag is not None: + hook = getattr(tag, hookname, None) + if hook is not None: + hooks[typ] = hook + return hook + + hooks[typ] = None + return None + + def get_extensions_used(self, version=default_version): + write_type_index = self._write_type_indices.get(str(version)) + if write_type_index is None: + return [] + + return list(write_type_index._extensions_used) diff --git a/asdf/versioning.py b/asdf/versioning.py index 3d2aff6e4..9bfbe1ba0 100644 --- a/asdf/versioning.py +++ b/asdf/versioning.py @@ -22,7 +22,23 @@ from .version import version as asdf_version -__all__ = ['AsdfVersion', 'AsdfSpec'] +__all__ = ['AsdfVersion', 'AsdfSpec', 'split_tag_version', 'join_tag_version'] + + +def split_tag_version(tag): + """ + Split a tag into its base and version. + """ + name, version = tag.rsplit('-', 1) + version = AsdfVersion(version) + return name, version + + +def join_tag_version(name, version): + """ + Join the root and version of a tag back together. + """ + return '{0}-{1}'.format(name, version) _version_map = {} diff --git a/asdf/yamlutil.py b/asdf/yamlutil.py index 8f64cd174..6c332ae3e 100644 --- a/asdf/yamlutil.py +++ b/asdf/yamlutil.py @@ -12,9 +12,9 @@ from . import tagged from . import treeutil from . import asdftypes -from . import versioning from . import util from .constants import YAML_TAG_PREFIX +from .versioning import split_tag_version from .exceptions import AsdfConversionWarning @@ -253,7 +253,7 @@ def walker(node): return node real_tag = ctx.type_index.get_real_tag(tag_name) - real_tag_name, real_tag_version = asdftypes.split_tag_version(real_tag) + real_tag_name, real_tag_version = split_tag_version(real_tag) # This means that there is an explicit description of versions that are # compatible with the associated tag class implementation, but the # version we found does not fit that description. From 3c2413a26460bb81d156e53c9133c7d1a61a3d37 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Mon, 19 Nov 2018 09:14:18 -0500 Subject: [PATCH 39/55] Refactor WriteTypeIndex init logic into real methods --- asdf/type_index.py | 110 ++++++++++++++++++++++----------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/asdf/type_index.py b/asdf/type_index.py index 155119254..5ce12273e 100644 --- a/asdf/type_index.py +++ b/asdf/type_index.py @@ -62,64 +62,13 @@ def __init__(self, version, index): "Don't know how to write out ASDF version {0}".format( self._version)) - def should_overwrite(cls, new_type): - existing_type = self._type_by_cls[cls] - - # Types that are provided by extensions from other packages should - # only override the type index corresponding to the latest version - # of ASDF. - if existing_type.tag_base() != new_type.tag_base(): - return self._version == default_version - - return True - - def add_type_to_index(cls, typ): - if cls in self._type_by_cls and not should_overwrite(cls, typ): - return - - self._type_by_cls[cls] = typ - self._extension_by_cls[cls] = index._extension_by_type[typ] - - def add_subclasses(typ, asdftype): - for subclass in util.iter_subclasses(typ): - # Do not overwrite the tag type for an existing subclass if the - # new tag serializes a class that is higher in the type - # hierarchy than the existing subclass. - if subclass in self._class_by_subclass: - if issubclass(self._class_by_subclass[subclass], typ): - # Allow for cases where a subclass tag is being - # overridden by a tag from another extension. - if (self._extension_by_cls[subclass] == - index._extension_by_type[asdftype]): - continue - self._class_by_subclass[subclass] = typ - self._type_by_subclasses[subclass] = asdftype - self._extension_by_cls[subclass] = index._extension_by_type[asdftype] - - def add_all_types(asdftype): - add_type_to_index(asdftype, asdftype) - for typ in asdftype.types: - add_type_to_index(typ, asdftype) - add_subclasses(typ, asdftype) - - if asdftype.handle_dynamic_subclasses: - for typ in asdftype.types: - self._types_with_dynamic_subclasses[typ] = asdftype - - def add_by_tag(name, version): - tag = join_tag_version(name, version) - if tag in index._type_by_tag: - asdftype = index._type_by_tag[tag] - self._type_by_name[name] = asdftype - add_all_types(asdftype) - # Process all types defined in the ASDF version map. It is important to # make sure that tags that are associated with the core part of the # standard are processed first in order to handle subclasses properly. for name, _version in core_version_map.items(): - add_by_tag(name, AsdfVersion(_version)) + self._add_by_tag(index, name, AsdfVersion(_version)) for name, _version in standard_version_map.items(): - add_by_tag(name, AsdfVersion(_version)) + self._add_by_tag(index, name, AsdfVersion(_version)) # Now add any extension types that aren't known to the ASDF standard. # This expects that all types defined by ASDF will be encountered @@ -130,10 +79,61 @@ def add_by_tag(name, version): # extension will always be processed first. for name, versions in index._versions_by_type_name.items(): if name not in self._type_by_name: - add_by_tag(name, versions[-1]) + self._add_by_tag(index, name, versions[-1]) for asdftype in index._unnamed_types: - add_all_types(asdftype) + self._add_all_types(index, asdftype) + + def _should_overwrite(self, cls, new_type): + existing_type = self._type_by_cls[cls] + + # Types that are provided by extensions from other packages should + # only override the type index corresponding to the latest version + # of ASDF. + if existing_type.tag_base() != new_type.tag_base(): + return self._version == default_version + + return True + + def _add_type_to_index(self, index, cls, typ): + if cls in self._type_by_cls and not self._should_overwrite(cls, typ): + return + + self._type_by_cls[cls] = typ + self._extension_by_cls[cls] = index._extension_by_type[typ] + + def _add_subclasses(self, index, typ, asdftype): + for subclass in util.iter_subclasses(typ): + # Do not overwrite the tag type for an existing subclass if the + # new tag serializes a class that is higher in the type + # hierarchy than the existing subclass. + if subclass in self._class_by_subclass: + if issubclass(self._class_by_subclass[subclass], typ): + # Allow for cases where a subclass tag is being + # overridden by a tag from another extension. + if (self._extension_by_cls[subclass] == + index._extension_by_type[asdftype]): + continue + self._class_by_subclass[subclass] = typ + self._type_by_subclasses[subclass] = asdftype + self._extension_by_cls[subclass] = index._extension_by_type[asdftype] + + def _add_all_types(self, index, asdftype): + self._add_type_to_index(index, asdftype, asdftype) + for typ in asdftype.types: + self._add_type_to_index(index, typ, asdftype) + self._add_subclasses(index, typ, asdftype) + + if asdftype.handle_dynamic_subclasses: + for typ in asdftype.types: + self._types_with_dynamic_subclasses[typ] = asdftype + + def _add_by_tag(self, index, name, version): + tag = join_tag_version(name, version) + if tag in index._type_by_tag: + asdftype = index._type_by_tag[tag] + self._type_by_name[name] = asdftype + self._add_all_types(index, asdftype) def _mark_used_extension(self, custom_type): self._extensions_used.add(self._extension_by_cls[custom_type]) From 00dc650286360190005ed38a8ccefd20b29ce835 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Mon, 19 Nov 2018 09:24:57 -0500 Subject: [PATCH 40/55] Deprecate asdf.asdftypes in favor of asdf.types --- asdf/asdftypes.py | 480 +--------------------------------------------- asdf/types.py | 476 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 485 insertions(+), 471 deletions(-) create mode 100644 asdf/types.py diff --git a/asdf/asdftypes.py b/asdf/asdftypes.py index e40a6f850..505abf498 100644 --- a/asdf/asdftypes.py +++ b/asdf/asdftypes.py @@ -1,476 +1,14 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst # -*- coding: utf-8 -*- +import warnings -import re -import importlib +from .exceptions import AsdfDeprecationWarning +# This is not exhaustive, but represents the public API +from .versioning import join_tag_version, split_tag_version +from .types import (AsdfType, CustomType, format_tag, ExtensionTypeMeta, + _all_asdftypes) -import six -from copy import copy - -from . import tagged -from . import util -from .versioning import AsdfVersion, AsdfSpec - - -__all__ = ['format_tag', 'CustomType'] - - -# regex used to parse module name from optional version string -MODULE_RE = re.compile(r'([a-zA-Z]+)(-(\d+\.\d+\.\d+))?') - - -def format_tag(organization, standard, version, tag_name): - """ - Format a YAML tag. - """ - tag = 'tag:{0}:{1}/{2}'.format(organization, standard, tag_name) - - if version is None: - return tag - - if isinstance(version, AsdfSpec): - version = str(version.spec) - - return "{0}-{1}".format(tag, version) - - -_all_asdftypes = set() - - -def _from_tree_tagged_missing_requirements(cls, tree, ctx): - # A special version of AsdfType.from_tree_tagged for when the - # required dependencies for an AsdfType are missing. - plural, verb = ('s', 'are') if len(cls.requires) else ('', 'is') - message = "{0} package{1} {2} required to instantiate '{3}'".format( - util.human_list(cls.requires), plural, verb, tree._tag) - # This error will be handled by yamlutil.tagged_tree_to_custom_tree, which - # will cause a warning to be issued indicating that the tree failed to be - # converted. - raise TypeError(message) - - -class ExtensionTypeMeta(type): - """ - Custom class constructor for tag types. - """ - _import_cache = {} - - @classmethod - def _has_required_modules(cls, requires): - for string in requires: - has_module = True - match = MODULE_RE.match(string) - modname, _, version = match.groups() - if modname in cls._import_cache: - if not cls._import_cache[modname]: - return False - try: - module = importlib.import_module(modname) - if version and hasattr(module, '__version__'): - if module.__version__ < version: - has_module = False - except ImportError: - has_module = False - finally: - cls._import_cache[modname] = has_module - if not has_module: - return False - return True - - @classmethod - def _find_in_bases(cls, attrs, bases, name, default=None): - if name in attrs: - return attrs[name] - for base in bases: - if hasattr(base, name): - return getattr(base, name) - return default - - @property - def versioned_siblings(mcls): - return getattr(mcls, '__versioned_siblings') or [] - - def __new__(mcls, name, bases, attrs): - requires = mcls._find_in_bases(attrs, bases, 'requires', []) - if not mcls._has_required_modules(requires): - attrs['from_tree_tagged'] = classmethod( - _from_tree_tagged_missing_requirements) - attrs['types'] = [] - attrs['has_required_modules'] = False - else: - attrs['has_required_modules'] = True - types = mcls._find_in_bases(attrs, bases, 'types', []) - new_types = [] - for typ in types: - if isinstance(typ, str): - typ = util.resolve_name(typ) - new_types.append(typ) - attrs['types'] = new_types - - cls = super(ExtensionTypeMeta, mcls).__new__(mcls, name, bases, attrs) - - if hasattr(cls, 'version'): - if not isinstance(cls.version, (AsdfVersion, AsdfSpec)): - cls.version = AsdfVersion(cls.version) - - if hasattr(cls, 'name'): - if isinstance(cls.name, str): - if 'yaml_tag' not in attrs: - cls.yaml_tag = cls.make_yaml_tag(cls.name) - elif isinstance(cls.name, list): - pass - elif cls.name is not None: - raise TypeError("name must be string or list") - - if hasattr(cls, 'supported_versions'): - if not isinstance(cls.supported_versions, (list, set)): - cls.supported_versions = [cls.supported_versions] - supported_versions = set() - for version in cls.supported_versions: - if not isinstance(version, (AsdfVersion, AsdfSpec)): - version = AsdfVersion(version) - # This should cause an exception for invalid input - supported_versions.add(version) - # We need to convert back to a list here so that the 'in' operator - # uses actual comparison instead of hash equality - cls.supported_versions = list(supported_versions) - siblings = list() - for version in cls.supported_versions: - if version != cls.version: - new_attrs = copy(attrs) - new_attrs['version'] = version - new_attrs['supported_versions'] = set() - new_attrs['_latest_version'] = cls.version - siblings.append( - ExtensionTypeMeta. __new__(mcls, name, bases, new_attrs)) - setattr(cls, '__versioned_siblings', siblings) - - return cls - - -class AsdfTypeMeta(ExtensionTypeMeta): - """ - Keeps track of `AsdfType` subclasses that are created, and stores them in - `AsdfTypeIndex`. - """ - def __new__(mcls, name, bases, attrs): - cls = super(AsdfTypeMeta, mcls).__new__(mcls, name, bases, attrs) - # Classes using this metaclass get added to the list of built-in - # extensions - _all_asdftypes.add(cls) - - return cls - - -class ExtensionType(object): - """ - The base class of all custom types in the tree. - - Besides the attributes defined below, most subclasses will also - override `to_tree` and `from_tree`. - """ - name = None - organization = 'stsci.edu' - standard = 'asdf' - version = (1, 0, 0) - supported_versions = set() - types = [] - handle_dynamic_subclasses = False - validators = {} - requires = [] - yaml_tag = None - - @classmethod - def names(cls): - """ - Returns the name(s) represented by this tag type as a list. - - While some tag types represent only a single custom type, others - represent multiple types. In the latter case, the `name` attribute of - the extension is actually a list, not simply a string. This method - normalizes the value of `name` by returning a list in all cases. - - Returns - ------- - `list` of names represented by this tag type - """ - if cls.name is None: - return None - - return cls.name if isinstance(cls.name, list) else [cls.name] - - @classmethod - def make_yaml_tag(cls, name, versioned=True): - """ - Given the name of a type, returns a string representing its YAML tag. - - Parameters - ---------- - name : str - The name of the type. In most cases this will correspond to the - `name` attribute of the tag type. However, it is passed as a - parameter since some tag types represent multiple custom - types. - - versioned : bool - If `True`, the tag will be versioned. Otherwise, a YAML tag without - a version will be returned. - - Returns - ------- - `str` representing the YAML tag - """ - return format_tag( - cls.organization, - cls.standard, - cls.version if versioned else None, - name) - - @classmethod - def tag_base(cls): - """ - Returns the base of the YAML tag for types represented by this class. - - This method returns the portion of the tag that represents the standard - and the organization of any type represented by this class. - - Returns - ------- - `str` representing the base of the YAML tag - """ - return cls.make_yaml_tag('', versioned=False) - - @classmethod - def to_tree(cls, node, ctx): - """ - Converts instances of custom types into YAML representations. - - This method should be overridden by custom extension classes in order - to define how custom types are serialized into YAML. The method must - return a single Python object corresponding to one of the basic YAML - types (dict, list, str, or number). However, the types can be nested - and combined in order to represent more complex custom types. - - This method is called as part of the process of writing an `AsdfFile` - object. Whenever a custom type (or a subclass of that type) that is - listed in the `types` attribute of this class is encountered, this - method will be used to serialize that type. - - The name `to_tree` refers to the act of converting a custom type into - part of a YAML object tree. - - Parameters - ---------- - node : `object` - Instance of a custom type to be serialized. Will be an instance (or - an instance of a subclass) of one of the types listed in the - `types` attribute of this class. - - ctx : `AsdfFile` - An instance of the `AsdfFile` object that is being written out. - - Returns - ------- - A basic YAML type (`dict`, `list`, `str`, `int`, `float`, or - `complex`) representing the properties of the custom type to be - serialized. These types can be nested in order to represent more - complex custom types. - """ - return node.__class__.__bases__[0](node) - - @classmethod - def to_tree_tagged(cls, node, ctx): - """ - Converts instances of custom types into tagged objects. - - It is more common for custom tag types to override `to_tree` instead of - this method. This method should only be overridden if it is necessary - to modify the YAML tag that will be used to tag this object. - - Parameters - ---------- - node : `object` - Instance of a custom type to be serialized. Will be an instance (or - an instance of a subclass) of one of the types listed in the - `types` attribute of this class. - - ctx : `AsdfFile` - An instance of the `AsdfFile` object that is being written out. - - Returns - ------- - An instance of `asdf.tagged.Tagged`. - """ - obj = cls.to_tree(node, ctx) - return tagged.tag_object(cls.yaml_tag, obj, ctx=ctx) - - @classmethod - def from_tree(cls, tree, ctx): - """ - Converts basic types representing YAML trees into custom types. - - This method should be overridden by custom extension classes in order - to define how custom types are deserialized from the YAML - representation back into their original types. The method will return - an instance of the original custom type. - - This method is called as part of the process of reading an ASDF file in - order to construct an `AsdfFile` object. Whenever a YAML subtree is - encountered that has a tag that corresponds to the `yaml_tag` property - of this class, this method will be used to deserialize that tree back - into an instance of the original custom type. - - Parameters - ---------- - tree : `object` representing YAML tree - An instance of a basic Python type (possibly nested) that - corresponds to a YAML subtree. - - ctx : `AsdfFile` - An instance of the `AsdfFile` object that is being constructed. - - Returns - ------- - An instance of the custom type represented by this extension class. - """ - return cls(tree) - - @classmethod - def from_tree_tagged(cls, tree, ctx): - """ - Converts from tagged tree into custom type. - - It is more common for extension classes to override `from_tree` instead - of this method. This method should only be overridden if it is - necessary to access the `_tag` property of the `Tagged` object - directly. - - Parameters - ---------- - tree : `asdf.tagged.Tagged` object representing YAML tree - - ctx : `AsdfFile` - An instance of the `AsdfFile` object that is being constructed. - - Returns - ------- - An instance of the custom type represented by this extension class. - """ - return cls.from_tree(tree.data, ctx) - - @classmethod - def incompatible_version(cls, version): - """ - Indicates if given version is known to be incompatible with this type. - - If this tag class explicitly identifies compatible versions then this - checks whether a given version is compatible or not (see - `supported_versions`). Otherwise, all versions are assumed to be - compatible. - - Child classes can override this method to affect how version - compatiblity for this type is determined. - - Parameters - ---------- - version : `str` or `~asdf.versioning.AsdfVersion` - The version to test for compatibility. - """ - if cls.supported_versions: - if version not in cls.supported_versions: - return True - return False - - -@six.add_metaclass(AsdfTypeMeta) -class AsdfType(ExtensionType): - """ - Base class for all built-in ASDF types. Types that inherit this class will - be automatically added to the list of built-ins. This should *not* be used - for user-defined extensions. - """ - -@six.add_metaclass(ExtensionTypeMeta) -class CustomType(ExtensionType): - """ - Base class for all user-defined types. - """ - - # These attributes are duplicated here with docstrings since a bug in - # sphinx prevents the docstrings of class attributes from being inherited - # properly (see https://github.com/sphinx-doc/sphinx/issues/741. The - # docstrings are not included anywhere else in the class hierarchy since - # this class is the only one exposed in the public API. - name = None - """ - `str` or `list`: The name of the type. - """ - - organization = 'stsci.edu' - """ - `str`: The organization responsible for the type. - """ - - standard = 'asdf' - """ - `str`: The standard the type is defined in. - """ - - version = (1, 0, 0) - """ - `str`, `tuple`, `AsdfVersion`, or `AsdfSpec`: The version of the type. - """ - - supported_versions = set() - """ - `set`: Versions that explicitly compatible with this extension class. - - If provided, indicates explicit compatibility with the given set - of versions. Other versions of the same schema that are not included in - this set will not be converted to custom types with this class. """ - - types = [] - """ - `list`: List of types that this extension class can convert to/from YAML. - - Custom Python types that, when found in the tree, will be converted into - basic types for YAML output. Can be either strings referring to the types - or the types themselves.""" - - handle_dynamic_subclasses = False - """ - `bool`: Indicates whether dynamically generated subclasses can be serialized - - Flag indicating whether this type is capable of serializing subclasses - of any of the types listed in ``types`` that are generated dynamically. - """ - - validators = {} - """ - `dict`: Mapping JSON Schema keywords to validation functions for jsonschema. - - Useful if the type defines extra types of validation that can be - performed. - """ - - requires = [] - """ - `list`: Python packages that are required to instantiate the object. - """ - - yaml_tag = None - """ - `str`: The YAML tag to use for the type. - - If not provided, it will be automatically generated from name, - organization, standard and version. - """ - - has_required_modules = True - """ - `bool`: Indicates whether modules specified by `requires` are available. - - NOTE: This value is automatically generated. Do not set it in subclasses as - it will be overwritten. - """ +warnings.warn( + "The module asdf.asdftypes has been deprecated and will be removed in 3.0. " + "Use asdf.types instead.", AsdfDeprecationWarning) diff --git a/asdf/types.py b/asdf/types.py new file mode 100644 index 000000000..e40a6f850 --- /dev/null +++ b/asdf/types.py @@ -0,0 +1,476 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +# -*- coding: utf-8 -*- + + +import re +import importlib + +import six +from copy import copy + +from . import tagged +from . import util +from .versioning import AsdfVersion, AsdfSpec + + +__all__ = ['format_tag', 'CustomType'] + + +# regex used to parse module name from optional version string +MODULE_RE = re.compile(r'([a-zA-Z]+)(-(\d+\.\d+\.\d+))?') + + +def format_tag(organization, standard, version, tag_name): + """ + Format a YAML tag. + """ + tag = 'tag:{0}:{1}/{2}'.format(organization, standard, tag_name) + + if version is None: + return tag + + if isinstance(version, AsdfSpec): + version = str(version.spec) + + return "{0}-{1}".format(tag, version) + + +_all_asdftypes = set() + + +def _from_tree_tagged_missing_requirements(cls, tree, ctx): + # A special version of AsdfType.from_tree_tagged for when the + # required dependencies for an AsdfType are missing. + plural, verb = ('s', 'are') if len(cls.requires) else ('', 'is') + message = "{0} package{1} {2} required to instantiate '{3}'".format( + util.human_list(cls.requires), plural, verb, tree._tag) + # This error will be handled by yamlutil.tagged_tree_to_custom_tree, which + # will cause a warning to be issued indicating that the tree failed to be + # converted. + raise TypeError(message) + + +class ExtensionTypeMeta(type): + """ + Custom class constructor for tag types. + """ + _import_cache = {} + + @classmethod + def _has_required_modules(cls, requires): + for string in requires: + has_module = True + match = MODULE_RE.match(string) + modname, _, version = match.groups() + if modname in cls._import_cache: + if not cls._import_cache[modname]: + return False + try: + module = importlib.import_module(modname) + if version and hasattr(module, '__version__'): + if module.__version__ < version: + has_module = False + except ImportError: + has_module = False + finally: + cls._import_cache[modname] = has_module + if not has_module: + return False + return True + + @classmethod + def _find_in_bases(cls, attrs, bases, name, default=None): + if name in attrs: + return attrs[name] + for base in bases: + if hasattr(base, name): + return getattr(base, name) + return default + + @property + def versioned_siblings(mcls): + return getattr(mcls, '__versioned_siblings') or [] + + def __new__(mcls, name, bases, attrs): + requires = mcls._find_in_bases(attrs, bases, 'requires', []) + if not mcls._has_required_modules(requires): + attrs['from_tree_tagged'] = classmethod( + _from_tree_tagged_missing_requirements) + attrs['types'] = [] + attrs['has_required_modules'] = False + else: + attrs['has_required_modules'] = True + types = mcls._find_in_bases(attrs, bases, 'types', []) + new_types = [] + for typ in types: + if isinstance(typ, str): + typ = util.resolve_name(typ) + new_types.append(typ) + attrs['types'] = new_types + + cls = super(ExtensionTypeMeta, mcls).__new__(mcls, name, bases, attrs) + + if hasattr(cls, 'version'): + if not isinstance(cls.version, (AsdfVersion, AsdfSpec)): + cls.version = AsdfVersion(cls.version) + + if hasattr(cls, 'name'): + if isinstance(cls.name, str): + if 'yaml_tag' not in attrs: + cls.yaml_tag = cls.make_yaml_tag(cls.name) + elif isinstance(cls.name, list): + pass + elif cls.name is not None: + raise TypeError("name must be string or list") + + if hasattr(cls, 'supported_versions'): + if not isinstance(cls.supported_versions, (list, set)): + cls.supported_versions = [cls.supported_versions] + supported_versions = set() + for version in cls.supported_versions: + if not isinstance(version, (AsdfVersion, AsdfSpec)): + version = AsdfVersion(version) + # This should cause an exception for invalid input + supported_versions.add(version) + # We need to convert back to a list here so that the 'in' operator + # uses actual comparison instead of hash equality + cls.supported_versions = list(supported_versions) + siblings = list() + for version in cls.supported_versions: + if version != cls.version: + new_attrs = copy(attrs) + new_attrs['version'] = version + new_attrs['supported_versions'] = set() + new_attrs['_latest_version'] = cls.version + siblings.append( + ExtensionTypeMeta. __new__(mcls, name, bases, new_attrs)) + setattr(cls, '__versioned_siblings', siblings) + + return cls + + +class AsdfTypeMeta(ExtensionTypeMeta): + """ + Keeps track of `AsdfType` subclasses that are created, and stores them in + `AsdfTypeIndex`. + """ + def __new__(mcls, name, bases, attrs): + cls = super(AsdfTypeMeta, mcls).__new__(mcls, name, bases, attrs) + # Classes using this metaclass get added to the list of built-in + # extensions + _all_asdftypes.add(cls) + + return cls + + +class ExtensionType(object): + """ + The base class of all custom types in the tree. + + Besides the attributes defined below, most subclasses will also + override `to_tree` and `from_tree`. + """ + name = None + organization = 'stsci.edu' + standard = 'asdf' + version = (1, 0, 0) + supported_versions = set() + types = [] + handle_dynamic_subclasses = False + validators = {} + requires = [] + yaml_tag = None + + @classmethod + def names(cls): + """ + Returns the name(s) represented by this tag type as a list. + + While some tag types represent only a single custom type, others + represent multiple types. In the latter case, the `name` attribute of + the extension is actually a list, not simply a string. This method + normalizes the value of `name` by returning a list in all cases. + + Returns + ------- + `list` of names represented by this tag type + """ + if cls.name is None: + return None + + return cls.name if isinstance(cls.name, list) else [cls.name] + + @classmethod + def make_yaml_tag(cls, name, versioned=True): + """ + Given the name of a type, returns a string representing its YAML tag. + + Parameters + ---------- + name : str + The name of the type. In most cases this will correspond to the + `name` attribute of the tag type. However, it is passed as a + parameter since some tag types represent multiple custom + types. + + versioned : bool + If `True`, the tag will be versioned. Otherwise, a YAML tag without + a version will be returned. + + Returns + ------- + `str` representing the YAML tag + """ + return format_tag( + cls.organization, + cls.standard, + cls.version if versioned else None, + name) + + @classmethod + def tag_base(cls): + """ + Returns the base of the YAML tag for types represented by this class. + + This method returns the portion of the tag that represents the standard + and the organization of any type represented by this class. + + Returns + ------- + `str` representing the base of the YAML tag + """ + return cls.make_yaml_tag('', versioned=False) + + @classmethod + def to_tree(cls, node, ctx): + """ + Converts instances of custom types into YAML representations. + + This method should be overridden by custom extension classes in order + to define how custom types are serialized into YAML. The method must + return a single Python object corresponding to one of the basic YAML + types (dict, list, str, or number). However, the types can be nested + and combined in order to represent more complex custom types. + + This method is called as part of the process of writing an `AsdfFile` + object. Whenever a custom type (or a subclass of that type) that is + listed in the `types` attribute of this class is encountered, this + method will be used to serialize that type. + + The name `to_tree` refers to the act of converting a custom type into + part of a YAML object tree. + + Parameters + ---------- + node : `object` + Instance of a custom type to be serialized. Will be an instance (or + an instance of a subclass) of one of the types listed in the + `types` attribute of this class. + + ctx : `AsdfFile` + An instance of the `AsdfFile` object that is being written out. + + Returns + ------- + A basic YAML type (`dict`, `list`, `str`, `int`, `float`, or + `complex`) representing the properties of the custom type to be + serialized. These types can be nested in order to represent more + complex custom types. + """ + return node.__class__.__bases__[0](node) + + @classmethod + def to_tree_tagged(cls, node, ctx): + """ + Converts instances of custom types into tagged objects. + + It is more common for custom tag types to override `to_tree` instead of + this method. This method should only be overridden if it is necessary + to modify the YAML tag that will be used to tag this object. + + Parameters + ---------- + node : `object` + Instance of a custom type to be serialized. Will be an instance (or + an instance of a subclass) of one of the types listed in the + `types` attribute of this class. + + ctx : `AsdfFile` + An instance of the `AsdfFile` object that is being written out. + + Returns + ------- + An instance of `asdf.tagged.Tagged`. + """ + obj = cls.to_tree(node, ctx) + return tagged.tag_object(cls.yaml_tag, obj, ctx=ctx) + + @classmethod + def from_tree(cls, tree, ctx): + """ + Converts basic types representing YAML trees into custom types. + + This method should be overridden by custom extension classes in order + to define how custom types are deserialized from the YAML + representation back into their original types. The method will return + an instance of the original custom type. + + This method is called as part of the process of reading an ASDF file in + order to construct an `AsdfFile` object. Whenever a YAML subtree is + encountered that has a tag that corresponds to the `yaml_tag` property + of this class, this method will be used to deserialize that tree back + into an instance of the original custom type. + + Parameters + ---------- + tree : `object` representing YAML tree + An instance of a basic Python type (possibly nested) that + corresponds to a YAML subtree. + + ctx : `AsdfFile` + An instance of the `AsdfFile` object that is being constructed. + + Returns + ------- + An instance of the custom type represented by this extension class. + """ + return cls(tree) + + @classmethod + def from_tree_tagged(cls, tree, ctx): + """ + Converts from tagged tree into custom type. + + It is more common for extension classes to override `from_tree` instead + of this method. This method should only be overridden if it is + necessary to access the `_tag` property of the `Tagged` object + directly. + + Parameters + ---------- + tree : `asdf.tagged.Tagged` object representing YAML tree + + ctx : `AsdfFile` + An instance of the `AsdfFile` object that is being constructed. + + Returns + ------- + An instance of the custom type represented by this extension class. + """ + return cls.from_tree(tree.data, ctx) + + @classmethod + def incompatible_version(cls, version): + """ + Indicates if given version is known to be incompatible with this type. + + If this tag class explicitly identifies compatible versions then this + checks whether a given version is compatible or not (see + `supported_versions`). Otherwise, all versions are assumed to be + compatible. + + Child classes can override this method to affect how version + compatiblity for this type is determined. + + Parameters + ---------- + version : `str` or `~asdf.versioning.AsdfVersion` + The version to test for compatibility. + """ + if cls.supported_versions: + if version not in cls.supported_versions: + return True + return False + + +@six.add_metaclass(AsdfTypeMeta) +class AsdfType(ExtensionType): + """ + Base class for all built-in ASDF types. Types that inherit this class will + be automatically added to the list of built-ins. This should *not* be used + for user-defined extensions. + """ + +@six.add_metaclass(ExtensionTypeMeta) +class CustomType(ExtensionType): + """ + Base class for all user-defined types. + """ + + # These attributes are duplicated here with docstrings since a bug in + # sphinx prevents the docstrings of class attributes from being inherited + # properly (see https://github.com/sphinx-doc/sphinx/issues/741. The + # docstrings are not included anywhere else in the class hierarchy since + # this class is the only one exposed in the public API. + name = None + """ + `str` or `list`: The name of the type. + """ + + organization = 'stsci.edu' + """ + `str`: The organization responsible for the type. + """ + + standard = 'asdf' + """ + `str`: The standard the type is defined in. + """ + + version = (1, 0, 0) + """ + `str`, `tuple`, `AsdfVersion`, or `AsdfSpec`: The version of the type. + """ + + supported_versions = set() + """ + `set`: Versions that explicitly compatible with this extension class. + + If provided, indicates explicit compatibility with the given set + of versions. Other versions of the same schema that are not included in + this set will not be converted to custom types with this class. """ + + types = [] + """ + `list`: List of types that this extension class can convert to/from YAML. + + Custom Python types that, when found in the tree, will be converted into + basic types for YAML output. Can be either strings referring to the types + or the types themselves.""" + + handle_dynamic_subclasses = False + """ + `bool`: Indicates whether dynamically generated subclasses can be serialized + + Flag indicating whether this type is capable of serializing subclasses + of any of the types listed in ``types`` that are generated dynamically. + """ + + validators = {} + """ + `dict`: Mapping JSON Schema keywords to validation functions for jsonschema. + + Useful if the type defines extra types of validation that can be + performed. + """ + + requires = [] + """ + `list`: Python packages that are required to instantiate the object. + """ + + yaml_tag = None + """ + `str`: The YAML tag to use for the type. + + If not provided, it will be automatically generated from name, + organization, standard and version. + """ + + has_required_modules = True + """ + `bool`: Indicates whether modules specified by `requires` are available. + + NOTE: This value is automatically generated. Do not set it in subclasses as + it will be overwritten. + """ From dc9f1fa0a0cca6dca4689f6e88e530796e4f2570 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Mon, 19 Nov 2018 09:43:06 -0500 Subject: [PATCH 41/55] Update all internal references to asdftypes --- asdf/__init__.py | 2 +- asdf/asdf.py | 4 ++-- asdf/extension.py | 6 ++--- asdf/fits_embed.py | 2 +- asdf/reference.py | 2 +- asdf/tags/core/__init__.py | 2 +- asdf/tags/core/complex.py | 2 +- asdf/tags/core/constant.py | 2 +- asdf/tags/core/external_reference.py | 2 +- asdf/tags/core/integer.py | 2 +- asdf/tags/core/ndarray.py | 2 +- asdf/tags/core/tests/test_history.py | 4 ++-- asdf/tests/schema_tester.py | 1 - asdf/tests/test_asdftypes.py | 36 ++++++++++++++-------------- asdf/tests/test_helpers.py | 4 ++-- asdf/tests/test_schema.py | 18 +++++++------- asdf/yamlutil.py | 1 - docs/asdf/developer_api.rst | 2 +- 18 files changed, 46 insertions(+), 48 deletions(-) diff --git a/asdf/__init__.py b/asdf/__init__.py index f6daae3a1..30a314e58 100644 --- a/asdf/__init__.py +++ b/asdf/__init__.py @@ -34,7 +34,7 @@ raise ImportError("asdf requires numpy") from .asdf import AsdfFile, open_asdf -from .asdftypes import CustomType +from .types import CustomType from .extension import AsdfExtension from .stream import Stream from . import commands diff --git a/asdf/asdf.py b/asdf/asdf.py index 36a1b3001..af5c965cc 100644 --- a/asdf/asdf.py +++ b/asdf/asdf.py @@ -67,7 +67,7 @@ def __init__(self, tree=None, uri=None, extensions=None, version=None, extensions : list of AsdfExtension A list of extensions to use when reading and writing ASDF files. - See `~asdf.asdftypes.AsdfExtension` for more information. + See `~asdf.types.AsdfExtension` for more information. version : str, optional The ASDF version to use when writing out. If not @@ -1276,7 +1276,7 @@ def open_asdf(fd, uri=None, mode=None, validate_checksums=False, extensions : list of AsdfExtension A list of extensions to use when reading and writing ASDF files. - See `~asdf.asdftypes.AsdfExtension` for more information. + See `~asdf.types.AsdfExtension` for more information. do_not_fill_defaults : bool, optional When `True`, do not fill in missing default values. diff --git a/asdf/extension.py b/asdf/extension.py index 7da26bad1..f9ebceeed 100644 --- a/asdf/extension.py +++ b/asdf/extension.py @@ -9,8 +9,8 @@ import six import importlib +from . import types from . import resolver -from . import asdftypes from .util import get_class_name from .type_index import AsdfTypeIndex from .version import version as asdf_version @@ -125,7 +125,7 @@ def __init__(self, extensions): for extension in extensions: if not isinstance(extension, AsdfExtension): raise TypeError( - "Extension must implement asdftypes.AsdfExtension " + "Extension must implement asdf.types.AsdfExtension " "interface") tag_mapping.extend(extension.tag_mapping) url_mapping.extend(extension.url_mapping) @@ -173,7 +173,7 @@ class BuiltinExtension(object): """ @property def types(self): - return asdftypes._all_asdftypes + return types._all_asdftypes @property def tag_mapping(self): diff --git a/asdf/fits_embed.py b/asdf/fits_embed.py index 08a109f6c..5964c1daa 100644 --- a/asdf/fits_embed.py +++ b/asdf/fits_embed.py @@ -190,7 +190,7 @@ def open(cls, fd, uri=None, validate_checksums=False, extensions=None, extensions : list of AsdfExtension, optional A list of extensions to the ASDF to support when reading - and writing ASDF files. See `asdftypes.AsdfExtension` for + and writing ASDF files. See `asdf.types.AsdfExtension` for more information. ignore_version_mismatch : bool, optional diff --git a/asdf/reference.py b/asdf/reference.py index 0e89c1e6f..e4ee759e0 100644 --- a/asdf/reference.py +++ b/asdf/reference.py @@ -15,7 +15,7 @@ from urllib import parse as urlparse -from .asdftypes import AsdfType +from .types import AsdfType from . import generic_io from . import treeutil from . import util diff --git a/asdf/tags/core/__init__.py b/asdf/tags/core/__init__.py index 7e0dadd02..28c58a6db 100644 --- a/asdf/tags/core/__init__.py +++ b/asdf/tags/core/__init__.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- -from ...asdftypes import AsdfType +from ...types import AsdfType from ...yamlutil import custom_tree_to_tagged_tree diff --git a/asdf/tags/core/complex.py b/asdf/tags/core/complex.py index 1e2dc6327..00f6ffb5f 100644 --- a/asdf/tags/core/complex.py +++ b/asdf/tags/core/complex.py @@ -3,7 +3,7 @@ import numpy as np -from ...asdftypes import AsdfType +from ...types import AsdfType from ... import util diff --git a/asdf/tags/core/constant.py b/asdf/tags/core/constant.py index 6a25e9cc5..e57003c07 100644 --- a/asdf/tags/core/constant.py +++ b/asdf/tags/core/constant.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- -from ...asdftypes import AsdfType +from ...types import AsdfType class Constant(object): diff --git a/asdf/tags/core/external_reference.py b/asdf/tags/core/external_reference.py index 991c3a601..e4ff9e263 100644 --- a/asdf/tags/core/external_reference.py +++ b/asdf/tags/core/external_reference.py @@ -1,4 +1,4 @@ -from ...asdftypes import AsdfType +from ...types import AsdfType class ExternalArrayReference(AsdfType): diff --git a/asdf/tags/core/integer.py b/asdf/tags/core/integer.py index 0d1d62972..6a7a060d1 100644 --- a/asdf/tags/core/integer.py +++ b/asdf/tags/core/integer.py @@ -5,7 +5,7 @@ import numpy as np -from ...asdftypes import AsdfType +from ...types import AsdfType from ...yamlutil import custom_tree_to_tagged_tree diff --git a/asdf/tags/core/ndarray.py b/asdf/tags/core/ndarray.py index 39e0b4423..b0ae4da5f 100644 --- a/asdf/tags/core/ndarray.py +++ b/asdf/tags/core/ndarray.py @@ -8,7 +8,7 @@ from jsonschema import ValidationError -from ...asdftypes import AsdfType +from ...types import AsdfType from ... import schema from ... import util from ... import yamlutil diff --git a/asdf/tags/core/tests/test_history.py b/asdf/tags/core/tests/test_history.py index 2ee8eedf1..2e7763907 100644 --- a/asdf/tags/core/tests/test_history.py +++ b/asdf/tags/core/tests/test_history.py @@ -11,7 +11,7 @@ import asdf from asdf import util -from asdf import asdftypes +from asdf import types from asdf.tests import helpers from asdf.tests.helpers import yaml_to_asdf, display_warnings from asdf.tags.core import HistoryEntry @@ -230,7 +230,7 @@ def test_strict_extension_check(): def test_metadata_with_custom_extension(tmpdir): - class FractionType(asdftypes.AsdfType): + class FractionType(types.AsdfType): name = 'fraction' organization = 'nowhere.org' version = (1, 0, 0) diff --git a/asdf/tests/schema_tester.py b/asdf/tests/schema_tester.py index 0f07a20dd..a0c39f41c 100644 --- a/asdf/tests/schema_tester.py +++ b/asdf/tests/schema_tester.py @@ -12,7 +12,6 @@ import asdf from asdf import AsdfFile -from asdf import asdftypes from asdf import block from asdf import schema from asdf import extension diff --git a/asdf/tests/test_asdftypes.py b/asdf/tests/test_asdftypes.py index 40ca4bc1d..0c0e283e5 100644 --- a/asdf/tests/test_asdftypes.py +++ b/asdf/tests/test_asdftypes.py @@ -9,7 +9,7 @@ import pytest import asdf -from asdf import asdftypes +from asdf import types from asdf import extension from asdf import util from asdf import versioning @@ -23,7 +23,7 @@ def test_custom_tag(): import fractions - class FractionType(asdftypes.CustomType): + class FractionType(types.CustomType): name = 'fraction' organization = 'nowhere.org' version = (1, 0, 0) @@ -232,7 +232,7 @@ def test_versioned_writing(monkeypatch): versioning.supported_versions + [versioning.AsdfVersion('42.0.0')] ) - class FancyComplexType(asdftypes.CustomType): + class FancyComplexType(types.CustomType): name = 'core/complex' organization = 'stsci.edu' standard = 'asdf' @@ -297,15 +297,15 @@ def url_mapping(self): def test_module_versioning(): - class NoModuleType(asdftypes.CustomType): + class NoModuleType(types.CustomType): # It seems highly unlikely that this would be a real module requires = ['qkjvqdja'] - class HasCorrectPytest(asdftypes.CustomType): + class HasCorrectPytest(types.CustomType): # This means it requires 1.0.0 or greater, so it should succeed requires = ['pytest-1.0.0'] - class DoesntHaveCorrectPytest(asdftypes.CustomType): + class DoesntHaveCorrectPytest(types.CustomType): requires = ['pytest-91984.1.7'] nmt = NoModuleType() @@ -373,7 +373,7 @@ def __init__(self, c=None, d=None): self.c = c self.d = d - class CustomFlowType(asdftypes.CustomType): + class CustomFlowType(types.CustomType): version = '1.1.0' name = 'custom_flow' organization = 'nowhere.org' @@ -435,26 +435,26 @@ def url_mapping(self): "tag:nowhere.org:custom/custom_flow-1.0.0 to custom type") def test_incompatible_version_check(): - class TestType0(asdftypes.CustomType): + class TestType0(types.CustomType): supported_versions = versioning.AsdfSpec('>=1.2.0') assert TestType0.incompatible_version('1.1.0') == True assert TestType0.incompatible_version('1.2.0') == False assert TestType0.incompatible_version('2.0.1') == False - class TestType1(asdftypes.CustomType): + class TestType1(types.CustomType): supported_versions = versioning.AsdfVersion('1.0.0') assert TestType1.incompatible_version('1.0.0') == False assert TestType1.incompatible_version('1.1.0') == True - class TestType2(asdftypes.CustomType): + class TestType2(types.CustomType): supported_versions = '1.0.0' assert TestType2.incompatible_version('1.0.0') == False assert TestType2.incompatible_version('1.1.0') == True - class TestType3(asdftypes.CustomType): + class TestType3(types.CustomType): # This doesn't make much sense, but it's just for the sake of example supported_versions = ['1.0.0', versioning.AsdfSpec('>=2.0.0')] @@ -463,7 +463,7 @@ class TestType3(asdftypes.CustomType): assert TestType3.incompatible_version('2.0.0') == False assert TestType3.incompatible_version('2.0.1') == False - class TestType4(asdftypes.CustomType): + class TestType4(types.CustomType): supported_versions = ['1.0.0', versioning.AsdfVersion('1.1.0')] assert TestType4.incompatible_version('1.0.0') == False @@ -471,7 +471,7 @@ class TestType4(asdftypes.CustomType): assert TestType4.incompatible_version('1.1.0') == False assert TestType4.incompatible_version('1.1.1') == True - class TestType5(asdftypes.CustomType): + class TestType5(types.CustomType): supported_versions = \ [versioning.AsdfSpec('<1.0.0'), versioning.AsdfSpec('>=2.0.0')] @@ -482,10 +482,10 @@ class TestType5(asdftypes.CustomType): assert TestType5.incompatible_version('1.1.0') == True with pytest.raises(ValueError): - class TestType6(asdftypes.CustomType): + class TestType6(types.CustomType): supported_versions = 'blue' with pytest.raises(ValueError): - class TestType6(asdftypes.CustomType): + class TestType6(types.CustomType): supported_versions = ['1.1.0', '2.2.0', 'blue'] def test_supported_versions(): @@ -494,7 +494,7 @@ def __init__(self, c=None, d=None): self.c = c self.d = d - class CustomFlowType(asdftypes.CustomType): + class CustomFlowType(types.CustomType): version = '1.1.0' supported_versions = [(1,0,0), versioning.AsdfSpec('>=1.1.0')] name = 'custom_flow' @@ -558,7 +558,7 @@ def test_unsupported_version_warning(): class CustomFlow(object): pass - class CustomFlowType(asdftypes.CustomType): + class CustomFlowType(types.CustomType): version = '1.0.0' supported_versions = [(1,0,0)] name = 'custom_flow' @@ -652,7 +652,7 @@ def test_tag_without_schema(tmpdir): tmpfile = str(tmpdir.join('foo.asdf')) - class FooType(asdftypes.CustomType): + class FooType(types.CustomType): name = 'foo' def __init__(self, a, b): diff --git a/asdf/tests/test_helpers.py b/asdf/tests/test_helpers.py index 289a99cca..4977dd5bf 100644 --- a/asdf/tests/test_helpers.py +++ b/asdf/tests/test_helpers.py @@ -3,14 +3,14 @@ import pytest -from asdf import asdftypes +from asdf import types from asdf.exceptions import AsdfConversionWarning from asdf.tests.helpers import assert_roundtrip_tree def test_conversion_error(tmpdir): - class FooType(asdftypes.CustomType): + class FooType(types.CustomType): name = 'foo' def __init__(self, a, b): diff --git a/asdf/tests/test_schema.py b/asdf/tests/test_schema.py index e1073eccd..9fc37554b 100644 --- a/asdf/tests/test_schema.py +++ b/asdf/tests/test_schema.py @@ -15,7 +15,7 @@ from numpy.testing import assert_array_equal import asdf -from asdf import asdftypes +from asdf import types from asdf import extension from asdf import resolver from asdf import schema @@ -46,7 +46,7 @@ def url_mapping(self): '/{url_suffix}.yaml')] -class TagReferenceType(asdftypes.CustomType): +class TagReferenceType(types.CustomType): """ This class is used by several tests below for validating foreign type references in schemas and ASDF files. @@ -202,7 +202,7 @@ def test_schema_caching(): def test_flow_style(): - class CustomFlowStyleType(dict, asdftypes.CustomType): + class CustomFlowStyleType(dict, types.CustomType): name = 'custom_flow' organization = 'nowhere.org' version = (1, 0, 0) @@ -225,7 +225,7 @@ def types(self): def test_style(): - class CustomStyleType(str, asdftypes.CustomType): + class CustomStyleType(str, types.CustomType): name = 'custom_style' organization = 'nowhere.org' version = (1, 0, 0) @@ -267,7 +267,7 @@ def test_property_order(): def test_invalid_nested(): - class CustomType(str, asdftypes.CustomType): + class CustomType(str, types.CustomType): name = 'custom' organization = 'nowhere.org' version = (1, 0, 0) @@ -361,7 +361,7 @@ def test_default_check_in_schema(): def test_fill_and_remove_defaults(): - class DefaultType(dict, asdftypes.CustomType): + class DefaultType(dict, types.CustomType): name = 'default' organization = 'nowhere.org' version = (1, 0, 0) @@ -419,7 +419,7 @@ def types(self): def test_foreign_tag_reference_validation(): - class ForeignTagReferenceType(asdftypes.CustomType): + class ForeignTagReferenceType(types.CustomType): name = 'foreign_tag_reference' organization = 'nowhere.org' version = (1, 0, 0) @@ -597,7 +597,7 @@ def test_nested_array_yaml(tmpdir): @pytest.mark.importorskip('astropy') def test_type_missing_dependencies(): - class MissingType(asdftypes.CustomType): + class MissingType(types.CustomType): name = 'missing' organization = 'nowhere.org' version = (1, 1, 0) @@ -625,7 +625,7 @@ def types(self): def test_assert_roundtrip_with_extension(tmpdir): called_custom_assert_equal = [False] - class CustomType(dict, asdftypes.CustomType): + class CustomType(dict, types.CustomType): name = 'custom_flow' organization = 'nowhere.org' version = (1, 0, 0) diff --git a/asdf/yamlutil.py b/asdf/yamlutil.py index 6c332ae3e..da8e3288d 100644 --- a/asdf/yamlutil.py +++ b/asdf/yamlutil.py @@ -11,7 +11,6 @@ from . import schema from . import tagged from . import treeutil -from . import asdftypes from . import util from .constants import YAML_TAG_PREFIX from .versioning import split_tag_version diff --git a/docs/asdf/developer_api.rst b/docs/asdf/developer_api.rst index e0ce75825..d8f3e2532 100644 --- a/docs/asdf/developer_api.rst +++ b/docs/asdf/developer_api.rst @@ -5,7 +5,7 @@ Developer API The classes and functions documented here will be of use to developers who wish to create their own custom ASDF types and extensions. -.. automodapi:: asdf.asdftypes +.. automodapi:: asdf.types .. automodapi:: asdf.extension From bf5997ceff61220a21ad7c8a39923056ab42ba00 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Mon, 19 Nov 2018 11:31:50 -0500 Subject: [PATCH 42/55] Update change log --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 3a7a91224..e3f9bfaed 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -10,6 +10,8 @@ - Remove WCS tags. These are now provided by the `gwcs package `_. [#593] +- Deprecate the ``asdf.asdftypes`` module in favor of ``asdf.types``. [#611] + 2.2.1 (2018-11-15) ------------------ From af0b3a7135f7c6f87872f8dadfbc5726cf871c89 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Mon, 19 Nov 2018 09:53:46 -0500 Subject: [PATCH 43/55] Remove all old-style class declarations --- asdf/block.py | 6 +++--- asdf/commands/diff.py | 6 +++--- asdf/commands/main.py | 2 +- asdf/compression.py | 4 ++-- asdf/extension.py | 6 +++--- asdf/fits_embed.py | 2 +- asdf/generic_io.py | 6 +++--- asdf/resolver.py | 2 +- asdf/tagged.py | 2 +- asdf/tags/core/constant.py | 2 +- asdf/tags/core/tests/test_ndarray.py | 2 +- asdf/tests/httpserver.py | 2 +- asdf/tests/test_asdftypes.py | 22 +++++++++++----------- asdf/tests/test_generic_io.py | 4 ++-- asdf/tests/test_yaml.py | 4 ++-- asdf/type_index.py | 4 ++-- asdf/types.py | 2 +- asdf/util.py | 4 ++-- asdf/versioning.py | 4 ++-- 19 files changed, 43 insertions(+), 43 deletions(-) diff --git a/asdf/block.py b/asdf/block.py index 6dd4ea885..d0cc37a5a 100644 --- a/asdf/block.py +++ b/asdf/block.py @@ -26,7 +26,7 @@ from . import yamlutil -class BlockManager(object): +class BlockManager: """ Manages the `Block`s associated with a ASDF file. """ @@ -789,7 +789,7 @@ def close(self): block.close() -class Block(object): +class Block: """ Represents a single block in a ASDF file. This is an implementation detail and should not be instantiated directly. @@ -1204,7 +1204,7 @@ def close(self): self._data = None -class UnloadedBlock(object): +class UnloadedBlock: """ Represents an indexed, but not yet loaded, internal block. All that is known about it is its offset. It converts itself to a diff --git a/asdf/commands/diff.py b/asdf/commands/diff.py index 6828456a7..d2a546085 100644 --- a/asdf/commands/diff.py +++ b/asdf/commands/diff.py @@ -70,7 +70,7 @@ def setup_arguments(cls, subparsers): def run(cls, args): return diff(args.filenames, args.minimal) -class ArrayNode(object): +class ArrayNode: """This class is used to represent unique dummy nodes in the diff tree. In general these dummy nodes will be list elements that we want to keep track of but not necessarily display. This allows the diff output to be @@ -81,7 +81,7 @@ def __init__(self, name): def __hash__(self): return hash(self.name) -class PrintTree(object): +class PrintTree: """This class is used to remember the nodes in the tree that have already been displayed in the diff output. """ @@ -114,7 +114,7 @@ def __setitem__(self, node_list, visit): current['children'][node] = dict(visited=True, children=dict()) current = current['children'][node] -class DiffContext(object): +class DiffContext: """Class that contains context data of the diff to be computed""" def __init__(self, asdf0, asdf1, iostream, minimal=False): self.asdf0 = asdf0 diff --git a/asdf/commands/main.py b/asdf/commands/main.py index 2fc7b1df7..cf4ba715e 100644 --- a/asdf/commands/main.py +++ b/asdf/commands/main.py @@ -12,7 +12,7 @@ command_order = [ 'Explode', 'Implode' ] -class Command(object): +class Command: @classmethod def setup_arguments(cls, subparsers): raise NotImplementedError() diff --git a/asdf/compression.py b/asdf/compression.py index c41d1604b..435d55e3e 100644 --- a/asdf/compression.py +++ b/asdf/compression.py @@ -40,7 +40,7 @@ def validate(compression): return compression -class Lz4Compressor(object): +class Lz4Compressor: def __init__(self, block_api): self._api = block_api @@ -50,7 +50,7 @@ def compress(self, data): return header + output -class Lz4Decompressor(object): +class Lz4Decompressor: def __init__(self, block_api): self._api = block_api self._size = 0 diff --git a/asdf/extension.py b/asdf/extension.py index f9ebceeed..4325fc538 100644 --- a/asdf/extension.py +++ b/asdf/extension.py @@ -24,7 +24,7 @@ @six.add_metaclass(abc.ABCMeta) -class AsdfExtension(object): +class AsdfExtension: """ Abstract base class defining an extension to ASDF. """ @@ -113,7 +113,7 @@ def url_mapping(self): pass -class AsdfExtensionList(object): +class AsdfExtensionList: """ Manage a set of extensions that are in effect. """ @@ -165,7 +165,7 @@ def validators(self): return self._validators -class BuiltinExtension(object): +class BuiltinExtension: """ This is the "extension" to ASDF that includes all the built-in tags. Even though it's not really an extension and it's always diff --git a/asdf/fits_embed.py b/asdf/fits_embed.py index 5964c1daa..d330b88e0 100644 --- a/asdf/fits_embed.py +++ b/asdf/fits_embed.py @@ -29,7 +29,7 @@ __all__ = ['AsdfInFits'] -class _FitsBlock(object): +class _FitsBlock: def __init__(self, hdu): self._hdu = hdu diff --git a/asdf/generic_io.py b/asdf/generic_io.py index 6eab89553..0f43c2571 100644 --- a/asdf/generic_io.py +++ b/asdf/generic_io.py @@ -182,7 +182,7 @@ def relative_uri(source, target): return relative -class _TruncatedReader(object): +class _TruncatedReader: """ Reads until a given delimiter is found. Only works with RandomAccessFile and InputStream, though as this is a private @@ -257,7 +257,7 @@ def read(self, nbytes=None): @six.add_metaclass(util.InheritDocstrings) -class GenericFile(object): +class GenericFile: """ Base class for an abstraction layer around a number of different file-like types. Each of its subclasses handles a particular kind @@ -656,7 +656,7 @@ def read_into_array(self, size): return np.frombuffer(buff, np.uint8, size, 0) -class GenericWrapper(object): +class GenericWrapper: """ A wrapper around a `GenericFile` object so that closing only happens in the very outer layer. diff --git a/asdf/resolver.py b/asdf/resolver.py index 98c6881f9..df71d4757 100644 --- a/asdf/resolver.py +++ b/asdf/resolver.py @@ -19,7 +19,7 @@ def find_schema_path(): return os.path.join(dirname, 'schemas') -class Resolver(object): +class Resolver: """ A class that can be used to map strings with a particular prefix to another. diff --git a/asdf/tagged.py b/asdf/tagged.py index 38e9594dc..ee6ea683d 100644 --- a/asdf/tagged.py +++ b/asdf/tagged.py @@ -37,7 +37,7 @@ __all__ = ['tag_object', 'get_tag'] -class Tagged(object): +class Tagged: """ Base class of classes that wrap a given object and store a tag with it. diff --git a/asdf/tags/core/constant.py b/asdf/tags/core/constant.py index e57003c07..f162b2aec 100644 --- a/asdf/tags/core/constant.py +++ b/asdf/tags/core/constant.py @@ -5,7 +5,7 @@ from ...types import AsdfType -class Constant(object): +class Constant: def __init__(self, value): self._value = value diff --git a/asdf/tags/core/tests/test_ndarray.py b/asdf/tags/core/tests/test_ndarray.py index 3a2436602..cc85f5e10 100644 --- a/asdf/tags/core/tests/test_ndarray.py +++ b/asdf/tags/core/tests/test_ndarray.py @@ -42,7 +42,7 @@ class CustomDatatype(CustomTestType): version = '1.0.0' -class CustomExtension(object): +class CustomExtension: @property def types(self): return [CustomNdim, CustomDatatype] diff --git a/asdf/tests/httpserver.py b/asdf/tests/httpserver.py index 8697fa057..2eab4ecab 100644 --- a/asdf/tests/httpserver.py +++ b/asdf/tests/httpserver.py @@ -50,7 +50,7 @@ def translate_path(self, path): server.server_close() -class HTTPServer(object): +class HTTPServer: handler_class = http.server.SimpleHTTPRequestHandler def __init__(self): diff --git a/asdf/tests/test_asdftypes.py b/asdf/tests/test_asdftypes.py index 0c0e283e5..8220a135a 100644 --- a/asdf/tests/test_asdftypes.py +++ b/asdf/tests/test_asdftypes.py @@ -38,7 +38,7 @@ def to_tree(cls, node, ctx): def from_tree(cls, tree, ctx): return fractions.Fraction(tree[0], tree[1]) - class FractionExtension(object): + class FractionExtension: @property def types(self): return [FractionType] @@ -164,7 +164,7 @@ def test_version_mismatch_with_supported_versions(): """Make sure that defining the supported_versions field does not affect whether or not schema mismatch warnings are triggered.""" - class CustomFlow(object): + class CustomFlow: pass class CustomFlowType(CustomTestType): @@ -175,7 +175,7 @@ class CustomFlowType(CustomTestType): standard = 'custom' types = [CustomFlow] - class CustomFlowExtension(object): + class CustomFlowExtension: @property def types(self): return [CustomFlowType] @@ -247,7 +247,7 @@ def to_tree(cls, node, ctx): def from_tree(cls, tree, ctx): return ComplexType.from_tree(tree, ctx) - class FancyComplexExtension(object): + class FancyComplexExtension: @property def types(self): return [FancyComplexType] @@ -273,7 +273,7 @@ def url_mapping(self): def test_longest_match(): - class FancyComplexExtension(object): + class FancyComplexExtension: @property def types(self): return [] @@ -368,7 +368,7 @@ def test_newer_tag(): # fairly contrived but we want to test whether ASDF can handle backwards # compatibility even when an explicit tag class for different versions of a # schema is not available. - class CustomFlow(object): + class CustomFlow: def __init__(self, c=None, d=None): self.c = c self.d = d @@ -391,7 +391,7 @@ def from_tree(cls, tree, ctx): def to_tree(cls, data, ctx): tree = dict(c=data.c, d=data.d) - class CustomFlowExtension(object): + class CustomFlowExtension: @property def types(self): return [CustomFlowType] @@ -489,7 +489,7 @@ class TestType6(types.CustomType): supported_versions = ['1.1.0', '2.2.0', 'blue'] def test_supported_versions(): - class CustomFlow(object): + class CustomFlow: def __init__(self, c=None, d=None): self.c = c self.d = d @@ -518,7 +518,7 @@ def to_tree(cls, data, ctx): else: tree = dict(c=data.c, d=data.d) - class CustomFlowExtension(object): + class CustomFlowExtension: @property def types(self): return [CustomFlowType] @@ -555,7 +555,7 @@ def url_mapping(self): assert type(old_data.tree['flow_thing']) == CustomFlow def test_unsupported_version_warning(): - class CustomFlow(object): + class CustomFlow: pass class CustomFlowType(types.CustomType): @@ -566,7 +566,7 @@ class CustomFlowType(types.CustomType): standard = 'custom' types = [CustomFlow] - class CustomFlowExtension(object): + class CustomFlowExtension: @property def types(self): return [CustomFlowType] diff --git a/asdf/tests/test_generic_io.py b/asdf/tests/test_generic_io.py index 3e51f1b5c..5bc2cc96c 100644 --- a/asdf/tests/test_generic_io.py +++ b/asdf/tests/test_generic_io.py @@ -465,11 +465,11 @@ def test_relative_uri(): def test_arbitrary_file_object(): - class Wrapper(object): + class Wrapper: def __init__(self, init): self._fd = init - class Random(object): + class Random: def seek(self, *args): return self._fd.seek(*args) diff --git a/asdf/tests/test_yaml.py b/asdf/tests/test_yaml.py index a960a39f4..b3a55b7fd 100644 --- a/asdf/tests/test_yaml.py +++ b/asdf/tests/test_yaml.py @@ -78,7 +78,7 @@ def test_arbitrary_python_object(): # Putting "just any old" Python object in the tree should raise an # exception. - class Foo(object): + class Foo: pass tree = {'object': Foo()} @@ -256,7 +256,7 @@ def test_yaml_nan_inf(): def test_tag_object(): - class SomeObject(object): + class SomeObject: pass tag = 'tag:nowhere.org:none/some/thing' diff --git a/asdf/type_index.py b/asdf/type_index.py index 5ce12273e..fdc7ea847 100644 --- a/asdf/type_index.py +++ b/asdf/type_index.py @@ -17,7 +17,7 @@ _BASIC_PYTHON_TYPES = [str, int, float, list, dict, tuple] -class _AsdfWriteTypeIndex(object): +class _AsdfWriteTypeIndex: """ The _AsdfWriteTypeIndex is a helper class for AsdfTypeIndex that manages an index of types for writing out ASDF files, i.e. from @@ -182,7 +182,7 @@ def from_custom_type(self, custom_type): return asdftype -class AsdfTypeIndex(object): +class AsdfTypeIndex: """ An index of the known `ExtensionType` classes. diff --git a/asdf/types.py b/asdf/types.py index e40a6f850..d7113f650 100644 --- a/asdf/types.py +++ b/asdf/types.py @@ -163,7 +163,7 @@ def __new__(mcls, name, bases, attrs): return cls -class ExtensionType(object): +class ExtensionType: """ The base class of all custom types in the tree. diff --git a/asdf/util.py b/asdf/util.py index f72fe3eae..b916ac87b 100644 --- a/asdf/util.py +++ b/asdf/util.py @@ -118,7 +118,7 @@ def calculate_padding(content_size, pad_blocks, block_size): return max(new_size - content_size, 0) -class BinaryStruct(object): +class BinaryStruct: """ A wrapper around the Python stdlib struct module to define a binary struct more like a dictionary than a tuple. @@ -368,7 +368,7 @@ class InheritDocstrings(type): >>> from asdf.util import InheritDocstrings >>> import six >>> @six.add_metaclass(InheritDocstrings) - ... class A(object): + ... class A: ... def wiggle(self): ... "Wiggle the thingamajig" ... pass diff --git a/asdf/versioning.py b/asdf/versioning.py index 9bfbe1ba0..08d9b9b07 100644 --- a/asdf/versioning.py +++ b/asdf/versioning.py @@ -71,7 +71,7 @@ def get_version_map(version): @total_ordering -class AsdfVersionMixin(object): +class AsdfVersionMixin: """This mix-in is required in order to impose the total ordering that we want for ``AsdfVersion``, rather than accepting the total ordering that is already provided by ``Version`` from ``semantic_version``. Defining these @@ -168,7 +168,7 @@ def __hash__(self): default_version = supported_versions[-1] -class VersionedMixin(object): +class VersionedMixin: _version = default_version @property From 8255c84634cf2ff3d01b155df73a4998832c7871 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Mon, 19 Nov 2018 13:37:08 -0500 Subject: [PATCH 44/55] Fix reference file tests --- asdf/tests/test_suite.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/asdf/tests/test_suite.py b/asdf/tests/test_suite.py index cfb73a385..fcac7dc31 100644 --- a/asdf/tests/test_suite.py +++ b/asdf/tests/test_suite.py @@ -12,6 +12,10 @@ from .helpers import assert_tree_match, display_warnings +_REFFILE_PATH = os.path.join(os.path.dirname(__file__), '..', '..', + 'asdf-standard', 'reference_files') + + def get_test_id(reference_file_path): """Helper function to return the informative part of a schema path""" path = os.path.normpath(str(reference_file_path)) @@ -19,9 +23,8 @@ def get_test_id(reference_file_path): def collect_reference_files(): """Function used by pytest to collect ASDF reference files for testing.""" - root = os.path.join(os.path.dirname(__file__), '..', "reference_files") for version in versioning.supported_versions: - version_dir = os.path.join(root, str(version)) + version_dir = os.path.join(_REFFILE_PATH, str(version)) if os.path.exists(version_dir): for filename in os.listdir(version_dir): if filename.endswith(".asdf"): From b5f92a53e64639a6202c1071bfd6abcd6de8c08c Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Mon, 19 Nov 2018 13:37:58 -0500 Subject: [PATCH 45/55] Rename test_suite to test_reference_files --- asdf/tests/{test_suite.py => test_reference_files.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename asdf/tests/{test_suite.py => test_reference_files.py} (100%) diff --git a/asdf/tests/test_suite.py b/asdf/tests/test_reference_files.py similarity index 100% rename from asdf/tests/test_suite.py rename to asdf/tests/test_reference_files.py From 19018ecb58b992323da9d0a203fedb9245bf907d Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Mon, 19 Nov 2018 13:45:18 -0500 Subject: [PATCH 46/55] Suppress warning from numpy when testing complex reference file --- asdf/tests/test_reference_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asdf/tests/test_reference_files.py b/asdf/tests/test_reference_files.py index fcac7dc31..8bf4add67 100644 --- a/asdf/tests/test_reference_files.py +++ b/asdf/tests/test_reference_files.py @@ -62,7 +62,7 @@ def test_reference_file(reference_file): name_without_ext, _ = os.path.splitext(reference_file) known_fail = False - expect_warnings = False + expect_warnings = 'complex' in reference_file if sys.maxunicode <= 65535: known_fail = known_fail or (basename in ('unicode_spp.asdf')) From 4f959e8406cd7d2b2ee26fb0ccaa50b8faaab374 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Mon, 19 Nov 2018 10:58:48 -0500 Subject: [PATCH 47/55] Move higher-level API tests into a new test module --- asdf/tests/test_api.py | 316 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 316 insertions(+) create mode 100644 asdf/tests/test_api.py diff --git a/asdf/tests/test_api.py b/asdf/tests/test_api.py new file mode 100644 index 000000000..093299a5c --- /dev/null +++ b/asdf/tests/test_api.py @@ -0,0 +1,316 @@ +# -*- coding: utf-8 -*- + +import os +import io + +import numpy as np +from numpy.testing import assert_array_equal +from astropy.modeling import models + +import pytest + +import asdf +from asdf import treeutil +from asdf import extension +from asdf import versioning +from asdf.exceptions import AsdfDeprecationWarning +from .helpers import assert_tree_match, assert_roundtrip_tree, display_warnings + + +def test_get_data_from_closed_file(tmpdir): + tmpdir = str(tmpdir) + path = os.path.join(tmpdir, 'test.asdf') + + my_array = np.arange(0, 64).reshape((8, 8)) + + tree = {'my_array': my_array} + ff = asdf.AsdfFile(tree) + ff.write_to(path) + + with asdf.open(path) as ff: + pass + + with pytest.raises(IOError): + assert_array_equal(my_array, ff.tree['my_array']) + + +def test_no_warning_nan_array(tmpdir): + """ + Tests for a regression that was introduced by + https://github.com/spacetelescope/asdf/pull/557 + """ + + tree = dict(array=np.array([1, 2, np.nan])) + + with pytest.warns(None) as w: + assert_roundtrip_tree(tree, tmpdir) + assert len(w) == 0, display_warnings(w) + + +def test_warning_deprecated_open(tmpdir): + + tmpfile = str(tmpdir.join('foo.asdf')) + + tree = dict(foo=42, bar='hello') + with asdf.AsdfFile(tree) as af: + af.write_to(tmpfile) + + with pytest.warns(AsdfDeprecationWarning): + with asdf.AsdfFile.open(tmpfile) as af: + assert_tree_match(tree, af.tree) + + +def test_open_readonly(tmpdir): + + tmpfile = str(tmpdir.join('readonly.asdf')) + + tree = dict(foo=42, bar='hello', baz=np.arange(20)) + with asdf.AsdfFile(tree) as af: + af.write_to(tmpfile, all_array_storage='internal') + + os.chmod(tmpfile, 0o440) + assert os.access(tmpfile, os.W_OK) == False + + with asdf.open(tmpfile) as af: + assert af['baz'].flags.writeable == False + + with pytest.raises(PermissionError): + with asdf.open(tmpfile, mode='rw'): + pass + + +def test_atomic_write(tmpdir, small_tree): + tmpfile = os.path.join(str(tmpdir), 'test.asdf') + + ff = asdf.AsdfFile(small_tree) + ff.write_to(tmpfile) + + with asdf.open(tmpfile, mode='r') as ff: + ff.write_to(tmpfile) + + +def test_overwrite(tmpdir): + # This is intended to reproduce the following issue: + # https://github.com/spacetelescope/asdf/issues/100 + tmpfile = os.path.join(str(tmpdir), 'test.asdf') + aff = models.AffineTransformation2D(matrix=[[1, 2], [3, 4]]) + f = asdf.AsdfFile() + f.tree['model'] = aff + f.write_to(tmpfile) + model = f.tree['model'] + + ff = asdf.AsdfFile() + ff.tree['model'] = model + ff.write_to(tmpfile) + + +def test_default_version(): + # See https://github.com/spacetelescope/asdf/issues/364 + + version_map = versioning.get_version_map(versioning.default_version) + + ff = asdf.AsdfFile() + assert ff.file_format_version == version_map['FILE_FORMAT'] + + +def test_update_exceptions(tmpdir): + tmpdir = str(tmpdir) + path = os.path.join(tmpdir, 'test.asdf') + + my_array = np.random.rand(8, 8) + tree = {'my_array': my_array} + ff = asdf.AsdfFile(tree) + ff.write_to(path) + + with asdf.open(path, mode='r', copy_arrays=True) as ff: + with pytest.raises(IOError): + ff.update() + + ff = asdf.AsdfFile(tree) + buff = io.BytesIO() + ff.write_to(buff) + + buff.seek(0) + with asdf.open(buff, mode='rw') as ff: + ff.update() + + with pytest.raises(ValueError): + asdf.AsdfFile().update() + + +def test_top_level_tree(small_tree): + tree = {'tree': small_tree} + ff = asdf.AsdfFile(tree) + assert_tree_match(ff.tree['tree'], ff['tree']) + + ff2 = asdf.AsdfFile() + ff2['tree'] = small_tree + assert_tree_match(ff2.tree['tree'], ff2['tree']) + + +def test_top_level_keys(small_tree): + tree = {'tree': small_tree} + ff = asdf.AsdfFile(tree) + assert ff.tree.keys() == ff.keys() + + +def test_walk_and_modify_remove_keys(): + tree = { + 'foo': 42, + 'bar': 43 + } + + def func(x): + if x == 42: + return None + return x + + tree2 = treeutil.walk_and_modify(tree, func) + + assert 'foo' not in tree2 + assert 'bar' in tree2 + + +def test_copy(tmpdir): + tmpdir = str(tmpdir) + + my_array = np.random.rand(8, 8) + tree = {'my_array': my_array, 'foo': {'bar': 'baz'}} + ff = asdf.AsdfFile(tree) + ff.write_to(os.path.join(tmpdir, 'test.asdf')) + + with asdf.open(os.path.join(tmpdir, 'test.asdf')) as ff: + ff2 = ff.copy() + ff2.tree['my_array'] *= 2 + ff2.tree['foo']['bar'] = 'boo' + + assert np.all(ff2.tree['my_array'] == + ff.tree['my_array'] * 2) + assert ff.tree['foo']['bar'] == 'baz' + + assert_array_equal(ff2.tree['my_array'], ff2.tree['my_array']) + + +def test_tag_to_schema_resolver_deprecation(): + ff = asdf.AsdfFile() + with pytest.warns(AsdfDeprecationWarning): + ff.tag_to_schema_resolver('foo') + + with pytest.warns(AsdfDeprecationWarning): + extension_list = extension.default_extensions.extension_list + extension_list.tag_to_schema_resolver('foo') + + +def test_access_tree_outside_handler(tmpdir): + tempname = str(tmpdir.join('test.asdf')) + + tree = {'random': np.random.random(10)} + + ff = asdf.AsdfFile(tree) + ff.write_to(str(tempname)) + + with asdf.open(tempname) as newf: + pass + + # Accessing array data outside of handler should fail + with pytest.raises(OSError): + newf.tree['random'][0] + + +def test_context_handler_resolve_and_inline(tmpdir): + # This reproduces the issue reported in + # https://github.com/spacetelescope/asdf/issues/406 + tempname = str(tmpdir.join('test.asdf')) + + tree = {'random': np.random.random(10)} + + ff = asdf.AsdfFile(tree) + ff.write_to(str(tempname)) + + with asdf.open(tempname) as newf: + newf.resolve_and_inline() + + with pytest.raises(OSError): + newf.tree['random'][0] + + +@pytest.mark.skip(reason='Until inline_threshold is added as a write option') +def test_inline_threshold(tmpdir): + + tree = { + 'small': np.ones(10), + 'large': np.ones(100) + } + + with asdf.AsdfFile(tree) as af: + assert len(list(af.blocks.inline_blocks)) == 1 + assert len(list(af.blocks.internal_blocks)) == 1 + + with asdf.AsdfFile(tree, inline_threshold=10) as af: + assert len(list(af.blocks.inline_blocks)) == 1 + assert len(list(af.blocks.internal_blocks)) == 1 + + with asdf.AsdfFile(tree, inline_threshold=5) as af: + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 2 + + with asdf.AsdfFile(tree, inline_threshold=100) as af: + assert len(list(af.blocks.inline_blocks)) == 2 + assert len(list(af.blocks.internal_blocks)) == 0 + + +@pytest.mark.skip(reason='Until inline_threshold is added as a write option') +def test_inline_threshold_masked(tmpdir): + + mask = np.random.randint(0, 1+1, 20) + masked_array = np.ma.masked_array(np.ones(20), mask=mask) + + tree = { + 'masked': masked_array + } + + # Make sure that masked arrays aren't automatically inlined, even if they + # are small enough + with asdf.AsdfFile(tree) as af: + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 2 + + tree = { + 'masked': masked_array, + 'normal': np.random.random(20) + } + + with asdf.AsdfFile(tree) as af: + assert len(list(af.blocks.inline_blocks)) == 1 + assert len(list(af.blocks.internal_blocks)) == 2 + + +@pytest.mark.skip(reason='Until inline_threshold is added as a write option') +def test_inline_threshold_override(tmpdir): + + tmpfile = str(tmpdir.join('inline.asdf')) + + tree = { + 'small': np.ones(10), + 'large': np.ones(100) + } + + with asdf.AsdfFile(tree) as af: + af.set_array_storage(tree['small'], 'internal') + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 2 + + with asdf.AsdfFile(tree) as af: + af.set_array_storage(tree['large'], 'inline') + assert len(list(af.blocks.inline_blocks)) == 2 + assert len(list(af.blocks.internal_blocks)) == 0 + + with asdf.AsdfFile(tree) as af: + af.write_to(tmpfile, all_array_storage='internal') + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 2 + + with asdf.AsdfFile(tree) as af: + af.write_to(tmpfile, all_array_storage='inline') + assert len(list(af.blocks.inline_blocks)) == 2 + assert len(list(af.blocks.internal_blocks)) == 0 From 2962d6c5f046341270c1da36314a75b72c112e55 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Mon, 19 Nov 2018 11:25:45 -0500 Subject: [PATCH 48/55] Move file format tests into a new test module --- asdf/tests/test_file_format.py | 236 +++++++++++++++++++++++++++++++++ asdf/tests/test_low_level.py | 236 --------------------------------- 2 files changed, 236 insertions(+), 236 deletions(-) create mode 100644 asdf/tests/test_file_format.py diff --git a/asdf/tests/test_file_format.py b/asdf/tests/test_file_format.py new file mode 100644 index 000000000..1fc0fd579 --- /dev/null +++ b/asdf/tests/test_file_format.py @@ -0,0 +1,236 @@ +# -*- coding: utf-8 -*- + +import os +import io + +import pytest + +import asdf +from asdf import generic_io + + +def test_no_yaml_end_marker(tmpdir): + content = b"""#ASDF 1.0.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.0.0 +foo: bar...baz +baz: 42 + """ + path = os.path.join(str(tmpdir), 'test.asdf') + + buff = io.BytesIO(content) + with pytest.raises(ValueError): + with asdf.open(buff): + pass + + buff.seek(0) + fd = generic_io.InputStream(buff, 'r') + with pytest.raises(ValueError): + with asdf.open(fd): + pass + + with open(path, 'wb') as fd: + fd.write(content) + + with open(path, 'rb') as fd: + with pytest.raises(ValueError): + with asdf.open(fd): + pass + + +def test_no_final_newline(tmpdir): + content = b"""#ASDF 1.0.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.0.0 +foo: ...bar... +baz: 42 +...""" + path = os.path.join(str(tmpdir), 'test.asdf') + + buff = io.BytesIO(content) + with asdf.open(buff) as ff: + assert len(ff.tree) == 2 + + buff.seek(0) + fd = generic_io.InputStream(buff, 'r') + with asdf.open(fd) as ff: + assert len(ff.tree) == 2 + + with open(path, 'wb') as fd: + fd.write(content) + + with open(path, 'rb') as fd: + with asdf.open(fd) as ff: + assert len(ff.tree) == 2 + + +def test_no_asdf_header(tmpdir): + content = b"What? This ain't no ASDF file" + + path = os.path.join(str(tmpdir), 'test.asdf') + + buff = io.BytesIO(content) + with pytest.raises(ValueError): + asdf.open(buff) + + with open(path, 'wb') as fd: + fd.write(content) + + with open(path, 'rb') as fd: + with pytest.raises(ValueError): + asdf.open(fd) + + +def test_no_asdf_blocks(tmpdir): + content = b"""#ASDF 1.0.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.0.0 +foo: bar +... +XXXXXXXX + """ + + path = os.path.join(str(tmpdir), 'test.asdf') + + buff = io.BytesIO(content) + with asdf.open(buff) as ff: + assert len(ff.blocks) == 0 + + buff.seek(0) + fd = generic_io.InputStream(buff, 'r') + with asdf.open(fd) as ff: + assert len(ff.blocks) == 0 + + with open(path, 'wb') as fd: + fd.write(content) + + with open(path, 'rb') as fd: + with asdf.open(fd) as ff: + assert len(ff.blocks) == 0 + + +def test_invalid_source(small_tree): + buff = io.BytesIO() + + ff = asdf.AsdfFile(small_tree) + # Since we're testing with small arrays, force all arrays to be stored + # in internal blocks rather than letting some of them be automatically put + # inline. + ff.write_to(buff, all_array_storage='internal') + + buff.seek(0) + with asdf.open(buff) as ff2: + ff2.blocks.get_block(0) + + with pytest.raises(ValueError): + ff2.blocks.get_block(2) + + with pytest.raises(IOError): + ff2.blocks.get_block("http://127.0.0.1/") + + with pytest.raises(TypeError): + ff2.blocks.get_block(42.0) + + with pytest.raises(ValueError): + ff2.blocks.get_source(42.0) + + block = ff2.blocks.get_block(0) + assert ff2.blocks.get_source(block) == 0 + + +def test_empty_file(): + buff = io.BytesIO(b"#ASDF 1.0.0\n") + buff.seek(0) + + with asdf.open(buff) as ff: + assert ff.tree == {} + assert len(ff.blocks) == 0 + + buff = io.BytesIO(b"#ASDF 1.0.0\n#ASDF_STANDARD 1.0.0") + buff.seek(0) + + with asdf.open(buff) as ff: + assert ff.tree == {} + assert len(ff.blocks) == 0 + + +def test_not_asdf_file(): + buff = io.BytesIO(b"SIMPLE") + buff.seek(0) + + with pytest.raises(ValueError): + with asdf.open(buff): + pass + + buff = io.BytesIO(b"SIMPLE\n") + buff.seek(0) + + with pytest.raises(ValueError): + with asdf.open(buff): + pass + + +def test_junk_file(): + buff = io.BytesIO(b"#ASDF 1.0.0\nFOO") + buff.seek(0) + + with pytest.raises(ValueError): + with asdf.open(buff): + pass + + +def test_block_mismatch(): + # This is a file with a single small block, followed by something + # that has an invalid block magic number. + + buff = io.BytesIO( + b'#ASDF 1.0.0\n\xd3BLK\x00\x28\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0FOOBAR') + + buff.seek(0) + with pytest.raises(ValueError): + with asdf.open(buff): + pass + + +def test_block_header_too_small(): + # The block header size must be at least 40 + + buff = io.BytesIO( + b'#ASDF 1.0.0\n\xd3BLK\0\0') + + buff.seek(0) + with pytest.raises(ValueError): + with asdf.open(buff): + pass + + +def test_invalid_version(tmpdir): + content = b"""#ASDF 0.1.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-0.1.0 +foo : bar +...""" + buff = io.BytesIO(content) + with pytest.raises(ValueError): + with asdf.open(buff) as ff: + pass + + +def test_valid_version(tmpdir): + content = b"""#ASDF 1.0.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.0.0 +foo : bar +...""" + buff = io.BytesIO(content) + with asdf.open(buff) as ff: + version = ff.file_format_version + + assert version.major == 1 + assert version.minor == 0 + assert version.patch == 0 diff --git a/asdf/tests/test_low_level.py b/asdf/tests/test_low_level.py index 52fecb6e2..0e65d1362 100644 --- a/asdf/tests/test_low_level.py +++ b/asdf/tests/test_low_level.py @@ -22,204 +22,6 @@ display_warnings) -def test_no_yaml_end_marker(tmpdir): - content = b"""#ASDF 1.0.0 -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- !core/asdf-1.0.0 -foo: bar...baz -baz: 42 - """ - path = os.path.join(str(tmpdir), 'test.asdf') - - buff = io.BytesIO(content) - with pytest.raises(ValueError): - with asdf.open(buff): - pass - - buff.seek(0) - fd = generic_io.InputStream(buff, 'r') - with pytest.raises(ValueError): - with asdf.open(fd): - pass - - with open(path, 'wb') as fd: - fd.write(content) - - with open(path, 'rb') as fd: - with pytest.raises(ValueError): - with asdf.open(fd): - pass - - -def test_no_final_newline(tmpdir): - content = b"""#ASDF 1.0.0 -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- !core/asdf-1.0.0 -foo: ...bar... -baz: 42 -...""" - path = os.path.join(str(tmpdir), 'test.asdf') - - buff = io.BytesIO(content) - with asdf.open(buff) as ff: - assert len(ff.tree) == 2 - - buff.seek(0) - fd = generic_io.InputStream(buff, 'r') - with asdf.open(fd) as ff: - assert len(ff.tree) == 2 - - with open(path, 'wb') as fd: - fd.write(content) - - with open(path, 'rb') as fd: - with asdf.open(fd) as ff: - assert len(ff.tree) == 2 - - -def test_no_asdf_header(tmpdir): - content = b"What? This ain't no ASDF file" - - path = os.path.join(str(tmpdir), 'test.asdf') - - buff = io.BytesIO(content) - with pytest.raises(ValueError): - asdf.open(buff) - - with open(path, 'wb') as fd: - fd.write(content) - - with open(path, 'rb') as fd: - with pytest.raises(ValueError): - asdf.open(fd) - - -def test_no_asdf_blocks(tmpdir): - content = b"""#ASDF 1.0.0 -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- !core/asdf-1.0.0 -foo: bar -... -XXXXXXXX - """ - - path = os.path.join(str(tmpdir), 'test.asdf') - - buff = io.BytesIO(content) - with asdf.open(buff) as ff: - assert len(ff.blocks) == 0 - - buff.seek(0) - fd = generic_io.InputStream(buff, 'r') - with asdf.open(fd) as ff: - assert len(ff.blocks) == 0 - - with open(path, 'wb') as fd: - fd.write(content) - - with open(path, 'rb') as fd: - with asdf.open(fd) as ff: - assert len(ff.blocks) == 0 - - -def test_invalid_source(small_tree): - buff = io.BytesIO() - - ff = asdf.AsdfFile(small_tree) - # Since we're testing with small arrays, force all arrays to be stored - # in internal blocks rather than letting some of them be automatically put - # inline. - ff.write_to(buff, all_array_storage='internal') - - buff.seek(0) - with asdf.open(buff) as ff2: - ff2.blocks.get_block(0) - - with pytest.raises(ValueError): - ff2.blocks.get_block(2) - - with pytest.raises(IOError): - ff2.blocks.get_block("http://127.0.0.1/") - - with pytest.raises(TypeError): - ff2.blocks.get_block(42.0) - - with pytest.raises(ValueError): - ff2.blocks.get_source(42.0) - - block = ff2.blocks.get_block(0) - assert ff2.blocks.get_source(block) == 0 - - -def test_empty_file(): - buff = io.BytesIO(b"#ASDF 1.0.0\n") - buff.seek(0) - - with asdf.open(buff) as ff: - assert ff.tree == {} - assert len(ff.blocks) == 0 - - buff = io.BytesIO(b"#ASDF 1.0.0\n#ASDF_STANDARD 1.0.0") - buff.seek(0) - - with asdf.open(buff) as ff: - assert ff.tree == {} - assert len(ff.blocks) == 0 - - -def test_not_asdf_file(): - buff = io.BytesIO(b"SIMPLE") - buff.seek(0) - - with pytest.raises(ValueError): - with asdf.open(buff): - pass - - buff = io.BytesIO(b"SIMPLE\n") - buff.seek(0) - - with pytest.raises(ValueError): - with asdf.open(buff): - pass - - -def test_junk_file(): - buff = io.BytesIO(b"#ASDF 1.0.0\nFOO") - buff.seek(0) - - with pytest.raises(ValueError): - with asdf.open(buff): - pass - - -def test_block_mismatch(): - # This is a file with a single small block, followed by something - # that has an invalid block magic number. - - buff = io.BytesIO( - b'#ASDF 1.0.0\n\xd3BLK\x00\x28\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0FOOBAR') - - buff.seek(0) - with pytest.raises(ValueError): - with asdf.open(buff): - pass - - -def test_block_header_too_small(): - # The block header size must be at least 40 - - buff = io.BytesIO( - b'#ASDF 1.0.0\n\xd3BLK\0\0') - - buff.seek(0) - with pytest.raises(ValueError): - with asdf.open(buff): - pass - - def test_external_block(tmpdir): tmpdir = str(tmpdir) @@ -1101,44 +903,6 @@ def test_open_no_memmap(tmpdir): assert not isinstance(array.block._data, np.memmap) -def test_invalid_version(tmpdir): - content = b"""#ASDF 0.1.0 -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- !core/asdf-0.1.0 -foo : bar -...""" - buff = io.BytesIO(content) - with pytest.raises(ValueError): - with asdf.open(buff) as ff: - pass - - -def test_valid_version(tmpdir): - content = b"""#ASDF 1.0.0 -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- !core/asdf-1.0.0 -foo : bar -...""" - buff = io.BytesIO(content) - with asdf.open(buff) as ff: - version = ff.file_format_version - - assert version.major == 1 - assert version.minor == 0 - assert version.patch == 0 - - -def test_default_version(): - # See https://github.com/spacetelescope/asdf/issues/364 - - version_map = versioning.get_version_map(versioning.default_version) - - ff = asdf.AsdfFile() - assert ff.file_format_version == version_map['FILE_FORMAT'] - - def test_fd_not_seekable(): data = np.ones(1024) b = block.Block(data=data) From cb028e37e2e0c03b74edff6b5c4011d7ef9cf5bb Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Mon, 19 Nov 2018 11:26:41 -0500 Subject: [PATCH 49/55] Rename test_low_level to test_array_blocks --- asdf/tests/{test_low_level.py => test_array_blocks.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename asdf/tests/{test_low_level.py => test_array_blocks.py} (100%) diff --git a/asdf/tests/test_low_level.py b/asdf/tests/test_array_blocks.py similarity index 100% rename from asdf/tests/test_low_level.py rename to asdf/tests/test_array_blocks.py From dc3f493a35cc15fad89f74eadd8ef6eaec2cdf61 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Wed, 21 Nov 2018 11:53:34 -0500 Subject: [PATCH 50/55] Add test for using pathlib paths to open/write ASDF files --- asdf/tests/test_api.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/asdf/tests/test_api.py b/asdf/tests/test_api.py index 093299a5c..b7a78ae7a 100644 --- a/asdf/tests/test_api.py +++ b/asdf/tests/test_api.py @@ -2,6 +2,7 @@ import os import io +import pathlib import numpy as np from numpy.testing import assert_array_equal @@ -234,6 +235,20 @@ def test_context_handler_resolve_and_inline(tmpdir): newf.tree['random'][0] +def test_open_pathlib_path(tmpdir): + + filename = str(tmpdir.join('pathlib.asdf')) + path = pathlib.Path(filename) + + tree = {'data': np.ones(10)} + + with asdf.AsdfFile(tree) as af: + af.write_to(path) + + with asdf.open(tree) as af: + assert af['data'] == tree['data'] + + @pytest.mark.skip(reason='Until inline_threshold is added as a write option') def test_inline_threshold(tmpdir): From e4e99f8465176da0c47fad4d362bd968653964dc Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Wed, 21 Nov 2018 12:08:12 -0500 Subject: [PATCH 51/55] Support use of pathlib.Path for asdf.open and write_to --- CHANGES.rst | 3 +++ asdf/generic_io.py | 9 +++++---- asdf/tests/test_api.py | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index e3f9bfaed..0f05d58d6 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,6 +12,9 @@ - Deprecate the ``asdf.asdftypes`` module in favor of ``asdf.types``. [#611] +- Support use of ``pathlib.Path`` with ``asdf.open`` and ``AsdfFile.write_to``. + [#617] + 2.2.1 (2018-11-15) ------------------ diff --git a/asdf/generic_io.py b/asdf/generic_io.py index 0f43c2571..7a4edfd45 100644 --- a/asdf/generic_io.py +++ b/asdf/generic_io.py @@ -10,12 +10,13 @@ """ import io -import math import os -import platform import re import sys +import math +import pathlib import tempfile +import platform from distutils.version import LooseVersion from os import SEEK_SET, SEEK_CUR, SEEK_END @@ -1171,8 +1172,8 @@ def get_file(init, mode='r', uri=None, close=False): init.mode, mode)) return GenericWrapper(init) - elif isinstance(init, str): - parsed = urlparse.urlparse(init) + elif isinstance(init, (str, pathlib.Path)): + parsed = urlparse.urlparse(str(init)) if parsed.scheme in ['http', 'https']: if 'w' in mode: raise ValueError( diff --git a/asdf/tests/test_api.py b/asdf/tests/test_api.py index b7a78ae7a..77cdf8db8 100644 --- a/asdf/tests/test_api.py +++ b/asdf/tests/test_api.py @@ -245,8 +245,8 @@ def test_open_pathlib_path(tmpdir): with asdf.AsdfFile(tree) as af: af.write_to(path) - with asdf.open(tree) as af: - assert af['data'] == tree['data'] + with asdf.open(path) as af: + assert (af['data'] == tree['data']).all() @pytest.mark.skip(reason='Until inline_threshold is added as a write option') From a009e77c5525e035e610b4064984de35cc86d4a8 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Tue, 27 Nov 2018 15:25:04 -0500 Subject: [PATCH 52/55] Add a test that exposes issues revealed in #599 --- asdf/tests/test_api.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/asdf/tests/test_api.py b/asdf/tests/test_api.py index 77cdf8db8..02f6472bb 100644 --- a/asdf/tests/test_api.py +++ b/asdf/tests/test_api.py @@ -249,6 +249,36 @@ def test_open_pathlib_path(tmpdir): assert (af['data'] == tree['data']).all() +@pytest.mark.xfail(reason='Setting auto_inline option modifies AsdfFile state') +def test_auto_inline(tmpdir): + + outfile = str(tmpdir.join('test.asdf')) + tree = dict(data=np.arange(6)) + + # Use the same object for each write in order to make sure that there + # aren't unanticipated side effects + with asdf.AsdfFile(tree) as af: + af.write_to(outfile) + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 1 + + af.write_to(outfile, auto_inline=10) + assert len(list(af.blocks.inline_blocks)) == 1 + assert len(list(af.blocks.internal_blocks)) == 0 + + af.write_to(outfile) + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 1 + + af.write_to(outfile, auto_inline=7) + assert len(list(af.blocks.inline_blocks)) == 1 + assert len(list(af.blocks.internal_blocks)) == 0 + + af.write_to(outfile, auto_inline=5) + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 1 + + @pytest.mark.skip(reason='Until inline_threshold is added as a write option') def test_inline_threshold(tmpdir): From 5af916700f4109a1c0aca2ecb74ddfe512243c01 Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Tue, 27 Nov 2018 16:01:14 -0500 Subject: [PATCH 53/55] Unpin pytest requirement on appveyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 3b6829125..75b38e5c7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -13,7 +13,7 @@ environment: ASTROPY_VERSION: "stable" GWCS_GIT: "git+git://github.com/spacetelescope/gwcs.git#egg=gwcs" GWCS_PIP: "gwcs" - CONDA_DEPENDENCIES: "semantic_version jsonschema pyyaml six lz4 pytest=3.6 pytest-astropy" + CONDA_DEPENDENCIES: "semantic_version jsonschema pyyaml six lz4 pytest-astropy" PIP_DEPENDENCIES: "pytest-faulthandler importlib_resources" PYTHON_ARCH: "64" From 9178651d0fc74e07b5ffe1284c0f5c7cc4f3812b Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Wed, 28 Nov 2018 13:05:07 -0500 Subject: [PATCH 54/55] Update ASDF Standard to v1.3.0 --- CHANGES.rst | 2 ++ asdf-standard | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 0f05d58d6..b5c9aab6d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -15,6 +15,8 @@ - Support use of ``pathlib.Path`` with ``asdf.open`` and ``AsdfFile.write_to``. [#617] +- Update ASDF Standard submodule to version 1.3.0. + 2.2.1 (2018-11-15) ------------------ diff --git a/asdf-standard b/asdf-standard index 0bc7aa7c6..b80568680 160000 --- a/asdf-standard +++ b/asdf-standard @@ -1 +1 @@ -Subproject commit 0bc7aa7c6b7b2099174f6ac40e2590f366f8b092 +Subproject commit b80568680ad4689f04e2f17ee8896a198dc76bbc From 41a1990a748ba437ca7c746950e1bbbc23e769ca Mon Sep 17 00:00:00 2001 From: Daniel D'Avella Date: Wed, 28 Nov 2018 13:48:53 -0500 Subject: [PATCH 55/55] Update What's New section of docs for 2.3 release --- docs/asdf/changes.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/asdf/changes.rst b/docs/asdf/changes.rst index 5bb267f26..68300e440 100644 --- a/docs/asdf/changes.rst +++ b/docs/asdf/changes.rst @@ -4,6 +4,26 @@ Changes ******* +What's New in ASDF 2.3? +======================= + +ASDF 2.3 reflects the update of ASDF Standard to v1.3.0, and contains a few +notable features and an API change: + +* Storage of arbitrary precision integers is now provided by + `asdf.IntegerType`. This new type is provided by version 1.3.0 of the ASDF + Standard. + +* Reading a file with integer literals that are too large now causes only a + warning instead of a validation error. This is to provide backwards + compatibility for files that were created with a buggy version of ASDF. + +* The functions `asdf.open` and `AsdfFile.write_to` now support the use of + `pathlib.Path`. + +* The `asdf.asdftypes` module has been deprecated in favor of `asdf.types`. The + old module will be removed entirely in the 3.0 release. + What's New in ASDF 2.2? =======================