diff --git a/.travis.yml b/.travis.yml index ed2f46dbe..bd30ed9e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,64 +22,65 @@ env: # overidden underneath. They are defined here in order to save having # to repeat them for all configurations. - PYTHON_VERSION=3.6 - - PYTEST_VERSION=3.7 + - PYTEST_VERSION=3.10 - ASTROPY_VERSION=stable - NUMPY_VERSION=stable - - PIP_DEPENDENCIES='pytest-faulthandler importlib_resources' - - CONDA_DEPENDENCIES='semantic_version jsonschema pyyaml six lz4 pytest-astropy' + - ALL_PIP_DEPENDENCIES='pytest-faulthandler importlib_resources' + - PIP_DEPENDENCIES="$ALL_PIP_DEPENDENCIES" + - ALL_CONDA_DEPENDENCIES='semantic_version jsonschema pyyaml six lz4 pytest-astropy' + - CONDA_DEPENDENCIES="$ALL_CONDA_DEPENDENCIES" - GWCS_GIT='git+git://github.com/spacetelescope/gwcs.git#egg=gwcs' - GWCS_PIP='gwcs' - - MAIN_CMD='python setup.py' - - SETUP_CMD='test --remote-data' + - RUN_CMD='pytest' matrix: # Make sure that installation does not fail - - SETUP_CMD='install' + - RUN_CMD='python setup.py install' # Make sure README will display properly on pypi - PIP_DEPENDENCIES='twine' TWINE_CHECK=1 - - PYTHON_VERSION=3.5 SETUP_CMD='test' - - PYTHON_VERSION=3.6 SETUP_CMD='test' - - PYTHON_VERSION=3.7 PYTEST_VERSION=3.8 SETUP_CMD='test' + - PYTHON_VERSION=3.5 + - PYTHON_VERSION=3.6 PYTEST_VERSION=3.8 + - PYTHON_VERSION=3.7 PYTEST_VERSION=3.9 matrix: fast_finish: true include: # Do a coverage test - - env: SETUP_CMD='test --coverage --open-files --remote-data' + - env: COVERAGE=1 + PIP_DEPENDENCIES="$ALL_PIP_DEPENDENCIES coverage coveralls" # Check for sphinx doc build warnings - we do this first because it # may run for a long time - - env: SETUP_CMD='build_docs -w' + - env: RUN_CMD='python setup.py build_docs -w' + CONDA_DEPENDENCIES="$ALL_CONDA_DEPENDENCIES sphinx-astropy" # Do a code style check - - env: MAIN_CMD="flake8 asdf --count" SETUP_CMD='' + - env: RUN_CMD="flake8 asdf --count" + PIP_DEPENDENCIES="$ALL_PIP_DEPENDENCIES flake8" # try older numpy versions - - env: PYTHON_VERSION=3.5 NUMPY_VERSION=1.11 SETUP_CMD='test' - - env: NUMPY_VERSION=1.12 SETUP_CMD='test' + - env: PYTHON_VERSION=3.5 NUMPY_VERSION=1.11 + - env: NUMPY_VERSION=1.12 - # run a test using native pytest # also test against development version of Astropy - - env: MAIN_CMD='pytest' SETUP_CMD='' ASTROPY_VERSION=development - GWCS_PIP="$GWCS_GIT" - PYTEST_VERSION=3.8 + - env: ASTROPY_VERSION=development GWCS_PIP="$GWCS_GIT" # latest stable versions - - env: NUMPY_VERSION=stable SETUP_CMD='test' + - env: NUMPY_VERSION=stable # Test against development version of numpy (this job can fail) - - env: NUMPY_VERSION=development SETUP_CMD='test' + - env: NUMPY_VERSION=development # Try a run on OSX - os: osx - env: NUMPY_VERSION=stable SETUP_CMD='test' + env: NUMPY_VERSION=stable # Test against latest version of jsonschema - env: PIP_DEPENDENCIES='jsonschema pytest-faulthandler importlib_resources' allow_failures: - - env: NUMPY_VERSION=development SETUP_CMD='test' + - env: NUMPY_VERSION=development - env: PIP_DEPENDENCIES='jsonschema pytest-faulthandler importlib_resources' install: @@ -92,12 +93,15 @@ script: - if [[ $TWINE_CHECK ]]; then python setup.py build sdist; twine check dist/*; + elif [[ $COVERAGE ]]; then + coverage run --source=asdf -m pytest --remote-data --open-files; + coverage report -m; else - $MAIN_CMD $SETUP_CMD; + $RUN_CMD; fi after_success: # If coveralls.io is set up for this package, uncomment the line # below and replace "packagename" with the name of your package. # The coveragerc file may be customized as needed for your package. - - if [[ $SETUP_CMD == 'test --coverage --open-files --remote-data' ]]; then coveralls --rcfile='asdf/tests/coveragerc'; fi + - if [[ $COVERAGE ]]; then coveralls --rcfile='asdf/tests/coveragerc'; fi diff --git a/CHANGES.rst b/CHANGES.rst index 099fd5b47..b5c9aab6d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,22 @@ +2.3.0 (unreleased) +------------------ + +- Storage of arbitrary precision integers is now provided by + ``asdf.IntegerType``. Reading a file with integer literals that are too + large now causes only a warning instead of a validation error. This is to + provide backwards compatibility for files that were created with a buggy + version of ASDF (see #553 below). [#566] + +- Remove WCS tags. These are now provided by the `gwcs package + `_. [#593] + +- Deprecate the ``asdf.asdftypes`` module in favor of ``asdf.types``. [#611] + +- Support use of ``pathlib.Path`` with ``asdf.open`` and ``AsdfFile.write_to``. + [#617] + +- Update ASDF Standard submodule to version 1.3.0. + 2.2.1 (2018-11-15) ------------------ @@ -31,6 +50,9 @@ index before any others. This fixes a bug that was related to the way that subclass tags were overwritten by external extensions. [#598] +- Remove WCS tags. These are now provided by the `gwcs package + `_. [#593] + 2.1.1 (2018-11-01) ------------------ diff --git a/appveyor.yml b/appveyor.yml index 4da97f6d6..75b38e5c7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -8,19 +8,19 @@ environment: PYTEST_VERSION: "3.7" MINICONDA_VERSION: "latest" CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci-helpers\\appveyor\\windows_sdk.cmd" - SETUP_CMD: "test" + RUN_CMD: "pytest" NUMPY_VERSION: "stable" ASTROPY_VERSION: "stable" GWCS_GIT: "git+git://github.com/spacetelescope/gwcs.git#egg=gwcs" GWCS_PIP: "gwcs" - CONDA_DEPENDENCIES: "semantic_version jsonschema pyyaml six lz4 pytest=3.6 pytest-astropy" + CONDA_DEPENDENCIES: "semantic_version jsonschema pyyaml six lz4 pytest-astropy" PIP_DEPENDENCIES: "pytest-faulthandler importlib_resources" PYTHON_ARCH: "64" matrix: # Make sure that installation does not fail - PYTHON_VERSION: "3.6" - SETUP_CMD: "install" + RUN_CMD: "python setup.py install" platform: x64 - PYTHON_VERSION: "3.5" @@ -65,4 +65,4 @@ install: build: false test_script: - - "%CMD_IN_ENV% python setup.py %SETUP_CMD%" + - "%CMD_IN_ENV% %RUN_CMD%" diff --git a/asdf-standard b/asdf-standard index 0bc7aa7c6..b80568680 160000 --- a/asdf-standard +++ b/asdf-standard @@ -1 +1 @@ -Subproject commit 0bc7aa7c6b7b2099174f6ac40e2590f366f8b092 +Subproject commit b80568680ad4689f04e2f17ee8896a198dc76bbc diff --git a/asdf/__init__.py b/asdf/__init__.py index 9b434e439..30a314e58 100644 --- a/asdf/__init__.py +++ b/asdf/__init__.py @@ -15,7 +15,7 @@ __all__ = [ 'AsdfFile', 'CustomType', 'AsdfExtension', 'Stream', 'open', 'test', - 'commands', 'ExternalArrayReference' + 'commands', 'IntegerType', 'ExternalArrayReference' ] try: @@ -34,10 +34,11 @@ raise ImportError("asdf requires numpy") from .asdf import AsdfFile, open_asdf -from .asdftypes import CustomType +from .types import CustomType from .extension import AsdfExtension from .stream import Stream from . import commands +from .tags.core import IntegerType from .tags.core.external_reference import ExternalArrayReference from jsonschema import ValidationError diff --git a/asdf/asdf.py b/asdf/asdf.py index 5cd804e4e..af5c965cc 100644 --- a/asdf/asdf.py +++ b/asdf/asdf.py @@ -67,7 +67,7 @@ def __init__(self, tree=None, uri=None, extensions=None, version=None, extensions : list of AsdfExtension A list of extensions to use when reading and writing ASDF files. - See `~asdf.asdftypes.AsdfExtension` for more information. + See `~asdf.types.AsdfExtension` for more information. version : str, optional The ASDF version to use when writing out. If not @@ -411,13 +411,14 @@ def comments(self): """ return self._comments - def _validate(self, tree, custom=True): + def _validate(self, tree, custom=True, reading=False): tagged_tree = yamlutil.custom_tree_to_tagged_tree( tree, self) - schema.validate(tagged_tree, self) + schema.validate(tagged_tree, self, reading=reading) # Perform secondary validation pass if requested if custom and self._custom_schema: - schema.validate(tagged_tree, self, self._custom_schema) + schema.validate(tagged_tree, self, self._custom_schema, + reading=reading) def validate(self): """ @@ -650,10 +651,10 @@ def _open_asdf(cls, self, fd, uri=None, mode='r', tree = reference.find_references(tree, self) if not do_not_fill_defaults: - schema.fill_defaults(tree, self) + schema.fill_defaults(tree, self, reading=True) try: - self._validate(tree) + self._validate(tree, reading=True) except ValidationError: self.close() raise @@ -1275,7 +1276,7 @@ def open_asdf(fd, uri=None, mode=None, validate_checksums=False, extensions : list of AsdfExtension A list of extensions to use when reading and writing ASDF files. - See `~asdf.asdftypes.AsdfExtension` for more information. + See `~asdf.types.AsdfExtension` for more information. do_not_fill_defaults : bool, optional When `True`, do not fill in missing default values. diff --git a/asdf/asdftypes.py b/asdf/asdftypes.py index 2b9c51297..505abf498 100644 --- a/asdf/asdftypes.py +++ b/asdf/asdftypes.py @@ -1,875 +1,14 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst # -*- coding: utf-8 -*- - -import re -import bisect import warnings -import importlib -from collections import OrderedDict - -import six -from copy import copy - -from functools import lru_cache - -from . import tagged -from . import util -from .versioning import AsdfVersion, AsdfSpec, get_version_map, default_version - - -__all__ = ['format_tag', 'CustomType', 'AsdfTypeIndex'] - - -_BASIC_PYTHON_TYPES = [str, int, float, list, dict, tuple] - -# regex used to parse module name from optional version string -MODULE_RE = re.compile(r'([a-zA-Z]+)(-(\d+\.\d+\.\d+))?') - - -def format_tag(organization, standard, version, tag_name): - """ - Format a YAML tag. - """ - tag = 'tag:{0}:{1}/{2}'.format(organization, standard, tag_name) - - if version is None: - return tag - - if isinstance(version, AsdfSpec): - version = str(version.spec) - - return "{0}-{1}".format(tag, version) - - -def split_tag_version(tag): - """ - Split a tag into its base and version. - """ - name, version = tag.rsplit('-', 1) - version = AsdfVersion(version) - return name, version - - -def join_tag_version(name, version): - """ - Join the root and version of a tag back together. - """ - return '{0}-{1}'.format(name, version) - - -class _AsdfWriteTypeIndex(object): - """ - The _AsdfWriteTypeIndex is a helper class for AsdfTypeIndex that - manages an index of types for writing out ASDF files, i.e. from - converting from custom types to tagged_types. It is not always - the inverse of the mapping from tags to custom types, since there - are likely multiple versions present for a given tag. - - This uses the `version_map.yaml` file that ships with the ASDF - standard to figure out which schemas correspond to a particular - version of the ASDF standard. - - An AsdfTypeIndex manages multiple _AsdfWriteTypeIndex instances - for each version the user may want to write out, and they are - instantiated on-demand. - - If version is ``'latest'``, it will just use the highest-numbered - versions of each of the schemas. This is currently only used to - aid in testing. - - In the future, this may be renamed to _ExtensionWriteTypeIndex since it is - not specific to classes that inherit `AsdfType`. - """ - _version_map = None - - def __init__(self, version, index): - self._version = version - - self._type_by_cls = {} - self._type_by_name = {} - self._type_by_subclasses = {} - self._class_by_subclass = {} - self._types_with_dynamic_subclasses = {} - self._extension_by_cls = {} - self._extensions_used = set() - - try: - version_map = get_version_map(self._version) - core_version_map = version_map['core'] - standard_version_map = version_map['standard'] - except ValueError: - raise ValueError( - "Don't know how to write out ASDF version {0}".format( - self._version)) - - def should_overwrite(cls, new_type): - existing_type = self._type_by_cls[cls] - - # Types that are provided by extensions from other packages should - # only override the type index corresponding to the latest version - # of ASDF. - if existing_type.tag_base() != new_type.tag_base(): - return self._version == default_version - - return True - - def add_type_to_index(cls, typ): - if cls in self._type_by_cls and not should_overwrite(cls, typ): - return - - self._type_by_cls[cls] = typ - self._extension_by_cls[cls] = index._extension_by_type[typ] - - def add_subclasses(typ, asdftype): - for subclass in util.iter_subclasses(typ): - # Do not overwrite the tag type for an existing subclass if the - # new tag serializes a class that is higher in the type - # hierarchy than the existing subclass. - if subclass in self._class_by_subclass: - if issubclass(self._class_by_subclass[subclass], typ): - # Allow for cases where a subclass tag is being - # overridden by a tag from another extension. - if (self._extension_by_cls[subclass] == - index._extension_by_type[asdftype]): - continue - self._class_by_subclass[subclass] = typ - self._type_by_subclasses[subclass] = asdftype - self._extension_by_cls[subclass] = index._extension_by_type[asdftype] - - def add_all_types(asdftype): - add_type_to_index(asdftype, asdftype) - for typ in asdftype.types: - add_type_to_index(typ, asdftype) - add_subclasses(typ, asdftype) - - if asdftype.handle_dynamic_subclasses: - for typ in asdftype.types: - self._types_with_dynamic_subclasses[typ] = asdftype - - def add_by_tag(name, version): - tag = join_tag_version(name, version) - if tag in index._type_by_tag: - asdftype = index._type_by_tag[tag] - self._type_by_name[name] = asdftype - add_all_types(asdftype) - - # Process all types defined in the ASDF version map. It is important to - # make sure that tags that are associated with the core part of the - # standard are processed first in order to handle subclasses properly. - for name, _version in core_version_map.items(): - add_by_tag(name, AsdfVersion(_version)) - for name, _version in standard_version_map.items(): - add_by_tag(name, AsdfVersion(_version)) - - # Now add any extension types that aren't known to the ASDF standard. - # This expects that all types defined by ASDF will be encountered - # before any types that are defined by external packages. This - # allows external packages to override types that are also defined - # by ASDF. The ordering is guaranteed due to the use of OrderedDict - # for _versions_by_type_name, and due to the fact that the built-in - # extension will always be processed first. - for name, versions in index._versions_by_type_name.items(): - if name not in self._type_by_name: - add_by_tag(name, versions[-1]) - - for asdftype in index._unnamed_types: - add_all_types(asdftype) - - def _mark_used_extension(self, custom_type): - self._extensions_used.add(self._extension_by_cls[custom_type]) - - def _process_dynamic_subclass(self, custom_type): - for key, val in self._types_with_dynamic_subclasses.items(): - if issubclass(custom_type, key): - self._type_by_cls[custom_type] = val - self._mark_used_extension(key) - return val - - return None - - def from_custom_type(self, custom_type): - """ - Given a custom type, return the corresponding `ExtensionType` - definition. - """ - asdftype = None - - # Try to find an exact class match first... - try: - asdftype = self._type_by_cls[custom_type] - except KeyError: - # ...failing that, match any subclasses - try: - asdftype = self._type_by_subclasses[custom_type] - except KeyError: - # ...failing that, try any subclasses that we couldn't - # cache in _type_by_subclasses. This generally only - # includes classes that are created dynamically post - # Python-import, e.g. astropy.modeling._CompoundModel - # subclasses. - return self._process_dynamic_subclass(custom_type) - - if asdftype is not None: - extension = self._extension_by_cls.get(custom_type) - if extension is not None: - self._mark_used_extension(custom_type) - else: - # Handle the case where the dynamic subclass was identified as - # a proper subclass above, but it has not yet been registered - # as such. - self._process_dynamic_subclass(custom_type) - - return asdftype - - -class AsdfTypeIndex(object): - """ - An index of the known `ExtensionType` classes. - - In the future this class may be renamed to ExtensionTypeIndex, since it is - not specific to classes that inherit `AsdfType`. - """ - def __init__(self): - self._write_type_indices = {} - self._type_by_tag = {} - # Use OrderedDict here to preserve the order in which types are added - # to the type index. Since the ASDF built-in extension is always - # processed first, this ensures that types defined by external packages - # will always override corresponding types that are defined by ASDF - # itself. However, if two different external packages define tags for - # the same type, the result is currently undefined. - self._versions_by_type_name = OrderedDict() - self._best_matches = {} - self._real_tag = {} - self._unnamed_types = set() - self._hooks_by_type = {} - self._all_types = set() - self._has_warned = {} - self._extension_by_type = {} - - def add_type(self, asdftype, extension): - """ - Add a type to the index. - """ - self._all_types.add(asdftype) - self._extension_by_type[asdftype] = extension - - if asdftype.yaml_tag is None and asdftype.name is None: - return - - if isinstance(asdftype.name, list): - yaml_tags = [asdftype.make_yaml_tag(name) for name in asdftype.name] - elif isinstance(asdftype.name, str): - yaml_tags = [asdftype.yaml_tag] - elif asdftype.name is None: - yaml_tags = [] - else: - raise TypeError("name must be a string, list or None") - - for yaml_tag in yaml_tags: - self._type_by_tag[yaml_tag] = asdftype - name, version = split_tag_version(yaml_tag) - versions = self._versions_by_type_name.get(name) - if versions is None: - self._versions_by_type_name[name] = [version] - else: - idx = bisect.bisect_left(versions, version) - if idx == len(versions) or versions[idx] != version: - versions.insert(idx, version) - - if not len(yaml_tags): - self._unnamed_types.add(asdftype) - - def from_custom_type(self, custom_type, version=default_version): - """ - Given a custom type, return the corresponding `ExtensionType` - definition. - """ - # Basic Python types should not ever have an AsdfType associated with - # them. - if custom_type in _BASIC_PYTHON_TYPES: - return None - - write_type_index = self._write_type_indices.get(str(version)) - if write_type_index is None: - write_type_index = _AsdfWriteTypeIndex(version, self) - self._write_type_indices[version] = write_type_index - - return write_type_index.from_custom_type(custom_type) - - def _get_version_mismatch(self, name, version, latest_version): - warning_string = None - - if (latest_version.major, latest_version.minor) != \ - (version.major, version.minor): - warning_string = \ - "'{}' with version {} found in file{{}}, but latest " \ - "supported version is {}".format( - name, version, latest_version) - - return warning_string - - def _warn_version_mismatch(self, ctx, tag, warning_string, fname): - if warning_string is not None: - # Ensure that only a single warning occurs per tag per AsdfFile - # TODO: If it is useful to only have a single warning per file on - # disk, then use `fname` in the key instead of `ctx`. - if not (ctx, tag) in self._has_warned: - warnings.warn(warning_string.format(fname)) - self._has_warned[(ctx, tag)] = True - - def fix_yaml_tag(self, ctx, tag, ignore_version_mismatch=True): - """ - Given a YAML tag, adjust it to the best supported version. - - If there is no exact match, this finds the newest version - understood that is still less than the version in file. Or, - the earliest understood version if none are less than the - version in the file. - - If ``ignore_version_mismatch==False``, this function raises a warning - if it could not find a match where the major and minor numbers are the - same. - """ - warning_string = None - - name, version = split_tag_version(tag) - - fname = " '{}'".format(ctx._fname) if ctx._fname else '' - - if tag in self._type_by_tag: - asdftype = self._type_by_tag[tag] - # Issue warnings for the case where there exists a class for the - # given tag due to the 'supported_versions' attribute being - # defined, but this tag is not the latest version of the type. - # This prevents 'supported_versions' from affecting the behavior of - # warnings that are purely related to YAML validation. - if not ignore_version_mismatch and hasattr(asdftype, '_latest_version'): - warning_string = self._get_version_mismatch( - name, version, asdftype._latest_version) - self._warn_version_mismatch(ctx, tag, warning_string, fname) - return tag - - if tag in self._best_matches: - best_tag, warning_string = self._best_matches[tag] - - if not ignore_version_mismatch: - self._warn_version_mismatch(ctx, tag, warning_string, fname) - - return best_tag - - versions = self._versions_by_type_name.get(name) - if versions is None: - return tag - - # The versions list is kept sorted, so bisect can be used to - # quickly find the best option. - i = bisect.bisect_left(versions, version) - i = max(0, i - 1) - - if not ignore_version_mismatch: - warning_string = self._get_version_mismatch( - name, version, versions[-1]) - self._warn_version_mismatch(ctx, tag, warning_string, fname) - - best_version = versions[i] - best_tag = join_tag_version(name, best_version) - self._best_matches[tag] = best_tag, warning_string - if tag != best_tag: - self._real_tag[best_tag] = tag - return best_tag - - def get_real_tag(self, tag): - if tag in self._real_tag: - return self._real_tag[tag] - elif tag in self._type_by_tag: - return tag - return None - - def from_yaml_tag(self, ctx, tag): - """ - From a given YAML tag string, return the corresponding - AsdfType definition. - """ - tag = self.fix_yaml_tag(ctx, tag) - return self._type_by_tag.get(tag) - - @lru_cache(5) - def has_hook(self, hook_name): - """ - Returns `True` if the given hook name exists on any of the managed - types. - """ - for cls in self._all_types: - if hasattr(cls, hook_name): - return True - return False - - def get_hook_for_type(self, hookname, typ, version=default_version): - """ - Get the hook function for the given type, if it exists, - else return None. - """ - hooks = self._hooks_by_type.setdefault(hookname, {}) - hook = hooks.get(typ, None) - if hook is not None: - return hook - - tag = self.from_custom_type(typ, version) - if tag is not None: - hook = getattr(tag, hookname, None) - if hook is not None: - hooks[typ] = hook - return hook - - hooks[typ] = None - return None - - def get_extensions_used(self, version=default_version): - write_type_index = self._write_type_indices.get(str(version)) - if write_type_index is None: - return [] - - return list(write_type_index._extensions_used) - - -_all_asdftypes = set() - - -def _from_tree_tagged_missing_requirements(cls, tree, ctx): - # A special version of AsdfType.from_tree_tagged for when the - # required dependencies for an AsdfType are missing. - plural, verb = ('s', 'are') if len(cls.requires) else ('', 'is') - message = "{0} package{1} {2} required to instantiate '{3}'".format( - util.human_list(cls.requires), plural, verb, tree._tag) - # This error will be handled by yamlutil.tagged_tree_to_custom_tree, which - # will cause a warning to be issued indicating that the tree failed to be - # converted. - raise TypeError(message) - - -class ExtensionTypeMeta(type): - """ - Custom class constructor for tag types. - """ - _import_cache = {} - - @classmethod - def _has_required_modules(cls, requires): - for string in requires: - has_module = True - match = MODULE_RE.match(string) - modname, _, version = match.groups() - if modname in cls._import_cache: - if not cls._import_cache[modname]: - return False - try: - module = importlib.import_module(modname) - if version and hasattr(module, '__version__'): - if module.__version__ < version: - has_module = False - except ImportError: - has_module = False - finally: - cls._import_cache[modname] = has_module - if not has_module: - return False - return True - - @classmethod - def _find_in_bases(cls, attrs, bases, name, default=None): - if name in attrs: - return attrs[name] - for base in bases: - if hasattr(base, name): - return getattr(base, name) - return default - - @property - def versioned_siblings(mcls): - return getattr(mcls, '__versioned_siblings') or [] - - def __new__(mcls, name, bases, attrs): - requires = mcls._find_in_bases(attrs, bases, 'requires', []) - if not mcls._has_required_modules(requires): - attrs['from_tree_tagged'] = classmethod( - _from_tree_tagged_missing_requirements) - attrs['types'] = [] - attrs['has_required_modules'] = False - else: - attrs['has_required_modules'] = True - types = mcls._find_in_bases(attrs, bases, 'types', []) - new_types = [] - for typ in types: - if isinstance(typ, str): - typ = util.resolve_name(typ) - new_types.append(typ) - attrs['types'] = new_types - - cls = super(ExtensionTypeMeta, mcls).__new__(mcls, name, bases, attrs) - - if hasattr(cls, 'version'): - if not isinstance(cls.version, (AsdfVersion, AsdfSpec)): - cls.version = AsdfVersion(cls.version) - - if hasattr(cls, 'name'): - if isinstance(cls.name, str): - if 'yaml_tag' not in attrs: - cls.yaml_tag = cls.make_yaml_tag(cls.name) - elif isinstance(cls.name, list): - pass - elif cls.name is not None: - raise TypeError("name must be string or list") - - if hasattr(cls, 'supported_versions'): - if not isinstance(cls.supported_versions, (list, set)): - cls.supported_versions = [cls.supported_versions] - supported_versions = set() - for version in cls.supported_versions: - if not isinstance(version, (AsdfVersion, AsdfSpec)): - version = AsdfVersion(version) - # This should cause an exception for invalid input - supported_versions.add(version) - # We need to convert back to a list here so that the 'in' operator - # uses actual comparison instead of hash equality - cls.supported_versions = list(supported_versions) - siblings = list() - for version in cls.supported_versions: - if version != cls.version: - new_attrs = copy(attrs) - new_attrs['version'] = version - new_attrs['supported_versions'] = set() - new_attrs['_latest_version'] = cls.version - siblings.append( - ExtensionTypeMeta. __new__(mcls, name, bases, new_attrs)) - setattr(cls, '__versioned_siblings', siblings) - - return cls - - -class AsdfTypeMeta(ExtensionTypeMeta): - """ - Keeps track of `AsdfType` subclasses that are created, and stores them in - `AsdfTypeIndex`. - """ - def __new__(mcls, name, bases, attrs): - cls = super(AsdfTypeMeta, mcls).__new__(mcls, name, bases, attrs) - # Classes using this metaclass get added to the list of built-in - # extensions - _all_asdftypes.add(cls) - - return cls - - -class ExtensionType(object): - """ - The base class of all custom types in the tree. - - Besides the attributes defined below, most subclasses will also - override `to_tree` and `from_tree`. - """ - name = None - organization = 'stsci.edu' - standard = 'asdf' - version = (1, 0, 0) - supported_versions = set() - types = [] - handle_dynamic_subclasses = False - validators = {} - requires = [] - yaml_tag = None - - @classmethod - def names(cls): - """ - Returns the name(s) represented by this tag type as a list. - - While some tag types represent only a single custom type, others - represent multiple types. In the latter case, the `name` attribute of - the extension is actually a list, not simply a string. This method - normalizes the value of `name` by returning a list in all cases. - - Returns - ------- - `list` of names represented by this tag type - """ - if cls.name is None: - return None - - return cls.name if isinstance(cls.name, list) else [cls.name] - - @classmethod - def make_yaml_tag(cls, name, versioned=True): - """ - Given the name of a type, returns a string representing its YAML tag. - - Parameters - ---------- - name : str - The name of the type. In most cases this will correspond to the - `name` attribute of the tag type. However, it is passed as a - parameter since some tag types represent multiple custom - types. - - versioned : bool - If `True`, the tag will be versioned. Otherwise, a YAML tag without - a version will be returned. - - Returns - ------- - `str` representing the YAML tag - """ - return format_tag( - cls.organization, - cls.standard, - cls.version if versioned else None, - name) - - @classmethod - def tag_base(cls): - """ - Returns the base of the YAML tag for types represented by this class. - - This method returns the portion of the tag that represents the standard - and the organization of any type represented by this class. - - Returns - ------- - `str` representing the base of the YAML tag - """ - return cls.make_yaml_tag('', versioned=False) - - @classmethod - def to_tree(cls, node, ctx): - """ - Converts instances of custom types into YAML representations. - - This method should be overridden by custom extension classes in order - to define how custom types are serialized into YAML. The method must - return a single Python object corresponding to one of the basic YAML - types (dict, list, str, or number). However, the types can be nested - and combined in order to represent more complex custom types. - - This method is called as part of the process of writing an `AsdfFile` - object. Whenever a custom type (or a subclass of that type) that is - listed in the `types` attribute of this class is encountered, this - method will be used to serialize that type. - - The name `to_tree` refers to the act of converting a custom type into - part of a YAML object tree. - - Parameters - ---------- - node : `object` - Instance of a custom type to be serialized. Will be an instance (or - an instance of a subclass) of one of the types listed in the - `types` attribute of this class. - - ctx : `AsdfFile` - An instance of the `AsdfFile` object that is being written out. - - Returns - ------- - A basic YAML type (`dict`, `list`, `str`, `int`, `float`, or - `complex`) representing the properties of the custom type to be - serialized. These types can be nested in order to represent more - complex custom types. - """ - return node.__class__.__bases__[0](node) - - @classmethod - def to_tree_tagged(cls, node, ctx): - """ - Converts instances of custom types into tagged objects. - - It is more common for custom tag types to override `to_tree` instead of - this method. This method should only be overridden if it is necessary - to modify the YAML tag that will be used to tag this object. - - Parameters - ---------- - node : `object` - Instance of a custom type to be serialized. Will be an instance (or - an instance of a subclass) of one of the types listed in the - `types` attribute of this class. - - ctx : `AsdfFile` - An instance of the `AsdfFile` object that is being written out. - - Returns - ------- - An instance of `asdf.tagged.Tagged`. - """ - obj = cls.to_tree(node, ctx) - return tagged.tag_object(cls.yaml_tag, obj, ctx=ctx) - - @classmethod - def from_tree(cls, tree, ctx): - """ - Converts basic types representing YAML trees into custom types. - - This method should be overridden by custom extension classes in order - to define how custom types are deserialized from the YAML - representation back into their original types. The method will return - an instance of the original custom type. - - This method is called as part of the process of reading an ASDF file in - order to construct an `AsdfFile` object. Whenever a YAML subtree is - encountered that has a tag that corresponds to the `yaml_tag` property - of this class, this method will be used to deserialize that tree back - into an instance of the original custom type. - - Parameters - ---------- - tree : `object` representing YAML tree - An instance of a basic Python type (possibly nested) that - corresponds to a YAML subtree. - - ctx : `AsdfFile` - An instance of the `AsdfFile` object that is being constructed. - - Returns - ------- - An instance of the custom type represented by this extension class. - """ - return cls(tree) - - @classmethod - def from_tree_tagged(cls, tree, ctx): - """ - Converts from tagged tree into custom type. - - It is more common for extension classes to override `from_tree` instead - of this method. This method should only be overridden if it is - necessary to access the `_tag` property of the `Tagged` object - directly. - - Parameters - ---------- - tree : `asdf.tagged.Tagged` object representing YAML tree - - ctx : `AsdfFile` - An instance of the `AsdfFile` object that is being constructed. - - Returns - ------- - An instance of the custom type represented by this extension class. - """ - return cls.from_tree(tree.data, ctx) - - @classmethod - def incompatible_version(cls, version): - """ - Indicates if given version is known to be incompatible with this type. - - If this tag class explicitly identifies compatible versions then this - checks whether a given version is compatible or not (see - `supported_versions`). Otherwise, all versions are assumed to be - compatible. - - Child classes can override this method to affect how version - compatiblity for this type is determined. - - Parameters - ---------- - version : `str` or `~asdf.versioning.AsdfVersion` - The version to test for compatibility. - """ - if cls.supported_versions: - if version not in cls.supported_versions: - return True - return False - - -@six.add_metaclass(AsdfTypeMeta) -class AsdfType(ExtensionType): - """ - Base class for all built-in ASDF types. Types that inherit this class will - be automatically added to the list of built-ins. This should *not* be used - for user-defined extensions. - """ - -@six.add_metaclass(ExtensionTypeMeta) -class CustomType(ExtensionType): - """ - Base class for all user-defined types. - """ - - # These attributes are duplicated here with docstrings since a bug in - # sphinx prevents the docstrings of class attributes from being inherited - # properly (see https://github.com/sphinx-doc/sphinx/issues/741. The - # docstrings are not included anywhere else in the class hierarchy since - # this class is the only one exposed in the public API. - name = None - """ - `str` or `list`: The name of the type. - """ - - organization = 'stsci.edu' - """ - `str`: The organization responsible for the type. - """ - - standard = 'asdf' - """ - `str`: The standard the type is defined in. - """ - - version = (1, 0, 0) - """ - `str`, `tuple`, `AsdfVersion`, or `AsdfSpec`: The version of the type. - """ - - supported_versions = set() - """ - `set`: Versions that explicitly compatible with this extension class. - - If provided, indicates explicit compatibility with the given set - of versions. Other versions of the same schema that are not included in - this set will not be converted to custom types with this class. """ - - types = [] - """ - `list`: List of types that this extension class can convert to/from YAML. - - Custom Python types that, when found in the tree, will be converted into - basic types for YAML output. Can be either strings referring to the types - or the types themselves.""" - - handle_dynamic_subclasses = False - """ - `bool`: Indicates whether dynamically generated subclasses can be serialized - - Flag indicating whether this type is capable of serializing subclasses - of any of the types listed in ``types`` that are generated dynamically. - """ - - validators = {} - """ - `dict`: Mapping JSON Schema keywords to validation functions for jsonschema. - - Useful if the type defines extra types of validation that can be - performed. - """ - - requires = [] - """ - `list`: Python packages that are required to instantiate the object. - """ - - yaml_tag = None - """ - `str`: The YAML tag to use for the type. - - If not provided, it will be automatically generated from name, - organization, standard and version. - """ - has_required_modules = True - """ - `bool`: Indicates whether modules specified by `requires` are available. +from .exceptions import AsdfDeprecationWarning +# This is not exhaustive, but represents the public API +from .versioning import join_tag_version, split_tag_version +from .types import (AsdfType, CustomType, format_tag, ExtensionTypeMeta, + _all_asdftypes) - NOTE: This value is automatically generated. Do not set it in subclasses as - it will be overwritten. - """ +warnings.warn( + "The module asdf.asdftypes has been deprecated and will be removed in 3.0. " + "Use asdf.types instead.", AsdfDeprecationWarning) diff --git a/asdf/block.py b/asdf/block.py index 45a5ac797..d0cc37a5a 100644 --- a/asdf/block.py +++ b/asdf/block.py @@ -12,6 +12,7 @@ from urllib import parse as urlparse import numpy as np +from numpy.ma.core import masked_array import yaml @@ -25,7 +26,7 @@ from . import yamlutil -class BlockManager(object): +class BlockManager: """ Manages the `Block`s associated with a ASDF file. """ @@ -715,6 +716,20 @@ def get_source(self, block): raise ValueError("block not found.") + def _should_inline(self, array): + + if not np.issubdtype(array.dtype, np.number): + return False + + if isinstance(array, masked_array): + return False + + # Make sure none of the values are too large to store as literals + if (array[~np.isnan(array)] > 2**52).any(): + return False + + return array.size <= self._inline_threshold_size + def find_or_create_block_for_array(self, arr, ctx): """ For a given array, looks for an existing block containing its @@ -774,7 +789,7 @@ def close(self): block.close() -class Block(object): +class Block: """ Represents a single block in a ASDF file. This is an implementation detail and should not be instantiated directly. @@ -1189,7 +1204,7 @@ def close(self): self._data = None -class UnloadedBlock(object): +class UnloadedBlock: """ Represents an indexed, but not yet loaded, internal block. All that is known about it is its offset. It converts itself to a diff --git a/asdf/commands/diff.py b/asdf/commands/diff.py index 6828456a7..d2a546085 100644 --- a/asdf/commands/diff.py +++ b/asdf/commands/diff.py @@ -70,7 +70,7 @@ def setup_arguments(cls, subparsers): def run(cls, args): return diff(args.filenames, args.minimal) -class ArrayNode(object): +class ArrayNode: """This class is used to represent unique dummy nodes in the diff tree. In general these dummy nodes will be list elements that we want to keep track of but not necessarily display. This allows the diff output to be @@ -81,7 +81,7 @@ def __init__(self, name): def __hash__(self): return hash(self.name) -class PrintTree(object): +class PrintTree: """This class is used to remember the nodes in the tree that have already been displayed in the diff output. """ @@ -114,7 +114,7 @@ def __setitem__(self, node_list, visit): current['children'][node] = dict(visited=True, children=dict()) current = current['children'][node] -class DiffContext(object): +class DiffContext: """Class that contains context data of the diff to be computed""" def __init__(self, asdf0, asdf1, iostream, minimal=False): self.asdf0 = asdf0 diff --git a/asdf/commands/main.py b/asdf/commands/main.py index 2fc7b1df7..cf4ba715e 100644 --- a/asdf/commands/main.py +++ b/asdf/commands/main.py @@ -12,7 +12,7 @@ command_order = [ 'Explode', 'Implode' ] -class Command(object): +class Command: @classmethod def setup_arguments(cls, subparsers): raise NotImplementedError() diff --git a/asdf/commands/tests/test_exploded.py b/asdf/commands/tests/test_exploded.py index e3b2855a6..47629f939 100644 --- a/asdf/commands/tests/test_exploded.py +++ b/asdf/commands/tests/test_exploded.py @@ -23,7 +23,10 @@ def test_explode_then_implode(tmpdir): path = os.path.join(str(tmpdir), 'original.asdf') ff = AsdfFile(tree) - ff.write_to(path) + # Since we're testing with small arrays, force all arrays to be stored + # in internal blocks rather than letting some of them be automatically put + # inline. + ff.write_to(path, all_array_storage='internal') assert len(ff.blocks) == 2 result = main.main_from_args(['explode', path]) diff --git a/asdf/compression.py b/asdf/compression.py index c41d1604b..435d55e3e 100644 --- a/asdf/compression.py +++ b/asdf/compression.py @@ -40,7 +40,7 @@ def validate(compression): return compression -class Lz4Compressor(object): +class Lz4Compressor: def __init__(self, block_api): self._api = block_api @@ -50,7 +50,7 @@ def compress(self, data): return header + output -class Lz4Decompressor(object): +class Lz4Decompressor: def __init__(self, block_api): self._api = block_api self._size = 0 diff --git a/asdf/extension.py b/asdf/extension.py index c9a244868..4325fc538 100644 --- a/asdf/extension.py +++ b/asdf/extension.py @@ -9,10 +9,11 @@ import six import importlib -from . import asdftypes +from . import types from . import resolver -from .version import version as asdf_version from .util import get_class_name +from .type_index import AsdfTypeIndex +from .version import version as asdf_version from .exceptions import AsdfDeprecationWarning @@ -23,7 +24,7 @@ @six.add_metaclass(abc.ABCMeta) -class AsdfExtension(object): +class AsdfExtension: """ Abstract base class defining an extension to ASDF. """ @@ -112,7 +113,7 @@ def url_mapping(self): pass -class AsdfExtensionList(object): +class AsdfExtensionList: """ Manage a set of extensions that are in effect. """ @@ -120,11 +121,11 @@ def __init__(self, extensions): tag_mapping = [] url_mapping = [] validators = {} - self._type_index = asdftypes.AsdfTypeIndex() + self._type_index = AsdfTypeIndex() for extension in extensions: if not isinstance(extension, AsdfExtension): raise TypeError( - "Extension must implement asdftypes.AsdfExtension " + "Extension must implement asdf.types.AsdfExtension " "interface") tag_mapping.extend(extension.tag_mapping) url_mapping.extend(extension.url_mapping) @@ -164,7 +165,7 @@ def validators(self): return self._validators -class BuiltinExtension(object): +class BuiltinExtension: """ This is the "extension" to ASDF that includes all the built-in tags. Even though it's not really an extension and it's always @@ -172,7 +173,7 @@ class BuiltinExtension(object): """ @property def types(self): - return asdftypes._all_asdftypes + return types._all_asdftypes @property def tag_mapping(self): diff --git a/asdf/fits_embed.py b/asdf/fits_embed.py index 08a109f6c..d330b88e0 100644 --- a/asdf/fits_embed.py +++ b/asdf/fits_embed.py @@ -29,7 +29,7 @@ __all__ = ['AsdfInFits'] -class _FitsBlock(object): +class _FitsBlock: def __init__(self, hdu): self._hdu = hdu @@ -190,7 +190,7 @@ def open(cls, fd, uri=None, validate_checksums=False, extensions=None, extensions : list of AsdfExtension, optional A list of extensions to the ASDF to support when reading - and writing ASDF files. See `asdftypes.AsdfExtension` for + and writing ASDF files. See `asdf.types.AsdfExtension` for more information. ignore_version_mismatch : bool, optional diff --git a/asdf/generic_io.py b/asdf/generic_io.py index 6eab89553..7a4edfd45 100644 --- a/asdf/generic_io.py +++ b/asdf/generic_io.py @@ -10,12 +10,13 @@ """ import io -import math import os -import platform import re import sys +import math +import pathlib import tempfile +import platform from distutils.version import LooseVersion from os import SEEK_SET, SEEK_CUR, SEEK_END @@ -182,7 +183,7 @@ def relative_uri(source, target): return relative -class _TruncatedReader(object): +class _TruncatedReader: """ Reads until a given delimiter is found. Only works with RandomAccessFile and InputStream, though as this is a private @@ -257,7 +258,7 @@ def read(self, nbytes=None): @six.add_metaclass(util.InheritDocstrings) -class GenericFile(object): +class GenericFile: """ Base class for an abstraction layer around a number of different file-like types. Each of its subclasses handles a particular kind @@ -656,7 +657,7 @@ def read_into_array(self, size): return np.frombuffer(buff, np.uint8, size, 0) -class GenericWrapper(object): +class GenericWrapper: """ A wrapper around a `GenericFile` object so that closing only happens in the very outer layer. @@ -1171,8 +1172,8 @@ def get_file(init, mode='r', uri=None, close=False): init.mode, mode)) return GenericWrapper(init) - elif isinstance(init, str): - parsed = urlparse.urlparse(init) + elif isinstance(init, (str, pathlib.Path)): + parsed = urlparse.urlparse(str(init)) if parsed.scheme in ['http', 'https']: if 'w' in mode: raise ValueError( diff --git a/asdf/reference.py b/asdf/reference.py index 0e89c1e6f..e4ee759e0 100644 --- a/asdf/reference.py +++ b/asdf/reference.py @@ -15,7 +15,7 @@ from urllib import parse as urlparse -from .asdftypes import AsdfType +from .types import AsdfType from . import generic_io from . import treeutil from . import util diff --git a/asdf/resolver.py b/asdf/resolver.py index 98c6881f9..df71d4757 100644 --- a/asdf/resolver.py +++ b/asdf/resolver.py @@ -19,7 +19,7 @@ def find_schema_path(): return os.path.join(dirname, 'schemas') -class Resolver(object): +class Resolver: """ A class that can be used to map strings with a particular prefix to another. diff --git a/asdf/schema.py b/asdf/schema.py index 477b3a9b0..231cf43fd 100644 --- a/asdf/schema.py +++ b/asdf/schema.py @@ -458,21 +458,33 @@ def get_validator(schema={}, ctx=None, validators=None, url_mapping=None, return validator -def validate_large_literals(instance): +def validate_large_literals(instance, reading=False): """ Validate that the tree has no large numeric literals. """ # We can count on 52 bits of precision for instance in treeutil.iter_tree(instance): - if (isinstance(instance, (Integral)) and ( - instance > ((1 << 51) - 1) or - instance < -((1 << 51) - 2))): + + if not isinstance(instance, Integral): + continue + + if instance <= ((1 << 51) - 1) and instance >= -((1 << 51) - 2): + continue + + if not reading: raise ValidationError( "Integer value {0} is too large to safely represent as a " "literal in ASDF".format(instance)) + warnings.warn( + "Invalid integer literal value {0} detected while reading file. " + "The value has been read safely, but the file should be " + "fixed.".format(instance) + ) -def validate(instance, ctx=None, schema={}, validators=None, *args, **kwargs): + +def validate(instance, ctx=None, schema={}, validators=None, reading=False, + *args, **kwargs): """ Validate the given instance (which must be a tagged tree) against the appropriate schema. The schema itself is located using the @@ -495,6 +507,11 @@ def validate(instance, ctx=None, schema={}, validators=None, *args, **kwargs): validators : dict, optional A dictionary mapping properties to validators to use (instead of the built-in ones and ones provided by extension types). + + reading: bool, optional + Indicates whether validation is being performed when the file is being + read. This is useful to allow for different validation behavior when + reading vs writing files. """ if ctx is None: from .asdf import AsdfFile @@ -504,10 +521,10 @@ def validate(instance, ctx=None, schema={}, validators=None, *args, **kwargs): *args, **kwargs) validator.validate(instance, _schema=(schema or None)) - validate_large_literals(instance) + validate_large_literals(instance, reading=reading) -def fill_defaults(instance, ctx): +def fill_defaults(instance, ctx, reading=False): """ For any default values in the schema, add them to the tree if they don't exist. @@ -518,8 +535,12 @@ def fill_defaults(instance, ctx): ctx : AsdfFile context Used to resolve tags and urls + + reading: bool, optional + Indicates whether the ASDF file is being read (in contrast to being + written). """ - validate(instance, ctx, validators=FILL_DEFAULTS) + validate(instance, ctx, validators=FILL_DEFAULTS, reading=reading) def remove_defaults(instance, ctx): diff --git a/asdf/tagged.py b/asdf/tagged.py index 38e9594dc..ee6ea683d 100644 --- a/asdf/tagged.py +++ b/asdf/tagged.py @@ -37,7 +37,7 @@ __all__ = ['tag_object', 'get_tag'] -class Tagged(object): +class Tagged: """ Base class of classes that wrap a given object and store a tag with it. diff --git a/asdf/tags/__init__.py b/asdf/tags/__init__.py index deb4c841e..055bc2cf8 100644 --- a/asdf/tags/__init__.py +++ b/asdf/tags/__init__.py @@ -4,4 +4,3 @@ # TODO: Import entire tree automatically and make these work like "plugins"? from . import core -from . import wcs diff --git a/asdf/tags/core/__init__.py b/asdf/tags/core/__init__.py index 5233f834e..28c58a6db 100644 --- a/asdf/tags/core/__init__.py +++ b/asdf/tags/core/__init__.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- -from ...asdftypes import AsdfType +from ...types import AsdfType from ...yamlutil import custom_tree_to_tagged_tree @@ -44,4 +44,5 @@ def to_tree(cls, node, ctx): from .constant import ConstantType from .ndarray import NDArrayType from .complex import ComplexType +from .integer import IntegerType from .external_reference import ExternalArrayReference diff --git a/asdf/tags/core/complex.py b/asdf/tags/core/complex.py index 1e2dc6327..00f6ffb5f 100644 --- a/asdf/tags/core/complex.py +++ b/asdf/tags/core/complex.py @@ -3,7 +3,7 @@ import numpy as np -from ...asdftypes import AsdfType +from ...types import AsdfType from ... import util diff --git a/asdf/tags/core/constant.py b/asdf/tags/core/constant.py index 6a25e9cc5..f162b2aec 100644 --- a/asdf/tags/core/constant.py +++ b/asdf/tags/core/constant.py @@ -2,10 +2,10 @@ # -*- coding: utf-8 -*- -from ...asdftypes import AsdfType +from ...types import AsdfType -class Constant(object): +class Constant: def __init__(self, value): self._value = value diff --git a/asdf/tags/core/external_reference.py b/asdf/tags/core/external_reference.py index 991c3a601..e4ff9e263 100644 --- a/asdf/tags/core/external_reference.py +++ b/asdf/tags/core/external_reference.py @@ -1,4 +1,4 @@ -from ...asdftypes import AsdfType +from ...types import AsdfType class ExternalArrayReference(AsdfType): diff --git a/asdf/tags/core/integer.py b/asdf/tags/core/integer.py new file mode 100644 index 000000000..6a7a060d1 --- /dev/null +++ b/asdf/tags/core/integer.py @@ -0,0 +1,121 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +# -*- coding: utf-8 -*- + +from numbers import Integral + +import numpy as np + +from ...types import AsdfType +from ...yamlutil import custom_tree_to_tagged_tree + + +class IntegerType(AsdfType): + """ + Enables the storage of arbitrarily large integer values + + The ASDF Standard mandates that integer literals in the tree can be no + larger than 52 bits. Use of this class enables the storage of arbitrarily + large integer values. + + When reading files that contain arbitrarily large integers, the values that + are restored in the tree will be raw Python `int` instances. + + Parameters + ---------- + + value: `numbers.Integral` + A Python integral value (e.g. `int` or `numpy.integer`) + + storage_type: `str`, optional + Optionally overrides the storage type of the array used to represent + the integer value. Valid values are "internal" (the default) and + "inline" + + Examples + -------- + + >>> import asdf + >>> import random + >>> # Create a large integer value + >>> largeval = random.getrandbits(100) + >>> # Store the large integer value to the tree using asdf.IntegerType + >>> tree = dict(largeval=asdf.IntegerType(largeval)) + >>> with asdf.AsdfFile(tree) as af: + ... af.write_to('largeval.asdf') + >>> with asdf.open('largeval.asdf') as aa: + ... assert aa['largeval'] == largeval + """ + + name = 'core/integer' + version = '1.0.0' + + _value_cache = dict() + + def __init__(self, value, storage_type='internal'): + assert storage_type in ['internal', 'inline'], "Invalid storage type given" + self._value = value + self._sign = '-' if value < 0 else '+' + self._storage = storage_type + + @classmethod + def to_tree(cls, node, ctx): + + if ctx not in cls._value_cache: + cls._value_cache[ctx] = dict() + + abs_value = int(np.abs(node._value)) + + # If the same value has already been stored, reuse the array + if abs_value in cls._value_cache[ctx]: + array = cls._value_cache[ctx][abs_value] + else: + # pack integer value into 32-bit words + words = [] + value = abs_value + while value > 0: + words.append(value & 0xffffffff) + value >>= 32 + + array = np.array(words, dtype=np.uint32) + if node._storage == 'internal': + cls._value_cache[ctx][abs_value] = array + + tree = dict() + ctx.set_array_storage(array, node._storage) + tree['words'] = custom_tree_to_tagged_tree(array, ctx) + tree['sign'] = node._sign + tree['string'] = str(int(node._value)) + + return tree + + @classmethod + def from_tree(cls, tree, ctx): + + value = 0 + for x in tree['words'][::-1]: + value <<= 32 + value |= int(x) + + if tree['sign'] == '-': + value = -value + + return IntegerType(value) + + def __int__(self): + return int(self._value) + + def __float__(self): + return float(self._value) + + def __eq__(self, other): + if isinstance(other, Integral): + return self._value == other + elif isinstance(other, IntegerType): + return self._value == other._value + else: + raise ValueError( + "Can't compare IntegralType to unknown type: {}".format( + type(other))) + + def __repr__(self): + return "IntegerType({})".format(self._value) diff --git a/asdf/tags/core/ndarray.py b/asdf/tags/core/ndarray.py index 39e0b4423..b0ae4da5f 100644 --- a/asdf/tags/core/ndarray.py +++ b/asdf/tags/core/ndarray.py @@ -8,7 +8,7 @@ from jsonschema import ValidationError -from ...asdftypes import AsdfType +from ...types import AsdfType from ... import schema from ... import util from ... import yamlutil diff --git a/asdf/tags/core/tests/test_history.py b/asdf/tags/core/tests/test_history.py index 2ee8eedf1..2e7763907 100644 --- a/asdf/tags/core/tests/test_history.py +++ b/asdf/tags/core/tests/test_history.py @@ -11,7 +11,7 @@ import asdf from asdf import util -from asdf import asdftypes +from asdf import types from asdf.tests import helpers from asdf.tests.helpers import yaml_to_asdf, display_warnings from asdf.tags.core import HistoryEntry @@ -230,7 +230,7 @@ def test_strict_extension_check(): def test_metadata_with_custom_extension(tmpdir): - class FractionType(asdftypes.AsdfType): + class FractionType(types.AsdfType): name = 'fraction' organization = 'nowhere.org' version = (1, 0, 0) diff --git a/asdf/tags/core/tests/test_integer.py b/asdf/tags/core/tests/test_integer.py new file mode 100644 index 000000000..5b171f966 --- /dev/null +++ b/asdf/tags/core/tests/test_integer.py @@ -0,0 +1,93 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +# -*- coding: utf-8 -*- + +import random + +import pytest + +import asdf +from asdf import IntegerType +from asdf.tests import helpers + + +# Make sure tests are deterministic +random.seed(0) + + +@pytest.mark.parametrize('sign', ['+', '-']) +@pytest.mark.parametrize('value', [ + random.getrandbits(64), + random.getrandbits(65), + random.getrandbits(100), + random.getrandbits(128), + random.getrandbits(129), + random.getrandbits(200), +]) +def test_integer_value(tmpdir, value, sign): + + if sign == '-': + value = -value + + integer = IntegerType(value) + tree = dict(integer=integer) + helpers.assert_roundtrip_tree(tree, tmpdir) + + +@pytest.mark.parametrize('inline', [False, True]) +def test_integer_storage(tmpdir, inline): + + tmpfile = str(tmpdir.join('integer.asdf')) + + kwargs = dict() + if inline: + kwargs['storage_type'] = 'inline' + + random.seed(0) + value = random.getrandbits(1000) + tree = dict(integer=IntegerType(value, **kwargs)) + + with asdf.AsdfFile(tree) as af: + af.write_to(tmpfile) + + with asdf.open(tmpfile, _force_raw_types=True) as rf: + if inline: + assert 'source' not in rf.tree['integer']['words'] + assert 'data' in rf.tree['integer']['words'] + else: + assert 'source' in rf.tree['integer']['words'] + assert 'data' not in rf.tree['integer']['words'] + + assert 'string' in rf.tree['integer'] + assert rf.tree['integer']['string'] == str(value) + + +def test_integer_storage_duplication(tmpdir): + + tmpfile = str(tmpdir.join('integer.asdf')) + + random.seed(0) + value = random.getrandbits(1000) + tree = dict(integer1=IntegerType(value), integer2=IntegerType(value)) + + with asdf.AsdfFile(tree) as af: + af.write_to(tmpfile) + assert len(af.blocks) == 1 + + with asdf.open(tmpfile, _force_raw_types=True) as rf: + assert rf.tree['integer1']['words']['source'] == 0 + assert rf.tree['integer2']['words']['source'] == 0 + + with asdf.open(tmpfile) as aa: + assert aa.tree['integer1'] == value + assert aa.tree['integer2'] == value + + +def test_integer_conversion(): + + random.seed(0) + value = random.getrandbits(1000) + + integer = asdf.IntegerType(value) + assert integer == value + assert int(integer) == int(value) + assert float(integer) == float(value) diff --git a/asdf/tags/core/tests/test_ndarray.py b/asdf/tags/core/tests/test_ndarray.py index 3a2436602..cc85f5e10 100644 --- a/asdf/tags/core/tests/test_ndarray.py +++ b/asdf/tags/core/tests/test_ndarray.py @@ -42,7 +42,7 @@ class CustomDatatype(CustomTestType): version = '1.0.0' -class CustomExtension(object): +class CustomExtension: @property def types(self): return [CustomNdim, CustomDatatype] diff --git a/asdf/tags/wcs/__init__.py b/asdf/tags/wcs/__init__.py deleted file mode 100644 index 00bdb2ec1..000000000 --- a/asdf/tags/wcs/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -# -*- coding: utf-8 -*- - - -from .wcs import * diff --git a/asdf/tags/wcs/tests/__init__.py b/asdf/tags/wcs/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/asdf/tags/wcs/tests/data/__init__.py b/asdf/tags/wcs/tests/data/__init__.py deleted file mode 100644 index 9dce85d06..000000000 --- a/asdf/tags/wcs/tests/data/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst diff --git a/asdf/tags/wcs/tests/data/test_frames-1.1.0.asdf b/asdf/tags/wcs/tests/data/test_frames-1.1.0.asdf deleted file mode 100644 index fb5d066e7..000000000 --- a/asdf/tags/wcs/tests/data/test_frames-1.1.0.asdf +++ /dev/null @@ -1,97 +0,0 @@ -#ASDF 1.0.0 -#ASDF_STANDARD 1.1.0 -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- !core/asdf-1.0.0 -asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf', - name: asdf, version: 1.3.3} -frames: -- !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: {type: ICRS} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: {equinox: !time/time-1.1.0 '2018-01-01 00:00:00.000', type: FK5} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: {equinox: !time/time-1.1.0 '2018-01-01 00:00:00.000', obstime: !time/time-1.1.0 '2015-01-01 - 00:00:00.000', type: FK4} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: {equinox: !time/time-1.1.0 '2018-01-01 00:00:00.000', obstime: !time/time-1.1.0 '2015-01-01 - 00:00:00.000', type: FK4_noeterms} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: {type: galactic} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [x, y, z] - axes_order: [0, 1, 2] - name: CelestialFrame - reference_frame: - galcen_coord: !wcs/icrs_coord-1.1.0 - dec: {value: -28.936175} - ra: - value: 266.4051 - wrap_angle: !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 deg, value: 360.0} - galcen_distance: !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: 5.0} - galcen_v_sun: - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 km s-1, value: 11.1} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 km s-1, value: 232.24} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 km s-1, value: 7.25} - roll: !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 deg, value: 3.0} - type: galactocentric - z_sun: !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 pc, value: 3.0} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: - obsgeoloc: - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: 3.0856775814671916e+16} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: 9.257032744401574e+16} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: 6.1713551629343834e+19} - obsgeovel: - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: 2.0} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: 1.0} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: 8.0} - obstime: !time/time-1.1.0 2018-01-01 00:00:00.000 - type: GCRS - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: {obstime: !time/time-1.1.0 '2018-01-01 00:00:00.000', type: CIRS} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [x, y, z] - axes_order: [0, 1, 2] - name: CelestialFrame - reference_frame: {obstime: !time/time-1.1.0 '2018-01-03 00:00:00.000', type: ITRS} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -- !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: - equinox: !time/time-1.1.0 J2000.000 - obsgeoloc: - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: 3.0856775814671916e+16} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: 9.257032744401574e+16} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: 6.1713551629343834e+19} - obsgeovel: - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: 2.0} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: 1.0} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: 8.0} - obstime: !time/time-1.1.0 2018-01-01 00:00:00.000 - type: precessed_geocentric - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -... diff --git a/asdf/tags/wcs/tests/data/test_wcs-1.0.0.asdf b/asdf/tags/wcs/tests/data/test_wcs-1.0.0.asdf deleted file mode 100644 index 195251394..000000000 --- a/asdf/tags/wcs/tests/data/test_wcs-1.0.0.asdf +++ /dev/null @@ -1,46 +0,0 @@ -#ASDF 1.0.0 -#ASDF_STANDARD 1.0.0 -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- !core/asdf-1.0.0 -asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf', - name: asdf, version: 1.3.3} -gw1: !wcs/wcs-1.0.0 - name: '' - steps: - - !wcs/step-1.0.0 - frame: detector - transform: !transform/concatenate-1.1.0 - forward: - - !transform/shift-1.1.0 {offset: 12.4} - - !transform/shift-1.1.0 {offset: -2.0} - - !wcs/step-1.0.0 {frame: icrs} -gw2: !wcs/wcs-1.0.0 - name: '' - steps: - - !wcs/step-1.0.0 - frame: detector - transform: !transform/concatenate-1.1.0 - forward: - - !transform/shift-1.1.0 {offset: 12.4} - - !transform/shift-1.1.0 {offset: -2.0} - - !wcs/step-1.0.0 {frame: icrs} -gw3: !wcs/wcs-1.0.0 - name: '' - steps: - - !wcs/step-1.0.0 - frame: !wcs/frame-1.1.0 - axes_names: [x, y] - name: detector - unit: [!unit/unit-1.0.0 pixel, !unit/unit-1.0.0 pixel] - transform: !transform/concatenate-1.1.0 - forward: - - !transform/shift-1.1.0 {offset: 12.4} - - !transform/shift-1.1.0 {offset: -2.0} - - !wcs/step-1.0.0 - frame: !wcs/celestial_frame-1.0.0 - axes_names: [lon, lat] - name: icrs - reference_frame: {type: ICRS} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -... diff --git a/asdf/tags/wcs/tests/data/test_wcs-1.1.0.asdf b/asdf/tags/wcs/tests/data/test_wcs-1.1.0.asdf deleted file mode 100644 index 9e0281edd..000000000 --- a/asdf/tags/wcs/tests/data/test_wcs-1.1.0.asdf +++ /dev/null @@ -1,46 +0,0 @@ -#ASDF 1.0.0 -#ASDF_STANDARD 1.1.0 -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- !core/asdf-1.0.0 -asdf_library: !core/software-1.0.0 {author: Space Telescope Science Institute, homepage: 'http://github.com/spacetelescope/asdf', - name: asdf, version: 1.3.3} -gw1: !wcs/wcs-1.0.0 - name: '' - steps: - - !wcs/step-1.0.0 - frame: detector - transform: !transform/concatenate-1.1.0 - forward: - - !transform/shift-1.1.0 {offset: 12.4} - - !transform/shift-1.1.0 {offset: -2.0} - - !wcs/step-1.0.0 {frame: icrs} -gw2: !wcs/wcs-1.0.0 - name: '' - steps: - - !wcs/step-1.0.0 - frame: detector - transform: !transform/concatenate-1.1.0 - forward: - - !transform/shift-1.1.0 {offset: 12.4} - - !transform/shift-1.1.0 {offset: -2.0} - - !wcs/step-1.0.0 {frame: icrs} -gw3: !wcs/wcs-1.0.0 - name: '' - steps: - - !wcs/step-1.0.0 - frame: !wcs/frame-1.1.0 - axes_names: [x, y] - name: detector - unit: [!unit/unit-1.0.0 pixel, !unit/unit-1.0.0 pixel] - transform: !transform/concatenate-1.1.0 - forward: - - !transform/shift-1.1.0 {offset: 12.4} - - !transform/shift-1.1.0 {offset: -2.0} - - !wcs/step-1.0.0 - frame: !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: icrs - reference_frame: {type: ICRS} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -... diff --git a/asdf/tags/wcs/tests/setup_package.py b/asdf/tags/wcs/tests/setup_package.py deleted file mode 100644 index 99c855276..000000000 --- a/asdf/tags/wcs/tests/setup_package.py +++ /dev/null @@ -1,5 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -# -*- coding: utf-8 -*- - -def get_package_data(): # pragma: no cover - return { str(_PACKAGE_NAME_ + '.tags.wcs.tests'): ['data/*.asdf'] } diff --git a/asdf/tags/wcs/tests/test_wcs.py b/asdf/tags/wcs/tests/test_wcs.py deleted file mode 100644 index d273993b0..000000000 --- a/asdf/tags/wcs/tests/test_wcs.py +++ /dev/null @@ -1,223 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -# -*- coding: utf-8 -*- - -import os -import pytest -import warnings -from functools import partial - -gwcs = pytest.importorskip('gwcs') -astropy = pytest.importorskip('astropy', minversion='3.0.0') - -_gwcs_version = gwcs.version.version -_astropy_version = astropy.version.version - -INCOMPATIBLE_VERSIONS = _gwcs_version == '0.9.0' and _astropy_version < '3.1.dev0' - - -from astropy.modeling import models -from astropy import coordinates as coord -from astropy import units as u -from astropy import time - -from gwcs import coordinate_frames as cf -from gwcs import wcs - -import asdf -from asdf import AsdfFile -from asdf.tests import helpers - -from . import data as test_data -get_test_data_path = partial(helpers.get_test_data_path, module=test_data) - - -@pytest.mark.parametrize('version', ['1.0.0', '1.1.0']) -def test_read_wcs(version): - """Simple test to make sure that we can read older versions of files - containing WCS objects. We do not test against versions of the ASDF format - more recent than 1.1.0 since the schemas and tags have moved to Astropy and - GWCS.""" - - filename = get_test_data_path("test_wcs-{}.asdf".format(version)) - with asdf.open(filename) as tree: - assert isinstance(tree['gw1'], wcs.WCS) - assert isinstance(tree['gw2'], wcs.WCS) - assert isinstance(tree['gw3'], wcs.WCS) - - -@pytest.mark.skipif(INCOMPATIBLE_VERSIONS, reason="Incompatible versions for GWCS and Astropy") -@pytest.mark.parametrize('version', ['1.0.0', '1.1.0', '1.2.0']) -def test_composite_frame(tmpdir, version): - icrs = coord.ICRS() - fk5 = coord.FK5() - cel1 = cf.CelestialFrame(reference_frame=icrs) - cel2 = cf.CelestialFrame(reference_frame=fk5) - - spec1 = cf.SpectralFrame(name='freq', unit=[u.Hz,], axes_order=(2,)) - spec2 = cf.SpectralFrame(name='wave', unit=[u.m,], axes_order=(2,)) - - comp1 = cf.CompositeFrame([cel1, spec1]) - comp2 = cf.CompositeFrame([cel2, spec2]) - comp = cf.CompositeFrame([comp1, cf.SpectralFrame(axes_order=(3,), unit=(u.m,))]) - - tree = { - 'comp1': comp1, - 'comp2': comp2, - 'comp': comp - } - - write_options = dict(version=version) - helpers.assert_roundtrip_tree(tree, tmpdir, write_options=write_options) - - -def test_frames(tmpdir): - """Simple check to make sure we can still read older files with frames. - Serialization of these frames was only introduced in v1.1.0, and we do not - test any subsequent ASDF versions since the schemas and tags for those - frames have moved to Astropy and gwcs.""" - - filename = get_test_data_path("test_frames-1.1.0.asdf") - with asdf.open(filename) as tree: - for frame in tree['frames']: - assert isinstance(frame, cf.CoordinateFrame) - - -def test_backwards_compat_galcen(): - # Hold these fields constant so that we can compare them - declination = 1.0208 # in degrees - right_ascension = 45.729 # in degrees - galcen_distance = 3.14 - roll = 4.0 - z_sun = 0.2084 - old_frame_yaml = """ -frames: - - !wcs/celestial_frame-1.0.0 - axes_names: [x, y, z] - axes_order: [0, 1, 2] - name: CelestialFrame - reference_frame: - type: galactocentric - galcen_dec: - - %f - - deg - galcen_ra: - - %f - - deg - galcen_distance: - - %f - - m - roll: - - %f - - deg - z_sun: - - %f - - pc - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -""" % (declination, right_ascension, galcen_distance, roll, z_sun) - - new_frame_yaml = """ -frames: - - !wcs/celestial_frame-1.1.0 - axes_names: [x, y, z] - axes_order: [0, 1, 2] - name: CelestialFrame - reference_frame: - type: galactocentric - galcen_coord: !wcs/icrs_coord-1.1.0 - dec: {value: %f} - ra: - value: %f - wrap_angle: - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 deg, value: 360.0} - galcen_distance: - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} - galcen_v_sun: - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 km s-1, value: 11.1} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 km s-1, value: 232.24} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 km s-1, value: 7.25} - roll: !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 deg, value: %f} - z_sun: !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 pc, value: %f} - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -""" % (declination, right_ascension, galcen_distance, roll, z_sun) - - old_buff = helpers.yaml_to_asdf(old_frame_yaml) - old_asdf = asdf.open(old_buff) - old_frame = old_asdf.tree['frames'][0] - new_buff = helpers.yaml_to_asdf(new_frame_yaml) - new_asdf = asdf.open(new_buff) - new_frame = new_asdf.tree['frames'][0] - - # Poor man's frame comparison since it's not implemented by astropy - assert old_frame.axes_names == new_frame.axes_names - assert old_frame.axes_order == new_frame.axes_order - assert old_frame.unit == new_frame.unit - - old_refframe = old_frame.reference_frame - new_refframe = new_frame.reference_frame - - # v1.0.0 frames have no representation of galcen_v_center, so do not compare - assert old_refframe.galcen_distance == new_refframe.galcen_distance - assert old_refframe.galcen_coord.dec == new_refframe.galcen_coord.dec - assert old_refframe.galcen_coord.ra == new_refframe.galcen_coord.ra - - -def test_backwards_compat_gcrs(): - obsgeoloc = ( - 3.0856775814671916e+16, - 9.257032744401574e+16, - 6.1713551629343834e+19 - ) - obsgeovel = (2.0, 1.0, 8.0) - - old_frame_yaml = """ -frames: - - !wcs/celestial_frame-1.0.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: - type: GCRS - obsgeoloc: - - [%f, %f, %f] - - !unit/unit-1.0.0 m - obsgeovel: - - [%f, %f, %f] - - !unit/unit-1.0.0 m s-1 - obstime: !time/time-1.0.0 2010-01-01 00:00:00.000 - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -""" % (obsgeovel + obsgeoloc) - - new_frame_yaml = """ -frames: - - !wcs/celestial_frame-1.1.0 - axes_names: [lon, lat] - name: CelestialFrame - reference_frame: - type: GCRS - obsgeoloc: - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m, value: %f} - obsgeovel: - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f} - - !unit/quantity-1.1.0 {unit: !unit/unit-1.0.0 m s-1, value: %f} - obstime: !time/time-1.1.0 2010-01-01 00:00:00.000 - unit: [!unit/unit-1.0.0 deg, !unit/unit-1.0.0 deg] -""" % (obsgeovel + obsgeoloc) - - old_buff = helpers.yaml_to_asdf(old_frame_yaml) - old_asdf = asdf.open(old_buff) - old_frame = old_asdf.tree['frames'][0] - old_loc = old_frame.reference_frame.obsgeoloc - old_vel = old_frame.reference_frame.obsgeovel - - new_buff = helpers.yaml_to_asdf(new_frame_yaml) - new_asdf = asdf.open(new_buff) - new_frame = new_asdf.tree['frames'][0] - new_loc = new_frame.reference_frame.obsgeoloc - new_vel = new_frame.reference_frame.obsgeovel - - assert (old_loc.x == new_loc.x and old_loc.y == new_loc.y and - old_loc.z == new_loc.z) - assert (old_vel.x == new_vel.x and old_vel.y == new_vel.y and - old_vel.z == new_vel.z) diff --git a/asdf/tags/wcs/wcs.py b/asdf/tags/wcs/wcs.py deleted file mode 100644 index 1a8c7de9e..000000000 --- a/asdf/tags/wcs/wcs.py +++ /dev/null @@ -1,390 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -# -*- coding: utf-8 -*- - -from ...asdftypes import AsdfType -from ... import yamlutil - - -_REQUIRES = ['gwcs', 'astropy'] - - -class WCSType(AsdfType): - name = "wcs/wcs" - requires = _REQUIRES - types = ['gwcs.WCS'] - version = '1.1.0' - - @classmethod - def from_tree(cls, node, ctx): - import gwcs - - steps = [(x['frame'], x.get('transform')) for x in node['steps']] - name = node['name'] - - return gwcs.WCS(steps, name=name) - - @classmethod - def to_tree(cls, gwcs, ctx): - def get_frame(frame_name): - frame = getattr(gwcs, frame_name) - if frame is None: - return frame_name - return frame - - frames = gwcs.available_frames - steps = [] - for i in range(len(frames) - 1): - frame_name = frames[i] - frame = get_frame(frame_name) - transform = gwcs.get_transform(frames[i], frames[i + 1]) - steps.append(StepType({'frame': frame, 'transform': transform})) - frame_name = frames[-1] - frame = get_frame(frame_name) - steps.append(StepType({'frame': frame})) - - return {'name': gwcs.name, - 'steps': yamlutil.custom_tree_to_tagged_tree(steps, ctx)} - - @classmethod - def assert_equal(cls, old, new): - from ...tests import helpers - - assert old.name == new.name - assert len(old.available_frames) == len(new.available_frames) - for (old_frame, old_transform), (new_frame, new_transform) in zip( - old.pipeline, new.pipeline): - helpers.assert_tree_match(old_frame, new_frame) - helpers.assert_tree_match(old_transform, new_transform) - - -class StepType(dict, AsdfType): - name = "wcs/step" - requires = _REQUIRES - version = '1.1.0' - - -class FrameType(AsdfType): - name = "wcs/frame" - requires = ['gwcs'] - types = ['gwcs.Frame2D'] - version = '1.1.0' - - @classmethod - def _get_reference_frame_mapping(cls): - if hasattr(cls, '_reference_frame_mapping'): - return cls._reference_frame_mapping - - from astropy.coordinates import builtin_frames - - cls._reference_frame_mapping = { - 'ICRS': builtin_frames.ICRS, - 'FK5': builtin_frames.FK5, - 'FK4': builtin_frames.FK4, - 'FK4_noeterms': builtin_frames.FK4NoETerms, - 'galactic': builtin_frames.Galactic, - 'galactocentric': builtin_frames.Galactocentric, - 'GCRS': builtin_frames.GCRS, - 'CIRS': builtin_frames.CIRS, - 'ITRS': builtin_frames.ITRS, - 'precessed_geocentric': builtin_frames.PrecessedGeocentric - } - - return cls._reference_frame_mapping - - @classmethod - def _get_inverse_reference_frame_mapping(cls): - if hasattr(cls, '_inverse_reference_frame_mapping'): - return cls._inverse_reference_frame_mapping - - reference_frame_mapping = cls._get_reference_frame_mapping() - - cls._inverse_reference_frame_mapping = {} - for key, val in reference_frame_mapping.items(): - cls._inverse_reference_frame_mapping[val] = key - - return cls._inverse_reference_frame_mapping - - @classmethod - def _reference_frame_from_tree(cls, node, ctx): - from astropy.units import Quantity - from astropy.io.misc.asdf.tags.unit.quantity import QuantityType - from astropy.coordinates import ICRS, CartesianRepresentation - - version = cls.version - reference_frame = node['reference_frame'] - reference_frame_name = reference_frame['type'] - - frame_cls = cls._get_reference_frame_mapping()[reference_frame_name] - - frame_kwargs = {} - for name in frame_cls.get_frame_attr_names().keys(): - val = reference_frame.get(name) - if val is not None: - # These are deprecated fields that must be handled as a special - # case for older versions of the schema - if name in ['galcen_ra', 'galcen_dec']: - continue - # There was no schema for quantities in v1.0.0 - if name in ['galcen_distance', 'roll', 'z_sun'] and version == '1.0.0': - val = Quantity(val[0], unit=val[1]) - # These fields are known to be CartesianRepresentations - if name in ['obsgeoloc', 'obsgeovel']: - if version == '1.0.0': - unit = val[1] - x = Quantity(val[0][0], unit=unit) - y = Quantity(val[0][1], unit=unit) - z = Quantity(val[0][2], unit=unit) - else: - x = QuantityType.from_tree(val[0], ctx) - y = QuantityType.from_tree(val[1], ctx) - z = QuantityType.from_tree(val[2], ctx) - val = CartesianRepresentation(x, y, z) - elif name == 'galcen_v_sun': - from astropy.coordinates import CartesianDifferential - # This field only exists since v1.1.0, and it only uses - # CartesianDifferential after v1.3.3 - d_x = QuantityType.from_tree(val[0], ctx) - d_y = QuantityType.from_tree(val[1], ctx) - d_z = QuantityType.from_tree(val[2], ctx) - val = CartesianDifferential(d_x, d_y, d_z) - else: - val = yamlutil.tagged_tree_to_custom_tree(val, ctx) - frame_kwargs[name] = val - has_ra_and_dec = reference_frame.get('galcen_dec') and \ - reference_frame.get('galcen_ra') - if version == '1.0.0' and has_ra_and_dec: - # Convert deprecated ra and dec fields into galcen_coord - galcen_dec = reference_frame['galcen_dec'] - galcen_ra = reference_frame['galcen_ra'] - dec = Quantity(galcen_dec[0], unit=galcen_dec[1]) - ra = Quantity(galcen_ra[0], unit=galcen_ra[1]) - frame_kwargs['galcen_coord'] = ICRS(dec=dec, ra=ra) - return frame_cls(**frame_kwargs) - - @classmethod - def _from_tree(cls, node, ctx): - kwargs = {'name': node['name']} - - if 'axes_names' in node: - kwargs['axes_names'] = node['axes_names'] - - if 'reference_frame' in node: - kwargs['reference_frame'] = \ - cls._reference_frame_from_tree(node, ctx) - - if 'axes_order' in node: - kwargs['axes_order'] = tuple(node['axes_order']) - - if 'unit' in node: - kwargs['unit'] = tuple( - yamlutil.tagged_tree_to_custom_tree(node['unit'], ctx)) - - return kwargs - - @classmethod - def _to_tree(cls, frame, ctx): - import numpy as np - from astropy.coordinates import CartesianRepresentation - from astropy.io.misc.asdf.tags.unit.quantity import QuantityType - from astropy.coordinates import CartesianDifferential - - node = {} - - node['name'] = frame.name - - if frame.axes_order != (0, 1): - node['axes_order'] = list(frame.axes_order) - - if frame.axes_names is not None: - node['axes_names'] = list(frame.axes_names) - - if frame.reference_frame is not None: - reference_frame = {} - reference_frame['type'] = cls._get_inverse_reference_frame_mapping()[ - type(frame.reference_frame)] - - for name in frame.reference_frame.get_frame_attr_names().keys(): - frameval = getattr(frame.reference_frame, name) - # CartesianRepresentation becomes a flat list of x,y,z - # coordinates with associated units - if isinstance(frameval, CartesianRepresentation): - value = [frameval.x, frameval.y, frameval.z] - frameval = value - elif isinstance(frameval, CartesianDifferential): - value = [frameval.d_x, frameval.d_y, frameval.d_z] - frameval = value - yamlval = yamlutil.custom_tree_to_tagged_tree(frameval, ctx) - reference_frame[name] = yamlval - - node['reference_frame'] = reference_frame - - if frame.unit is not None: - node['unit'] = yamlutil.custom_tree_to_tagged_tree( - list(frame.unit), ctx) - - return node - - @classmethod - def _assert_equal(cls, old, new): - from ...tests import helpers - - assert old.name == new.name - assert old.axes_order == new.axes_order - assert old.axes_names == new.axes_names - assert type(old.reference_frame) == type(new.reference_frame) - assert old.unit == new.unit - - if old.reference_frame is not None: - for name in old.reference_frame.get_frame_attr_names().keys(): - helpers.assert_tree_match( - getattr(old.reference_frame, name), - getattr(new.reference_frame, name)) - - @classmethod - def assert_equal(cls, old, new): - cls._assert_equal(old, new) - - @classmethod - def from_tree(cls, node, ctx): - import gwcs - - node = cls._from_tree(node, ctx) - - return gwcs.Frame2D(**node) - - @classmethod - def to_tree(cls, frame, ctx): - return cls._to_tree(frame, ctx) - -class CelestialFrameType(FrameType): - name = "wcs/celestial_frame" - types = ['gwcs.CelestialFrame'] - supported_versions = [(1,0,0), (1,1,0)] - - @classmethod - def from_tree(cls, node, ctx): - import gwcs - - node = cls._from_tree(node, ctx) - - return gwcs.CelestialFrame(**node) - - @classmethod - def to_tree(cls, frame, ctx): - return cls._to_tree(frame, ctx) - - @classmethod - def assert_equal(cls, old, new): - cls._assert_equal(old, new) - - assert old.reference_position == new.reference_position - - -class SpectralFrame(FrameType): - name = "wcs/spectral_frame" - types = ['gwcs.SpectralFrame'] - - @classmethod - def from_tree(cls, node, ctx): - import gwcs - - node = cls._from_tree(node, ctx) - - if 'reference_position' in node: - node['reference_position'] = node['reference_position'].upper() - - return gwcs.SpectralFrame(**node) - - @classmethod - def to_tree(cls, frame, ctx): - node = cls._to_tree(frame, ctx) - - if frame.reference_position is not None: - node['reference_position'] = frame.reference_position.lower() - - return node - - -class CompositeFrame(FrameType): - name = "wcs/composite_frame" - types = ['gwcs.CompositeFrame'] - version = '1.1.0' - - @classmethod - def from_tree(cls, node, ctx): - import gwcs - - if len(node) != 2: - raise ValueError("CompositeFrame has extra properties") - - name = node['name'] - frames = node['frames'] - - return gwcs.CompositeFrame(frames, name) - - @classmethod - def to_tree(cls, frame, ctx): - return { - 'name': frame.name, - 'frames': yamlutil.custom_tree_to_tagged_tree(frame.frames, ctx) - } - - @classmethod - def assert_equal(cls, old, new): - from ...tests import helpers - - assert old.name == new.name - for old_frame, new_frame in zip(old.frames, new.frames): - helpers.assert_tree_match(old_frame, new_frame) - -class ICRSCoord(AsdfType): - """The newest version of this tag and the associated schema have moved to - Astropy. This implementation is retained here for the purposes of backwards - compatibility with older files. - """ - name = "wcs/icrs_coord" - types = ['astropy.coordinates.ICRS'] - requires = ['astropy'] - version = "1.1.0" - - @classmethod - def from_tree(cls, node, ctx): - from astropy.io.misc.asdf.tags.unit.quantity import QuantityType - from astropy.coordinates import ICRS, Longitude, Latitude, Angle - - angle = QuantityType.from_tree(node['ra']['wrap_angle'], ctx) - wrap_angle = Angle(angle.value, unit=angle.unit) - ra = Longitude( - node['ra']['value'], - unit=node['ra']['unit'], - wrap_angle=wrap_angle) - dec = Latitude(node['dec']['value'], unit=node['dec']['unit']) - - return ICRS(ra=ra, dec=dec) - - @classmethod - def to_tree(cls, frame, ctx): # pragma: no cover - # We do not run coverage analysis since new ICRS objects will be - # serialized by the tag implementation in Astropy. Eventually if we - # have a better way to write older versions of tags, we can re-add - # tests for this code. - from astropy.units import Quantity - from astropy.coordinates import ICRS - from astropy.io.misc.asdf.tags.unit.quantity import QuantityType - - node = {} - - wrap_angle = Quantity( - frame.ra.wrap_angle.value, - unit=frame.ra.wrap_angle.unit) - node['ra'] = { - 'value': frame.ra.value, - 'unit': frame.ra.unit.to_string(), - 'wrap_angle': yamlutil.custom_tree_to_tagged_tree(wrap_angle, ctx) - } - node['dec'] = { - 'value': frame.dec.value, - 'unit': frame.dec.unit.to_string() - } - - return node diff --git a/asdf/tests/httpserver.py b/asdf/tests/httpserver.py index 8697fa057..2eab4ecab 100644 --- a/asdf/tests/httpserver.py +++ b/asdf/tests/httpserver.py @@ -50,7 +50,7 @@ def translate_path(self, path): server.server_close() -class HTTPServer(object): +class HTTPServer: handler_class = http.server.SimpleHTTPRequestHandler def __init__(self): diff --git a/asdf/tests/schema_tester.py b/asdf/tests/schema_tester.py index 85efa2c28..a0c39f41c 100644 --- a/asdf/tests/schema_tester.py +++ b/asdf/tests/schema_tester.py @@ -12,7 +12,6 @@ import asdf from asdf import AsdfFile -from asdf import asdftypes from asdf import block from asdf import schema from asdf import extension @@ -120,7 +119,7 @@ def should_skip(name, version): def parse_schema_filename(filename): components = filename[filename.find('schemas') + 1:].split(os.path.sep) tag = 'tag:{}:{}'.format(components[1], '/'.join(components[2:])) - name, version = asdftypes.split_tag_version(tag.replace('.yaml', '')) + name, version = versioning.split_tag_version(tag.replace('.yaml', '')) return name, version diff --git a/asdf/tests/test_api.py b/asdf/tests/test_api.py new file mode 100644 index 000000000..02f6472bb --- /dev/null +++ b/asdf/tests/test_api.py @@ -0,0 +1,361 @@ +# -*- coding: utf-8 -*- + +import os +import io +import pathlib + +import numpy as np +from numpy.testing import assert_array_equal +from astropy.modeling import models + +import pytest + +import asdf +from asdf import treeutil +from asdf import extension +from asdf import versioning +from asdf.exceptions import AsdfDeprecationWarning +from .helpers import assert_tree_match, assert_roundtrip_tree, display_warnings + + +def test_get_data_from_closed_file(tmpdir): + tmpdir = str(tmpdir) + path = os.path.join(tmpdir, 'test.asdf') + + my_array = np.arange(0, 64).reshape((8, 8)) + + tree = {'my_array': my_array} + ff = asdf.AsdfFile(tree) + ff.write_to(path) + + with asdf.open(path) as ff: + pass + + with pytest.raises(IOError): + assert_array_equal(my_array, ff.tree['my_array']) + + +def test_no_warning_nan_array(tmpdir): + """ + Tests for a regression that was introduced by + https://github.com/spacetelescope/asdf/pull/557 + """ + + tree = dict(array=np.array([1, 2, np.nan])) + + with pytest.warns(None) as w: + assert_roundtrip_tree(tree, tmpdir) + assert len(w) == 0, display_warnings(w) + + +def test_warning_deprecated_open(tmpdir): + + tmpfile = str(tmpdir.join('foo.asdf')) + + tree = dict(foo=42, bar='hello') + with asdf.AsdfFile(tree) as af: + af.write_to(tmpfile) + + with pytest.warns(AsdfDeprecationWarning): + with asdf.AsdfFile.open(tmpfile) as af: + assert_tree_match(tree, af.tree) + + +def test_open_readonly(tmpdir): + + tmpfile = str(tmpdir.join('readonly.asdf')) + + tree = dict(foo=42, bar='hello', baz=np.arange(20)) + with asdf.AsdfFile(tree) as af: + af.write_to(tmpfile, all_array_storage='internal') + + os.chmod(tmpfile, 0o440) + assert os.access(tmpfile, os.W_OK) == False + + with asdf.open(tmpfile) as af: + assert af['baz'].flags.writeable == False + + with pytest.raises(PermissionError): + with asdf.open(tmpfile, mode='rw'): + pass + + +def test_atomic_write(tmpdir, small_tree): + tmpfile = os.path.join(str(tmpdir), 'test.asdf') + + ff = asdf.AsdfFile(small_tree) + ff.write_to(tmpfile) + + with asdf.open(tmpfile, mode='r') as ff: + ff.write_to(tmpfile) + + +def test_overwrite(tmpdir): + # This is intended to reproduce the following issue: + # https://github.com/spacetelescope/asdf/issues/100 + tmpfile = os.path.join(str(tmpdir), 'test.asdf') + aff = models.AffineTransformation2D(matrix=[[1, 2], [3, 4]]) + f = asdf.AsdfFile() + f.tree['model'] = aff + f.write_to(tmpfile) + model = f.tree['model'] + + ff = asdf.AsdfFile() + ff.tree['model'] = model + ff.write_to(tmpfile) + + +def test_default_version(): + # See https://github.com/spacetelescope/asdf/issues/364 + + version_map = versioning.get_version_map(versioning.default_version) + + ff = asdf.AsdfFile() + assert ff.file_format_version == version_map['FILE_FORMAT'] + + +def test_update_exceptions(tmpdir): + tmpdir = str(tmpdir) + path = os.path.join(tmpdir, 'test.asdf') + + my_array = np.random.rand(8, 8) + tree = {'my_array': my_array} + ff = asdf.AsdfFile(tree) + ff.write_to(path) + + with asdf.open(path, mode='r', copy_arrays=True) as ff: + with pytest.raises(IOError): + ff.update() + + ff = asdf.AsdfFile(tree) + buff = io.BytesIO() + ff.write_to(buff) + + buff.seek(0) + with asdf.open(buff, mode='rw') as ff: + ff.update() + + with pytest.raises(ValueError): + asdf.AsdfFile().update() + + +def test_top_level_tree(small_tree): + tree = {'tree': small_tree} + ff = asdf.AsdfFile(tree) + assert_tree_match(ff.tree['tree'], ff['tree']) + + ff2 = asdf.AsdfFile() + ff2['tree'] = small_tree + assert_tree_match(ff2.tree['tree'], ff2['tree']) + + +def test_top_level_keys(small_tree): + tree = {'tree': small_tree} + ff = asdf.AsdfFile(tree) + assert ff.tree.keys() == ff.keys() + + +def test_walk_and_modify_remove_keys(): + tree = { + 'foo': 42, + 'bar': 43 + } + + def func(x): + if x == 42: + return None + return x + + tree2 = treeutil.walk_and_modify(tree, func) + + assert 'foo' not in tree2 + assert 'bar' in tree2 + + +def test_copy(tmpdir): + tmpdir = str(tmpdir) + + my_array = np.random.rand(8, 8) + tree = {'my_array': my_array, 'foo': {'bar': 'baz'}} + ff = asdf.AsdfFile(tree) + ff.write_to(os.path.join(tmpdir, 'test.asdf')) + + with asdf.open(os.path.join(tmpdir, 'test.asdf')) as ff: + ff2 = ff.copy() + ff2.tree['my_array'] *= 2 + ff2.tree['foo']['bar'] = 'boo' + + assert np.all(ff2.tree['my_array'] == + ff.tree['my_array'] * 2) + assert ff.tree['foo']['bar'] == 'baz' + + assert_array_equal(ff2.tree['my_array'], ff2.tree['my_array']) + + +def test_tag_to_schema_resolver_deprecation(): + ff = asdf.AsdfFile() + with pytest.warns(AsdfDeprecationWarning): + ff.tag_to_schema_resolver('foo') + + with pytest.warns(AsdfDeprecationWarning): + extension_list = extension.default_extensions.extension_list + extension_list.tag_to_schema_resolver('foo') + + +def test_access_tree_outside_handler(tmpdir): + tempname = str(tmpdir.join('test.asdf')) + + tree = {'random': np.random.random(10)} + + ff = asdf.AsdfFile(tree) + ff.write_to(str(tempname)) + + with asdf.open(tempname) as newf: + pass + + # Accessing array data outside of handler should fail + with pytest.raises(OSError): + newf.tree['random'][0] + + +def test_context_handler_resolve_and_inline(tmpdir): + # This reproduces the issue reported in + # https://github.com/spacetelescope/asdf/issues/406 + tempname = str(tmpdir.join('test.asdf')) + + tree = {'random': np.random.random(10)} + + ff = asdf.AsdfFile(tree) + ff.write_to(str(tempname)) + + with asdf.open(tempname) as newf: + newf.resolve_and_inline() + + with pytest.raises(OSError): + newf.tree['random'][0] + + +def test_open_pathlib_path(tmpdir): + + filename = str(tmpdir.join('pathlib.asdf')) + path = pathlib.Path(filename) + + tree = {'data': np.ones(10)} + + with asdf.AsdfFile(tree) as af: + af.write_to(path) + + with asdf.open(path) as af: + assert (af['data'] == tree['data']).all() + + +@pytest.mark.xfail(reason='Setting auto_inline option modifies AsdfFile state') +def test_auto_inline(tmpdir): + + outfile = str(tmpdir.join('test.asdf')) + tree = dict(data=np.arange(6)) + + # Use the same object for each write in order to make sure that there + # aren't unanticipated side effects + with asdf.AsdfFile(tree) as af: + af.write_to(outfile) + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 1 + + af.write_to(outfile, auto_inline=10) + assert len(list(af.blocks.inline_blocks)) == 1 + assert len(list(af.blocks.internal_blocks)) == 0 + + af.write_to(outfile) + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 1 + + af.write_to(outfile, auto_inline=7) + assert len(list(af.blocks.inline_blocks)) == 1 + assert len(list(af.blocks.internal_blocks)) == 0 + + af.write_to(outfile, auto_inline=5) + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 1 + + +@pytest.mark.skip(reason='Until inline_threshold is added as a write option') +def test_inline_threshold(tmpdir): + + tree = { + 'small': np.ones(10), + 'large': np.ones(100) + } + + with asdf.AsdfFile(tree) as af: + assert len(list(af.blocks.inline_blocks)) == 1 + assert len(list(af.blocks.internal_blocks)) == 1 + + with asdf.AsdfFile(tree, inline_threshold=10) as af: + assert len(list(af.blocks.inline_blocks)) == 1 + assert len(list(af.blocks.internal_blocks)) == 1 + + with asdf.AsdfFile(tree, inline_threshold=5) as af: + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 2 + + with asdf.AsdfFile(tree, inline_threshold=100) as af: + assert len(list(af.blocks.inline_blocks)) == 2 + assert len(list(af.blocks.internal_blocks)) == 0 + + +@pytest.mark.skip(reason='Until inline_threshold is added as a write option') +def test_inline_threshold_masked(tmpdir): + + mask = np.random.randint(0, 1+1, 20) + masked_array = np.ma.masked_array(np.ones(20), mask=mask) + + tree = { + 'masked': masked_array + } + + # Make sure that masked arrays aren't automatically inlined, even if they + # are small enough + with asdf.AsdfFile(tree) as af: + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 2 + + tree = { + 'masked': masked_array, + 'normal': np.random.random(20) + } + + with asdf.AsdfFile(tree) as af: + assert len(list(af.blocks.inline_blocks)) == 1 + assert len(list(af.blocks.internal_blocks)) == 2 + + +@pytest.mark.skip(reason='Until inline_threshold is added as a write option') +def test_inline_threshold_override(tmpdir): + + tmpfile = str(tmpdir.join('inline.asdf')) + + tree = { + 'small': np.ones(10), + 'large': np.ones(100) + } + + with asdf.AsdfFile(tree) as af: + af.set_array_storage(tree['small'], 'internal') + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 2 + + with asdf.AsdfFile(tree) as af: + af.set_array_storage(tree['large'], 'inline') + assert len(list(af.blocks.inline_blocks)) == 2 + assert len(list(af.blocks.internal_blocks)) == 0 + + with asdf.AsdfFile(tree) as af: + af.write_to(tmpfile, all_array_storage='internal') + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 2 + + with asdf.AsdfFile(tree) as af: + af.write_to(tmpfile, all_array_storage='inline') + assert len(list(af.blocks.inline_blocks)) == 2 + assert len(list(af.blocks.internal_blocks)) == 0 diff --git a/asdf/tests/test_low_level.py b/asdf/tests/test_array_blocks.py similarity index 84% rename from asdf/tests/test_low_level.py rename to asdf/tests/test_array_blocks.py index 51e054185..0e65d1362 100644 --- a/asdf/tests/test_low_level.py +++ b/asdf/tests/test_array_blocks.py @@ -18,202 +18,8 @@ from asdf import versioning from asdf.exceptions import AsdfDeprecationWarning -from ..tests.helpers import assert_tree_match - - -def test_no_yaml_end_marker(tmpdir): - content = b"""#ASDF 1.0.0 -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- !core/asdf-1.0.0 -foo: bar...baz -baz: 42 - """ - path = os.path.join(str(tmpdir), 'test.asdf') - - buff = io.BytesIO(content) - with pytest.raises(ValueError): - with asdf.open(buff): - pass - - buff.seek(0) - fd = generic_io.InputStream(buff, 'r') - with pytest.raises(ValueError): - with asdf.open(fd): - pass - - with open(path, 'wb') as fd: - fd.write(content) - - with open(path, 'rb') as fd: - with pytest.raises(ValueError): - with asdf.open(fd): - pass - - -def test_no_final_newline(tmpdir): - content = b"""#ASDF 1.0.0 -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- !core/asdf-1.0.0 -foo: ...bar... -baz: 42 -...""" - path = os.path.join(str(tmpdir), 'test.asdf') - - buff = io.BytesIO(content) - with asdf.open(buff) as ff: - assert len(ff.tree) == 2 - - buff.seek(0) - fd = generic_io.InputStream(buff, 'r') - with asdf.open(fd) as ff: - assert len(ff.tree) == 2 - - with open(path, 'wb') as fd: - fd.write(content) - - with open(path, 'rb') as fd: - with asdf.open(fd) as ff: - assert len(ff.tree) == 2 - - -def test_no_asdf_header(tmpdir): - content = b"What? This ain't no ASDF file" - - path = os.path.join(str(tmpdir), 'test.asdf') - - buff = io.BytesIO(content) - with pytest.raises(ValueError): - asdf.open(buff) - - with open(path, 'wb') as fd: - fd.write(content) - - with open(path, 'rb') as fd: - with pytest.raises(ValueError): - asdf.open(fd) - - -def test_no_asdf_blocks(tmpdir): - content = b"""#ASDF 1.0.0 -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- !core/asdf-1.0.0 -foo: bar -... -XXXXXXXX - """ - - path = os.path.join(str(tmpdir), 'test.asdf') - - buff = io.BytesIO(content) - with asdf.open(buff) as ff: - assert len(ff.blocks) == 0 - - buff.seek(0) - fd = generic_io.InputStream(buff, 'r') - with asdf.open(fd) as ff: - assert len(ff.blocks) == 0 - - with open(path, 'wb') as fd: - fd.write(content) - - with open(path, 'rb') as fd: - with asdf.open(fd) as ff: - assert len(ff.blocks) == 0 - - -def test_invalid_source(small_tree): - buff = io.BytesIO() - - ff = asdf.AsdfFile(small_tree) - ff.write_to(buff) - - buff.seek(0) - with asdf.open(buff) as ff2: - ff2.blocks.get_block(0) - - with pytest.raises(ValueError): - ff2.blocks.get_block(2) - - with pytest.raises(IOError): - ff2.blocks.get_block("http://127.0.0.1/") - - with pytest.raises(TypeError): - ff2.blocks.get_block(42.0) - - with pytest.raises(ValueError): - ff2.blocks.get_source(42.0) - - block = ff2.blocks.get_block(0) - assert ff2.blocks.get_source(block) == 0 - - -def test_empty_file(): - buff = io.BytesIO(b"#ASDF 1.0.0\n") - buff.seek(0) - - with asdf.open(buff) as ff: - assert ff.tree == {} - assert len(ff.blocks) == 0 - - buff = io.BytesIO(b"#ASDF 1.0.0\n#ASDF_STANDARD 1.0.0") - buff.seek(0) - - with asdf.open(buff) as ff: - assert ff.tree == {} - assert len(ff.blocks) == 0 - - -def test_not_asdf_file(): - buff = io.BytesIO(b"SIMPLE") - buff.seek(0) - - with pytest.raises(ValueError): - with asdf.open(buff): - pass - - buff = io.BytesIO(b"SIMPLE\n") - buff.seek(0) - - with pytest.raises(ValueError): - with asdf.open(buff): - pass - - -def test_junk_file(): - buff = io.BytesIO(b"#ASDF 1.0.0\nFOO") - buff.seek(0) - - with pytest.raises(ValueError): - with asdf.open(buff): - pass - - -def test_block_mismatch(): - # This is a file with a single small block, followed by something - # that has an invalid block magic number. - - buff = io.BytesIO( - b'#ASDF 1.0.0\n\xd3BLK\x00\x28\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0FOOBAR') - - buff.seek(0) - with pytest.raises(ValueError): - with asdf.open(buff): - pass - - -def test_block_header_too_small(): - # The block header size must be at least 40 - - buff = io.BytesIO( - b'#ASDF 1.0.0\n\xd3BLK\0\0') - - buff.seek(0) - with pytest.raises(ValueError): - with asdf.open(buff): - pass +from ..tests.helpers import (assert_tree_match, assert_roundtrip_tree, + display_warnings) def test_external_block(tmpdir): @@ -802,7 +608,10 @@ def test_deferred_block_loading(small_tree): buff = io.BytesIO() ff = asdf.AsdfFile(small_tree) - ff.write_to(buff, include_block_index=False) + # Since we're testing with small arrays, force all arrays to be stored + # in internal blocks rather than letting some of them be automatically put + # inline. + ff.write_to(buff, include_block_index=False, all_array_storage='internal') buff.seek(0) with asdf.open(buff) as ff2: @@ -869,7 +678,10 @@ def test_large_block_index(): } ff = asdf.AsdfFile(tree) - ff.write_to(buff) + # Since we're testing with small arrays, force all arrays to be stored + # in internal blocks rather than letting some of them be automatically put + # inline. + ff.write_to(buff, all_array_storage='internal') buff.seek(0) with asdf.open(buff) as ff2: @@ -927,7 +739,10 @@ def test_short_file_find_block_index(): buff = io.BytesIO() ff = asdf.AsdfFile({'arr': np.ndarray([1]), 'arr2': np.ndarray([2])}) - ff.write_to(buff, include_block_index=False) + # Since we're testing with small arrays, force all arrays to be stored + # in internal blocks rather than letting some of them be automatically put + # inline. + ff.write_to(buff, include_block_index=False, all_array_storage='internal') buff.write(b'#ASDF BLOCK INDEX\n') buff.write(b'0' * (io.DEFAULT_BUFFER_SIZE * 4)) @@ -1088,44 +903,6 @@ def test_open_no_memmap(tmpdir): assert not isinstance(array.block._data, np.memmap) -def test_invalid_version(tmpdir): - content = b"""#ASDF 0.1.0 -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- !core/asdf-0.1.0 -foo : bar -...""" - buff = io.BytesIO(content) - with pytest.raises(ValueError): - with asdf.open(buff) as ff: - pass - - -def test_valid_version(tmpdir): - content = b"""#ASDF 1.0.0 -%YAML 1.1 -%TAG ! tag:stsci.edu:asdf/ ---- !core/asdf-1.0.0 -foo : bar -...""" - buff = io.BytesIO(content) - with asdf.open(buff) as ff: - version = ff.file_format_version - - assert version.major == 1 - assert version.minor == 0 - assert version.patch == 0 - - -def test_default_version(): - # See https://github.com/spacetelescope/asdf/issues/364 - - version_map = versioning.get_version_map(versioning.default_version) - - ff = asdf.AsdfFile() - assert ff.file_format_version == version_map['FILE_FORMAT'] - - def test_fd_not_seekable(): data = np.ones(1024) b = block.Block(data=data) @@ -1233,3 +1010,97 @@ def test_open_readonly(tmpdir): with pytest.raises(PermissionError): with asdf.open(tmpfile, mode='rw'): pass + +@pytest.mark.skip(reason='Until inline_threshold is added as a write option') +def test_inline_threshold(tmpdir): + + tree = { + 'small': np.ones(10), + 'large': np.ones(100) + } + + with asdf.AsdfFile(tree) as af: + assert len(list(af.blocks.inline_blocks)) == 1 + assert len(list(af.blocks.internal_blocks)) == 1 + + with asdf.AsdfFile(tree, inline_threshold=10) as af: + assert len(list(af.blocks.inline_blocks)) == 1 + assert len(list(af.blocks.internal_blocks)) == 1 + + with asdf.AsdfFile(tree, inline_threshold=5) as af: + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 2 + + with asdf.AsdfFile(tree, inline_threshold=100) as af: + assert len(list(af.blocks.inline_blocks)) == 2 + assert len(list(af.blocks.internal_blocks)) == 0 + + +@pytest.mark.skip(reason='Until inline_threshold is added as a write option') +def test_inline_threshold_masked(tmpdir): + + mask = np.random.randint(0, 1+1, 20) + masked_array = np.ma.masked_array(np.ones(20), mask=mask) + + tree = { + 'masked': masked_array + } + + # Make sure that masked arrays aren't automatically inlined, even if they + # are small enough + with asdf.AsdfFile(tree) as af: + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 2 + + tree = { + 'masked': masked_array, + 'normal': np.random.random(20) + } + + with asdf.AsdfFile(tree) as af: + assert len(list(af.blocks.inline_blocks)) == 1 + assert len(list(af.blocks.internal_blocks)) == 2 + + +@pytest.mark.skip(reason='Until inline_threshold is added as a write option') +def test_inline_threshold_override(tmpdir): + + tmpfile = str(tmpdir.join('inline.asdf')) + + tree = { + 'small': np.ones(10), + 'large': np.ones(100) + } + + with asdf.AsdfFile(tree) as af: + af.set_array_storage(tree['small'], 'internal') + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 2 + + with asdf.AsdfFile(tree) as af: + af.set_array_storage(tree['large'], 'inline') + assert len(list(af.blocks.inline_blocks)) == 2 + assert len(list(af.blocks.internal_blocks)) == 0 + + with asdf.AsdfFile(tree) as af: + af.write_to(tmpfile, all_array_storage='internal') + assert len(list(af.blocks.inline_blocks)) == 0 + assert len(list(af.blocks.internal_blocks)) == 2 + + with asdf.AsdfFile(tree) as af: + af.write_to(tmpfile, all_array_storage='inline') + assert len(list(af.blocks.inline_blocks)) == 2 + assert len(list(af.blocks.internal_blocks)) == 0 + + +def test_no_warning_nan_array(tmpdir): + """ + Tests for a regression that was introduced by + https://github.com/spacetelescope/asdf/pull/557 + """ + + tree = dict(array=np.array([1, 2, np.nan])) + + with pytest.warns(None) as w: + assert_roundtrip_tree(tree, tmpdir) + assert len(w) == 0, display_warnings(w) diff --git a/asdf/tests/test_asdftypes.py b/asdf/tests/test_asdftypes.py index d880426b2..8220a135a 100644 --- a/asdf/tests/test_asdftypes.py +++ b/asdf/tests/test_asdftypes.py @@ -9,7 +9,7 @@ import pytest import asdf -from asdf import asdftypes +from asdf import types from asdf import extension from asdf import util from asdf import versioning @@ -23,7 +23,7 @@ def test_custom_tag(): import fractions - class FractionType(asdftypes.CustomType): + class FractionType(types.CustomType): name = 'fraction' organization = 'nowhere.org' version = (1, 0, 0) @@ -38,7 +38,7 @@ def to_tree(cls, node, ctx): def from_tree(cls, tree, ctx): return fractions.Fraction(tree[0], tree[1]) - class FractionExtension(object): + class FractionExtension: @property def types(self): return [FractionType] @@ -164,7 +164,7 @@ def test_version_mismatch_with_supported_versions(): """Make sure that defining the supported_versions field does not affect whether or not schema mismatch warnings are triggered.""" - class CustomFlow(object): + class CustomFlow: pass class CustomFlowType(CustomTestType): @@ -175,7 +175,7 @@ class CustomFlowType(CustomTestType): standard = 'custom' types = [CustomFlow] - class CustomFlowExtension(object): + class CustomFlowExtension: @property def types(self): return [CustomFlowType] @@ -232,7 +232,7 @@ def test_versioned_writing(monkeypatch): versioning.supported_versions + [versioning.AsdfVersion('42.0.0')] ) - class FancyComplexType(asdftypes.CustomType): + class FancyComplexType(types.CustomType): name = 'core/complex' organization = 'stsci.edu' standard = 'asdf' @@ -247,7 +247,7 @@ def to_tree(cls, node, ctx): def from_tree(cls, tree, ctx): return ComplexType.from_tree(tree, ctx) - class FancyComplexExtension(object): + class FancyComplexExtension: @property def types(self): return [FancyComplexType] @@ -273,7 +273,7 @@ def url_mapping(self): def test_longest_match(): - class FancyComplexExtension(object): + class FancyComplexExtension: @property def types(self): return [] @@ -297,15 +297,15 @@ def url_mapping(self): def test_module_versioning(): - class NoModuleType(asdftypes.CustomType): + class NoModuleType(types.CustomType): # It seems highly unlikely that this would be a real module requires = ['qkjvqdja'] - class HasCorrectPytest(asdftypes.CustomType): + class HasCorrectPytest(types.CustomType): # This means it requires 1.0.0 or greater, so it should succeed requires = ['pytest-1.0.0'] - class DoesntHaveCorrectPytest(asdftypes.CustomType): + class DoesntHaveCorrectPytest(types.CustomType): requires = ['pytest-91984.1.7'] nmt = NoModuleType() @@ -368,12 +368,12 @@ def test_newer_tag(): # fairly contrived but we want to test whether ASDF can handle backwards # compatibility even when an explicit tag class for different versions of a # schema is not available. - class CustomFlow(object): + class CustomFlow: def __init__(self, c=None, d=None): self.c = c self.d = d - class CustomFlowType(asdftypes.CustomType): + class CustomFlowType(types.CustomType): version = '1.1.0' name = 'custom_flow' organization = 'nowhere.org' @@ -391,7 +391,7 @@ def from_tree(cls, tree, ctx): def to_tree(cls, data, ctx): tree = dict(c=data.c, d=data.d) - class CustomFlowExtension(object): + class CustomFlowExtension: @property def types(self): return [CustomFlowType] @@ -435,26 +435,26 @@ def url_mapping(self): "tag:nowhere.org:custom/custom_flow-1.0.0 to custom type") def test_incompatible_version_check(): - class TestType0(asdftypes.CustomType): + class TestType0(types.CustomType): supported_versions = versioning.AsdfSpec('>=1.2.0') assert TestType0.incompatible_version('1.1.0') == True assert TestType0.incompatible_version('1.2.0') == False assert TestType0.incompatible_version('2.0.1') == False - class TestType1(asdftypes.CustomType): + class TestType1(types.CustomType): supported_versions = versioning.AsdfVersion('1.0.0') assert TestType1.incompatible_version('1.0.0') == False assert TestType1.incompatible_version('1.1.0') == True - class TestType2(asdftypes.CustomType): + class TestType2(types.CustomType): supported_versions = '1.0.0' assert TestType2.incompatible_version('1.0.0') == False assert TestType2.incompatible_version('1.1.0') == True - class TestType3(asdftypes.CustomType): + class TestType3(types.CustomType): # This doesn't make much sense, but it's just for the sake of example supported_versions = ['1.0.0', versioning.AsdfSpec('>=2.0.0')] @@ -463,7 +463,7 @@ class TestType3(asdftypes.CustomType): assert TestType3.incompatible_version('2.0.0') == False assert TestType3.incompatible_version('2.0.1') == False - class TestType4(asdftypes.CustomType): + class TestType4(types.CustomType): supported_versions = ['1.0.0', versioning.AsdfVersion('1.1.0')] assert TestType4.incompatible_version('1.0.0') == False @@ -471,7 +471,7 @@ class TestType4(asdftypes.CustomType): assert TestType4.incompatible_version('1.1.0') == False assert TestType4.incompatible_version('1.1.1') == True - class TestType5(asdftypes.CustomType): + class TestType5(types.CustomType): supported_versions = \ [versioning.AsdfSpec('<1.0.0'), versioning.AsdfSpec('>=2.0.0')] @@ -482,19 +482,19 @@ class TestType5(asdftypes.CustomType): assert TestType5.incompatible_version('1.1.0') == True with pytest.raises(ValueError): - class TestType6(asdftypes.CustomType): + class TestType6(types.CustomType): supported_versions = 'blue' with pytest.raises(ValueError): - class TestType6(asdftypes.CustomType): + class TestType6(types.CustomType): supported_versions = ['1.1.0', '2.2.0', 'blue'] def test_supported_versions(): - class CustomFlow(object): + class CustomFlow: def __init__(self, c=None, d=None): self.c = c self.d = d - class CustomFlowType(asdftypes.CustomType): + class CustomFlowType(types.CustomType): version = '1.1.0' supported_versions = [(1,0,0), versioning.AsdfSpec('>=1.1.0')] name = 'custom_flow' @@ -518,7 +518,7 @@ def to_tree(cls, data, ctx): else: tree = dict(c=data.c, d=data.d) - class CustomFlowExtension(object): + class CustomFlowExtension: @property def types(self): return [CustomFlowType] @@ -555,10 +555,10 @@ def url_mapping(self): assert type(old_data.tree['flow_thing']) == CustomFlow def test_unsupported_version_warning(): - class CustomFlow(object): + class CustomFlow: pass - class CustomFlowType(asdftypes.CustomType): + class CustomFlowType(types.CustomType): version = '1.0.0' supported_versions = [(1,0,0)] name = 'custom_flow' @@ -566,7 +566,7 @@ class CustomFlowType(asdftypes.CustomType): standard = 'custom' types = [CustomFlow] - class CustomFlowExtension(object): + class CustomFlowExtension: @property def types(self): return [CustomFlowType] @@ -617,7 +617,6 @@ def test_extension_override(tmpdir): with open(tmpfile, 'rb') as ff: contents = str(ff.read()) assert gwcs.tags.WCSType.yaml_tag in contents - assert asdf.tags.wcs.WCSType.yaml_tag not in contents def test_extension_override_subclass(tmpdir): @@ -647,14 +646,13 @@ class SubclassWCS(gwcs.WCS): with open(tmpfile, 'rb') as ff: contents = str(ff.read()) assert gwcs.tags.WCSType.yaml_tag in contents - assert asdf.tags.wcs.WCSType.yaml_tag not in contents def test_tag_without_schema(tmpdir): tmpfile = str(tmpdir.join('foo.asdf')) - class FooType(asdftypes.CustomType): + class FooType(types.CustomType): name = 'foo' def __init__(self, a, b): diff --git a/asdf/tests/test_file_format.py b/asdf/tests/test_file_format.py new file mode 100644 index 000000000..1fc0fd579 --- /dev/null +++ b/asdf/tests/test_file_format.py @@ -0,0 +1,236 @@ +# -*- coding: utf-8 -*- + +import os +import io + +import pytest + +import asdf +from asdf import generic_io + + +def test_no_yaml_end_marker(tmpdir): + content = b"""#ASDF 1.0.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.0.0 +foo: bar...baz +baz: 42 + """ + path = os.path.join(str(tmpdir), 'test.asdf') + + buff = io.BytesIO(content) + with pytest.raises(ValueError): + with asdf.open(buff): + pass + + buff.seek(0) + fd = generic_io.InputStream(buff, 'r') + with pytest.raises(ValueError): + with asdf.open(fd): + pass + + with open(path, 'wb') as fd: + fd.write(content) + + with open(path, 'rb') as fd: + with pytest.raises(ValueError): + with asdf.open(fd): + pass + + +def test_no_final_newline(tmpdir): + content = b"""#ASDF 1.0.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.0.0 +foo: ...bar... +baz: 42 +...""" + path = os.path.join(str(tmpdir), 'test.asdf') + + buff = io.BytesIO(content) + with asdf.open(buff) as ff: + assert len(ff.tree) == 2 + + buff.seek(0) + fd = generic_io.InputStream(buff, 'r') + with asdf.open(fd) as ff: + assert len(ff.tree) == 2 + + with open(path, 'wb') as fd: + fd.write(content) + + with open(path, 'rb') as fd: + with asdf.open(fd) as ff: + assert len(ff.tree) == 2 + + +def test_no_asdf_header(tmpdir): + content = b"What? This ain't no ASDF file" + + path = os.path.join(str(tmpdir), 'test.asdf') + + buff = io.BytesIO(content) + with pytest.raises(ValueError): + asdf.open(buff) + + with open(path, 'wb') as fd: + fd.write(content) + + with open(path, 'rb') as fd: + with pytest.raises(ValueError): + asdf.open(fd) + + +def test_no_asdf_blocks(tmpdir): + content = b"""#ASDF 1.0.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.0.0 +foo: bar +... +XXXXXXXX + """ + + path = os.path.join(str(tmpdir), 'test.asdf') + + buff = io.BytesIO(content) + with asdf.open(buff) as ff: + assert len(ff.blocks) == 0 + + buff.seek(0) + fd = generic_io.InputStream(buff, 'r') + with asdf.open(fd) as ff: + assert len(ff.blocks) == 0 + + with open(path, 'wb') as fd: + fd.write(content) + + with open(path, 'rb') as fd: + with asdf.open(fd) as ff: + assert len(ff.blocks) == 0 + + +def test_invalid_source(small_tree): + buff = io.BytesIO() + + ff = asdf.AsdfFile(small_tree) + # Since we're testing with small arrays, force all arrays to be stored + # in internal blocks rather than letting some of them be automatically put + # inline. + ff.write_to(buff, all_array_storage='internal') + + buff.seek(0) + with asdf.open(buff) as ff2: + ff2.blocks.get_block(0) + + with pytest.raises(ValueError): + ff2.blocks.get_block(2) + + with pytest.raises(IOError): + ff2.blocks.get_block("http://127.0.0.1/") + + with pytest.raises(TypeError): + ff2.blocks.get_block(42.0) + + with pytest.raises(ValueError): + ff2.blocks.get_source(42.0) + + block = ff2.blocks.get_block(0) + assert ff2.blocks.get_source(block) == 0 + + +def test_empty_file(): + buff = io.BytesIO(b"#ASDF 1.0.0\n") + buff.seek(0) + + with asdf.open(buff) as ff: + assert ff.tree == {} + assert len(ff.blocks) == 0 + + buff = io.BytesIO(b"#ASDF 1.0.0\n#ASDF_STANDARD 1.0.0") + buff.seek(0) + + with asdf.open(buff) as ff: + assert ff.tree == {} + assert len(ff.blocks) == 0 + + +def test_not_asdf_file(): + buff = io.BytesIO(b"SIMPLE") + buff.seek(0) + + with pytest.raises(ValueError): + with asdf.open(buff): + pass + + buff = io.BytesIO(b"SIMPLE\n") + buff.seek(0) + + with pytest.raises(ValueError): + with asdf.open(buff): + pass + + +def test_junk_file(): + buff = io.BytesIO(b"#ASDF 1.0.0\nFOO") + buff.seek(0) + + with pytest.raises(ValueError): + with asdf.open(buff): + pass + + +def test_block_mismatch(): + # This is a file with a single small block, followed by something + # that has an invalid block magic number. + + buff = io.BytesIO( + b'#ASDF 1.0.0\n\xd3BLK\x00\x28\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0FOOBAR') + + buff.seek(0) + with pytest.raises(ValueError): + with asdf.open(buff): + pass + + +def test_block_header_too_small(): + # The block header size must be at least 40 + + buff = io.BytesIO( + b'#ASDF 1.0.0\n\xd3BLK\0\0') + + buff.seek(0) + with pytest.raises(ValueError): + with asdf.open(buff): + pass + + +def test_invalid_version(tmpdir): + content = b"""#ASDF 0.1.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-0.1.0 +foo : bar +...""" + buff = io.BytesIO(content) + with pytest.raises(ValueError): + with asdf.open(buff) as ff: + pass + + +def test_valid_version(tmpdir): + content = b"""#ASDF 1.0.0 +%YAML 1.1 +%TAG ! tag:stsci.edu:asdf/ +--- !core/asdf-1.0.0 +foo : bar +...""" + buff = io.BytesIO(content) + with asdf.open(buff) as ff: + version = ff.file_format_version + + assert version.major == 1 + assert version.minor == 0 + assert version.patch == 0 diff --git a/asdf/tests/test_generic_io.py b/asdf/tests/test_generic_io.py index 8237cc3a8..5bc2cc96c 100644 --- a/asdf/tests/test_generic_io.py +++ b/asdf/tests/test_generic_io.py @@ -26,6 +26,12 @@ def tree(request): def _roundtrip(tree, get_write_fd, get_read_fd, write_options={}, read_options={}): + + # Since we're testing with small arrays, force all arrays to be stored + # in internal blocks rather than letting some of them be automatically put + # inline. + write_options.setdefault('all_array_storage', 'internal') + with get_write_fd() as fd: asdf.AsdfFile(tree).write_to(fd, **write_options) # Work around the fact that generic_io's get_file doesn't have a way of @@ -459,11 +465,11 @@ def test_relative_uri(): def test_arbitrary_file_object(): - class Wrapper(object): + class Wrapper: def __init__(self, init): self._fd = init - class Random(object): + class Random: def seek(self, *args): return self._fd.seek(*args) diff --git a/asdf/tests/test_helpers.py b/asdf/tests/test_helpers.py index 289a99cca..4977dd5bf 100644 --- a/asdf/tests/test_helpers.py +++ b/asdf/tests/test_helpers.py @@ -3,14 +3,14 @@ import pytest -from asdf import asdftypes +from asdf import types from asdf.exceptions import AsdfConversionWarning from asdf.tests.helpers import assert_roundtrip_tree def test_conversion_error(tmpdir): - class FooType(asdftypes.CustomType): + class FooType(types.CustomType): name = 'foo' def __init__(self, a, b): diff --git a/asdf/tests/test_reference.py b/asdf/tests/test_reference.py index 993c22204..43b7d0f39 100644 --- a/asdf/tests/test_reference.py +++ b/asdf/tests/test_reference.py @@ -32,11 +32,14 @@ def test_external_reference(tmpdir): } external_path = os.path.join(str(tmpdir), 'external.asdf') ext = asdf.AsdfFile(exttree) - ext.write_to(external_path) + # Since we're testing with small arrays, force all arrays to be stored + # in internal blocks rather than letting some of them be automatically put + # inline. + ext.write_to(external_path, all_array_storage='internal') external_path = os.path.join(str(tmpdir), 'external2.asdf') ff = asdf.AsdfFile(exttree) - ff.write_to(external_path) + ff.write_to(external_path, all_array_storage='internal') tree = { # The special name "data" here must be an array. This is diff --git a/asdf/tests/test_suite.py b/asdf/tests/test_reference_files.py similarity index 90% rename from asdf/tests/test_suite.py rename to asdf/tests/test_reference_files.py index cfb73a385..8bf4add67 100644 --- a/asdf/tests/test_suite.py +++ b/asdf/tests/test_reference_files.py @@ -12,6 +12,10 @@ from .helpers import assert_tree_match, display_warnings +_REFFILE_PATH = os.path.join(os.path.dirname(__file__), '..', '..', + 'asdf-standard', 'reference_files') + + def get_test_id(reference_file_path): """Helper function to return the informative part of a schema path""" path = os.path.normpath(str(reference_file_path)) @@ -19,9 +23,8 @@ def get_test_id(reference_file_path): def collect_reference_files(): """Function used by pytest to collect ASDF reference files for testing.""" - root = os.path.join(os.path.dirname(__file__), '..', "reference_files") for version in versioning.supported_versions: - version_dir = os.path.join(root, str(version)) + version_dir = os.path.join(_REFFILE_PATH, str(version)) if os.path.exists(version_dir): for filename in os.listdir(version_dir): if filename.endswith(".asdf"): @@ -59,7 +62,7 @@ def test_reference_file(reference_file): name_without_ext, _ = os.path.splitext(reference_file) known_fail = False - expect_warnings = False + expect_warnings = 'complex' in reference_file if sys.maxunicode <= 65535: known_fail = known_fail or (basename in ('unicode_spp.asdf')) diff --git a/asdf/tests/test_schema.py b/asdf/tests/test_schema.py index 243370d7b..9fc37554b 100644 --- a/asdf/tests/test_schema.py +++ b/asdf/tests/test_schema.py @@ -15,7 +15,7 @@ from numpy.testing import assert_array_equal import asdf -from asdf import asdftypes +from asdf import types from asdf import extension from asdf import resolver from asdf import schema @@ -46,7 +46,7 @@ def url_mapping(self): '/{url_suffix}.yaml')] -class TagReferenceType(asdftypes.CustomType): +class TagReferenceType(types.CustomType): """ This class is used by several tests below for validating foreign type references in schemas and ASDF files. @@ -202,7 +202,7 @@ def test_schema_caching(): def test_flow_style(): - class CustomFlowStyleType(dict, asdftypes.CustomType): + class CustomFlowStyleType(dict, types.CustomType): name = 'custom_flow' organization = 'nowhere.org' version = (1, 0, 0) @@ -225,7 +225,7 @@ def types(self): def test_style(): - class CustomStyleType(str, asdftypes.CustomType): + class CustomStyleType(str, types.CustomType): name = 'custom_style' organization = 'nowhere.org' version = (1, 0, 0) @@ -267,7 +267,7 @@ def test_property_order(): def test_invalid_nested(): - class CustomType(str, asdftypes.CustomType): + class CustomType(str, types.CustomType): name = 'custom' organization = 'nowhere.org' version = (1, 0, 0) @@ -361,7 +361,7 @@ def test_default_check_in_schema(): def test_fill_and_remove_defaults(): - class DefaultType(dict, asdftypes.CustomType): + class DefaultType(dict, types.CustomType): name = 'default' organization = 'nowhere.org' version = (1, 0, 0) @@ -419,7 +419,7 @@ def types(self): def test_foreign_tag_reference_validation(): - class ForeignTagReferenceType(asdftypes.CustomType): + class ForeignTagReferenceType(types.CustomType): name = 'foreign_tag_reference' organization = 'nowhere.org' version = (1, 0, 0) @@ -500,6 +500,24 @@ def test_large_literals(use_numpy): print(buff.getvalue()) +def test_read_large_literal(): + + value = 1 << 64 + yaml = """integer: {}""".format(value) + + buff = helpers.yaml_to_asdf(yaml) + + with pytest.warns(UserWarning) as w: + with asdf.open(buff) as af: + assert af['integer'] == value + + # We get two warnings: one for validation time, and one when defaults + # are filled. It seems like we could improve this architecture, though... + assert len(w) == 2 + assert str(w[0].message).startswith('Invalid integer literal value') + assert str(w[1].message).startswith('Invalid integer literal value') + + def test_nested_array(): s = { 'type': 'object', @@ -579,7 +597,7 @@ def test_nested_array_yaml(tmpdir): @pytest.mark.importorskip('astropy') def test_type_missing_dependencies(): - class MissingType(asdftypes.CustomType): + class MissingType(types.CustomType): name = 'missing' organization = 'nowhere.org' version = (1, 1, 0) @@ -607,7 +625,7 @@ def types(self): def test_assert_roundtrip_with_extension(tmpdir): called_custom_assert_equal = [False] - class CustomType(dict, asdftypes.CustomType): + class CustomType(dict, types.CustomType): name = 'custom_flow' organization = 'nowhere.org' version = (1, 0, 0) diff --git a/asdf/tests/test_stream.py b/asdf/tests/test_stream.py index 2c12135fe..2c0eabecb 100644 --- a/asdf/tests/test_stream.py +++ b/asdf/tests/test_stream.py @@ -87,6 +87,9 @@ def test_stream_with_nonstream(): } ff = asdf.AsdfFile(tree) + # Since we're testing with small arrays, force this array to be stored in + # an internal block rather than letting it be automatically put inline. + ff.set_array_storage(ff['nonstream'], 'internal') ff.write_to(buff) for i in range(100): buff.write(np.array([i] * 12, np.float64).tostring()) @@ -112,6 +115,10 @@ def test_stream_real_file(tmpdir): with open(path, 'wb') as fd: ff = asdf.AsdfFile(tree) + # Since we're testing with small arrays, force this array to be stored + # in an internal block rather than letting it be automatically put + # inline. + ff.set_array_storage(ff['nonstream'], 'internal') ff.write_to(fd) for i in range(100): fd.write(np.array([i] * 12, np.float64).tostring()) diff --git a/asdf/tests/test_yaml.py b/asdf/tests/test_yaml.py index a960a39f4..b3a55b7fd 100644 --- a/asdf/tests/test_yaml.py +++ b/asdf/tests/test_yaml.py @@ -78,7 +78,7 @@ def test_arbitrary_python_object(): # Putting "just any old" Python object in the tree should raise an # exception. - class Foo(object): + class Foo: pass tree = {'object': Foo()} @@ -256,7 +256,7 @@ def test_yaml_nan_inf(): def test_tag_object(): - class SomeObject(object): + class SomeObject: pass tag = 'tag:nowhere.org:none/some/thing' diff --git a/asdf/type_index.py b/asdf/type_index.py new file mode 100644 index 000000000..fdc7ea847 --- /dev/null +++ b/asdf/type_index.py @@ -0,0 +1,393 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +# -*- coding: utf-8 -*- + +import bisect +import warnings +from functools import lru_cache +from collections import OrderedDict + +from . import util +from .versioning import (AsdfVersion, get_version_map, default_version, + split_tag_version, join_tag_version) + + +__all__ = ['AsdfTypeIndex'] + + +_BASIC_PYTHON_TYPES = [str, int, float, list, dict, tuple] + + +class _AsdfWriteTypeIndex: + """ + The _AsdfWriteTypeIndex is a helper class for AsdfTypeIndex that + manages an index of types for writing out ASDF files, i.e. from + converting from custom types to tagged_types. It is not always + the inverse of the mapping from tags to custom types, since there + are likely multiple versions present for a given tag. + + This uses the `version_map.yaml` file that ships with the ASDF + standard to figure out which schemas correspond to a particular + version of the ASDF standard. + + An AsdfTypeIndex manages multiple _AsdfWriteTypeIndex instances + for each version the user may want to write out, and they are + instantiated on-demand. + + If version is ``'latest'``, it will just use the highest-numbered + versions of each of the schemas. This is currently only used to + aid in testing. + + In the future, this may be renamed to _ExtensionWriteTypeIndex since it is + not specific to classes that inherit `AsdfType`. + """ + _version_map = None + + def __init__(self, version, index): + self._version = version + + self._type_by_cls = {} + self._type_by_name = {} + self._type_by_subclasses = {} + self._class_by_subclass = {} + self._types_with_dynamic_subclasses = {} + self._extension_by_cls = {} + self._extensions_used = set() + + try: + version_map = get_version_map(self._version) + core_version_map = version_map['core'] + standard_version_map = version_map['standard'] + except ValueError: + raise ValueError( + "Don't know how to write out ASDF version {0}".format( + self._version)) + + # Process all types defined in the ASDF version map. It is important to + # make sure that tags that are associated with the core part of the + # standard are processed first in order to handle subclasses properly. + for name, _version in core_version_map.items(): + self._add_by_tag(index, name, AsdfVersion(_version)) + for name, _version in standard_version_map.items(): + self._add_by_tag(index, name, AsdfVersion(_version)) + + # Now add any extension types that aren't known to the ASDF standard. + # This expects that all types defined by ASDF will be encountered + # before any types that are defined by external packages. This + # allows external packages to override types that are also defined + # by ASDF. The ordering is guaranteed due to the use of OrderedDict + # for _versions_by_type_name, and due to the fact that the built-in + # extension will always be processed first. + for name, versions in index._versions_by_type_name.items(): + if name not in self._type_by_name: + self._add_by_tag(index, name, versions[-1]) + + for asdftype in index._unnamed_types: + self._add_all_types(index, asdftype) + + def _should_overwrite(self, cls, new_type): + existing_type = self._type_by_cls[cls] + + # Types that are provided by extensions from other packages should + # only override the type index corresponding to the latest version + # of ASDF. + if existing_type.tag_base() != new_type.tag_base(): + return self._version == default_version + + return True + + def _add_type_to_index(self, index, cls, typ): + if cls in self._type_by_cls and not self._should_overwrite(cls, typ): + return + + self._type_by_cls[cls] = typ + self._extension_by_cls[cls] = index._extension_by_type[typ] + + def _add_subclasses(self, index, typ, asdftype): + for subclass in util.iter_subclasses(typ): + # Do not overwrite the tag type for an existing subclass if the + # new tag serializes a class that is higher in the type + # hierarchy than the existing subclass. + if subclass in self._class_by_subclass: + if issubclass(self._class_by_subclass[subclass], typ): + # Allow for cases where a subclass tag is being + # overridden by a tag from another extension. + if (self._extension_by_cls[subclass] == + index._extension_by_type[asdftype]): + continue + self._class_by_subclass[subclass] = typ + self._type_by_subclasses[subclass] = asdftype + self._extension_by_cls[subclass] = index._extension_by_type[asdftype] + + def _add_all_types(self, index, asdftype): + self._add_type_to_index(index, asdftype, asdftype) + for typ in asdftype.types: + self._add_type_to_index(index, typ, asdftype) + self._add_subclasses(index, typ, asdftype) + + if asdftype.handle_dynamic_subclasses: + for typ in asdftype.types: + self._types_with_dynamic_subclasses[typ] = asdftype + + def _add_by_tag(self, index, name, version): + tag = join_tag_version(name, version) + if tag in index._type_by_tag: + asdftype = index._type_by_tag[tag] + self._type_by_name[name] = asdftype + self._add_all_types(index, asdftype) + + def _mark_used_extension(self, custom_type): + self._extensions_used.add(self._extension_by_cls[custom_type]) + + def _process_dynamic_subclass(self, custom_type): + for key, val in self._types_with_dynamic_subclasses.items(): + if issubclass(custom_type, key): + self._type_by_cls[custom_type] = val + self._mark_used_extension(key) + return val + + return None + + def from_custom_type(self, custom_type): + """ + Given a custom type, return the corresponding `ExtensionType` + definition. + """ + asdftype = None + + # Try to find an exact class match first... + try: + asdftype = self._type_by_cls[custom_type] + except KeyError: + # ...failing that, match any subclasses + try: + asdftype = self._type_by_subclasses[custom_type] + except KeyError: + # ...failing that, try any subclasses that we couldn't + # cache in _type_by_subclasses. This generally only + # includes classes that are created dynamically post + # Python-import, e.g. astropy.modeling._CompoundModel + # subclasses. + return self._process_dynamic_subclass(custom_type) + + if asdftype is not None: + extension = self._extension_by_cls.get(custom_type) + if extension is not None: + self._mark_used_extension(custom_type) + else: + # Handle the case where the dynamic subclass was identified as + # a proper subclass above, but it has not yet been registered + # as such. + self._process_dynamic_subclass(custom_type) + + return asdftype + + +class AsdfTypeIndex: + """ + An index of the known `ExtensionType` classes. + + In the future this class may be renamed to ExtensionTypeIndex, since it is + not specific to classes that inherit `AsdfType`. + """ + def __init__(self): + self._write_type_indices = {} + self._type_by_tag = {} + # Use OrderedDict here to preserve the order in which types are added + # to the type index. Since the ASDF built-in extension is always + # processed first, this ensures that types defined by external packages + # will always override corresponding types that are defined by ASDF + # itself. However, if two different external packages define tags for + # the same type, the result is currently undefined. + self._versions_by_type_name = OrderedDict() + self._best_matches = {} + self._real_tag = {} + self._unnamed_types = set() + self._hooks_by_type = {} + self._all_types = set() + self._has_warned = {} + self._extension_by_type = {} + + def add_type(self, asdftype, extension): + """ + Add a type to the index. + """ + self._all_types.add(asdftype) + self._extension_by_type[asdftype] = extension + + if asdftype.yaml_tag is None and asdftype.name is None: + return + + if isinstance(asdftype.name, list): + yaml_tags = [asdftype.make_yaml_tag(name) for name in asdftype.name] + elif isinstance(asdftype.name, str): + yaml_tags = [asdftype.yaml_tag] + elif asdftype.name is None: + yaml_tags = [] + else: + raise TypeError("name must be a string, list or None") + + for yaml_tag in yaml_tags: + self._type_by_tag[yaml_tag] = asdftype + name, version = split_tag_version(yaml_tag) + versions = self._versions_by_type_name.get(name) + if versions is None: + self._versions_by_type_name[name] = [version] + else: + idx = bisect.bisect_left(versions, version) + if idx == len(versions) or versions[idx] != version: + versions.insert(idx, version) + + if not len(yaml_tags): + self._unnamed_types.add(asdftype) + + def from_custom_type(self, custom_type, version=default_version): + """ + Given a custom type, return the corresponding `ExtensionType` + definition. + """ + # Basic Python types should not ever have an AsdfType associated with + # them. + if custom_type in _BASIC_PYTHON_TYPES: + return None + + write_type_index = self._write_type_indices.get(str(version)) + if write_type_index is None: + write_type_index = _AsdfWriteTypeIndex(version, self) + self._write_type_indices[version] = write_type_index + + return write_type_index.from_custom_type(custom_type) + + def _get_version_mismatch(self, name, version, latest_version): + warning_string = None + + if (latest_version.major, latest_version.minor) != \ + (version.major, version.minor): + warning_string = \ + "'{}' with version {} found in file{{}}, but latest " \ + "supported version is {}".format( + name, version, latest_version) + + return warning_string + + def _warn_version_mismatch(self, ctx, tag, warning_string, fname): + if warning_string is not None: + # Ensure that only a single warning occurs per tag per AsdfFile + # TODO: If it is useful to only have a single warning per file on + # disk, then use `fname` in the key instead of `ctx`. + if not (ctx, tag) in self._has_warned: + warnings.warn(warning_string.format(fname)) + self._has_warned[(ctx, tag)] = True + + def fix_yaml_tag(self, ctx, tag, ignore_version_mismatch=True): + """ + Given a YAML tag, adjust it to the best supported version. + + If there is no exact match, this finds the newest version + understood that is still less than the version in file. Or, + the earliest understood version if none are less than the + version in the file. + + If ``ignore_version_mismatch==False``, this function raises a warning + if it could not find a match where the major and minor numbers are the + same. + """ + warning_string = None + + name, version = split_tag_version(tag) + + fname = " '{}'".format(ctx._fname) if ctx._fname else '' + + if tag in self._type_by_tag: + asdftype = self._type_by_tag[tag] + # Issue warnings for the case where there exists a class for the + # given tag due to the 'supported_versions' attribute being + # defined, but this tag is not the latest version of the type. + # This prevents 'supported_versions' from affecting the behavior of + # warnings that are purely related to YAML validation. + if not ignore_version_mismatch and hasattr(asdftype, '_latest_version'): + warning_string = self._get_version_mismatch( + name, version, asdftype._latest_version) + self._warn_version_mismatch(ctx, tag, warning_string, fname) + return tag + + if tag in self._best_matches: + best_tag, warning_string = self._best_matches[tag] + + if not ignore_version_mismatch: + self._warn_version_mismatch(ctx, tag, warning_string, fname) + + return best_tag + + versions = self._versions_by_type_name.get(name) + if versions is None: + return tag + + # The versions list is kept sorted, so bisect can be used to + # quickly find the best option. + i = bisect.bisect_left(versions, version) + i = max(0, i - 1) + + if not ignore_version_mismatch: + warning_string = self._get_version_mismatch( + name, version, versions[-1]) + self._warn_version_mismatch(ctx, tag, warning_string, fname) + + best_version = versions[i] + best_tag = join_tag_version(name, best_version) + self._best_matches[tag] = best_tag, warning_string + if tag != best_tag: + self._real_tag[best_tag] = tag + return best_tag + + def get_real_tag(self, tag): + if tag in self._real_tag: + return self._real_tag[tag] + elif tag in self._type_by_tag: + return tag + return None + + def from_yaml_tag(self, ctx, tag): + """ + From a given YAML tag string, return the corresponding + AsdfType definition. + """ + tag = self.fix_yaml_tag(ctx, tag) + return self._type_by_tag.get(tag) + + @lru_cache(5) + def has_hook(self, hook_name): + """ + Returns `True` if the given hook name exists on any of the managed + types. + """ + for cls in self._all_types: + if hasattr(cls, hook_name): + return True + return False + + def get_hook_for_type(self, hookname, typ, version=default_version): + """ + Get the hook function for the given type, if it exists, + else return None. + """ + hooks = self._hooks_by_type.setdefault(hookname, {}) + hook = hooks.get(typ, None) + if hook is not None: + return hook + + tag = self.from_custom_type(typ, version) + if tag is not None: + hook = getattr(tag, hookname, None) + if hook is not None: + hooks[typ] = hook + return hook + + hooks[typ] = None + return None + + def get_extensions_used(self, version=default_version): + write_type_index = self._write_type_indices.get(str(version)) + if write_type_index is None: + return [] + + return list(write_type_index._extensions_used) diff --git a/asdf/types.py b/asdf/types.py new file mode 100644 index 000000000..d7113f650 --- /dev/null +++ b/asdf/types.py @@ -0,0 +1,476 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +# -*- coding: utf-8 -*- + + +import re +import importlib + +import six +from copy import copy + +from . import tagged +from . import util +from .versioning import AsdfVersion, AsdfSpec + + +__all__ = ['format_tag', 'CustomType'] + + +# regex used to parse module name from optional version string +MODULE_RE = re.compile(r'([a-zA-Z]+)(-(\d+\.\d+\.\d+))?') + + +def format_tag(organization, standard, version, tag_name): + """ + Format a YAML tag. + """ + tag = 'tag:{0}:{1}/{2}'.format(organization, standard, tag_name) + + if version is None: + return tag + + if isinstance(version, AsdfSpec): + version = str(version.spec) + + return "{0}-{1}".format(tag, version) + + +_all_asdftypes = set() + + +def _from_tree_tagged_missing_requirements(cls, tree, ctx): + # A special version of AsdfType.from_tree_tagged for when the + # required dependencies for an AsdfType are missing. + plural, verb = ('s', 'are') if len(cls.requires) else ('', 'is') + message = "{0} package{1} {2} required to instantiate '{3}'".format( + util.human_list(cls.requires), plural, verb, tree._tag) + # This error will be handled by yamlutil.tagged_tree_to_custom_tree, which + # will cause a warning to be issued indicating that the tree failed to be + # converted. + raise TypeError(message) + + +class ExtensionTypeMeta(type): + """ + Custom class constructor for tag types. + """ + _import_cache = {} + + @classmethod + def _has_required_modules(cls, requires): + for string in requires: + has_module = True + match = MODULE_RE.match(string) + modname, _, version = match.groups() + if modname in cls._import_cache: + if not cls._import_cache[modname]: + return False + try: + module = importlib.import_module(modname) + if version and hasattr(module, '__version__'): + if module.__version__ < version: + has_module = False + except ImportError: + has_module = False + finally: + cls._import_cache[modname] = has_module + if not has_module: + return False + return True + + @classmethod + def _find_in_bases(cls, attrs, bases, name, default=None): + if name in attrs: + return attrs[name] + for base in bases: + if hasattr(base, name): + return getattr(base, name) + return default + + @property + def versioned_siblings(mcls): + return getattr(mcls, '__versioned_siblings') or [] + + def __new__(mcls, name, bases, attrs): + requires = mcls._find_in_bases(attrs, bases, 'requires', []) + if not mcls._has_required_modules(requires): + attrs['from_tree_tagged'] = classmethod( + _from_tree_tagged_missing_requirements) + attrs['types'] = [] + attrs['has_required_modules'] = False + else: + attrs['has_required_modules'] = True + types = mcls._find_in_bases(attrs, bases, 'types', []) + new_types = [] + for typ in types: + if isinstance(typ, str): + typ = util.resolve_name(typ) + new_types.append(typ) + attrs['types'] = new_types + + cls = super(ExtensionTypeMeta, mcls).__new__(mcls, name, bases, attrs) + + if hasattr(cls, 'version'): + if not isinstance(cls.version, (AsdfVersion, AsdfSpec)): + cls.version = AsdfVersion(cls.version) + + if hasattr(cls, 'name'): + if isinstance(cls.name, str): + if 'yaml_tag' not in attrs: + cls.yaml_tag = cls.make_yaml_tag(cls.name) + elif isinstance(cls.name, list): + pass + elif cls.name is not None: + raise TypeError("name must be string or list") + + if hasattr(cls, 'supported_versions'): + if not isinstance(cls.supported_versions, (list, set)): + cls.supported_versions = [cls.supported_versions] + supported_versions = set() + for version in cls.supported_versions: + if not isinstance(version, (AsdfVersion, AsdfSpec)): + version = AsdfVersion(version) + # This should cause an exception for invalid input + supported_versions.add(version) + # We need to convert back to a list here so that the 'in' operator + # uses actual comparison instead of hash equality + cls.supported_versions = list(supported_versions) + siblings = list() + for version in cls.supported_versions: + if version != cls.version: + new_attrs = copy(attrs) + new_attrs['version'] = version + new_attrs['supported_versions'] = set() + new_attrs['_latest_version'] = cls.version + siblings.append( + ExtensionTypeMeta. __new__(mcls, name, bases, new_attrs)) + setattr(cls, '__versioned_siblings', siblings) + + return cls + + +class AsdfTypeMeta(ExtensionTypeMeta): + """ + Keeps track of `AsdfType` subclasses that are created, and stores them in + `AsdfTypeIndex`. + """ + def __new__(mcls, name, bases, attrs): + cls = super(AsdfTypeMeta, mcls).__new__(mcls, name, bases, attrs) + # Classes using this metaclass get added to the list of built-in + # extensions + _all_asdftypes.add(cls) + + return cls + + +class ExtensionType: + """ + The base class of all custom types in the tree. + + Besides the attributes defined below, most subclasses will also + override `to_tree` and `from_tree`. + """ + name = None + organization = 'stsci.edu' + standard = 'asdf' + version = (1, 0, 0) + supported_versions = set() + types = [] + handle_dynamic_subclasses = False + validators = {} + requires = [] + yaml_tag = None + + @classmethod + def names(cls): + """ + Returns the name(s) represented by this tag type as a list. + + While some tag types represent only a single custom type, others + represent multiple types. In the latter case, the `name` attribute of + the extension is actually a list, not simply a string. This method + normalizes the value of `name` by returning a list in all cases. + + Returns + ------- + `list` of names represented by this tag type + """ + if cls.name is None: + return None + + return cls.name if isinstance(cls.name, list) else [cls.name] + + @classmethod + def make_yaml_tag(cls, name, versioned=True): + """ + Given the name of a type, returns a string representing its YAML tag. + + Parameters + ---------- + name : str + The name of the type. In most cases this will correspond to the + `name` attribute of the tag type. However, it is passed as a + parameter since some tag types represent multiple custom + types. + + versioned : bool + If `True`, the tag will be versioned. Otherwise, a YAML tag without + a version will be returned. + + Returns + ------- + `str` representing the YAML tag + """ + return format_tag( + cls.organization, + cls.standard, + cls.version if versioned else None, + name) + + @classmethod + def tag_base(cls): + """ + Returns the base of the YAML tag for types represented by this class. + + This method returns the portion of the tag that represents the standard + and the organization of any type represented by this class. + + Returns + ------- + `str` representing the base of the YAML tag + """ + return cls.make_yaml_tag('', versioned=False) + + @classmethod + def to_tree(cls, node, ctx): + """ + Converts instances of custom types into YAML representations. + + This method should be overridden by custom extension classes in order + to define how custom types are serialized into YAML. The method must + return a single Python object corresponding to one of the basic YAML + types (dict, list, str, or number). However, the types can be nested + and combined in order to represent more complex custom types. + + This method is called as part of the process of writing an `AsdfFile` + object. Whenever a custom type (or a subclass of that type) that is + listed in the `types` attribute of this class is encountered, this + method will be used to serialize that type. + + The name `to_tree` refers to the act of converting a custom type into + part of a YAML object tree. + + Parameters + ---------- + node : `object` + Instance of a custom type to be serialized. Will be an instance (or + an instance of a subclass) of one of the types listed in the + `types` attribute of this class. + + ctx : `AsdfFile` + An instance of the `AsdfFile` object that is being written out. + + Returns + ------- + A basic YAML type (`dict`, `list`, `str`, `int`, `float`, or + `complex`) representing the properties of the custom type to be + serialized. These types can be nested in order to represent more + complex custom types. + """ + return node.__class__.__bases__[0](node) + + @classmethod + def to_tree_tagged(cls, node, ctx): + """ + Converts instances of custom types into tagged objects. + + It is more common for custom tag types to override `to_tree` instead of + this method. This method should only be overridden if it is necessary + to modify the YAML tag that will be used to tag this object. + + Parameters + ---------- + node : `object` + Instance of a custom type to be serialized. Will be an instance (or + an instance of a subclass) of one of the types listed in the + `types` attribute of this class. + + ctx : `AsdfFile` + An instance of the `AsdfFile` object that is being written out. + + Returns + ------- + An instance of `asdf.tagged.Tagged`. + """ + obj = cls.to_tree(node, ctx) + return tagged.tag_object(cls.yaml_tag, obj, ctx=ctx) + + @classmethod + def from_tree(cls, tree, ctx): + """ + Converts basic types representing YAML trees into custom types. + + This method should be overridden by custom extension classes in order + to define how custom types are deserialized from the YAML + representation back into their original types. The method will return + an instance of the original custom type. + + This method is called as part of the process of reading an ASDF file in + order to construct an `AsdfFile` object. Whenever a YAML subtree is + encountered that has a tag that corresponds to the `yaml_tag` property + of this class, this method will be used to deserialize that tree back + into an instance of the original custom type. + + Parameters + ---------- + tree : `object` representing YAML tree + An instance of a basic Python type (possibly nested) that + corresponds to a YAML subtree. + + ctx : `AsdfFile` + An instance of the `AsdfFile` object that is being constructed. + + Returns + ------- + An instance of the custom type represented by this extension class. + """ + return cls(tree) + + @classmethod + def from_tree_tagged(cls, tree, ctx): + """ + Converts from tagged tree into custom type. + + It is more common for extension classes to override `from_tree` instead + of this method. This method should only be overridden if it is + necessary to access the `_tag` property of the `Tagged` object + directly. + + Parameters + ---------- + tree : `asdf.tagged.Tagged` object representing YAML tree + + ctx : `AsdfFile` + An instance of the `AsdfFile` object that is being constructed. + + Returns + ------- + An instance of the custom type represented by this extension class. + """ + return cls.from_tree(tree.data, ctx) + + @classmethod + def incompatible_version(cls, version): + """ + Indicates if given version is known to be incompatible with this type. + + If this tag class explicitly identifies compatible versions then this + checks whether a given version is compatible or not (see + `supported_versions`). Otherwise, all versions are assumed to be + compatible. + + Child classes can override this method to affect how version + compatiblity for this type is determined. + + Parameters + ---------- + version : `str` or `~asdf.versioning.AsdfVersion` + The version to test for compatibility. + """ + if cls.supported_versions: + if version not in cls.supported_versions: + return True + return False + + +@six.add_metaclass(AsdfTypeMeta) +class AsdfType(ExtensionType): + """ + Base class for all built-in ASDF types. Types that inherit this class will + be automatically added to the list of built-ins. This should *not* be used + for user-defined extensions. + """ + +@six.add_metaclass(ExtensionTypeMeta) +class CustomType(ExtensionType): + """ + Base class for all user-defined types. + """ + + # These attributes are duplicated here with docstrings since a bug in + # sphinx prevents the docstrings of class attributes from being inherited + # properly (see https://github.com/sphinx-doc/sphinx/issues/741. The + # docstrings are not included anywhere else in the class hierarchy since + # this class is the only one exposed in the public API. + name = None + """ + `str` or `list`: The name of the type. + """ + + organization = 'stsci.edu' + """ + `str`: The organization responsible for the type. + """ + + standard = 'asdf' + """ + `str`: The standard the type is defined in. + """ + + version = (1, 0, 0) + """ + `str`, `tuple`, `AsdfVersion`, or `AsdfSpec`: The version of the type. + """ + + supported_versions = set() + """ + `set`: Versions that explicitly compatible with this extension class. + + If provided, indicates explicit compatibility with the given set + of versions. Other versions of the same schema that are not included in + this set will not be converted to custom types with this class. """ + + types = [] + """ + `list`: List of types that this extension class can convert to/from YAML. + + Custom Python types that, when found in the tree, will be converted into + basic types for YAML output. Can be either strings referring to the types + or the types themselves.""" + + handle_dynamic_subclasses = False + """ + `bool`: Indicates whether dynamically generated subclasses can be serialized + + Flag indicating whether this type is capable of serializing subclasses + of any of the types listed in ``types`` that are generated dynamically. + """ + + validators = {} + """ + `dict`: Mapping JSON Schema keywords to validation functions for jsonschema. + + Useful if the type defines extra types of validation that can be + performed. + """ + + requires = [] + """ + `list`: Python packages that are required to instantiate the object. + """ + + yaml_tag = None + """ + `str`: The YAML tag to use for the type. + + If not provided, it will be automatically generated from name, + organization, standard and version. + """ + + has_required_modules = True + """ + `bool`: Indicates whether modules specified by `requires` are available. + + NOTE: This value is automatically generated. Do not set it in subclasses as + it will be overwritten. + """ diff --git a/asdf/util.py b/asdf/util.py index f72fe3eae..b916ac87b 100644 --- a/asdf/util.py +++ b/asdf/util.py @@ -118,7 +118,7 @@ def calculate_padding(content_size, pad_blocks, block_size): return max(new_size - content_size, 0) -class BinaryStruct(object): +class BinaryStruct: """ A wrapper around the Python stdlib struct module to define a binary struct more like a dictionary than a tuple. @@ -368,7 +368,7 @@ class InheritDocstrings(type): >>> from asdf.util import InheritDocstrings >>> import six >>> @six.add_metaclass(InheritDocstrings) - ... class A(object): + ... class A: ... def wiggle(self): ... "Wiggle the thingamajig" ... pass diff --git a/asdf/versioning.py b/asdf/versioning.py index 312e6ac1b..08d9b9b07 100644 --- a/asdf/versioning.py +++ b/asdf/versioning.py @@ -22,7 +22,23 @@ from .version import version as asdf_version -__all__ = ['AsdfVersion', 'AsdfSpec'] +__all__ = ['AsdfVersion', 'AsdfSpec', 'split_tag_version', 'join_tag_version'] + + +def split_tag_version(tag): + """ + Split a tag into its base and version. + """ + name, version = tag.rsplit('-', 1) + version = AsdfVersion(version) + return name, version + + +def join_tag_version(name, version): + """ + Join the root and version of a tag back together. + """ + return '{0}-{1}'.format(name, version) _version_map = {} @@ -55,7 +71,7 @@ def get_version_map(version): @total_ordering -class AsdfVersionMixin(object): +class AsdfVersionMixin: """This mix-in is required in order to impose the total ordering that we want for ``AsdfVersion``, rather than accepting the total ordering that is already provided by ``Version`` from ``semantic_version``. Defining these @@ -142,17 +158,17 @@ def __hash__(self): return super(AsdfSpec, self).__hash__() -default_version = AsdfVersion('1.2.0') - - supported_versions = [ AsdfVersion('1.0.0'), AsdfVersion('1.1.0'), - AsdfVersion('1.2.0') + AsdfVersion('1.2.0'), + AsdfVersion('1.3.0') ] +default_version = supported_versions[-1] + -class VersionedMixin(object): +class VersionedMixin: _version = default_version @property diff --git a/asdf/yamlutil.py b/asdf/yamlutil.py index 8f64cd174..da8e3288d 100644 --- a/asdf/yamlutil.py +++ b/asdf/yamlutil.py @@ -11,10 +11,9 @@ from . import schema from . import tagged from . import treeutil -from . import asdftypes -from . import versioning from . import util from .constants import YAML_TAG_PREFIX +from .versioning import split_tag_version from .exceptions import AsdfConversionWarning @@ -253,7 +252,7 @@ def walker(node): return node real_tag = ctx.type_index.get_real_tag(tag_name) - real_tag_name, real_tag_version = asdftypes.split_tag_version(real_tag) + real_tag_name, real_tag_version = split_tag_version(real_tag) # This means that there is an explicit description of versions that are # compatible with the associated tag class implementation, but the # version we found does not fit that description. diff --git a/docs/asdf/changes.rst b/docs/asdf/changes.rst index 5bb267f26..68300e440 100644 --- a/docs/asdf/changes.rst +++ b/docs/asdf/changes.rst @@ -4,6 +4,26 @@ Changes ******* +What's New in ASDF 2.3? +======================= + +ASDF 2.3 reflects the update of ASDF Standard to v1.3.0, and contains a few +notable features and an API change: + +* Storage of arbitrary precision integers is now provided by + `asdf.IntegerType`. This new type is provided by version 1.3.0 of the ASDF + Standard. + +* Reading a file with integer literals that are too large now causes only a + warning instead of a validation error. This is to provide backwards + compatibility for files that were created with a buggy version of ASDF. + +* The functions `asdf.open` and `AsdfFile.write_to` now support the use of + `pathlib.Path`. + +* The `asdf.asdftypes` module has been deprecated in favor of `asdf.types`. The + old module will be removed entirely in the 3.0 release. + What's New in ASDF 2.2? ======================= diff --git a/docs/asdf/developer_api.rst b/docs/asdf/developer_api.rst index e0ce75825..d8f3e2532 100644 --- a/docs/asdf/developer_api.rst +++ b/docs/asdf/developer_api.rst @@ -5,7 +5,7 @@ Developer API The classes and functions documented here will be of use to developers who wish to create their own custom ASDF types and extensions. -.. automodapi:: asdf.asdftypes +.. automodapi:: asdf.types .. automodapi:: asdf.extension diff --git a/docs/asdf/features.rst b/docs/asdf/features.rst index 3d19b565e..0f4cb42c4 100644 --- a/docs/asdf/features.rst +++ b/docs/asdf/features.rst @@ -18,6 +18,19 @@ respectively. The top-level tree object behaves like a Python dictionary and supports arbitrary nesting of data structures. For simple examples of creating and reading trees, see :ref:`overview`. +.. note:: + + The ASDF Standard imposes a maximum size of 52 bits for integer literals in + the tree (see `the docs `_ + for details and justification). Attempting to store a larger value will + result in a validation error. + + Integers and floats of up to 64 bits can be stored inside of :mod:`numpy` + arrays (see below). + + For arbitrary precision integer support, see `IntegerType`. + + One of the key features of ASDF is its ability to serialize :mod:`numpy` arrays. This is discussed in detail in :ref:`array-data`. diff --git a/setup.cfg b/setup.cfg index f26212bbc..c1839ade6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,7 +17,7 @@ open_files_ignore = test.fits asdf.fits # Account for both the astropy test runner case and the native pytest case asdf_schema_root = asdf-standard/schemas asdf/schemas asdf_schema_skip_names = asdf-schema-1.0.0 draft-01 -asdf_schema_skip_examples = domain-1.0.0 +asdf_schema_skip_examples = domain-1.0.0 frame-1.0.0 frame-1.1.0 #addopts = --doctest-rst [ah_bootstrap]