From 10a4e5325f640176311773875f0fb293049c8112 Mon Sep 17 00:00:00 2001 From: Ed Slavich Date: Fri, 11 Feb 2022 12:38:42 -0500 Subject: [PATCH 1/2] Fix some of the packaged asdf-standard issues --- asdf/resolver.py | 20 ++------------------ asdf/resource.py | 24 +----------------------- asdf/schema.py | 11 +++++++++-- asdf/tests/helpers.py | 31 ++++++++++--------------------- asdf/tests/test_api.py | 2 +- asdf/tests/test_config.py | 3 ++- asdf/tests/test_resource.py | 9 +++------ setup.cfg | 2 ++ 8 files changed, 30 insertions(+), 72 deletions(-) diff --git a/asdf/resolver.py b/asdf/resolver.py index 2b653cd05..6b552fd21 100644 --- a/asdf/resolver.py +++ b/asdf/resolver.py @@ -1,23 +1,10 @@ import sys -import os.path import warnings from . import constants -from . import util from .exceptions import AsdfDeprecationWarning -def find_schema_path(): - dirname = os.path.dirname(__file__) - - # This means we are working within a development build - if os.path.exists(os.path.join(dirname, '..', 'asdf-standard')): - return os.path.join(dirname, '..', 'asdf-standard', 'schemas') - - # Otherwise, we return the installed location - return os.path.join(dirname, 'schemas') - - class Resolver: """ A class that can be used to map strings with a particular prefix @@ -149,11 +136,8 @@ def __eq__(self, other): return self._resolvers == other._resolvers -DEFAULT_URL_MAPPING = [ - (constants.STSCI_SCHEMA_URI_BASE, - util.filepath_to_url( - os.path.join(find_schema_path(), 'stsci.edu')) + - '/{url_suffix}.yaml')] +DEFAULT_URL_MAPPING = [] + DEFAULT_TAG_TO_URL_MAPPING = [ (constants.STSCI_SCHEMA_TAG_BASE, 'http://stsci.edu/schemas/asdf{tag_suffix}') diff --git a/asdf/resource.py b/asdf/resource.py index 00c43181e..3270b65ea 100644 --- a/asdf/resource.py +++ b/asdf/resource.py @@ -272,29 +272,7 @@ def __repr__(self): return "JsonschemaResourceMapping()" -def get_core_resource_mappings(): - """ - Get the resource mapping instances for the core schemas. - This method is registered with the asdf.resource_mappings entry point. - """ - core_schemas_root = importlib_resources.files(asdf)/"schemas"/"stsci.edu" - if not core_schemas_root.is_dir(): - # In an editable install, the schemas can be found in the - # asdf-standard submodule. - core_schemas_root = Path(__file__).parent.parent/"asdf-standard"/"schemas"/"stsci.edu" - if not core_schemas_root.is_dir(): - raise RuntimeError("Unable to locate core schemas") - - resources_root = importlib_resources.files(asdf)/"resources" - if not resources_root.is_dir(): - # In an editable install, the resources can be found in the - # asdf-standard submodule. - resources_root = Path(__file__).parent.parent/"asdf-standard"/"resources" - if not resources_root.is_dir(): - raise RuntimeError("Unable to locate core resources") - +def get_json_schema_resource_mappings(): return [ - DirectoryResourceMapping(core_schemas_root, "http://stsci.edu/schemas", recursive=True), - DirectoryResourceMapping(resources_root / "asdf-format.org", "asdf://asdf-format.org", recursive=True), JsonschemaResourceMapping(), ] diff --git a/asdf/schema.py b/asdf/schema.py index 2993c4545..792f244c8 100644 --- a/asdf/schema.py +++ b/asdf/schema.py @@ -362,6 +362,13 @@ def load_schema(url): # Check if this is a URI provided by the new # Mapping API: resource_manager = get_config().resource_manager + + if url not in resource_manager: + # Allow the resolvers to do their thing, in case they know + # how to turn this string into a URI that the resource manager + # recognizes. + url = resolver(str(url)) + if url in resource_manager: content = resource_manager[url] # The jsonschema metaschemas are JSON, but pyyaml @@ -371,8 +378,8 @@ def load_schema(url): result = yaml.load(content, Loader=yamlutil.AsdfLoader) # nosec return result, url - # If not, fall back to fetching the schema the old way: - url = resolver(str(url)) + # If not, this must be a URL (or missing). Fall back to fetching + # the schema the old way: return _load_schema(url) return load_schema diff --git a/asdf/tests/helpers.py b/asdf/tests/helpers.py index b8f5c1406..b15f231cf 100644 --- a/asdf/tests/helpers.py +++ b/asdf/tests/helpers.py @@ -431,24 +431,13 @@ def _assert_extension_type_correctness(extension, extension_type, resolver): "properties on the related extension ({}).".format(extension_type.__name__) ) - try: - with generic_io.get_file(schema_location) as f: - schema = yaml.safe_load(f.read()) - except Exception: - assert False, ( - "{} supports tag, {}, ".format(extension_type.__name__, check_type.yaml_tag) + - "which resolves to schema at {}, but ".format(schema_location) + - "schema cannot be read." - ) - - assert "tag" in schema, ( - "{} supports tag, {}, ".format(extension_type.__name__, check_type.yaml_tag) + - "but tag resolves to a schema at {} that is ".format(schema_location) + - "missing its tag field." - ) - - assert schema["tag"] == check_type.yaml_tag, ( - "{} supports tag, {}, ".format(extension_type.__name__, check_type.yaml_tag) + - "but tag resolves to a schema at {} that ".format(schema_location) + - "describes a different tag: {}".format(schema["tag"]) - ) + if schema_location not in asdf.get_config().resource_manager: + try: + with generic_io.get_file(schema_location) as f: + schema = yaml.safe_load(f.read()) + except Exception: + assert False, ( + "{} supports tag, {}, ".format(extension_type.__name__, check_type.yaml_tag) + + "which resolves to schema at {}, but ".format(schema_location) + + "schema cannot be read." + ) diff --git a/asdf/tests/test_api.py b/asdf/tests/test_api.py index e6cf8f9c5..3ef994b37 100644 --- a/asdf/tests/test_api.py +++ b/asdf/tests/test_api.py @@ -484,7 +484,7 @@ def test_get_default_resolver(): result = resolver('tag:stsci.edu:asdf/core/ndarray-1.0.0') - assert result.endswith("/schemas/stsci.edu/asdf/core/ndarray-1.0.0.yaml") + assert result == 'http://stsci.edu/schemas/asdf/core/ndarray-1.0.0' def test_info_module(capsys, tmpdir): diff --git a/asdf/tests/test_config.py b/asdf/tests/test_config.py index 376960690..0ead15fe3 100644 --- a/asdf/tests/test_config.py +++ b/asdf/tests/test_config.py @@ -1,6 +1,7 @@ import threading import pytest +import asdf_standard.integration import asdf from asdf import get_config @@ -110,7 +111,7 @@ def test_array_inline_threshold(): def test_resource_mappings(): with asdf.config_context() as config: - core_mappings = resource.get_core_resource_mappings() + core_mappings = resource.get_json_schema_resource_mappings() + asdf_standard.integration.get_resource_mappings() default_mappings = config.resource_mappings assert len(default_mappings) >= len(core_mappings) diff --git a/asdf/tests/test_resource.py b/asdf/tests/test_resource.py index 5b04f287e..064c66a7f 100644 --- a/asdf/tests/test_resource.py +++ b/asdf/tests/test_resource.py @@ -14,7 +14,7 @@ DirectoryResourceMapping, ResourceManager, ResourceMappingProxy, - get_core_resource_mappings, + get_json_schema_resource_mappings, JsonschemaResourceMapping, ) @@ -232,12 +232,9 @@ def test_jsonschema_resource_mapping(): @pytest.mark.parametrize("uri", [ "http://json-schema.org/draft-04/schema", - "http://stsci.edu/schemas/yaml-schema/draft-01", - "http://stsci.edu/schemas/asdf/core/asdf-1.1.0", - "asdf://asdf-format.org/core/schemas/extension_manifest-1.0.0", ]) -def test_get_core_resource_mappings(uri): - mappings = get_core_resource_mappings() +def test_get_json_schema_resource_mappings(uri): + mappings = get_json_schema_resource_mappings() mapping = next(m for m in mappings if uri in m) assert mapping is not None diff --git a/setup.cfg b/setup.cfg index dc7191c2c..8b9d2d08d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -57,6 +57,8 @@ console_scripts = asdftool = asdf.commands.main:main asdf_extensions = builtin = asdf.extension:BuiltinExtension +asdf.resource_mappings = + asdf = asdf.resource:get_json_schema_resource_mappings pytest11 = asdf_schema_tester = pytest_asdf.plugin From a658be9800f8616a02204bfed6914487d7bd5700 Mon Sep 17 00:00:00 2001 From: Ed Slavich Date: Fri, 11 Feb 2022 12:49:02 -0500 Subject: [PATCH 2/2] Catch IOError instead of FileNotFoundError --- asdf/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asdf/schema.py b/asdf/schema.py index 792f244c8..b836877c2 100644 --- a/asdf/schema.py +++ b/asdf/schema.py @@ -318,7 +318,7 @@ def iter_errors(self, instance, _schema=None): for schema_uri in schema_uris: try: s = _load_schema_cached(schema_uri, self.ctx.resolver, False, False) - except FileNotFoundError: + except IOError: msg = "Unable to locate schema file for '{}': '{}'" warnings.warn(msg.format(tag, schema_uri), AsdfWarning) s = {}