diff --git a/changelog/324.feature.rst b/changelog/324.feature.rst new file mode 100644 index 00000000..54fdaf2a --- /dev/null +++ b/changelog/324.feature.rst @@ -0,0 +1 @@ +Update ASDF schemas for upcoming ASDF standard 1.6.0. diff --git a/dkist/io/asdf/converters/models.py b/dkist/io/asdf/converters/models.py index ca04b22e..5998d0f7 100644 --- a/dkist/io/asdf/converters/models.py +++ b/dkist/io/asdf/converters/models.py @@ -4,10 +4,8 @@ class VaryingCelestialConverter(TransformConverterBase): tags = [ - "asdf://dkist.nso.edu/tags/varying_celestial_transform-1.1.0", - "asdf://dkist.nso.edu/tags/varying_celestial_transform-1.0.0", - "asdf://dkist.nso.edu/tags/inverse_varying_celestial_transform-1.1.0", - "asdf://dkist.nso.edu/tags/inverse_varying_celestial_transform-1.0.0", + "asdf://dkist.nso.edu/tags/varying_celestial_transform-*", + "asdf://dkist.nso.edu/tags/inverse_varying_celestial_transform-*", # Old slit tags must be kept so we can read old files, but not types as # we will not save slit classes any more "asdf://dkist.nso.edu/tags/varying_celestial_transform_slit-1.0.0", @@ -23,10 +21,13 @@ class VaryingCelestialConverter(TransformConverterBase): ] def select_tag(self, obj, tags, ctx): + tag_pattern = "asdf://dkist.nso.edu/tags/varying_celestial_transform" if obj._is_inverse: - return "asdf://dkist.nso.edu/tags/inverse_varying_celestial_transform-1.1.0" + tag_pattern = "asdf://dkist.nso.edu/tags/inverse_varying_celestial_transform" - return "asdf://dkist.nso.edu/tags/varying_celestial_transform-1.1.0" + for tag in tags: + if tag.startswith(tag_pattern): + return tag def from_yaml_tree_transform(self, node, tag, ctx): from dkist.wcs.models import varying_celestial_transform_from_tables @@ -74,7 +75,7 @@ class CoupledCompoundConverter(TransformConverterBase): ASDF serialization support for CompoundModel. """ tags = [ - "asdf://dkist.nso.edu/tags/coupled_compound_model-1.0.0", + "asdf://dkist.nso.edu/tags/coupled_compound_model-*", ] types = ["dkist.wcs.models.CoupledCompoundModel"] @@ -122,7 +123,7 @@ class RavelConverter(TransformConverterBase): """ tags = [ - "asdf://dkist.nso.edu/tags/ravel_model-1.0.0" + "asdf://dkist.nso.edu/tags/ravel_model-*" ] types = ["dkist.wcs.models.Ravel"] @@ -142,7 +143,7 @@ class AsymmetricMappingConverter(TransformConverterBase): """ tags = [ - "asdf://dkist.nso.edu/tags/asymmetric_mapping_model-1.0.0" + "asdf://dkist.nso.edu/tags/asymmetric_mapping_model-*" ] types = ["dkist.wcs.models.AsymmetricMapping"] diff --git a/dkist/io/asdf/entry_points.py b/dkist/io/asdf/entry_points.py index f4eb87ca..8d86cf0f 100644 --- a/dkist/io/asdf/entry_points.py +++ b/dkist/io/asdf/entry_points.py @@ -45,6 +45,8 @@ def get_extensions(): converters=dkist_converters), ManifestExtension.from_uri("asdf://dkist.nso.edu/manifests/dkist-1.0.0", converters=dkist_converters), + ManifestExtension.from_uri("asdf://dkist.nso.edu/manifests/dkist-wcs-1.3.0", + converters=wcs_converters), ManifestExtension.from_uri("asdf://dkist.nso.edu/manifests/dkist-wcs-1.2.0", converters=wcs_converters), ManifestExtension.from_uri("asdf://dkist.nso.edu/manifests/dkist-wcs-1.1.0", diff --git a/dkist/io/asdf/resources/manifests/dkist-0.9.0.yaml b/dkist/io/asdf/resources/manifests/dkist-0.9.0.yaml index e876e11d..d33380b0 100644 --- a/dkist/io/asdf/resources/manifests/dkist-0.9.0.yaml +++ b/dkist/io/asdf/resources/manifests/dkist-0.9.0.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -id: asdf://dkist.nso.edu/dkist/manifests/dkist-0.9.0 +id: asdf://dkist.nso.edu/manifests/dkist-0.9.0 extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-0.9.0 title: DKIST extension description: ASDF schemas and tags for pre-1.0 DKIST tags. diff --git a/dkist/io/asdf/resources/manifests/dkist-1.0.0.yaml b/dkist/io/asdf/resources/manifests/dkist-1.0.0.yaml index 7b7641a6..4a2804c1 100644 --- a/dkist/io/asdf/resources/manifests/dkist-1.0.0.yaml +++ b/dkist/io/asdf/resources/manifests/dkist-1.0.0.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -id: asdf://dkist.nso.edu/dkist/manifests/dkist-1.0.0 +id: asdf://dkist.nso.edu/manifests/dkist-1.0.0 extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-1.0.0 title: DKIST extension description: ASDF schemas and tags for DKIST classes. diff --git a/dkist/io/asdf/resources/manifests/dkist-1.1.0.yaml b/dkist/io/asdf/resources/manifests/dkist-1.1.0.yaml index a729d1d4..e9832331 100644 --- a/dkist/io/asdf/resources/manifests/dkist-1.1.0.yaml +++ b/dkist/io/asdf/resources/manifests/dkist-1.1.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- -id: asdf://dkist.nso.edu/dkist/manifests/dkist-1.1.0 -extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-1.0.0 +id: asdf://dkist.nso.edu/manifests/dkist-1.1.0 +extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-1.1.0 title: DKIST extension description: ASDF schemas and tags for DKIST classes. diff --git a/dkist/io/asdf/resources/manifests/dkist-1.2.0.yaml b/dkist/io/asdf/resources/manifests/dkist-1.2.0.yaml index 58076ebc..830047a3 100644 --- a/dkist/io/asdf/resources/manifests/dkist-1.2.0.yaml +++ b/dkist/io/asdf/resources/manifests/dkist-1.2.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- -id: asdf://dkist.nso.edu/dkist/manifests/dkist-1.2.0 -extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-1.0.0 +id: asdf://dkist.nso.edu/manifests/dkist-1.2.0 +extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-1.2.0 title: DKIST extension description: ASDF schemas and tags for DKIST classes. diff --git a/dkist/io/asdf/resources/manifests/dkist-wcs-1.0.0.yaml b/dkist/io/asdf/resources/manifests/dkist-wcs-1.0.0.yaml index 44f24d02..1335fddb 100644 --- a/dkist/io/asdf/resources/manifests/dkist-wcs-1.0.0.yaml +++ b/dkist/io/asdf/resources/manifests/dkist-wcs-1.0.0.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -id: asdf://dkist.nso.edu/dkist/manifests/dkist-wcs-1.0.0 +id: asdf://dkist.nso.edu/manifests/dkist-wcs-1.0.0 extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-wcs-1.0.0 title: DKIST WCS extension description: ASDF schemas and tags for models and WCS related classes. diff --git a/dkist/io/asdf/resources/manifests/dkist-wcs-1.1.0.yaml b/dkist/io/asdf/resources/manifests/dkist-wcs-1.1.0.yaml index ead05544..db9b669d 100644 --- a/dkist/io/asdf/resources/manifests/dkist-wcs-1.1.0.yaml +++ b/dkist/io/asdf/resources/manifests/dkist-wcs-1.1.0.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -id: asdf://dkist.nso.edu/dkist/manifests/dkist-wcs-1.1.0 +id: asdf://dkist.nso.edu/manifests/dkist-wcs-1.1.0 extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-wcs-1.1.0 title: DKIST WCS extension description: ASDF schemas and tags for models and WCS related classes. diff --git a/dkist/io/asdf/resources/manifests/dkist-wcs-1.2.0.yaml b/dkist/io/asdf/resources/manifests/dkist-wcs-1.2.0.yaml index e6c7f08a..75ed7746 100644 --- a/dkist/io/asdf/resources/manifests/dkist-wcs-1.2.0.yaml +++ b/dkist/io/asdf/resources/manifests/dkist-wcs-1.2.0.yaml @@ -1,6 +1,6 @@ %YAML 1.1 --- -id: asdf://dkist.nso.edu/dkist/manifests/dkist-wcs-1.2.0 +id: asdf://dkist.nso.edu/manifests/dkist-wcs-1.2.0 extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-wcs-1.2.0 title: DKIST WCS extension description: ASDF schemas and tags for models and WCS related classes. diff --git a/dkist/io/asdf/resources/manifests/dkist-wcs-1.3.0.yaml b/dkist/io/asdf/resources/manifests/dkist-wcs-1.3.0.yaml new file mode 100644 index 00000000..ebc6b713 --- /dev/null +++ b/dkist/io/asdf/resources/manifests/dkist-wcs-1.3.0.yaml @@ -0,0 +1,27 @@ +%YAML 1.1 +--- +id: asdf://dkist.nso.edu/manifests/dkist-wcs-1.3.0 +extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-wcs-1.3.0 +asdf_standard_requirement: + gte: 1.6.0 + +title: DKIST WCS extension +description: ASDF schemas and tags for models and WCS related classes. + +tags: + # the tag version does not match the schema version + - schema_uri: "asdf://dkist.nso.edu/schemas/varying_celestial_transform-1.1.0" + tag_uri: "asdf://dkist.nso.edu/tags/varying_celestial_transform-1.2.0" + + # the varying_celestial_transform schema is reused here + - schema_uri: "asdf://dkist.nso.edu/schemas/varying_celestial_transform-1.1.0" + tag_uri: "asdf://dkist.nso.edu/tags/inverse_varying_celestial_transform-1.2.0" + + - schema_uri: "asdf://dkist.nso.edu/schemas/coupled_compound_model-1.1.0" + tag_uri: "asdf://dkist.nso.edu/tags/coupled_compound_model-1.1.0" + + - schema_uri: "asdf://dkist.nso.edu/schemas/ravel_model-1.1.0" + tag_uri: "asdf://dkist.nso.edu/tags/ravel_model-1.1.0" + + - schema_uri: "asdf://dkist.nso.edu/schemas/asymmetric_mapping_model-1.1.0" + tag_uri: "asdf://dkist.nso.edu/tags/asymmetric_mapping_model-1.1.0" diff --git a/dkist/io/asdf/resources/schemas/asymmetric_mapping_model-1.1.0.yaml b/dkist/io/asdf/resources/schemas/asymmetric_mapping_model-1.1.0.yaml new file mode 100644 index 00000000..3c128301 --- /dev/null +++ b/dkist/io/asdf/resources/schemas/asymmetric_mapping_model-1.1.0.yaml @@ -0,0 +1,30 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://dkist.nso.edu/schemas/asymmetric_mapping_model-1.1.0" +title: > + Reorder, add and drop axes with different mappings in forward and reverse transforms. + +definitions: + mapping: + type: array + items: + type: integer + +allOf: + - $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.3.0" + - properties: + forward_n_inputs: + description: | + Explicitly set the number of input axes in the forward direction. + type: integer + backward_n_inputs: + description: | + Explicitly set the number of input axes in the backward direction. + type: integer + forward_mapping: + $ref: "#/definitions/mapping" + backward_mapping: + $ref: "#/definitions/mapping" + required: [forward_mapping, backward_mapping] +... diff --git a/dkist/io/asdf/resources/schemas/coupled_compound_model-1.1.0.yaml b/dkist/io/asdf/resources/schemas/coupled_compound_model-1.1.0.yaml new file mode 100644 index 00000000..122afce5 --- /dev/null +++ b/dkist/io/asdf/resources/schemas/coupled_compound_model-1.1.0.yaml @@ -0,0 +1,85 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://dkist.nso.edu/schemas/coupled_compound_model-1.1.0" +title: > + Send axes to different subtransforms. +description: | + This transform takes two models which share one or more inputs on the forward + transform, and where the left hand model's inverse is dependent on the + output of the right hand model's inverse output. + + Take the following example with a time dependent celestial transform + (modelled as dependent upon the pixel coordinate for time rather than the + world coordinate). + + The forward transform uses the "z" pixel dimension as input to both the + Celestial and Temporal models, this leads to the following transform in the + forward direction: + + ``` + : x y z + : │ │ │ + : │ │ ┌────────┤ + : │ │ │ │ + : ▼ ▼ ▼ ▼ + : ┌─────────┐ ┌────────┐ + : │Celestial│ │Temporal│ + : └─┬───┬───┘ └───┬────┘ + : │ │ │ + : │ │ │ + : │ │ │ + : ▼ ▼ ▼ + : lon lat time + ``` + + The complexity is in the reverse transform, where the inverse Celestial + transform is also dependent upon the pixel coordinate z. + This means that the output of the inverse Temporal transform has to be + duplicated as an input to the Celestial transform's inverse. + This is achieved by the use of the ``Mapping`` models in + ``CoupledCompoundModel.inverse`` to create a multi-stage compound model + which duplicates the output of the right hand side model:: + + ``` + : lon lat time + : │ │ │ + : │ │ ▼ + : │ │ ┌─────────┐ + : │ │ │Temporal'│ + : │ │ └──┬──┬───┘ + : │ │ z │ │ + : │ │ ┌─────┘ │ + : │ │ │ │ + : ▼ ▼ ▼ │ + : ┌──────────┐ │ + : │Celestial'│ │ + : └─┬───┬────┘ │ + : │ │ │ + : ▼ ▼ ▼ + : x y z + ``` + +examples: + - + - A set of transforms + - asdf-standard-1.6.0 + - | + ! + shared_inputs: 1 + forward: + - !transform/shift-1.3.0 + offset: 2.0 + - !transform/shift-1.3.0 + offset: 3.0 +allOf: + - $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.3.0" + - properties: + forward: + type: array + items: + $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.3.0" + shared_inputs: + type: number + required: [forward, shared_inputs] +... diff --git a/dkist/io/asdf/resources/schemas/ravel_model-1.1.0.yaml b/dkist/io/asdf/resources/schemas/ravel_model-1.1.0.yaml new file mode 100644 index 00000000..05e27bd5 --- /dev/null +++ b/dkist/io/asdf/resources/schemas/ravel_model-1.1.0.yaml @@ -0,0 +1,17 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://dkist.nso.edu/schemas/ravel_model-1.1.0" + +title: A model to flatten 2D indices into 1D +description: + A model which takes a pair of indices and flattens them into the corresponding index for a 1D array. This can be used as a compound with Tabular1D to enable it to be indexed as if it were Tabular2D. + +allOf: + - $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.3.0" + - properties: + array_shape: + type: array + order: + type: string + required: [array_shape, order] diff --git a/dkist/io/asdf/resources/schemas/varying_celestial_transform-1.0.0.yaml b/dkist/io/asdf/resources/schemas/varying_celestial_transform-1.0.0.yaml index 892e8f99..990a0995 100644 --- a/dkist/io/asdf/resources/schemas/varying_celestial_transform-1.0.0.yaml +++ b/dkist/io/asdf/resources/schemas/varying_celestial_transform-1.0.0.yaml @@ -12,11 +12,11 @@ allOf: - properties: crpix: anyOf: - - tag: "core/ndarray-1.*" + - tag: "tag:stsci.edu:asdf/core/ndarray-1.*" - tag: "tag:stsci.edu:asdf/unit/quantity-1.*" cdelt: anyOf: - - tag: "core/ndarray-1.*" + - tag: "tag:stsci.edu:asdf/core/ndarray-1.*" - tag: "tag:stsci.edu:asdf/unit/quantity-1.*" lon_pole: anyOf: @@ -24,11 +24,11 @@ allOf: - tag: "tag:stsci.edu:asdf/unit/quantity-1.*" crval_table: anyOf: - - tag: "core/ndarray-1.*" + - tag: "tag:stsci.edu:asdf/core/ndarray-1.*" - tag: "tag:stsci.edu:asdf/unit/quantity-1.*" pc_table: anyOf: - - tag: "core/ndarray-1.*" + - tag: "tag:stsci.edu:asdf/core/ndarray-1.*" - tag: "tag:stsci.edu:asdf/unit/quantity-1.*" projection: $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.2.0" diff --git a/dkist/io/asdf/resources/schemas/varying_celestial_transform-1.1.0.yaml b/dkist/io/asdf/resources/schemas/varying_celestial_transform-1.1.0.yaml new file mode 100644 index 00000000..bb660a0c --- /dev/null +++ b/dkist/io/asdf/resources/schemas/varying_celestial_transform-1.1.0.yaml @@ -0,0 +1,38 @@ +%YAML 1.1 +--- +$schema: "http://stsci.edu/schemas/yaml-schema/draft-01" +id: "asdf://dkist.nso.edu/schemas/varying_celestial_transform-1.1.0" + +title: A varying FITS-like celestial transform. +description: + A model which represents a FITS-like celestial WCS transform which varies over a third pixel input. + +allOf: + - $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.3.0" + - properties: + crpix: + anyOf: + - tag: "tag:stsci.edu:asdf/core/ndarray-1.*" + - tag: "tag:stsci.edu:asdf/unit/quantity-1.*" + cdelt: + anyOf: + - tag: "tag:stsci.edu:asdf/core/ndarray-1.*" + - tag: "tag:stsci.edu:asdf/unit/quantity-1.*" + lon_pole: + anyOf: + - type: number + - tag: "tag:stsci.edu:asdf/unit/quantity-1.*" + crval_table: + anyOf: + - tag: "tag:stsci.edu:asdf/core/ndarray-1.*" + - tag: "tag:stsci.edu:asdf/unit/quantity-1.*" + pc_table: + anyOf: + - tag: "tag:stsci.edu:asdf/core/ndarray-1.*" + - tag: "tag:stsci.edu:asdf/unit/quantity-1.*" + projection: + $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.3.0" + + required: [crpix, cdelt, lon_pole, crval_table, pc_table, projection] + additionalProperties: true +... diff --git a/pyproject.toml b/pyproject.toml index b3294386..83b13da9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,12 +22,11 @@ dependencies = [ # Some of these schema dependencies are minimums because we generated asdf # files with them unpinned so they are now required to read generated asdf # files. - "asdf>=2.11.2", # Pick up jsonschema bug fix + "asdf>=2.11.2", "asdf-astropy>=0.2.0", "asdf-coordinates-schemas>=0.1.0", "asdf-standard>=1.0.3", "asdf-transform-schemas>=0.3.0", - "asdf-unit-schemas>=0.1.0", "asdf-wcs-schemas>=0.3.0", "astropy>=5.3", "dask[array]>=2021.8.0", diff --git a/tox.ini b/tox.ini index 282c91f2..a05a7de7 100644 --- a/tox.ini +++ b/tox.ini @@ -60,8 +60,12 @@ commands = # !online: {env:PYTEST_COMMAND} {posargs} # Then specify a specific one like this # online: {env:PYTEST_COMMAND} --remote-data=any {posargs} - !online: {env:PYTEST_COMMAND} {posargs} - online: {env:PYTEST_COMMAND} --remote-data=any {posargs} + {env:PYTEST_COMMAND} \ + online: --remote-data=any \ + # It's not possible to test the new schemas with the oldest dependencies + # as the new schemas require new dependent schemas + oldestdeps: -o asdf_schema_tests_enabled=false + {posargs} [testenv:codestyle] pypi_filter =