updates for upcoming ASDF standard 1.6.0

changelog/324.feature.rst

@@ -0,0 +1 @@
+Update ASDF schemas for upcoming ASDF standard 1.6.0.

dkist/io/asdf/converters/models.py

@@ -4,10 +4,8 @@
 
 class VaryingCelestialConverter(TransformConverterBase):
     tags = [
-        "asdf://dkist.nso.edu/tags/varying_celestial_transform-1.1.0",
-        "asdf://dkist.nso.edu/tags/varying_celestial_transform-1.0.0",
-        "asdf://dkist.nso.edu/tags/inverse_varying_celestial_transform-1.1.0",
-        "asdf://dkist.nso.edu/tags/inverse_varying_celestial_transform-1.0.0",
+        "asdf://dkist.nso.edu/tags/varying_celestial_transform-*",
+        "asdf://dkist.nso.edu/tags/inverse_varying_celestial_transform-*",
         # Old slit tags must be kept so we can read old files, but not types as
         # we will not save slit classes any more
         "asdf://dkist.nso.edu/tags/varying_celestial_transform_slit-1.0.0",
@@ -23,10 +21,13 @@
     ]
 
     def select_tag(self, obj, tags, ctx):
+        tag_pattern = "asdf://dkist.nso.edu/tags/varying_celestial_transform"
         if obj._is_inverse:
-            return "asdf://dkist.nso.edu/tags/inverse_varying_celestial_transform-1.1.0"
+            tag_pattern = "asdf://dkist.nso.edu/tags/inverse_varying_celestial_transform"
 
-        return "asdf://dkist.nso.edu/tags/varying_celestial_transform-1.1.0"
+        for tag in tags:
+            if tag.startswith(tag_pattern):
+                return tag
 
     def from_yaml_tree_transform(self, node, tag, ctx):
         from dkist.wcs.models import varying_celestial_transform_from_tables
@@ -74,7 +75,7 @@
     ASDF serialization support for CompoundModel.
     """
     tags = [
-        "asdf://dkist.nso.edu/tags/coupled_compound_model-1.0.0",
+        "asdf://dkist.nso.edu/tags/coupled_compound_model-*",
     ]
 
     types = ["dkist.wcs.models.CoupledCompoundModel"]
@@ -122,7 +123,7 @@
     """
 
     tags  = [
-        "asdf://dkist.nso.edu/tags/ravel_model-1.0.0"
+        "asdf://dkist.nso.edu/tags/ravel_model-*"
     ]
 
     types = ["dkist.wcs.models.Ravel"]
@@ -142,7 +143,7 @@
     """
 
     tags  = [
-        "asdf://dkist.nso.edu/tags/asymmetric_mapping_model-1.0.0"
+        "asdf://dkist.nso.edu/tags/asymmetric_mapping_model-*"
     ]
 
     types = ["dkist.wcs.models.AsymmetricMapping"]

dkist/io/asdf/entry_points.py

@@ -45,6 +45,8 @@ def get_extensions():
                                    converters=dkist_converters),
         ManifestExtension.from_uri("asdf://dkist.nso.edu/manifests/dkist-1.0.0",
                                    converters=dkist_converters),
+        ManifestExtension.from_uri("asdf://dkist.nso.edu/manifests/dkist-wcs-1.3.0",
+                                   converters=wcs_converters),
         ManifestExtension.from_uri("asdf://dkist.nso.edu/manifests/dkist-wcs-1.2.0",
                                    converters=wcs_converters),
         ManifestExtension.from_uri("asdf://dkist.nso.edu/manifests/dkist-wcs-1.1.0",

dkist/io/asdf/resources/manifests/dkist-0.9.0.yaml

@@ -1,6 +1,6 @@
 %YAML 1.1
 ---
-id: asdf://dkist.nso.edu/dkist/manifests/dkist-0.9.0
+id: asdf://dkist.nso.edu/manifests/dkist-0.9.0
 extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-0.9.0
 title: DKIST extension
 description: ASDF schemas and tags for pre-1.0 DKIST tags.

dkist/io/asdf/resources/manifests/dkist-1.0.0.yaml

@@ -1,6 +1,6 @@
 %YAML 1.1
 ---
-id: asdf://dkist.nso.edu/dkist/manifests/dkist-1.0.0
+id: asdf://dkist.nso.edu/manifests/dkist-1.0.0
 extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-1.0.0
 title: DKIST extension
 description: ASDF schemas and tags for DKIST classes.

dkist/io/asdf/resources/manifests/dkist-1.1.0.yaml

@@ -1,7 +1,7 @@
 %YAML 1.1
 ---
-id: asdf://dkist.nso.edu/dkist/manifests/dkist-1.1.0
-extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-1.0.0
+id: asdf://dkist.nso.edu/manifests/dkist-1.1.0
+extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-1.1.0
 title: DKIST extension
 description: ASDF schemas and tags for DKIST classes.
 

dkist/io/asdf/resources/manifests/dkist-1.2.0.yaml

@@ -1,7 +1,7 @@
 %YAML 1.1
 ---
-id: asdf://dkist.nso.edu/dkist/manifests/dkist-1.2.0
-extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-1.0.0
+id: asdf://dkist.nso.edu/manifests/dkist-1.2.0
+extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-1.2.0
 title: DKIST extension
 description: ASDF schemas and tags for DKIST classes.
 

dkist/io/asdf/resources/manifests/dkist-wcs-1.0.0.yaml

@@ -1,6 +1,6 @@
 %YAML 1.1
 ---
-id: asdf://dkist.nso.edu/dkist/manifests/dkist-wcs-1.0.0
+id: asdf://dkist.nso.edu/manifests/dkist-wcs-1.0.0
 extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-wcs-1.0.0
 title: DKIST WCS extension
 description: ASDF schemas and tags for models and WCS related classes.

dkist/io/asdf/resources/manifests/dkist-wcs-1.1.0.yaml

@@ -1,6 +1,6 @@
 %YAML 1.1
 ---
-id: asdf://dkist.nso.edu/dkist/manifests/dkist-wcs-1.1.0
+id: asdf://dkist.nso.edu/manifests/dkist-wcs-1.1.0
 extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-wcs-1.1.0
 title: DKIST WCS extension
 description: ASDF schemas and tags for models and WCS related classes.

dkist/io/asdf/resources/manifests/dkist-wcs-1.2.0.yaml

@@ -1,6 +1,6 @@
 %YAML 1.1
 ---
-id: asdf://dkist.nso.edu/dkist/manifests/dkist-wcs-1.2.0
+id: asdf://dkist.nso.edu/manifests/dkist-wcs-1.2.0
 extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-wcs-1.2.0
 title: DKIST WCS extension
 description: ASDF schemas and tags for models and WCS related classes.

dkist/io/asdf/resources/manifests/dkist-wcs-1.3.0.yaml

@@ -0,0 +1,27 @@
+%YAML 1.1
+---
+id: asdf://dkist.nso.edu/manifests/dkist-wcs-1.3.0
+extension_uri: asdf://dkist.nso.edu/dkist/extensions/dkist-wcs-1.3.0
+asdf_standard_requirement:
+  gte: 1.6.0
+
+title: DKIST WCS extension
+description: ASDF schemas and tags for models and WCS related classes.
+
+tags:
+  # the tag version does not match the schema version
+  - schema_uri: "asdf://dkist.nso.edu/schemas/varying_celestial_transform-1.1.0"
+    tag_uri: "asdf://dkist.nso.edu/tags/varying_celestial_transform-1.2.0"
+
+  # the varying_celestial_transform schema is reused here
+  - schema_uri: "asdf://dkist.nso.edu/schemas/varying_celestial_transform-1.1.0"
+    tag_uri: "asdf://dkist.nso.edu/tags/inverse_varying_celestial_transform-1.2.0"
+
+  - schema_uri: "asdf://dkist.nso.edu/schemas/coupled_compound_model-1.1.0"
+    tag_uri: "asdf://dkist.nso.edu/tags/coupled_compound_model-1.1.0"
+
+  - schema_uri: "asdf://dkist.nso.edu/schemas/ravel_model-1.1.0"
+    tag_uri: "asdf://dkist.nso.edu/tags/ravel_model-1.1.0"
+
+  - schema_uri: "asdf://dkist.nso.edu/schemas/asymmetric_mapping_model-1.1.0"
+    tag_uri: "asdf://dkist.nso.edu/tags/asymmetric_mapping_model-1.1.0"

dkist/io/asdf/resources/schemas/asymmetric_mapping_model-1.1.0.yaml

@@ -0,0 +1,30 @@
+%YAML 1.1
+---
+$schema: "http://stsci.edu/schemas/yaml-schema/draft-01"
+id: "asdf://dkist.nso.edu/schemas/asymmetric_mapping_model-1.1.0"
+title: >
+  Reorder, add and drop axes with different mappings in forward and reverse transforms.
+
+definitions:
+  mapping:
+    type: array
+    items:
+      type: integer
+
+allOf:
+  - $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.3.0"
+  - properties:
+      forward_n_inputs:
+        description: |
+          Explicitly set the number of input axes in the forward direction.
+        type: integer
+      backward_n_inputs:
+        description: |
+          Explicitly set the number of input axes in the backward direction.
+        type: integer
+      forward_mapping:
+        $ref: "#/definitions/mapping"
+      backward_mapping:
+        $ref: "#/definitions/mapping"
+    required: [forward_mapping, backward_mapping]
+...

dkist/io/asdf/resources/schemas/coupled_compound_model-1.1.0.yaml

@@ -0,0 +1,85 @@
+%YAML 1.1
+---
+$schema: "http://stsci.edu/schemas/yaml-schema/draft-01"
+id: "asdf://dkist.nso.edu/schemas/coupled_compound_model-1.1.0"
+title: >
+  Send axes to different subtransforms.
+description: |
+    This transform takes two models which share one or more inputs on the forward
+    transform, and where the left hand model's inverse is dependent on the
+    output of the right hand model's inverse output.
+
+    Take the following example with a time dependent celestial transform
+    (modelled as dependent upon the pixel coordinate for time rather than the
+    world coordinate).
+
+    The forward transform uses the "z" pixel dimension as input to both the
+    Celestial and Temporal models, this leads to the following transform in the
+    forward direction:
+
+    ```
+      :    x  y           z
+      :    │  │           │
+      :    │  │  ┌────────┤
+      :    │  │  │        │
+      :    ▼  ▼  ▼        ▼
+      :  ┌─────────┐  ┌────────┐
+      :  │Celestial│  │Temporal│
+      :  └─┬───┬───┘  └───┬────┘
+      :    │   │          │
+      :    │   │          │
+      :    │   │          │
+      :    ▼   ▼          ▼
+      :   lon lat       time
+    ```
+
+    The complexity is in the reverse transform, where the inverse Celestial
+    transform is also dependent upon the pixel coordinate z.
+    This means that the output of the inverse Temporal transform has to be
+    duplicated as an input to the Celestial transform's inverse.
+    This is achieved by the use of the ``Mapping`` models in
+    ``CoupledCompoundModel.inverse`` to create a multi-stage compound model
+    which duplicates the output of the right hand side model::
+
+    ```
+      :   lon lat       time
+      :    │   │         │
+      :    │   │         ▼
+      :    │   │     ┌─────────┐
+      :    │   │     │Temporal'│
+      :    │   │     └──┬──┬───┘
+      :    │   │    z   │  │
+      :    │   │  ┌─────┘  │
+      :    │   │  │        │
+      :    ▼   ▼  ▼        │
+      :  ┌──────────┐      │
+      :  │Celestial'│      │
+      :  └─┬───┬────┘      │
+      :    │   │           │
+      :    ▼   ▼           ▼
+      :    x   y           z
+    ```
+
+examples:
+  -
+    - A set of transforms
+    - asdf-standard-1.6.0
+    - |
+      !<asdf://dkist.nso.edu/tags/coupled_compound_model-1.1.0>
+        shared_inputs: 1
+        forward:
+          - !transform/shift-1.3.0
+            offset: 2.0
+          - !transform/shift-1.3.0
+            offset: 3.0
+allOf:
+  - $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.3.0"
+  - properties:
+      forward:
+        type: array
+        items:
+          $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.3.0"
+      shared_inputs:
+        type: number
+    required: [forward, shared_inputs]
+...

dkist/io/asdf/resources/schemas/ravel_model-1.1.0.yaml

@@ -0,0 +1,17 @@
+%YAML 1.1
+---
+$schema: "http://stsci.edu/schemas/yaml-schema/draft-01"
+id: "asdf://dkist.nso.edu/schemas/ravel_model-1.1.0"
+
+title: A model to flatten 2D indices into 1D
+description:
+  A model which takes a pair of indices and flattens them into the corresponding index for a 1D array. This can be used as a compound with Tabular1D to enable it to be indexed as if it were Tabular2D.
+
+allOf:
+  - $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.3.0"
+  - properties:
+      array_shape:
+        type: array
+      order:
+        type: string
+    required: [array_shape, order]

dkist/io/asdf/resources/schemas/varying_celestial_transform-1.0.0.yaml

@@ -12,23 +12,23 @@ allOf:
   - properties:
       crpix:
         anyOf:
-          - tag: "core/ndarray-1.*"
+          - tag: "tag:stsci.edu:asdf/core/ndarray-1.*"
           - tag: "tag:stsci.edu:asdf/unit/quantity-1.*"
       cdelt:
         anyOf:
-          - tag: "core/ndarray-1.*"
+          - tag: "tag:stsci.edu:asdf/core/ndarray-1.*"
           - tag: "tag:stsci.edu:asdf/unit/quantity-1.*"
       lon_pole:
         anyOf:
           - type: number
           - tag: "tag:stsci.edu:asdf/unit/quantity-1.*"
       crval_table:
         anyOf:
-          - tag: "core/ndarray-1.*"
+          - tag: "tag:stsci.edu:asdf/core/ndarray-1.*"
           - tag: "tag:stsci.edu:asdf/unit/quantity-1.*"
       pc_table:
         anyOf:
-          - tag: "core/ndarray-1.*"
+          - tag: "tag:stsci.edu:asdf/core/ndarray-1.*"
           - tag: "tag:stsci.edu:asdf/unit/quantity-1.*"
       projection:
         $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.2.0"

dkist/io/asdf/resources/schemas/varying_celestial_transform-1.1.0.yaml

@@ -0,0 +1,38 @@
+%YAML 1.1
+---
+$schema: "http://stsci.edu/schemas/yaml-schema/draft-01"
+id: "asdf://dkist.nso.edu/schemas/varying_celestial_transform-1.1.0"
+
+title: A varying FITS-like celestial transform.
+description:
+  A model which represents a FITS-like celestial WCS transform which varies over a third pixel input.
+
+allOf:
+  - $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.3.0"
+  - properties:
+      crpix:
+        anyOf:
+          - tag: "tag:stsci.edu:asdf/core/ndarray-1.*"
+          - tag: "tag:stsci.edu:asdf/unit/quantity-1.*"
+      cdelt:
+        anyOf:
+          - tag: "tag:stsci.edu:asdf/core/ndarray-1.*"
+          - tag: "tag:stsci.edu:asdf/unit/quantity-1.*"
+      lon_pole:
+        anyOf:
+          - type: number
+          - tag: "tag:stsci.edu:asdf/unit/quantity-1.*"
+      crval_table:
+        anyOf:
+          - tag: "tag:stsci.edu:asdf/core/ndarray-1.*"
+          - tag: "tag:stsci.edu:asdf/unit/quantity-1.*"
+      pc_table:
+        anyOf:
+          - tag: "tag:stsci.edu:asdf/core/ndarray-1.*"
+          - tag: "tag:stsci.edu:asdf/unit/quantity-1.*"
+      projection:
+        $ref: "http://stsci.edu/schemas/asdf/transform/transform-1.3.0"
+
+    required: [crpix, cdelt, lon_pole, crval_table, pc_table, projection]
+    additionalProperties: true
+...

pyproject.toml

@@ -22,12 +22,11 @@ dependencies = [
   # Some of these schema dependencies are minimums because we generated asdf
   # files with them unpinned so they are now required to read generated asdf
   # files.
-  "asdf>=2.11.2",  # Pick up jsonschema bug fix
+  "asdf>=2.11.2",
   "asdf-astropy>=0.2.0",
   "asdf-coordinates-schemas>=0.1.0",
   "asdf-standard>=1.0.3",
   "asdf-transform-schemas>=0.3.0",
-  "asdf-unit-schemas>=0.1.0",
   "asdf-wcs-schemas>=0.3.0",
   "astropy>=5.3",
   "dask[array]>=2021.8.0",

tox.ini

@@ -60,8 +60,12 @@ commands =
     # !online: {env:PYTEST_COMMAND} {posargs}
     # Then specify a specific one like this
     # online: {env:PYTEST_COMMAND} --remote-data=any {posargs}
-    !online: {env:PYTEST_COMMAND} {posargs}
-    online: {env:PYTEST_COMMAND} --remote-data=any {posargs}
+    {env:PYTEST_COMMAND} \
+    online: --remote-data=any \
+    # It's not possible to test the new schemas with the oldest dependencies
+    # as the new schemas require new dependent schemas
+    oldestdeps: -o asdf_schema_tests_enabled=false
+    {posargs}
 
 [testenv:codestyle]
 pypi_filter =