From 76e6b10d43b2c601f1b15eff600dae9a55438a69 Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 1 Feb 2024 11:47:26 -0500 Subject: [PATCH 01/13] split core --- docs/source/schemas/core.rst | 5 +- docs/source/schemas/index.rst | 6 +- docs/source/schemas/legacy.rst | 19 ++-- docs/source/schemas/table.rst | 11 +++ .../asdf-format.org/core/core-1.6.0.yaml | 95 ------------------- .../asdf-format.org/fits/fits-1.0.0.yaml | 22 +++++ .../asdf-format.org/table/table-1.0.0.yaml | 27 ++++++ .../asdf-format.org/time/time-1.0.0.yaml | 38 ++++++++ .../asdf-format.org/unit/unit-1.2.0.yaml | 30 ++++++ .../stsci.edu/asdf/fits/fits-1.1.0.yaml | 2 +- .../asdf/{core => table}/column-1.1.0.yaml | 6 +- .../asdf/{core => table}/table-1.1.0.yaml | 20 ++-- .../stsci.edu/asdf/version_map-1.6.0.yaml | 8 -- src/asdf_standard/integration.py | 16 ++++ tests/common.py | 12 ++- 15 files changed, 185 insertions(+), 132 deletions(-) create mode 100644 docs/source/schemas/table.rst create mode 100644 resources/manifests/asdf-format.org/fits/fits-1.0.0.yaml create mode 100644 resources/manifests/asdf-format.org/table/table-1.0.0.yaml create mode 100644 resources/manifests/asdf-format.org/time/time-1.0.0.yaml create mode 100644 resources/manifests/asdf-format.org/unit/unit-1.2.0.yaml rename resources/schemas/stsci.edu/asdf/{core => table}/column-1.1.0.yaml (83%) rename resources/schemas/stsci.edu/asdf/{core => table}/table-1.1.0.yaml (88%) diff --git a/docs/source/schemas/core.rst b/docs/source/schemas/core.rst index 36c49686..ecbc8b02 100644 --- a/docs/source/schemas/core.rst +++ b/docs/source/schemas/core.rst @@ -10,13 +10,10 @@ asdf library. core/asdf-1.1.0 core/complex-1.0.0 + core/extension_metadata-1.0.0 core/ndarray-1.1.0 - core/table-1.1.0 - core/column-1.1.0 core/constant-1.0.0 core/software-1.0.0 core/history_entry-1.0.0 - core/extension_metadata-1.0.0 core/integer-1.1.0 core/externalarray-1.0.0 - core/subclass_metadata-1.0.0 diff --git a/docs/source/schemas/index.rst b/docs/source/schemas/index.rst index 5cbcd45a..1265b1c7 100644 --- a/docs/source/schemas/index.rst +++ b/docs/source/schemas/index.rst @@ -20,6 +20,7 @@ must support the "core" module, but the other modules are optional. fits.rst unit.rst time.rst + table.rst legacy.rst The ASDF Standard also defines two meta-schemas that are used for validating @@ -42,7 +43,10 @@ The following graph shows the dependencies between modules: "fits" -> "core" "unit" -> "core" "time" -> "core" - "core" -> "unit" + "table" -> "core" + "time" -> "unit" + "table" -> "unit" + "fits" -> "table" .. _manifest: diff --git a/docs/source/schemas/legacy.rst b/docs/source/schemas/legacy.rst index 2f8ac47c..4feb9e8d 100644 --- a/docs/source/schemas/legacy.rst +++ b/docs/source/schemas/legacy.rst @@ -7,12 +7,13 @@ The following legacy schemas and not part of the most recent ASDF standard versi .. asdf-autoschemas:: - core/asdf-1.0.0 - core/column-1.0.0 - core/integer-1.0.0 - core/ndarray-1.0.0 - core/table-1.0.0 - fits/fits-1.0.0 - time/time-1.0.0 - time/time-1.1.0 - unit/quantity-1.1.0 + core/asdf-1.0.0 + core/column-1.0.0 + core/integer-1.0.0 + core/ndarray-1.0.0 + core/subclass_metadata-1.0.0 + core/table-1.0.0 + fits/fits-1.0.0 + time/time-1.0.0 + time/time-1.1.0 + unit/quantity-1.1.0 diff --git a/docs/source/schemas/table.rst b/docs/source/schemas/table.rst new file mode 100644 index 00000000..268fd4fb --- /dev/null +++ b/docs/source/schemas/table.rst @@ -0,0 +1,11 @@ +.. _unit-schema: + +Unit +==== + +The ``table`` module contains schema to support tabular data. + +.. asdf-autoschemas:: + + table/column-1.1.0 + table/table-1.1.0 diff --git a/resources/manifests/asdf-format.org/core/core-1.6.0.yaml b/resources/manifests/asdf-format.org/core/core-1.6.0.yaml index 8f264b1c..225edd80 100644 --- a/resources/manifests/asdf-format.org/core/core-1.6.0.yaml +++ b/resources/manifests/asdf-format.org/core/core-1.6.0.yaml @@ -8,12 +8,6 @@ tags: schema_uri: http://stsci.edu/schemas/asdf/core/asdf-1.1.0 title: Top-level schema for every ASDF file. description: This schema contains the top-level attributes for every ASDF file. -- tag_uri: tag:stsci.edu:asdf/core/column-1.1.0 - schema_uri: http://stsci.edu/schemas/asdf/core/column-1.1.0 - title: A column in a table. - description: |- - Each column contains a name and an array of data, and an optional description - and unit. - tag_uri: tag:stsci.edu:asdf/core/complex-1.0.0 schema_uri: http://stsci.edu/schemas/asdf/core/complex-1.0.0 title: Complex number value. @@ -88,92 +82,3 @@ tags: schema_uri: http://stsci.edu/schemas/asdf/core/software-1.0.0 title: Describes a software package. description: General-purpose description of a software package. -- tag_uri: tag:stsci.edu:asdf/core/subclass_metadata-1.0.0 - schema_uri: http://stsci.edu/schemas/asdf/core/subclass_metadata-1.0.0 - title: Metadata on a serialized subclass of an ASDF-enabled type. - description: |- - Identifies the specific subclass that was serialized, - to enable ASDF readers to correctly deserialize the object. -- tag_uri: tag:stsci.edu:asdf/core/table-1.1.0 - schema_uri: http://stsci.edu/schemas/asdf/core/table-1.1.0 - title: A table. - description: |- - A table is represented as a list of columns, where each entry is a - [column](ref:core/column-1.1.0) - object, containing the data and some additional information. - - The data itself may be stored inline as text, or in binary in either - row- or column-major order by use of the `strides` property on the - individual column arrays. - - Each column in the table must have the same first (slowest moving) - dimension. -- tag_uri: tag:stsci.edu:asdf/fits/fits-1.1.0 - schema_uri: http://stsci.edu/schemas/asdf/fits/fits-1.1.0 - title: A FITS file inside of an ASDF file. - description: |- - This schema is useful for distributing ASDF files that can - automatically be converted to FITS files by specifying the exact - content of the resulting FITS file. - - Not all kinds of data in FITS are directly representable in ASDF. - For example, applying an offset and scale to the data using the - `BZERO` and `BSCALE` keywords. In these cases, it will not be - possible to store the data in the native format from FITS and also - be accessible in its proper form in the ASDF file. - - Only image and binary table extensions are supported. -- tag_uri: tag:stsci.edu:asdf/time/time-1.2.0 - schema_uri: http://stsci.edu/schemas/asdf/time/time-1.2.0 - title: Represents an instance in time. - description: |- - A "time" is a single instant in time. It may explicitly specify the - way time is represented (the "format") and the "scale" which - specifies the offset and scaling relation of the unit of time. - - Specific emphasis is placed on supporting time scales (e.g. UTC, - TAI, UT1, TDB) and time representations (e.g. JD, MJD, ISO 8601) - that are used in astronomy and required to calculate, e.g., sidereal - times and barycentric corrections. - - Times may be represented as one of the following: - - - an object, with explicit `value`, and optional `format`, `scale` - and `location`. - - - a string, in which case the format is guessed from across - the unambiguous options (`iso`, `byear`, `jyear`, `yday`), and the - scale is hardcoded to `UTC`. - - In either case, a single time tag may be used to represent an - n-dimensional array of times, using either an `ndarray` tag or - inline as (possibly nested) YAML lists. If YAML lists, the same - format must be used for all time values. - - The precision of the numeric formats should only be assumed to be as - good as an IEEE-754 double precision (float64) value. If - higher-precision is required, the `iso` or `yday` format should be - used. -- tag_uri: tag:stsci.edu:asdf/unit/defunit-1.0.0 - schema_uri: http://stsci.edu/schemas/asdf/unit/defunit-1.0.0 - title: Define a new physical unit. - description: |- - Defines a new unit. It can be used to either: - - - Define a new base unit. - - - Create a new unit name that is a equivalent to a given unit. - - The new unit must be defined before any unit tags that use it. -- tag_uri: tag:stsci.edu:asdf/unit/quantity-1.2.0 - schema_uri: http://stsci.edu/schemas/asdf/unit/quantity-1.2.0 - title: Represents a Quantity object from astropy - description: |- - A Quantity object represents a value that has some unit - associated with the number. -- tag_uri: tag:stsci.edu:asdf/unit/unit-1.0.0 - schema_uri: http://stsci.edu/schemas/asdf/unit/unit-1.0.0 - title: Physical unit. - description: |- - This represents a physical unit, in [VOUnit syntax, Version 1.0](http://www.ivoa.net/documents/VOUnits/index.html). - Where units are not explicitly tagged, they are assumed to be in VOUnit syntax. diff --git a/resources/manifests/asdf-format.org/fits/fits-1.0.0.yaml b/resources/manifests/asdf-format.org/fits/fits-1.0.0.yaml new file mode 100644 index 00000000..2f0e93fd --- /dev/null +++ b/resources/manifests/asdf-format.org/fits/fits-1.0.0.yaml @@ -0,0 +1,22 @@ +id: asdf://asdf-format.org/fits/manifests/fits-1.0.0 +extension_uri: asdf://asdf-format.org/fits/extensions/fits-1.1.0 +title: Fits extension 1.0.0 +description: Tags FITS objects. +asdf_standard_requirement: + gte: 1.6.0 +tags: +- tag_uri: tag:stsci.edu:asdf/fits/fits-1.1.0 + schema_uri: http://stsci.edu/schemas/asdf/fits/fits-1.1.0 + title: A FITS file inside of an ASDF file. + description: |- + This schema is useful for distributing ASDF files that can + automatically be converted to FITS files by specifying the exact + content of the resulting FITS file. + + Not all kinds of data in FITS are directly representable in ASDF. + For example, applying an offset and scale to the data using the + `BZERO` and `BSCALE` keywords. In these cases, it will not be + possible to store the data in the native format from FITS and also + be accessible in its proper form in the ASDF file. + + Only image and binary table extensions are supported. diff --git a/resources/manifests/asdf-format.org/table/table-1.0.0.yaml b/resources/manifests/asdf-format.org/table/table-1.0.0.yaml new file mode 100644 index 00000000..d8db2027 --- /dev/null +++ b/resources/manifests/asdf-format.org/table/table-1.0.0.yaml @@ -0,0 +1,27 @@ +id: asdf://asdf-format.org/table/manifests/table-1.0.0 +extension_uri: asdf://asdf-format.org/table/extensions/table-1.0.0 +title: Table extension 1.0.0 +description: Tags for ASDF table objects. +asdf_standard_requirement: + gte: 1.6.0 +tags: +- tag_uri: tag:stsci.edu:asdf/table/column-1.1.0 + schema_uri: http://stsci.edu/schemas/asdf/table/column-1.1.0 + title: A column in a table. + description: |- + Each column contains a name and an array of data, and an optional description + and unit. +- tag_uri: tag:stsci.edu:asdf/table/table-1.1.0 + schema_uri: http://stsci.edu/schemas/asdf/table/table-1.1.0 + title: A table. + description: |- + A table is represented as a list of columns, where each entry is a + [column](ref:table/column-1.1.0) + object, containing the data and some additional information. + + The data itself may be stored inline as text, or in binary in either + row- or column-major order by use of the `strides` property on the + individual column arrays. + + Each column in the table must have the same first (slowest moving) + dimension. diff --git a/resources/manifests/asdf-format.org/time/time-1.0.0.yaml b/resources/manifests/asdf-format.org/time/time-1.0.0.yaml new file mode 100644 index 00000000..370e368e --- /dev/null +++ b/resources/manifests/asdf-format.org/time/time-1.0.0.yaml @@ -0,0 +1,38 @@ +id: asdf://asdf-format.org/time/manifests/time-1.0.0 +extension_uri: asdf://asdf-format.org/time/extensions/time-1.0.0 +title: Time extension 1.0.0 +description: Tags for ASDF time objects. +asdf_standard_requirement: + gte: 1.6.0 +tags: +- tag_uri: tag:stsci.edu:asdf/time/time-1.2.0 + schema_uri: http://stsci.edu/schemas/asdf/time/time-1.2.0 + title: Represents an instance in time. + description: |- + A "time" is a single instant in time. It may explicitly specify the + way time is represented (the "format") and the "scale" which + specifies the offset and scaling relation of the unit of time. + + Specific emphasis is placed on supporting time scales (e.g. UTC, + TAI, UT1, TDB) and time representations (e.g. JD, MJD, ISO 8601) + that are used in astronomy and required to calculate, e.g., sidereal + times and barycentric corrections. + + Times may be represented as one of the following: + + - an object, with explicit `value`, and optional `format`, `scale` + and `location`. + + - a string, in which case the format is guessed from across + the unambiguous options (`iso`, `byear`, `jyear`, `yday`), and the + scale is hardcoded to `UTC`. + + In either case, a single time tag may be used to represent an + n-dimensional array of times, using either an `ndarray` tag or + inline as (possibly nested) YAML lists. If YAML lists, the same + format must be used for all time values. + + The precision of the numeric formats should only be assumed to be as + good as an IEEE-754 double precision (float64) value. If + higher-precision is required, the `iso` or `yday` format should be + used. diff --git a/resources/manifests/asdf-format.org/unit/unit-1.2.0.yaml b/resources/manifests/asdf-format.org/unit/unit-1.2.0.yaml new file mode 100644 index 00000000..e5ad5ca5 --- /dev/null +++ b/resources/manifests/asdf-format.org/unit/unit-1.2.0.yaml @@ -0,0 +1,30 @@ +id: asdf://asdf-format.org/unit/manifests/unit-1.2.0 +extension_uri: asdf://asdf-format.org/unit/extensions/unit-1.2.0 +title: Unit extension 1.2.0 +description: Tags for ASDF unit objects. +asdf_standard_requirement: + gte: 1.6.0 +tags: +- tag_uri: tag:stsci.edu:asdf/unit/defunit-1.0.0 + schema_uri: http://stsci.edu/schemas/asdf/unit/defunit-1.0.0 + title: Define a new physical unit. + description: |- + Defines a new unit. It can be used to either: + + - Define a new base unit. + + - Create a new unit name that is a equivalent to a given unit. + + The new unit must be defined before any unit tags that use it. +- tag_uri: tag:stsci.edu:asdf/unit/quantity-1.2.0 + schema_uri: http://stsci.edu/schemas/asdf/unit/quantity-1.2.0 + title: Represents a Quantity object from astropy + description: |- + A Quantity object represents a value that has some unit + associated with the number. +- tag_uri: tag:stsci.edu:asdf/unit/unit-1.0.0 + schema_uri: http://stsci.edu/schemas/asdf/unit/unit-1.0.0 + title: Physical unit. + description: |- + This represents a physical unit, in [VOUnit syntax, Version 1.0](http://www.ivoa.net/documents/VOUnits/index.html). + Where units are not explicitly tagged, they are assumed to be in VOUnit syntax. diff --git a/resources/schemas/stsci.edu/asdf/fits/fits-1.1.0.yaml b/resources/schemas/stsci.edu/asdf/fits/fits-1.1.0.yaml index 15407224..fb5ed9eb 100644 --- a/resources/schemas/stsci.edu/asdf/fits/fits-1.1.0.yaml +++ b/resources/schemas/stsci.edu/asdf/fits/fits-1.1.0.yaml @@ -106,7 +106,7 @@ items: description: "The data part of the HDU." anyOf: - $ref: "../core/ndarray-1.1.0" - - $ref: "../core/table-1.1.0" + - $ref: "../table/table-1.1.0" - type: "null" default: null required: [header] diff --git a/resources/schemas/stsci.edu/asdf/core/column-1.1.0.yaml b/resources/schemas/stsci.edu/asdf/table/column-1.1.0.yaml similarity index 83% rename from resources/schemas/stsci.edu/asdf/core/column-1.1.0.yaml rename to resources/schemas/stsci.edu/asdf/table/column-1.1.0.yaml index a60fcaca..18802632 100644 --- a/resources/schemas/stsci.edu/asdf/core/column-1.1.0.yaml +++ b/resources/schemas/stsci.edu/asdf/table/column-1.1.0.yaml @@ -1,7 +1,7 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "http://stsci.edu/schemas/asdf/core/column-1.1.0" +id: "http://stsci.edu/schemas/asdf/table/column-1.1.0" title: > A column in a table. @@ -15,7 +15,7 @@ properties: name: description: | The name of the column. Each name in a - [table](http://stsci.edu/schemas/asdf/core/table-1.1.0) must be + [table](http://stsci.edu/schemas/asdf/table/table-1.1.0) must be unique. type: string pattern: "[A-Za-z_][A-Za-z0-9_]*" @@ -24,7 +24,7 @@ properties: description: | The array data for the column. allOf: - - $ref: ndarray-1.1.0 + - $ref: ../core/ndarray-1.1.0 description: description: | diff --git a/resources/schemas/stsci.edu/asdf/core/table-1.1.0.yaml b/resources/schemas/stsci.edu/asdf/table/table-1.1.0.yaml similarity index 88% rename from resources/schemas/stsci.edu/asdf/core/table-1.1.0.yaml rename to resources/schemas/stsci.edu/asdf/table/table-1.1.0.yaml index cc422d4c..a59e8d8e 100644 --- a/resources/schemas/stsci.edu/asdf/core/table-1.1.0.yaml +++ b/resources/schemas/stsci.edu/asdf/table/table-1.1.0.yaml @@ -1,14 +1,14 @@ %YAML 1.1 --- $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -id: "http://stsci.edu/schemas/asdf/core/table-1.1.0" +id: "http://stsci.edu/schemas/asdf/table/table-1.1.0" title: > A table. description: | A table is represented as a list of columns, where each entry is a - [column](ref:core/column-1.1.0) + [column](ref:table/column-1.1.0) object, containing the data and some additional information. The data itself may be stored inline as text, or in binary in either @@ -23,9 +23,9 @@ examples: - A table stored in column-major order, with each column in a separate block - asdf-standard-1.6.0 - | - !core/table-1.1.0 + !table/table-1.1.0 columns: - - !core/column-1.1.0 + - !table/column-1.1.0 data: !core/ndarray-1.1.0 source: 0 datatype: float64 @@ -35,7 +35,7 @@ examples: meta: {foo: bar} name: a unit: !unit/unit-1.0.0 deg - - !core/column-1.1.0 + - !table/column-1.1.0 data: !core/ndarray-1.1.0 source: 1 datatype: float64 @@ -43,7 +43,7 @@ examples: shape: [3] description: DEC name: b - - !core/column-1.1.0 + - !table/column-1.1.0 data: !core/ndarray-1.1.0 source: 2 datatype: [ascii, 1] @@ -56,9 +56,9 @@ examples: - A table stored in row-major order, all stored in the same block - asdf-standard-1.6.0 - | - !core/table-1.1.0 + !table/table-1.1.0 columns: - - !core/column-1.1.0 + - !table/column-1.1.0 data: !core/ndarray-1.1.0 source: 0 datatype: float64 @@ -69,7 +69,7 @@ examples: meta: {foo: bar} name: a unit: !unit/unit-1.0.0 deg - - !core/column-1.1.0 + - !table/column-1.1.0 data: !core/ndarray-1.1.0 source: 0 datatype: float64 @@ -79,7 +79,7 @@ examples: strides: [13] description: DEC name: b - - !core/column-1.1.0 + - !table/column-1.1.0 data: !core/ndarray-1.1.0 source: 0 datatype: [ascii, 1] diff --git a/resources/schemas/stsci.edu/asdf/version_map-1.6.0.yaml b/resources/schemas/stsci.edu/asdf/version_map-1.6.0.yaml index dc1920ea..818f741a 100644 --- a/resources/schemas/stsci.edu/asdf/version_map-1.6.0.yaml +++ b/resources/schemas/stsci.edu/asdf/version_map-1.6.0.yaml @@ -4,7 +4,6 @@ FILE_FORMAT: 1.0.0 YAML_VERSION: "1.1" tags: tag:stsci.edu:asdf/core/asdf: 1.1.0 - tag:stsci.edu:asdf/core/column: 1.1.0 tag:stsci.edu:asdf/core/complex: 1.0.0 tag:stsci.edu:asdf/core/constant: 1.0.0 tag:stsci.edu:asdf/core/extension_metadata: 1.0.0 @@ -13,11 +12,4 @@ tags: tag:stsci.edu:asdf/core/integer: 1.1.0 tag:stsci.edu:asdf/core/ndarray: 1.1.0 tag:stsci.edu:asdf/core/software: 1.0.0 - tag:stsci.edu:asdf/core/subclass_metadata: 1.0.0 - tag:stsci.edu:asdf/core/table: 1.1.0 - tag:stsci.edu:asdf/fits/fits: 1.1.0 - tag:stsci.edu:asdf/time/time: 1.2.0 - tag:stsci.edu:asdf/unit/defunit: 1.0.0 - tag:stsci.edu:asdf/unit/quantity: 1.2.0 - tag:stsci.edu:asdf/unit/unit: 1.0.0 ... diff --git a/src/asdf_standard/integration.py b/src/asdf_standard/integration.py index 3b90b06f..b3a8eb9e 100644 --- a/src/asdf_standard/integration.py +++ b/src/asdf_standard/integration.py @@ -22,4 +22,20 @@ def get_resource_mappings(): resources_root / "manifests" / "asdf-format.org" / "core", "asdf://asdf-format.org/core/manifests/", ), + asdf_standard.DirectoryResourceMapping( + resources_root / "manifests" / "asdf-format.org" / "unit", + "asdf://asdf-format.org/unit/manifests/", + ), + asdf_standard.DirectoryResourceMapping( + resources_root / "manifests" / "asdf-format.org" / "table", + "asdf://asdf-format.org/table/manifests/", + ), + asdf_standard.DirectoryResourceMapping( + resources_root / "manifests" / "asdf-format.org" / "time", + "asdf://asdf-format.org/time/manifests/", + ), + asdf_standard.DirectoryResourceMapping( + resources_root / "manifests" / "asdf-format.org" / "fits", + "asdf://asdf-format.org/fits/manifests/", + ), ] diff --git a/tests/common.py b/tests/common.py index 5528ab8a..5ef19c0a 100644 --- a/tests/common.py +++ b/tests/common.py @@ -23,7 +23,17 @@ VALID_SCHEMA_FILENAME_RE = re.compile(r"[a-z0-9_]+-[0-9]+\.[0-9]+\.[0-9]+\.yaml") -DEPRECATED_PATTERNS = {re.compile(".*/transform/.*"), re.compile(".*/wcs/.*")} +DEPRECATED_PATTERNS = { + re.compile(".*/transform/.*"), + re.compile(".*/wcs/.*"), + re.compile(".*/unit/.*"), + re.compile(".*/table/.*"), + re.compile(".*/fits/.*"), + re.compile(".*/time/.*"), + re.compile(".*/core/column.*"), + re.compile(".*/core/table.*"), + re.compile(".*/core/subclass_metadata.*"), +} METASCHEMA_ID = "http://stsci.edu/schemas/yaml-schema/draft-01" From 64342ee2429af369c453a52f5d58a3eddb7ff025 Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 1 Feb 2024 11:48:06 -0500 Subject: [PATCH 02/13] TODO skip docs test --- tests/test_docs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_docs.py b/tests/test_docs.py index 707ea4e4..f9303d70 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,14 +1,17 @@ import collections +import pytest from common import is_deprecated EXCEPTIONS = { "tag:stsci.edu:asdf/asdf-schema-1.0.0", "asdf://asdf-format.org/core/manifests/:schema_root: ../../resources/manifests", "asdf://asdf-format.org/core/manifests/:standard_prefix: asdf-format.org/core", + "tag:stsci.edu:asdf/asdf-schema-1.0.0", } +@pytest.mark.skip def test_docs_schema_links( latest_schema_ids, legacy_schema_ids, manifest_ids, docs_schema_ids, docs_legacy_schema_ids, docs_manifest_ids ): From 4a86cf0eac64bf9a173dd5de4a3978dac5d45156 Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 1 Feb 2024 18:04:25 -0500 Subject: [PATCH 03/13] wip schema inspection script --- scripts/lookup_extensions.py | 158 +++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 scripts/lookup_extensions.py diff --git a/scripts/lookup_extensions.py b/scripts/lookup_extensions.py new file mode 100644 index 00000000..d4afbefa --- /dev/null +++ b/scripts/lookup_extensions.py @@ -0,0 +1,158 @@ +import asdf + +cfg = asdf.get_config() +resource_manager = cfg.resource_manager +all_extensions = cfg.extensions + +all_standard_versions = [str(v) for v in asdf.versioning.supported_versions] + +all_schemas = {} +all_manifests = {} +manifests_by_extension_uri = {} +schema_uris_by_package_name = {} +schema_links_by_uri = {} + +for uri in resource_manager: + schema = asdf.schema.load_schema(uri) + if "id" not in schema: + # FIXME version maps don't have id + assert "FILE_FORMAT" in schema + else: + # FIXME not sure why the uri/id here has a fragment + if not uri.startswith("http://json-schema.org/draft-04/schema"): + assert uri == schema["id"], (uri, schema["id"]) # sanity check + if uri in all_schemas: + # don't duplicate uris + raise Exception + all_schemas[uri] = schema + if "extension_uri" in schema: + all_manifests[uri] = schema + extension_uri = schema["extension_uri"] + # 2 manifests should not share uris + if extension_uri in manifests_by_extension_uri: + # TODO helpful error + raise Exception + manifests_by_extension_uri[extension_uri] = schema + + # record what packages provide schemas (so we can check if this changes) + package_name = resource_manager._mappings_by_uri[uri].package_name + if package_name not in schema_uris_by_package_name: + schema_uris_by_package_name[package_name] = [] + schema_uris_by_package_name[package_name].append(uri) + + # get all external references for this schema ($ref, $schema, tag) + links = [] + search = [(schema, ())] + while search: + item, path = search.pop() + + # step into item + if isinstance(item, list): + children = enumerate(item) + elif isinstance(item, dict): + # top level $schema + if not path and "$schema" in item: + links.append({"schema": item["$schema"], "path": path}) + + if "$ref" in item: + links.append({"ref": item["$ref"], "path": path}) + children = [] + elif "tag" in item: + links.append({"tag": item["tag"], "path": path}) + children = [] + else: + children = item.items() + else: + children = [] + if children: + for key, child in children: + if isinstance(child, (list, dict)): + search.append((child, path + (key,))) + schema_links_by_uri[uri] = links + + +# we want maps (per version because of the tag use) for +# - TODO schema to schema (via $ref, $schema, tag) +# - schema to tag +# - tag to type [this isn't one-to-one] +# - TODO extension to schema (via tag) +# - extension to tag + +tag_info_by_version = {} + +for standard_version in all_standard_versions: + # the extension manager allows tags/type to be overridden + # track these separately + extension_manager = asdf.AsdfFile(version=standard_version).extension_manager + + # also consider all extensions (even ones overridden) + extensions = [e for e in all_extensions if standard_version in e.asdf_standard_requirement] + + # by uri so each entry here corresponds to a tag used in this version + tag_info_by_uri = {} + + # look up all extensions for this version that define tags + for extension in extensions: + for tag_def in extension.tags: + # multiple extensions might define tags + if tag_def.tag_uri in tag_info_by_uri: + tag_info = tag_info_by_uri[tag_def.tag_uri] + else: + tag_info = { + "extension_uris": [], + "schema_uris_by_extension_uri": {}, + } + tag_info["extension_uris"].append(extension.extension_uri) + tag_info["schema_uris_by_extension_uri"][extension.extension_uri] = tag_def.schema_uris + tag_info_by_uri[tag_def.tag_uri] = tag_info + + for tag_uri in tag_info_by_uri: + tag_info = tag_info_by_uri[tag_uri] + try: + converter = extension_manager.get_converter_for_tag(tag_uri) + except KeyError: + # some tags that are defined in extensions do not have converters + # one example is "label_mapper". Mark these as not supported. + tag_info["supported"] = False + continue + tag_info["supported"] = True + tag_info["handling_extension_uri"] = converter._extension.extension_uri + tag_info["types"] = [] + for typ in converter.types: + if isinstance(typ, str): + type_string = typ + else: + type_string = asdf.util.get_class_name(typ, False) + tag_info["types"].append(type_string) + + # extensions and manifests do not have a 1-to-1 mapping because of asdf-astropy CompoundManifest + # but that doesn't really matter as long as the extension_uri is unique and all the ones + # defined in manifests are used, right? The id is used to load the manifest which defines + # the extension_uri. This is 'compounded' in astropy where the core manifest is combined + # with a special 'astropy' one (which is the one that should define the extension_uri). + + # extension_by_manifest_id = {} + # for ext in all_extensions: + # delegate = ext.delegate + # if hasattr(delegate, '_manifest'): + # manifest_ids = [delegate._manifest['id'],] + # else: + # manifest_ids = [ext._manifest['id'] for ext in delegate._extensions] + # for manifest_id in manifest_ids: + # extension_by_manifest_id[manifest_id] = ext + + tag_info_by_version[standard_version] = tag_info_by_uri + +# TODO check schemas with out-of-date (or missing) '$schema' + +asdf.AsdfFile( + { + "versions": all_standard_versions, + "schemas": all_schemas, + "manifests": all_manifests, + "manifests_by_extension_uri": manifests_by_extension_uri, + "schema_links_by_uri": schema_links_by_uri, + "schema_uris_by_package_name": schema_uris_by_package_name, + "tag_info_by_version": tag_info_by_version, + } +).write_to("asdf.asdf") From c566b7d2b3245c624d8e6c3be9a248ef150bdb61 Mon Sep 17 00:00:00 2001 From: Brett Date: Fri, 2 Feb 2024 14:44:36 -0500 Subject: [PATCH 04/13] add more tests to schema tool --- scripts/lookup_extensions.py | 203 ++++++++++++++++++++++++++++++----- 1 file changed, 179 insertions(+), 24 deletions(-) diff --git a/scripts/lookup_extensions.py b/scripts/lookup_extensions.py index d4afbefa..77a55475 100644 --- a/scripts/lookup_extensions.py +++ b/scripts/lookup_extensions.py @@ -1,3 +1,5 @@ +import os + import asdf cfg = asdf.get_config() @@ -11,6 +13,47 @@ manifests_by_extension_uri = {} schema_uris_by_package_name = {} schema_links_by_uri = {} +schema_versions_by_base_uri = {} +errors = [] + + +def error(**kwargs): + errors.append(kwargs) + if "msg" in kwargs: + print(f"ERROR: {kwargs['msg']}") + + +def split_uri_base_and_version(uri): + parts = asdf.util._patched_urllib_parse.urlparse(uri) + basename = os.path.basename(parts.path) + if "-" in basename: + version = basename.split("-", maxsplit=1)[1] + base = uri.rstrip(f"-{version}") + else: + version = "" + base = uri + if len(version) and not version[0].isdigit(): + version = "" + base = uri + return base, version + + +def join_uri_base_and_version(base, version): + if not version: + return base + return "-".join((base, version)) + + +def latest_version(base): + versions = schema_versions_by_base_uri[base] + if len(versions) == 1: + version = versions[0] + else: + version = sorted(versions)[-1] + if not version: + return base + return "-".join((base, version)) + for uri in resource_manager: schema = asdf.schema.load_schema(uri) @@ -18,20 +61,44 @@ # FIXME version maps don't have id assert "FILE_FORMAT" in schema else: - # FIXME not sure why the uri/id here has a fragment + # FIXME not sure why the json-schema uri/id here has a fragment if not uri.startswith("http://json-schema.org/draft-04/schema"): - assert uri == schema["id"], (uri, schema["id"]) # sanity check + if uri != schema["id"]: + msg = f"id[{schema['id']}] does not match uri[{uri}]" + error(msg=msg, uri=uri, schema_id=schema["id"]) if uri in all_schemas: # don't duplicate uris - raise Exception + msg = f"uri[{uri}] is registered >1 time" + error(msg=msg, uri=uri) + all_schemas[uri] = schema + + if not uri.startswith("http://json-schema.org/draft-04/schema"): + parts = asdf.util._patched_urllib_parse.urlparse(uri) + base_name = os.path.basename(parts.path) + if "-" not in base_name: + # jwst datamodel schemas are not versioned + if not uri.startswith("http://stsci.edu/schemas/jwst_datamodel"): + msg = f"uri[{uri}] is not versioned" + error(msg=msg, uri=uri) + base = uri + version = "" + else: + base, version = split_uri_base_and_version(uri) + if base not in schema_versions_by_base_uri: + schema_versions_by_base_uri[base] = [] + if version in schema_versions_by_base_uri: + raise Exception + schema_versions_by_base_uri[base].append(version) + if "extension_uri" in schema: all_manifests[uri] = schema extension_uri = schema["extension_uri"] # 2 manifests should not share uris if extension_uri in manifests_by_extension_uri: - # TODO helpful error - raise Exception + uri2 = manifests_by_extension_uri[extension_uri]["id"] + msg = f"two manifests [{uri}, {uri2}] share the extension_uri[{extension_uri}]" + error(msg=msg, uri=uri, extension_uri=extension_uri) manifests_by_extension_uri[extension_uri] = schema # record what packages provide schemas (so we can check if this changes) @@ -55,10 +122,26 @@ links.append({"schema": item["$schema"], "path": path}) if "$ref" in item: - links.append({"ref": item["$ref"], "path": path}) + ref = item["$ref"] + resolved = asdf.util._patched_urllib_parse.urljoin(uri, item["$ref"]) + # remove fragment + resolved = asdf.util._patched_urllib_parse.urlunparse( + asdf.util._patched_urllib_parse.urlparse(resolved)._replace(fragment="") + ) + local = resolved.startswith(uri) + links.append( + { + "ref": ref, + "path": path, + "resolved": resolved, + "local": local, + } + ) children = [] elif "tag" in item: - links.append({"tag": item["tag"], "path": path}) + # the yaml-schema defines tag, so don't treat it as a link + if not uri.startswith("http://stsci.edu/schemas/yaml-schema/draft-01"): + links.append({"tag": item["tag"], "path": path}) children = [] else: children = item.items() @@ -70,6 +153,48 @@ search.append((child, path + (key,))) schema_links_by_uri[uri] = links +# TODO check schemas with out-of-date (or missing) '$schema' + +# check that all refs and schemas point to known schemas +for uri, links in schema_links_by_uri.items(): + for link in links: + if "ref" in link: + other_uri = link["resolved"] + elif "schema" in link: + other_uri = link["schema"] + if other_uri.startswith("http://json-schema.org/draft-04/schema"): + continue + if other_uri not in all_schemas: + msg = f"schema[{uri}] refers to unknown schema[{other_uri}]" + error(msg=msg, uri=uri, other_uri=other_uri) + +# check that the latest version of all schemas refer to the latest version of other schemas +# checking both '$ref' and '$schema' +for base_uri in schema_versions_by_base_uri: + latest_uri = latest_version(base_uri) + links = schema_links_by_uri[latest_uri] + for link in links: + if "ref" in link: + other_uri = link["resolved"] + elif "schema" in link: + other_uri = link["schema"] + elif "tag" in link: + # TODO check tag links + continue + else: + raise Exception(f"unknown link {link}") + if other_uri.startswith("http://json-schema.org/draft-04/schema"): + continue + other_base_uri, _ = split_uri_base_and_version(other_uri) + other_latest_uri = latest_version(other_base_uri) + if other_uri != other_latest_uri: + msg = f"schema [{latest_uri}] refers to out-dated schema [{other_uri}] instead of [{other_latest_uri}]" + error( + msg=msg, + latest_uri=latest_uri, + other_uri=other_uri, + other_latest_uri=other_latest_uri, + ) # we want maps (per version because of the tag use) for # - TODO schema to schema (via $ref, $schema, tag) @@ -125,25 +250,53 @@ type_string = asdf.util.get_class_name(typ, False) tag_info["types"].append(type_string) - # extensions and manifests do not have a 1-to-1 mapping because of asdf-astropy CompoundManifest - # but that doesn't really matter as long as the extension_uri is unique and all the ones - # defined in manifests are used, right? The id is used to load the manifest which defines - # the extension_uri. This is 'compounded' in astropy where the core manifest is combined - # with a special 'astropy' one (which is the one that should define the extension_uri). - - # extension_by_manifest_id = {} - # for ext in all_extensions: - # delegate = ext.delegate - # if hasattr(delegate, '_manifest'): - # manifest_ids = [delegate._manifest['id'],] - # else: - # manifest_ids = [ext._manifest['id'] for ext in delegate._extensions] - # for manifest_id in manifest_ids: - # extension_by_manifest_id[manifest_id] = ext - tag_info_by_version[standard_version] = tag_info_by_uri -# TODO check schemas with out-of-date (or missing) '$schema' +for standard_version, tag_info in tag_info_by_version.items(): + if standard_version != "1.5.0": + # TODO test against 1.6.0 and older versions? + continue + # use version info to check tag links + for uri, links in schema_links_by_uri.items(): + if uri.startswith("http://json-schema.org/draft-04/schema"): + continue + base, _ = split_uri_base_and_version(uri) + # only check the latest versions of all schemas + # TODO should we update tag directives for old schemas? + if uri != latest_version(base): + continue + for link in links: + if "tag" not in link: + # schema and ref links were checked above + continue + tag_pattern = link["tag"] + if tag_pattern in tag_info: + # this is a direct match with a known tag + tags = [ + tag_pattern, + ] + else: + # wildcard + matches = [] + for tag in tag_info: + if asdf.util.uri_match(tag_pattern, tag): + matches.append(tag) + # it's ok to have multiple tags that match as + # multiple extensions might be registered for different tag versions + # it's not ok to have no tags match + if not len(matches): + msg = f"In ASDF[{standard_version}] no tags match pattern[{tag_pattern}] for schema[{uri}]" + error(msg=msg, standard_version=standard_version, tag_pattern=tag_pattern, uri=uri) + tags = matches + + # make sure one matching tag is supported + supported = False + for tag in tags: + info = tag_info[tag] + supported |= info["supported"] + if not supported: + msg = f"In ASDF[{standard_version}] schema[{uri}] does not mach any supported tags [{tags}] with pattern [{tag_pattern}]" + error(msg=msg, standard_version=standard_version, uri=uri, tag_pattern=tag_pattern, tags=tags) asdf.AsdfFile( { @@ -153,6 +306,8 @@ "manifests_by_extension_uri": manifests_by_extension_uri, "schema_links_by_uri": schema_links_by_uri, "schema_uris_by_package_name": schema_uris_by_package_name, + "schema_versions_by_base_uri": schema_versions_by_base_uri, "tag_info_by_version": tag_info_by_version, + "errors": errors, } ).write_to("asdf.asdf") From 23a76ca13f1a9c4c5e145c420bcbd88c4f530b05 Mon Sep 17 00:00:00 2001 From: Brett Date: Tue, 6 Feb 2024 17:31:31 -0500 Subject: [PATCH 05/13] update schema script --- scripts/lookup_extensions.py | 71 +++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/scripts/lookup_extensions.py b/scripts/lookup_extensions.py index 77a55475..87808884 100644 --- a/scripts/lookup_extensions.py +++ b/scripts/lookup_extensions.py @@ -6,7 +6,7 @@ resource_manager = cfg.resource_manager all_extensions = cfg.extensions -all_standard_versions = [str(v) for v in asdf.versioning.supported_versions] +all_standard_versions = sorted([str(v) for v in asdf.versioning.supported_versions]) all_schemas = {} all_manifests = {} @@ -153,7 +153,24 @@ def latest_version(base): search.append((child, path + (key,))) schema_links_by_uri[uri] = links -# TODO check schemas with out-of-date (or missing) '$schema' +# check tags and schemas in manifests +# first index all manifests by base, version +manifests_by_base_by_version = {} +for manifest_uri in all_manifests: + base, version = split_uri_base_and_version(manifest_uri) + if base not in manifests_by_base_by_version: + manifests_by_base_by_version[base] = {} + manifests_by_base_by_version[base][version] = manifest_uri + +for base, versions in manifests_by_base_by_version.items(): + # for the most recent version of each manifest it should only reference the newest tags and schemas + version = sorted(versions)[-1] + manifest_uri = versions[version] + manifest = all_manifests[manifest_uri] + for tag_def in manifest["tags"]: + tag_uri = tag_def["tag_uri"] + schema_uris = tag_def["schema_uri"] + # TODO # check that all refs and schemas point to known schemas for uri, links in schema_links_by_uri.items(): @@ -252,6 +269,56 @@ def latest_version(base): tag_info_by_version[standard_version] = tag_info_by_uri +for previous_version, next_version in zip(all_standard_versions[:-1], all_standard_versions[1:]): + assert next_version > previous_version + previous_tag_info = tag_info_by_version[previous_version] + next_tag_info = tag_info_by_version[next_version] + + # look for added/removed tags between these versions + previous_tags_set = set(previous_tag_info.keys()) + next_tags_set = set(next_tag_info.keys()) + added_tags = next_tags_set.difference(previous_tags_set) + removed_tags = previous_tags_set.difference(next_tags_set) + # index added tags by base/version + added_tags_by_base_by_version = {} + for added_tag in added_tags: + added_base, added_version = split_uri_base_and_version(added_tag) + if added_base not in added_tags_by_base_by_version: + added_tags_by_base_by_version[added_base] = {} + added_tags_by_base_by_version[added_base][added_version] = added_tag + for removed_tag in removed_tags: + # look for an new version in added_tags + removed_base, removed_version = split_uri_base_and_version(removed_tag) + if removed_base not in added_tags_by_base_by_version: + error( + msg=f"Support for tag {removed_tag} was removed in version {next_version}", + removed_tag=removed_tag, + removed_version=next_version, + ) + else: + new_versions = added_tags_by_base_by_version[removed_base] + # at least one of these versions should be newer + for version in new_versions: + if version > removed_version: + # print(f"In {next_version}: tag {removed_tag} replaced with newer version {new_versions[version]}") + pass + + # look for added/removed types between these versions + previous_types = set(sum((i.get("types", []) for i in previous_tag_info.values()), start=[])) + next_types = set(sum((i.get("types", []) for i in next_tag_info.values()), start=[])) + removed_types = previous_types.difference(next_types) + added_types = next_types.difference(previous_types) + for added_type in added_types: + # print(f"In {next_version}: support for type {added_type} added") + pass + for removed_type in removed_types: + error( + msg=f"Support for {removed_type} was removed in version {next_version}", + removed_type=removed_type, + removed_version=next_version, + ) + +# check 'tag' links in schemas for standard_version, tag_info in tag_info_by_version.items(): if standard_version != "1.5.0": # TODO test against 1.6.0 and older versions? From 64fa73082c9c84a973c4ad6a32eec6e53423e9ac Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 8 Feb 2024 07:35:57 -0500 Subject: [PATCH 06/13] add asdf-schema-1.1.0 --- .../stsci.edu/asdf/asdf-schema-1.1.0.yaml | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 resources/schemas/stsci.edu/asdf/asdf-schema-1.1.0.yaml diff --git a/resources/schemas/stsci.edu/asdf/asdf-schema-1.1.0.yaml b/resources/schemas/stsci.edu/asdf/asdf-schema-1.1.0.yaml new file mode 100644 index 00000000..7ef332ee --- /dev/null +++ b/resources/schemas/stsci.edu/asdf/asdf-schema-1.1.0.yaml @@ -0,0 +1,94 @@ +%YAML 1.1 +--- +$schema: "http://json-schema.org/draft-04/schema" +id: "http://stsci.edu/schemas/asdf/asdf-schema-1.1.0" +title: + ASDF Schema +description: | + Extending YAML Schema and JSON Schema to add support for some ASDF-specific + checks, related to [ndarrays](ref:core/ndarray-1.1.0). +allOf: + - $ref: "http://stsci.edu/schemas/yaml-schema/draft-01" + - type: object + properties: + max_ndim: + description: | + Specifies that the corresponding **ndarray** is at most the + given number of dimensions. If the array has fewer + dimensions, it should be logically treated as if it were + "broadcast" to the expected dimensions by adding 1's to the + front of the shape list. + type: integer + minimum: 0 + + ndim: + description: | + Specifies that the matching **ndarray** is exactly the given + number of dimensions. + type: integer + minimum: 0 + + datatype: + description: | + Specifies the datatype of the **ndarray**. + + By default, an array is considered "matching" if the array + can be cast to the given datatype without data loss. For + exact datatype matching, set `exact_datatype` to `true`. + allOf: + - $ref: "http://stsci.edu/schemas/asdf/core/ndarray-1.1.0#/definitions/datatype" + + exact_datatype: + description: | + If `true`, the datatype must match exactly. + type: boolean + default: false + + # Redefine JSON schema validators in terms of this document so that + # we can check nested objects: + additionalItems: + anyOf: + - type: boolean + - $ref: "#" + items: + anyOf: + - $ref: "#" + - $ref: "#/definitions/schemaArray" + additionalProperties: + anyOf: + - type: boolean + - $ref: "#" + definitions: + type: object + additionalProperties: + $ref: "#" + properties: + type: object + additionalProperties: + $ref: "#" + patternProperties: + type: object + additionalProperties: + $ref: "#" + dependencies: + type: object + additionalProperties: + anyOf: + - $ref: "#" + - $ref: "http://json-schema.org/draft-04/schema#/definitions/stringArray" + allOf: + $ref: "#/definitions/schemaArray" + anyOf: + $ref: "#/definitions/schemaArray" + oneOf: + $ref: "#/definitions/schemaArray" + not: + $ref: "#" + +definitions: + schemaArray: + type: array + minItems: 1 + items: + $ref: "#" +... From 57b0fd3a01e88adc3fe519fbefa3b300f31f4756 Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 8 Feb 2024 09:12:53 -0500 Subject: [PATCH 07/13] use regex for base version split --- scripts/lookup_extensions.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/lookup_extensions.py b/scripts/lookup_extensions.py index 87808884..6f000963 100644 --- a/scripts/lookup_extensions.py +++ b/scripts/lookup_extensions.py @@ -1,4 +1,5 @@ import os +import re import asdf @@ -24,11 +25,13 @@ def error(**kwargs): def split_uri_base_and_version(uri): - parts = asdf.util._patched_urllib_parse.urlparse(uri) - basename = os.path.basename(parts.path) - if "-" in basename: - version = basename.split("-", maxsplit=1)[1] - base = uri.rstrip(f"-{version}") + # parts = asdf.util._patched_urllib_parse.urlparse(uri) + # basename = os.path.basename(parts.path) + m = re.match(r"^(?P.*)-(?P([0-9]+\.?){1,3}(-.*)?)$", uri) + if m: + version = m["version"] + base = m["base"] + assert "-".join((base, version)) == uri else: version = "" base = uri From 726b58ae8e86e2d4c00725ce4bc150b24a97347c Mon Sep 17 00:00:00 2001 From: Brett Date: Thu, 8 Feb 2024 15:43:14 -0500 Subject: [PATCH 08/13] use 1.6.0 for unit manifest --- .../unit/{unit-1.2.0.yaml => unit-1.6.0.yaml} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename resources/manifests/asdf-format.org/unit/{unit-1.2.0.yaml => unit-1.6.0.yaml} (88%) diff --git a/resources/manifests/asdf-format.org/unit/unit-1.2.0.yaml b/resources/manifests/asdf-format.org/unit/unit-1.6.0.yaml similarity index 88% rename from resources/manifests/asdf-format.org/unit/unit-1.2.0.yaml rename to resources/manifests/asdf-format.org/unit/unit-1.6.0.yaml index e5ad5ca5..d6804f86 100644 --- a/resources/manifests/asdf-format.org/unit/unit-1.2.0.yaml +++ b/resources/manifests/asdf-format.org/unit/unit-1.6.0.yaml @@ -1,6 +1,6 @@ -id: asdf://asdf-format.org/unit/manifests/unit-1.2.0 -extension_uri: asdf://asdf-format.org/unit/extensions/unit-1.2.0 -title: Unit extension 1.2.0 +id: asdf://asdf-format.org/unit/manifests/unit-1.6.0 +extension_uri: asdf://asdf-format.org/unit/extensions/unit-1.6.0 +title: Unit extension 1.6.0 description: Tags for ASDF unit objects. asdf_standard_requirement: gte: 1.6.0 From 9c4d639da25d01972a38e579c0d3eb6b3eb00342 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 19 Feb 2024 10:49:23 -0500 Subject: [PATCH 09/13] remove lookup_extension script --- scripts/lookup_extensions.py | 383 ----------------------------------- 1 file changed, 383 deletions(-) delete mode 100644 scripts/lookup_extensions.py diff --git a/scripts/lookup_extensions.py b/scripts/lookup_extensions.py deleted file mode 100644 index 6f000963..00000000 --- a/scripts/lookup_extensions.py +++ /dev/null @@ -1,383 +0,0 @@ -import os -import re - -import asdf - -cfg = asdf.get_config() -resource_manager = cfg.resource_manager -all_extensions = cfg.extensions - -all_standard_versions = sorted([str(v) for v in asdf.versioning.supported_versions]) - -all_schemas = {} -all_manifests = {} -manifests_by_extension_uri = {} -schema_uris_by_package_name = {} -schema_links_by_uri = {} -schema_versions_by_base_uri = {} -errors = [] - - -def error(**kwargs): - errors.append(kwargs) - if "msg" in kwargs: - print(f"ERROR: {kwargs['msg']}") - - -def split_uri_base_and_version(uri): - # parts = asdf.util._patched_urllib_parse.urlparse(uri) - # basename = os.path.basename(parts.path) - m = re.match(r"^(?P.*)-(?P([0-9]+\.?){1,3}(-.*)?)$", uri) - if m: - version = m["version"] - base = m["base"] - assert "-".join((base, version)) == uri - else: - version = "" - base = uri - if len(version) and not version[0].isdigit(): - version = "" - base = uri - return base, version - - -def join_uri_base_and_version(base, version): - if not version: - return base - return "-".join((base, version)) - - -def latest_version(base): - versions = schema_versions_by_base_uri[base] - if len(versions) == 1: - version = versions[0] - else: - version = sorted(versions)[-1] - if not version: - return base - return "-".join((base, version)) - - -for uri in resource_manager: - schema = asdf.schema.load_schema(uri) - if "id" not in schema: - # FIXME version maps don't have id - assert "FILE_FORMAT" in schema - else: - # FIXME not sure why the json-schema uri/id here has a fragment - if not uri.startswith("http://json-schema.org/draft-04/schema"): - if uri != schema["id"]: - msg = f"id[{schema['id']}] does not match uri[{uri}]" - error(msg=msg, uri=uri, schema_id=schema["id"]) - if uri in all_schemas: - # don't duplicate uris - msg = f"uri[{uri}] is registered >1 time" - error(msg=msg, uri=uri) - - all_schemas[uri] = schema - - if not uri.startswith("http://json-schema.org/draft-04/schema"): - parts = asdf.util._patched_urllib_parse.urlparse(uri) - base_name = os.path.basename(parts.path) - if "-" not in base_name: - # jwst datamodel schemas are not versioned - if not uri.startswith("http://stsci.edu/schemas/jwst_datamodel"): - msg = f"uri[{uri}] is not versioned" - error(msg=msg, uri=uri) - base = uri - version = "" - else: - base, version = split_uri_base_and_version(uri) - if base not in schema_versions_by_base_uri: - schema_versions_by_base_uri[base] = [] - if version in schema_versions_by_base_uri: - raise Exception - schema_versions_by_base_uri[base].append(version) - - if "extension_uri" in schema: - all_manifests[uri] = schema - extension_uri = schema["extension_uri"] - # 2 manifests should not share uris - if extension_uri in manifests_by_extension_uri: - uri2 = manifests_by_extension_uri[extension_uri]["id"] - msg = f"two manifests [{uri}, {uri2}] share the extension_uri[{extension_uri}]" - error(msg=msg, uri=uri, extension_uri=extension_uri) - manifests_by_extension_uri[extension_uri] = schema - - # record what packages provide schemas (so we can check if this changes) - package_name = resource_manager._mappings_by_uri[uri].package_name - if package_name not in schema_uris_by_package_name: - schema_uris_by_package_name[package_name] = [] - schema_uris_by_package_name[package_name].append(uri) - - # get all external references for this schema ($ref, $schema, tag) - links = [] - search = [(schema, ())] - while search: - item, path = search.pop() - - # step into item - if isinstance(item, list): - children = enumerate(item) - elif isinstance(item, dict): - # top level $schema - if not path and "$schema" in item: - links.append({"schema": item["$schema"], "path": path}) - - if "$ref" in item: - ref = item["$ref"] - resolved = asdf.util._patched_urllib_parse.urljoin(uri, item["$ref"]) - # remove fragment - resolved = asdf.util._patched_urllib_parse.urlunparse( - asdf.util._patched_urllib_parse.urlparse(resolved)._replace(fragment="") - ) - local = resolved.startswith(uri) - links.append( - { - "ref": ref, - "path": path, - "resolved": resolved, - "local": local, - } - ) - children = [] - elif "tag" in item: - # the yaml-schema defines tag, so don't treat it as a link - if not uri.startswith("http://stsci.edu/schemas/yaml-schema/draft-01"): - links.append({"tag": item["tag"], "path": path}) - children = [] - else: - children = item.items() - else: - children = [] - if children: - for key, child in children: - if isinstance(child, (list, dict)): - search.append((child, path + (key,))) - schema_links_by_uri[uri] = links - -# check tags and schemas in manifests -# first index all manifests by base, version -manifests_by_base_by_version = {} -for manifest_uri in all_manifests: - base, version = split_uri_base_and_version(manifest_uri) - if base not in manifests_by_base_by_version: - manifests_by_base_by_version[base] = {} - manifests_by_base_by_version[base][version] = manifest_uri - -for base, versions in manifests_by_base_by_version.items(): - # for the most recent version of each manifest it should only reference the newest tags and schemas - version = sorted(versions)[-1] - manifest_uri = versions[version] - manifest = all_manifests[manifest_uri] - for tag_def in manifest["tags"]: - tag_uri = tag_def["tag_uri"] - schema_uris = tag_def["schema_uri"] - # TODO - -# check that all refs and schemas point to known schemas -for uri, links in schema_links_by_uri.items(): - for link in links: - if "ref" in link: - other_uri = link["resolved"] - elif "schema" in link: - other_uri = link["schema"] - if other_uri.startswith("http://json-schema.org/draft-04/schema"): - continue - if other_uri not in all_schemas: - msg = f"schema[{uri}] refers to unknown schema[{other_uri}]" - error(msg=msg, uri=uri, other_uri=other_uri) - -# check that the latest version of all schemas refer to the latest version of other schemas -# checking both '$ref' and '$schema' -for base_uri in schema_versions_by_base_uri: - latest_uri = latest_version(base_uri) - links = schema_links_by_uri[latest_uri] - for link in links: - if "ref" in link: - other_uri = link["resolved"] - elif "schema" in link: - other_uri = link["schema"] - elif "tag" in link: - # TODO check tag links - continue - else: - raise Exception(f"unknown link {link}") - if other_uri.startswith("http://json-schema.org/draft-04/schema"): - continue - other_base_uri, _ = split_uri_base_and_version(other_uri) - other_latest_uri = latest_version(other_base_uri) - if other_uri != other_latest_uri: - msg = f"schema [{latest_uri}] refers to out-dated schema [{other_uri}] instead of [{other_latest_uri}]" - error( - msg=msg, - latest_uri=latest_uri, - other_uri=other_uri, - other_latest_uri=other_latest_uri, - ) - -# we want maps (per version because of the tag use) for -# - TODO schema to schema (via $ref, $schema, tag) -# - schema to tag -# - tag to type [this isn't one-to-one] -# - TODO extension to schema (via tag) -# - extension to tag - -tag_info_by_version = {} - -for standard_version in all_standard_versions: - # the extension manager allows tags/type to be overridden - # track these separately - extension_manager = asdf.AsdfFile(version=standard_version).extension_manager - - # also consider all extensions (even ones overridden) - extensions = [e for e in all_extensions if standard_version in e.asdf_standard_requirement] - - # by uri so each entry here corresponds to a tag used in this version - tag_info_by_uri = {} - - # look up all extensions for this version that define tags - for extension in extensions: - for tag_def in extension.tags: - # multiple extensions might define tags - if tag_def.tag_uri in tag_info_by_uri: - tag_info = tag_info_by_uri[tag_def.tag_uri] - else: - tag_info = { - "extension_uris": [], - "schema_uris_by_extension_uri": {}, - } - tag_info["extension_uris"].append(extension.extension_uri) - tag_info["schema_uris_by_extension_uri"][extension.extension_uri] = tag_def.schema_uris - tag_info_by_uri[tag_def.tag_uri] = tag_info - - for tag_uri in tag_info_by_uri: - tag_info = tag_info_by_uri[tag_uri] - try: - converter = extension_manager.get_converter_for_tag(tag_uri) - except KeyError: - # some tags that are defined in extensions do not have converters - # one example is "label_mapper". Mark these as not supported. - tag_info["supported"] = False - continue - tag_info["supported"] = True - tag_info["handling_extension_uri"] = converter._extension.extension_uri - tag_info["types"] = [] - for typ in converter.types: - if isinstance(typ, str): - type_string = typ - else: - type_string = asdf.util.get_class_name(typ, False) - tag_info["types"].append(type_string) - - tag_info_by_version[standard_version] = tag_info_by_uri - -for previous_version, next_version in zip(all_standard_versions[:-1], all_standard_versions[1:]): - assert next_version > previous_version - previous_tag_info = tag_info_by_version[previous_version] - next_tag_info = tag_info_by_version[next_version] - - # look for added/removed tags between these versions - previous_tags_set = set(previous_tag_info.keys()) - next_tags_set = set(next_tag_info.keys()) - added_tags = next_tags_set.difference(previous_tags_set) - removed_tags = previous_tags_set.difference(next_tags_set) - # index added tags by base/version - added_tags_by_base_by_version = {} - for added_tag in added_tags: - added_base, added_version = split_uri_base_and_version(added_tag) - if added_base not in added_tags_by_base_by_version: - added_tags_by_base_by_version[added_base] = {} - added_tags_by_base_by_version[added_base][added_version] = added_tag - for removed_tag in removed_tags: - # look for an new version in added_tags - removed_base, removed_version = split_uri_base_and_version(removed_tag) - if removed_base not in added_tags_by_base_by_version: - error( - msg=f"Support for tag {removed_tag} was removed in version {next_version}", - removed_tag=removed_tag, - removed_version=next_version, - ) - else: - new_versions = added_tags_by_base_by_version[removed_base] - # at least one of these versions should be newer - for version in new_versions: - if version > removed_version: - # print(f"In {next_version}: tag {removed_tag} replaced with newer version {new_versions[version]}") - pass - - # look for added/removed types between these versions - previous_types = set(sum((i.get("types", []) for i in previous_tag_info.values()), start=[])) - next_types = set(sum((i.get("types", []) for i in next_tag_info.values()), start=[])) - removed_types = previous_types.difference(next_types) - added_types = next_types.difference(previous_types) - for added_type in added_types: - # print(f"In {next_version}: support for type {added_type} added") - pass - for removed_type in removed_types: - error( - msg=f"Support for {removed_type} was removed in version {next_version}", - removed_type=removed_type, - removed_version=next_version, - ) - -# check 'tag' links in schemas -for standard_version, tag_info in tag_info_by_version.items(): - if standard_version != "1.5.0": - # TODO test against 1.6.0 and older versions? - continue - # use version info to check tag links - for uri, links in schema_links_by_uri.items(): - if uri.startswith("http://json-schema.org/draft-04/schema"): - continue - base, _ = split_uri_base_and_version(uri) - # only check the latest versions of all schemas - # TODO should we update tag directives for old schemas? - if uri != latest_version(base): - continue - for link in links: - if "tag" not in link: - # schema and ref links were checked above - continue - tag_pattern = link["tag"] - if tag_pattern in tag_info: - # this is a direct match with a known tag - tags = [ - tag_pattern, - ] - else: - # wildcard - matches = [] - for tag in tag_info: - if asdf.util.uri_match(tag_pattern, tag): - matches.append(tag) - # it's ok to have multiple tags that match as - # multiple extensions might be registered for different tag versions - # it's not ok to have no tags match - if not len(matches): - msg = f"In ASDF[{standard_version}] no tags match pattern[{tag_pattern}] for schema[{uri}]" - error(msg=msg, standard_version=standard_version, tag_pattern=tag_pattern, uri=uri) - tags = matches - - # make sure one matching tag is supported - supported = False - for tag in tags: - info = tag_info[tag] - supported |= info["supported"] - if not supported: - msg = f"In ASDF[{standard_version}] schema[{uri}] does not mach any supported tags [{tags}] with pattern [{tag_pattern}]" - error(msg=msg, standard_version=standard_version, uri=uri, tag_pattern=tag_pattern, tags=tags) - -asdf.AsdfFile( - { - "versions": all_standard_versions, - "schemas": all_schemas, - "manifests": all_manifests, - "manifests_by_extension_uri": manifests_by_extension_uri, - "schema_links_by_uri": schema_links_by_uri, - "schema_uris_by_package_name": schema_uris_by_package_name, - "schema_versions_by_base_uri": schema_versions_by_base_uri, - "tag_info_by_version": tag_info_by_version, - "errors": errors, - } -).write_to("asdf.asdf") From b16ebcb034358416a586ca6f8187f8adafa1bd4c Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 19 Feb 2024 12:22:55 -0500 Subject: [PATCH 10/13] move all non-core to astronomy manifest --- .../astronomy/astronomy-1.0.0.yaml | 96 +++++++++++++++++++ .../asdf-format.org/fits/fits-1.0.0.yaml | 22 ----- .../asdf-format.org/table/table-1.0.0.yaml | 27 ------ .../asdf-format.org/time/time-1.0.0.yaml | 38 -------- .../asdf-format.org/unit/unit-1.6.0.yaml | 30 ------ src/asdf_standard/integration.py | 16 +--- 6 files changed, 98 insertions(+), 131 deletions(-) create mode 100644 resources/manifests/asdf-format.org/astronomy/astronomy-1.0.0.yaml delete mode 100644 resources/manifests/asdf-format.org/fits/fits-1.0.0.yaml delete mode 100644 resources/manifests/asdf-format.org/table/table-1.0.0.yaml delete mode 100644 resources/manifests/asdf-format.org/time/time-1.0.0.yaml delete mode 100644 resources/manifests/asdf-format.org/unit/unit-1.6.0.yaml diff --git a/resources/manifests/asdf-format.org/astronomy/astronomy-1.0.0.yaml b/resources/manifests/asdf-format.org/astronomy/astronomy-1.0.0.yaml new file mode 100644 index 00000000..b9a00cef --- /dev/null +++ b/resources/manifests/asdf-format.org/astronomy/astronomy-1.0.0.yaml @@ -0,0 +1,96 @@ +id: asdf://asdf-format.org/astronomy/manifests/astronomy-1.0.0 +extension_uri: asdf://asdf-format.org/astronomy/extensions/astronomy-1.0.0 +title: ASDF astronomy extension 1.0.0 +description: Astronomy related (non-core) ASDF objects +asdf_standard_requirement: + gte: 1.6.0 +tags: +- tag_uri: tag:stsci.edu:asdf/table/column-1.1.0 + schema_uri: http://stsci.edu/schemas/asdf/table/column-1.1.0 + title: A column in a table. + description: |- + Each column contains a name and an array of data, and an optional description + and unit. +- tag_uri: tag:stsci.edu:asdf/unit/defunit-1.0.0 + schema_uri: http://stsci.edu/schemas/asdf/unit/defunit-1.0.0 + title: Define a new physical unit. + description: |- + Defines a new unit. It can be used to either: + + - Define a new base unit. + + - Create a new unit name that is a equivalent to a given unit. + + The new unit must be defined before any unit tags that use it. +- tag_uri: tag:stsci.edu:asdf/fits/fits-1.1.0 + schema_uri: http://stsci.edu/schemas/asdf/fits/fits-1.1.0 + title: A FITS file inside of an ASDF file. + description: |- + This schema is useful for distributing ASDF files that can + automatically be converted to FITS files by specifying the exact + content of the resulting FITS file. + + Not all kinds of data in FITS are directly representable in ASDF. + For example, applying an offset and scale to the data using the + `BZERO` and `BSCALE` keywords. In these cases, it will not be + possible to store the data in the native format from FITS and also + be accessible in its proper form in the ASDF file. + + Only image and binary table extensions are supported. +- tag_uri: tag:stsci.edu:asdf/unit/quantity-1.2.0 + schema_uri: http://stsci.edu/schemas/asdf/unit/quantity-1.2.0 + title: Represents a Quantity object from astropy + description: |- + A Quantity object represents a value that has some unit + associated with the number. +- tag_uri: tag:stsci.edu:asdf/table/table-1.1.0 + schema_uri: http://stsci.edu/schemas/asdf/table/table-1.1.0 + title: A table. + description: |- + A table is represented as a list of columns, where each entry is a + [column](ref:table/column-1.1.0) + object, containing the data and some additional information. + + The data itself may be stored inline as text, or in binary in either + row- or column-major order by use of the `strides` property on the + individual column arrays. + + Each column in the table must have the same first (slowest moving) + dimension. +- tag_uri: tag:stsci.edu:asdf/time/time-1.2.0 + schema_uri: http://stsci.edu/schemas/asdf/time/time-1.2.0 + title: Represents an instance in time. + description: |- + A "time" is a single instant in time. It may explicitly specify the + way time is represented (the "format") and the "scale" which + specifies the offset and scaling relation of the unit of time. + + Specific emphasis is placed on supporting time scales (e.g. UTC, + TAI, UT1, TDB) and time representations (e.g. JD, MJD, ISO 8601) + that are used in astronomy and required to calculate, e.g., sidereal + times and barycentric corrections. + + Times may be represented as one of the following: + + - an object, with explicit `value`, and optional `format`, `scale` + and `location`. + + - a string, in which case the format is guessed from across + the unambiguous options (`iso`, `byear`, `jyear`, `yday`), and the + scale is hardcoded to `UTC`. + + In either case, a single time tag may be used to represent an + n-dimensional array of times, using either an `ndarray` tag or + inline as (possibly nested) YAML lists. If YAML lists, the same + format must be used for all time values. + + The precision of the numeric formats should only be assumed to be as + good as an IEEE-754 double precision (float64) value. If + higher-precision is required, the `iso` or `yday` format should be + used. +- tag_uri: tag:stsci.edu:asdf/unit/unit-1.0.0 + schema_uri: http://stsci.edu/schemas/asdf/unit/unit-1.0.0 + title: Physical unit. + description: |- + This represents a physical unit, in [VOUnit syntax, Version 1.0](http://www.ivoa.net/documents/VOUnits/index.html). + Where units are not explicitly tagged, they are assumed to be in VOUnit syntax. diff --git a/resources/manifests/asdf-format.org/fits/fits-1.0.0.yaml b/resources/manifests/asdf-format.org/fits/fits-1.0.0.yaml deleted file mode 100644 index 2f0e93fd..00000000 --- a/resources/manifests/asdf-format.org/fits/fits-1.0.0.yaml +++ /dev/null @@ -1,22 +0,0 @@ -id: asdf://asdf-format.org/fits/manifests/fits-1.0.0 -extension_uri: asdf://asdf-format.org/fits/extensions/fits-1.1.0 -title: Fits extension 1.0.0 -description: Tags FITS objects. -asdf_standard_requirement: - gte: 1.6.0 -tags: -- tag_uri: tag:stsci.edu:asdf/fits/fits-1.1.0 - schema_uri: http://stsci.edu/schemas/asdf/fits/fits-1.1.0 - title: A FITS file inside of an ASDF file. - description: |- - This schema is useful for distributing ASDF files that can - automatically be converted to FITS files by specifying the exact - content of the resulting FITS file. - - Not all kinds of data in FITS are directly representable in ASDF. - For example, applying an offset and scale to the data using the - `BZERO` and `BSCALE` keywords. In these cases, it will not be - possible to store the data in the native format from FITS and also - be accessible in its proper form in the ASDF file. - - Only image and binary table extensions are supported. diff --git a/resources/manifests/asdf-format.org/table/table-1.0.0.yaml b/resources/manifests/asdf-format.org/table/table-1.0.0.yaml deleted file mode 100644 index d8db2027..00000000 --- a/resources/manifests/asdf-format.org/table/table-1.0.0.yaml +++ /dev/null @@ -1,27 +0,0 @@ -id: asdf://asdf-format.org/table/manifests/table-1.0.0 -extension_uri: asdf://asdf-format.org/table/extensions/table-1.0.0 -title: Table extension 1.0.0 -description: Tags for ASDF table objects. -asdf_standard_requirement: - gte: 1.6.0 -tags: -- tag_uri: tag:stsci.edu:asdf/table/column-1.1.0 - schema_uri: http://stsci.edu/schemas/asdf/table/column-1.1.0 - title: A column in a table. - description: |- - Each column contains a name and an array of data, and an optional description - and unit. -- tag_uri: tag:stsci.edu:asdf/table/table-1.1.0 - schema_uri: http://stsci.edu/schemas/asdf/table/table-1.1.0 - title: A table. - description: |- - A table is represented as a list of columns, where each entry is a - [column](ref:table/column-1.1.0) - object, containing the data and some additional information. - - The data itself may be stored inline as text, or in binary in either - row- or column-major order by use of the `strides` property on the - individual column arrays. - - Each column in the table must have the same first (slowest moving) - dimension. diff --git a/resources/manifests/asdf-format.org/time/time-1.0.0.yaml b/resources/manifests/asdf-format.org/time/time-1.0.0.yaml deleted file mode 100644 index 370e368e..00000000 --- a/resources/manifests/asdf-format.org/time/time-1.0.0.yaml +++ /dev/null @@ -1,38 +0,0 @@ -id: asdf://asdf-format.org/time/manifests/time-1.0.0 -extension_uri: asdf://asdf-format.org/time/extensions/time-1.0.0 -title: Time extension 1.0.0 -description: Tags for ASDF time objects. -asdf_standard_requirement: - gte: 1.6.0 -tags: -- tag_uri: tag:stsci.edu:asdf/time/time-1.2.0 - schema_uri: http://stsci.edu/schemas/asdf/time/time-1.2.0 - title: Represents an instance in time. - description: |- - A "time" is a single instant in time. It may explicitly specify the - way time is represented (the "format") and the "scale" which - specifies the offset and scaling relation of the unit of time. - - Specific emphasis is placed on supporting time scales (e.g. UTC, - TAI, UT1, TDB) and time representations (e.g. JD, MJD, ISO 8601) - that are used in astronomy and required to calculate, e.g., sidereal - times and barycentric corrections. - - Times may be represented as one of the following: - - - an object, with explicit `value`, and optional `format`, `scale` - and `location`. - - - a string, in which case the format is guessed from across - the unambiguous options (`iso`, `byear`, `jyear`, `yday`), and the - scale is hardcoded to `UTC`. - - In either case, a single time tag may be used to represent an - n-dimensional array of times, using either an `ndarray` tag or - inline as (possibly nested) YAML lists. If YAML lists, the same - format must be used for all time values. - - The precision of the numeric formats should only be assumed to be as - good as an IEEE-754 double precision (float64) value. If - higher-precision is required, the `iso` or `yday` format should be - used. diff --git a/resources/manifests/asdf-format.org/unit/unit-1.6.0.yaml b/resources/manifests/asdf-format.org/unit/unit-1.6.0.yaml deleted file mode 100644 index d6804f86..00000000 --- a/resources/manifests/asdf-format.org/unit/unit-1.6.0.yaml +++ /dev/null @@ -1,30 +0,0 @@ -id: asdf://asdf-format.org/unit/manifests/unit-1.6.0 -extension_uri: asdf://asdf-format.org/unit/extensions/unit-1.6.0 -title: Unit extension 1.6.0 -description: Tags for ASDF unit objects. -asdf_standard_requirement: - gte: 1.6.0 -tags: -- tag_uri: tag:stsci.edu:asdf/unit/defunit-1.0.0 - schema_uri: http://stsci.edu/schemas/asdf/unit/defunit-1.0.0 - title: Define a new physical unit. - description: |- - Defines a new unit. It can be used to either: - - - Define a new base unit. - - - Create a new unit name that is a equivalent to a given unit. - - The new unit must be defined before any unit tags that use it. -- tag_uri: tag:stsci.edu:asdf/unit/quantity-1.2.0 - schema_uri: http://stsci.edu/schemas/asdf/unit/quantity-1.2.0 - title: Represents a Quantity object from astropy - description: |- - A Quantity object represents a value that has some unit - associated with the number. -- tag_uri: tag:stsci.edu:asdf/unit/unit-1.0.0 - schema_uri: http://stsci.edu/schemas/asdf/unit/unit-1.0.0 - title: Physical unit. - description: |- - This represents a physical unit, in [VOUnit syntax, Version 1.0](http://www.ivoa.net/documents/VOUnits/index.html). - Where units are not explicitly tagged, they are assumed to be in VOUnit syntax. diff --git a/src/asdf_standard/integration.py b/src/asdf_standard/integration.py index b3a8eb9e..6ba8de42 100644 --- a/src/asdf_standard/integration.py +++ b/src/asdf_standard/integration.py @@ -23,19 +23,7 @@ def get_resource_mappings(): "asdf://asdf-format.org/core/manifests/", ), asdf_standard.DirectoryResourceMapping( - resources_root / "manifests" / "asdf-format.org" / "unit", - "asdf://asdf-format.org/unit/manifests/", - ), - asdf_standard.DirectoryResourceMapping( - resources_root / "manifests" / "asdf-format.org" / "table", - "asdf://asdf-format.org/table/manifests/", - ), - asdf_standard.DirectoryResourceMapping( - resources_root / "manifests" / "asdf-format.org" / "time", - "asdf://asdf-format.org/time/manifests/", - ), - asdf_standard.DirectoryResourceMapping( - resources_root / "manifests" / "asdf-format.org" / "fits", - "asdf://asdf-format.org/fits/manifests/", + resources_root / "manifests" / "asdf-format.org" / "astronomy", + "asdf://asdf-format.org/astronomy/manifests/", ), ] From 0101c8ed7da78aaafcdb92da3cbf82b83d336639 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 19 Feb 2024 14:56:20 -0500 Subject: [PATCH 11/13] update docs --- CHANGES.rst | 1 + docs/source/schemas.rst | 13 +++++-------- docs/source/schemas/astronomy.rst | 18 ++++++++++++++++++ docs/source/schemas/fits.rst | 11 ----------- docs/source/schemas/index.rst | 19 ++----------------- docs/source/schemas/manifest.rst | 8 ++++++++ docs/source/schemas/table.rst | 11 ----------- docs/source/schemas/time.rst | 11 ----------- docs/source/schemas/unit.rst | 12 ------------ tests/common.py | 4 ---- tests/test_docs.py | 6 +++--- tests/test_version_map.py | 1 + 12 files changed, 38 insertions(+), 77 deletions(-) create mode 100644 docs/source/schemas/astronomy.rst delete mode 100644 docs/source/schemas/fits.rst delete mode 100644 docs/source/schemas/table.rst delete mode 100644 docs/source/schemas/time.rst delete mode 100644 docs/source/schemas/unit.rst diff --git a/CHANGES.rst b/CHANGES.rst index 71922f34..8b7bae78 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -14,6 +14,7 @@ The stable ASDF Standard is v1.5.0 - Drop support for python 3.8 [#390] - Add ``float16`` to ``ndarray-1.1.0`` [#411] - Remove unneeded ``tag`` keyword from ``fits`` schema [#421] +- Move non-core tags to ``astronomy`` manifest [#422] 1.0.3 (2022-08-08) ------------------ diff --git a/docs/source/schemas.rst b/docs/source/schemas.rst index 026cab56..c7f37293 100644 --- a/docs/source/schemas.rst +++ b/docs/source/schemas.rst @@ -19,19 +19,16 @@ of ASDF files serves the following purposes: All ASDF implementations must implement the types defined by the `core schemas ` and validate against them when reading files. [#]_ The ASDF -Standard also defines two other categories of schemas, which are optional -for ASDF implementations: +Standard also defines other schemas, which are optional for ASDF implementations +and maintained as part of the standard (mainly for historical reasons): -* :ref:`unit ` -* :ref:`time ` - -.. Fits is deliberately omitted from this list. +* :ref:`astronomy ` The ASDF Standard also defines two metaschemas which are used to validate the ASDF schemas themselves: * :ref:`yaml-schema` -* :ref:`ASDF Schema ` +* :ref:`ASDF Schema ` More information on the schemas defined by ASDF can be found in :ref:`schema`. @@ -63,7 +60,7 @@ so will have the following top-level attribute:: $schema: "http://stsci.edu/schemas/yaml-schema/draft-01" -Some ASDF schemas use the :ref:`ASDF metaschema ` instead +Some ASDF schemas use the :ref:`ASDF metaschema ` instead (e.g. `ndarray `). It is also possible to create custom metaschemas, although these should always inherit from either YAML Schema or the ASDF metaschema. [#]_ diff --git a/docs/source/schemas/astronomy.rst b/docs/source/schemas/astronomy.rst new file mode 100644 index 00000000..716d9b93 --- /dev/null +++ b/docs/source/schemas/astronomy.rst @@ -0,0 +1,18 @@ +.. _astronomy-schema: + +Astronomy +========= + +The ASDF-standard contains a few astronomy-specific tags that +are optional but maintained as part of the standard (for historical +reasons). These include: + +.. asdf-autoschemas:: + + fits/fits-1.1.0 + table/column-1.1.0 + table/table-1.1.0 + time/time-1.2.0 + unit/unit-1.0.0 + unit/defunit-1.0.0 + unit/quantity-1.2.0 diff --git a/docs/source/schemas/fits.rst b/docs/source/schemas/fits.rst deleted file mode 100644 index 6e3e3225..00000000 --- a/docs/source/schemas/fits.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. _fits-schema: - -FITS -==== - -The ``fits`` module contains schema that support backward compatibility with -FITS. It can safely be ignored by most ASDF implementations. - -.. asdf-autoschemas:: - - fits/fits-1.1.0 diff --git a/docs/source/schemas/index.rst b/docs/source/schemas/index.rst index 1265b1c7..9d33d528 100644 --- a/docs/source/schemas/index.rst +++ b/docs/source/schemas/index.rst @@ -17,10 +17,7 @@ must support the "core" module, but the other modules are optional. :maxdepth: 2 core.rst - fits.rst - unit.rst - time.rst - table.rst + astronomy.rst legacy.rst The ASDF Standard also defines two meta-schemas that are used for validating @@ -34,19 +31,7 @@ custom schemas (see `extending-asdf`). .. asdf-autoschemas:: - asdf-schema-1.0.0 - -The following graph shows the dependencies between modules: - -.. digraph:: modules - - "fits" -> "core" - "unit" -> "core" - "time" -> "core" - "table" -> "core" - "time" -> "unit" - "table" -> "unit" - "fits" -> "table" + asdf-schema-1.1.0 .. _manifest: diff --git a/docs/source/schemas/manifest.rst b/docs/source/schemas/manifest.rst index 717b2d59..94cb16d0 100644 --- a/docs/source/schemas/manifest.rst +++ b/docs/source/schemas/manifest.rst @@ -16,3 +16,11 @@ by a single manifest document for that ASDF standard version. core-1.4.0 core-1.5.0 core-1.6.0 + +The non-core (astronomy) tags are described separately: + +.. asdf-autoschemas:: + :schema_root: ../../resources/manifests + :standard_prefix: asdf-format.org/astronomy + + astronomy-1.0.0 diff --git a/docs/source/schemas/table.rst b/docs/source/schemas/table.rst deleted file mode 100644 index 268fd4fb..00000000 --- a/docs/source/schemas/table.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. _unit-schema: - -Unit -==== - -The ``table`` module contains schema to support tabular data. - -.. asdf-autoschemas:: - - table/column-1.1.0 - table/table-1.1.0 diff --git a/docs/source/schemas/time.rst b/docs/source/schemas/time.rst deleted file mode 100644 index 6d95747d..00000000 --- a/docs/source/schemas/time.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. _time-schema: - -Time -==== - -The ``time`` module contains schema to support representing instances -in time and time deltas. - -.. asdf-autoschemas:: - - time/time-1.2.0 diff --git a/docs/source/schemas/unit.rst b/docs/source/schemas/unit.rst deleted file mode 100644 index 3dec1c3e..00000000 --- a/docs/source/schemas/unit.rst +++ /dev/null @@ -1,12 +0,0 @@ -.. _unit-schema: - -Unit -==== - -The ``unit`` module contains schema to support the units of physical quantities. - -.. asdf-autoschemas:: - - unit/unit-1.0.0 - unit/defunit-1.0.0 - unit/quantity-1.2.0 diff --git a/tests/common.py b/tests/common.py index 5ef19c0a..0b2d5990 100644 --- a/tests/common.py +++ b/tests/common.py @@ -26,10 +26,6 @@ DEPRECATED_PATTERNS = { re.compile(".*/transform/.*"), re.compile(".*/wcs/.*"), - re.compile(".*/unit/.*"), - re.compile(".*/table/.*"), - re.compile(".*/fits/.*"), - re.compile(".*/time/.*"), re.compile(".*/core/column.*"), re.compile(".*/core/table.*"), re.compile(".*/core/subclass_metadata.*"), diff --git a/tests/test_docs.py b/tests/test_docs.py index f9303d70..1126e10e 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,17 +1,17 @@ import collections -import pytest from common import is_deprecated EXCEPTIONS = { "tag:stsci.edu:asdf/asdf-schema-1.0.0", "asdf://asdf-format.org/core/manifests/:schema_root: ../../resources/manifests", "asdf://asdf-format.org/core/manifests/:standard_prefix: asdf-format.org/core", - "tag:stsci.edu:asdf/asdf-schema-1.0.0", + "http://stsci.edu/schemas/asdf/core/column-1.0.0", + "http://stsci.edu/schemas/asdf/core/table-1.0.0", + "http://stsci.edu/schemas/asdf/core/subclass_metadata-1.0.0", } -@pytest.mark.skip def test_docs_schema_links( latest_schema_ids, legacy_schema_ids, manifest_ids, docs_schema_ids, docs_legacy_schema_ids, docs_manifest_ids ): diff --git a/tests/test_version_map.py b/tests/test_version_map.py index 47671738..ee4a8099 100644 --- a/tests/test_version_map.py +++ b/tests/test_version_map.py @@ -42,6 +42,7 @@ def test_version_map(path): assert False, message +@pytest.mark.skip @pytest.mark.parametrize("path, previous_path", zip(SORTED_PATHS[1:], SORTED_PATHS[0:-1])) def test_version_map_tags_retained(path, previous_path): """ From 5ff17535f5af4ab4be3bdb92c50879f013d69243 Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 19 Feb 2024 15:10:07 -0500 Subject: [PATCH 12/13] unskip test --- tests/common.py | 4 ++++ tests/test_docs.py | 7 +++++++ tests/test_version_map.py | 1 - 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/common.py b/tests/common.py index 0b2d5990..ea749495 100644 --- a/tests/common.py +++ b/tests/common.py @@ -29,6 +29,10 @@ re.compile(".*/core/column.*"), re.compile(".*/core/table.*"), re.compile(".*/core/subclass_metadata.*"), + re.compile(".*/fits/.*"), + re.compile(".*/table/.*"), + re.compile(".*/unit/.*"), + re.compile(".*/time/.*"), } METASCHEMA_ID = "http://stsci.edu/schemas/yaml-schema/draft-01" diff --git a/tests/test_docs.py b/tests/test_docs.py index 1126e10e..bc370b54 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -9,6 +9,13 @@ "http://stsci.edu/schemas/asdf/core/column-1.0.0", "http://stsci.edu/schemas/asdf/core/table-1.0.0", "http://stsci.edu/schemas/asdf/core/subclass_metadata-1.0.0", + "http://stsci.edu/schemas/asdf/fits/fits-1.1.0", + "http://stsci.edu/schemas/asdf/table/column-1.1.0", + "http://stsci.edu/schemas/asdf/table/table-1.1.0", + "http://stsci.edu/schemas/asdf/time/time-1.2.0", + "http://stsci.edu/schemas/asdf/unit/defunit-1.0.0", + "http://stsci.edu/schemas/asdf/unit/quantity-1.2.0", + "http://stsci.edu/schemas/asdf/unit/unit-1.0.0", } diff --git a/tests/test_version_map.py b/tests/test_version_map.py index ee4a8099..47671738 100644 --- a/tests/test_version_map.py +++ b/tests/test_version_map.py @@ -42,7 +42,6 @@ def test_version_map(path): assert False, message -@pytest.mark.skip @pytest.mark.parametrize("path, previous_path", zip(SORTED_PATHS[1:], SORTED_PATHS[0:-1])) def test_version_map_tags_retained(path, previous_path): """ From c17232833f8d4366ed7621e073bd01c2eec9cbba Mon Sep 17 00:00:00 2001 From: Brett Date: Mon, 19 Feb 2024 15:20:31 -0500 Subject: [PATCH 13/13] fix minor formatting in ndarray-1.1.0 schema --- resources/schemas/stsci.edu/asdf/core/ndarray-1.1.0.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/schemas/stsci.edu/asdf/core/ndarray-1.1.0.yaml b/resources/schemas/stsci.edu/asdf/core/ndarray-1.1.0.yaml index 8533df23..5eae86a5 100644 --- a/resources/schemas/stsci.edu/asdf/core/ndarray-1.1.0.yaml +++ b/resources/schemas/stsci.edu/asdf/core/ndarray-1.1.0.yaml @@ -170,7 +170,7 @@ definitions: - `uint8`, `uint16`, `uint32`, `uint64`: Unsigned integer types, with the given bit size. - - `float16 `: Half-precision floating-point type or "binary16", + - `float16`: Half-precision floating-point type or "binary16", as defined in IEEE 754. - `float32`: Single-precision floating-point type or "binary32",