From 73b816c9495a94f571855232d2669ffb35f1267b Mon Sep 17 00:00:00 2001 From: "Keto D. Zhang" Date: Fri, 28 Jun 2024 11:08:39 -0700 Subject: [PATCH 1/5] feat: Replace ASDF schema generator with custom GenerateAsdfSchema --- asdf_pydantic/model.py | 25 ++++++++++++++++++++++--- asdf_pydantic/schema.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) create mode 100644 asdf_pydantic/schema.py diff --git a/asdf_pydantic/model.py b/asdf_pydantic/model.py index 0bdc49a..69d644a 100644 --- a/asdf_pydantic/model.py +++ b/asdf_pydantic/model.py @@ -42,8 +42,29 @@ def asdf_yaml_tree(self) -> dict: return d @classmethod + def model_asdf_schema( + cls, + by_alias: bool = True, + ref_template: str = DEFAULT_ASDF_SCHEMA_REF_TEMPLATE, + schema_generator: type[GenerateAsdfSchema] = GenerateAsdfSchema, + ): + """Get the ASDF schema definition for this model.""" + # Implementation follows closely with the `BaseModel.model_json_schema` + schema_generator_instance = schema_generator( + by_alias=by_alias, + ref_template=ref_template, + tag=cls._tag, + ) + json_schema = schema_generator_instance.generate(cls.__pydantic_core_schema__) + + header = "%YAML 1.1\n---\n" + + return f"{header}\n{yaml.safe_dump(json_schema)}" def schema_asdf( - cls, *, metaschema: str = "http://stsci.edu/schemas/asdf/asdf-schema-1.0.0" + cls, + *, + metaschema: str = "http://stsci.edu/schemas/asdf/asdf-schema-1.0.0", + **kwargs, ) -> str: """Get the ASDF schema definition for this model. @@ -54,8 +75,6 @@ def schema_asdf( See https://asdf.readthedocs.io/en/stable/asdf/extending/schemas.html#anatomy-of-a-schema for more options. """ # noqa: E501 - # TODO: Function signature should follow BaseModel.schema() or - # BaseModel.schema_json() header = textwrap.dedent( f""" %YAML 1.1 diff --git a/asdf_pydantic/schema.py b/asdf_pydantic/schema.py new file mode 100644 index 0000000..d68b50e --- /dev/null +++ b/asdf_pydantic/schema.py @@ -0,0 +1,38 @@ +from pydantic.json_schema import GenerateJsonSchema + +DEFAULT_ASDF_SCHEMA_REF_TEMPLATE = "#/definitions/{model}" + + +class GenerateAsdfSchema(GenerateJsonSchema): + """Generates ASDF-compatible schema from Pydantic's default JSON schema generator. + + ```{caution} Experimental + This schema generator is not complete. It currently creates JSON 2020-12 + schema (despite `$schema` says it's `asdf-schema-1.0.0`) which are not + compatible with ASDF. + ``` + """ + + # HACK: When we can support tree models, then not all schema should have tag + tag: str | None + schema_dialect = "http://stsci.edu/schemas/asdf/asdf-schema-1.0.0" + + def __init__( + self, + by_alias: bool = True, + ref_template: str = DEFAULT_ASDF_SCHEMA_REF_TEMPLATE, + tag: str | None = None, + ): + super().__init__(by_alias=by_alias, ref_template=ref_template) + self.tag = tag + + def generate(self, schema, mode="validation"): + json_schema = super().generate(schema, mode) # noqa: F841 + + if self.tag: + json_schema["$schema"] = self.schema_dialect + json_schema["id"] = self.tag + json_schema["tag"] = f"tag:{self.tag.split('://', maxsplit=2)[-1]}" + + # TODO: Convert jsonschema 2020-12 to ASDF schema + return json_schema From 2260efe0dcf80a8cd3322b25a4d1381db401818d Mon Sep 17 00:00:00 2001 From: "Keto D. Zhang" Date: Fri, 28 Jun 2024 11:10:17 -0700 Subject: [PATCH 2/5] feat: Deprecate AsdfPydanticModel.schema_asdf(), use model_schema_asdf() instead. --- asdf_pydantic/model.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/asdf_pydantic/model.py b/asdf_pydantic/model.py index 69d644a..ccba39a 100644 --- a/asdf_pydantic/model.py +++ b/asdf_pydantic/model.py @@ -1,8 +1,11 @@ -import textwrap from typing import ClassVar import yaml from pydantic import BaseModel +from pydantic.json_schema import GenerateJsonSchema +from typing_extensions import deprecated + +from asdf_pydantic.schema import DEFAULT_ASDF_SCHEMA_REF_TEMPLATE, GenerateAsdfSchema class AsdfPydanticModel(BaseModel): @@ -60,10 +63,15 @@ def model_asdf_schema( header = "%YAML 1.1\n---\n" return f"{header}\n{yaml.safe_dump(json_schema)}" + + @classmethod + @deprecated( + "The `schema_asdf` method is deprecated; use `model_asdf_schema` instead." + ) def schema_asdf( cls, *, - metaschema: str = "http://stsci.edu/schemas/asdf/asdf-schema-1.0.0", + metaschema: str = GenerateAsdfSchema.schema_dialect, **kwargs, ) -> str: """Get the ASDF schema definition for this model. @@ -71,19 +79,11 @@ def schema_asdf( Parameters ---------- metaschema, optional - A metaschema URI, by default "http://stsci.edu/schemas/asdf/asdf-schema-1.0.0". - See https://asdf.readthedocs.io/en/stable/asdf/extending/schemas.html#anatomy-of-a-schema - for more options. + A metaschema URI """ # noqa: E501 - header = textwrap.dedent( - f""" - %YAML 1.1 - --- - $schema: {metaschema} - id: {cls._tag} - tag: tag:{cls._tag.split('://', maxsplit=2)[-1]} - - """ - ) - body = yaml.safe_dump(cls.model_json_schema()) - return header + body + if metaschema != GenerateAsdfSchema.schema_dialect: + raise NotImplementedError( + f"Only {GenerateAsdfSchema.schema_dialect} is supported as metaschema." + ) + + return cls.model_asdf_schema(**kwargs) From 57ec924109fe2118f1bc83493bcab8b044a940a6 Mon Sep 17 00:00:00 2001 From: "Keto D. Zhang" Date: Fri, 28 Jun 2024 11:10:48 -0700 Subject: [PATCH 3/5] test: Refactor --- asdf_pydantic/schema.py | 6 ++++-- tests/examples/test_rectangle.py | 12 ++++++++---- tests/schema_validation_test.py | 4 ++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/asdf_pydantic/schema.py b/asdf_pydantic/schema.py index d68b50e..321144f 100644 --- a/asdf_pydantic/schema.py +++ b/asdf_pydantic/schema.py @@ -1,3 +1,5 @@ +from typing import Optional + from pydantic.json_schema import GenerateJsonSchema DEFAULT_ASDF_SCHEMA_REF_TEMPLATE = "#/definitions/{model}" @@ -14,14 +16,14 @@ class GenerateAsdfSchema(GenerateJsonSchema): """ # HACK: When we can support tree models, then not all schema should have tag - tag: str | None + tag: Optional[str] schema_dialect = "http://stsci.edu/schemas/asdf/asdf-schema-1.0.0" def __init__( self, by_alias: bool = True, ref_template: str = DEFAULT_ASDF_SCHEMA_REF_TEMPLATE, - tag: str | None = None, + tag: Optional[str] = None, ): super().__init__(by_alias=by_alias, ref_template=ref_template) self.tag = tag diff --git a/tests/examples/test_rectangle.py b/tests/examples/test_rectangle.py index b52ea49..bd24368 100644 --- a/tests/examples/test_rectangle.py +++ b/tests/examples/test_rectangle.py @@ -1,6 +1,8 @@ import asdf +import pytest from asdf.extension import Extension from asdf.schema import check_schema, load_schema +from yaml.scanner import ScannerError from asdf_pydantic import AsdfPydanticConverter from asdf_pydantic.examples.shapes import AsdfRectangle @@ -23,7 +25,7 @@ class TestExtension(Extension): asdf.get_config().add_resource_mapping( { "asdf://asdf-pydantic/shapes/schemas/rectangle-1.0.0": ( - AsdfRectangle.schema_asdf().encode("utf-8") + AsdfRectangle.model_asdf_schema().encode("utf-8") ) } ) @@ -31,9 +33,11 @@ class TestExtension(Extension): def test_schema(): - schema = load_schema("asdf://asdf-pydantic/shapes/schemas/rectangle-1.0.0") - - check_schema(schema) + try: + schema = load_schema("asdf://asdf-pydantic/shapes/schemas/rectangle-1.0.0") + check_schema(schema) + except ScannerError as e: + pytest.fail(f"{e}\n{AsdfRectangle.model_asdf_schema()}") assert schema["$schema"] == "http://stsci.edu/schemas/asdf/asdf-schema-1.0.0" assert schema["title"] == "AsdfRectangle" diff --git a/tests/schema_validation_test.py b/tests/schema_validation_test.py index f62ac0d..85cc05e 100644 --- a/tests/schema_validation_test.py +++ b/tests/schema_validation_test.py @@ -28,7 +28,7 @@ class TestExtension(Extension): asdf.get_config().add_resource_mapping( { "asdf://asdf-pydantic/shapes/schemas/rectangle-1.0.0": ( - AsdfRectangle.schema_asdf().encode("utf-8") + AsdfRectangle.model_asdf_schema().encode("utf-8") ) } ) @@ -130,7 +130,7 @@ def test_validate_fail_on_bad_yaml_file(): def test_given_child_field_contains_asdf_object_then_schema_has_child_tag(): from asdf.schema import check_schema - schema = yaml.safe_load(AsdfNode.schema_asdf()) # type: ignore + schema = yaml.safe_load(AsdfNode.model_asdf_schema()) # type: ignore check_schema(schema) child_schema = schema["definitions"]["AsdfNode"]["properties"]["child"] From 9c4c1dc6fc78b7959aa038b19c4220430502c118 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 28 Jun 2024 19:26:49 +0000 Subject: [PATCH 4/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- asdf_pydantic/model.py | 1 - 1 file changed, 1 deletion(-) diff --git a/asdf_pydantic/model.py b/asdf_pydantic/model.py index ccba39a..7de635d 100644 --- a/asdf_pydantic/model.py +++ b/asdf_pydantic/model.py @@ -2,7 +2,6 @@ import yaml from pydantic import BaseModel -from pydantic.json_schema import GenerateJsonSchema from typing_extensions import deprecated from asdf_pydantic.schema import DEFAULT_ASDF_SCHEMA_REF_TEMPLATE, GenerateAsdfSchema From a3d3d2d7000084f954c0a44f2d5dee645f94035f Mon Sep 17 00:00:00 2001 From: "Keto D. Zhang" Date: Fri, 28 Jun 2024 13:39:43 -0700 Subject: [PATCH 5/5] feat: drop support for Python 3.9 with numpy v2 --- pyproject.toml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ef5869a..a8598ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,8 @@ classifiers = [ dependencies = [ "asdf>=3", "pydantic>=2", - "numpy>=1.25" + "numpy>=1.25", + "numpy<2;python_version<'3.10'", ] dynamic = ["version"] @@ -63,13 +64,20 @@ matrix-name-format = "{variable}={value}" test = "pytest {args}" [[tool.hatch.envs.test.matrix]] -python = ["3.9", "3.10", "3.11", "3.12"] +# Only test with numpy v1 on Python 3.9 +python = ["3.9"] +numpy-version = ["1"] + +[[tool.hatch.envs.test.matrix]] +python = ["3.10", "3.11", "3.12"] numpy-version = ["1", "2"] + [tool.hatch.envs.test.overrides] matrix.numpy-version.dependencies = [ { value = "numpy>=1,<2", if = ["1"] }, { value = "numpy>=2,<3", if = ["2"] }, + { value = "astropy>=6.1", if = ["2"] }, ] [tool.hatch.envs.docs]