From e9c28d057413aa801ec9af86b89f3c4d5b3de8e5 Mon Sep 17 00:00:00 2001 From: Thomas Falch Johansen Date: Fri, 15 Sep 2023 15:53:04 +0200 Subject: [PATCH] fix: handle dates in yaml correctly Support YYYY-MM-DD and YYYY-MM-DD HH:MM:SS for temp models If we dont explicitly handle dates with our own date type, then PyYAML will change to datetime.date for what we want to have datetime.datetime. To make sure that all are converted to datetime.datetime, we set all dates in yaml as our date type. --- .../input/yaml_types/yaml_temporal_model.py | 9 +- .../input/yaml_types/yaml_variable.py | 17 ++++ .../test_json_schema_changed/schemas.json | 99 +------------------ .../test_validation_json_schemas.py | 11 ++- .../yaml_types/test_yaml_temporal_model.py | 6 +- 5 files changed, 29 insertions(+), 113 deletions(-) diff --git a/src/ecalc/libraries/libecalc/common/libecalc/input/yaml_types/yaml_temporal_model.py b/src/ecalc/libraries/libecalc/common/libecalc/input/yaml_types/yaml_temporal_model.py index 1ee23c64e3..31f254ee0c 100644 --- a/src/ecalc/libraries/libecalc/common/libecalc/input/yaml_types/yaml_temporal_model.py +++ b/src/ecalc/libraries/libecalc/common/libecalc/input/yaml_types/yaml_temporal_model.py @@ -1,11 +1,6 @@ from typing import Dict, TypeVar, Union -from pydantic import ConstrainedStr - - -class DatetimeString(ConstrainedStr): - regex = "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$" - +from libecalc.input.yaml_types.yaml_variable import YamlDefaultDatetime TModel = TypeVar("TModel") -YamlTemporalModel = Union[TModel, Dict[DatetimeString, TModel]] +YamlTemporalModel = Union[TModel, Dict[YamlDefaultDatetime, TModel]] diff --git a/src/ecalc/libraries/libecalc/common/libecalc/input/yaml_types/yaml_variable.py b/src/ecalc/libraries/libecalc/common/libecalc/input/yaml_types/yaml_variable.py index 6a25b059bf..852f4e47e1 100644 --- a/src/ecalc/libraries/libecalc/common/libecalc/input/yaml_types/yaml_variable.py +++ b/src/ecalc/libraries/libecalc/common/libecalc/input/yaml_types/yaml_variable.py @@ -9,6 +9,10 @@ class YamlSingleVariable(YamlBase): + """ + A variable in YAML that can be set to a valid eCalc Expression + """ + value: Expression def to_dto(self): @@ -16,12 +20,25 @@ def to_dto(self): class YamlDefaultDatetime(datetime): + """ + PyYAML is smart and detects datetime.date and datetime.datetime differently in YAML, and parses usually + dates to datetime.date. However, in eCalc we required datetime.datetime, and there is a subtle difference + in behaviour between those too. Therefore we need to cast to datetime.datetime as early as possible to make + sure eCalc behaves correctly. + """ + @classmethod def __get_validators__(cls): yield cls.date_to_datetime @classmethod def date_to_datetime(cls, value) -> datetime: + """ + Handles both datetimes and dates supported formats, and converts them to + datetime.datetime internally to avoid problems with mixing datetime.date and datetime.datetime + :param value: string with a PyYAML supported date or datetime format + :return: the corresponding datetime.datetime. datetime.date will have HH:MM:SS set to 00:00:00 + """ try: date = parse_date(value) return convert_date_to_datetime(date) diff --git a/src/ecalc/libraries/libecalc/common/tests/input/validation/snapshots/test_validation_json_schemas/test_json_schema_changed/schemas.json b/src/ecalc/libraries/libecalc/common/tests/input/validation/snapshots/test_validation_json_schemas/test_json_schema_changed/schemas.json index ad7298e3a1..dd31730662 100644 --- a/src/ecalc/libraries/libecalc/common/tests/input/validation/snapshots/test_validation_json_schemas/test_json_schema_changed/schemas.json +++ b/src/ecalc/libraries/libecalc/common/tests/input/validation/snapshots/test_validation_json_schemas/test_json_schema_changed/schemas.json @@ -64,11 +64,6 @@ "additionalProperties": { "type": "string" }, - "patternProperties": { - "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$": { - "type": "string" - } - }, "type": "object" } ], @@ -122,14 +117,6 @@ }, "type": "array" }, - "patternProperties": { - "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$": { - "items": { - "$ref": "#/definitions/YamlCompressorSystemOperationalSettings" - }, - "type": "array" - } - }, "type": "object" } ], @@ -407,11 +394,6 @@ "additionalProperties": { "$ref": "https://test.ecalc.equinor.com/api/v1/schema-validation/energy-usage-model.json#properties/ENERGY_USAGE_MODEL" }, - "patternProperties": { - "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$": { - "$ref": "https://test.ecalc.equinor.com/api/v1/schema-validation/energy-usage-model.json#properties/ENERGY_USAGE_MODEL" - } - }, "type": "object" } ], @@ -514,11 +496,6 @@ "additionalProperties": { "$ref": "https://test.ecalc.equinor.com/api/v1/schema-validation/energy-usage-model.json#properties/ENERGY_USAGE_MODEL" }, - "patternProperties": { - "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$": { - "$ref": "https://test.ecalc.equinor.com/api/v1/schema-validation/energy-usage-model.json#properties/ENERGY_USAGE_MODEL" - } - }, "type": "object" } ], @@ -534,11 +511,6 @@ "additionalProperties": { "type": "string" }, - "patternProperties": { - "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$": { - "type": "string" - } - }, "type": "object" } ], @@ -651,11 +623,6 @@ "additionalProperties": { "$ref": "https://test.ecalc.equinor.com/api/v1/schema-validation/energy-usage-model-common.json#definitions/number_or_string" }, - "patternProperties": { - "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$": { - "$ref": "https://test.ecalc.equinor.com/api/v1/schema-validation/energy-usage-model-common.json#definitions/number_or_string" - } - }, "type": "object" } ], @@ -671,11 +638,6 @@ "additionalProperties": { "type": "string" }, - "patternProperties": { - "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$": { - "type": "string" - } - }, "type": "object" } ], @@ -723,11 +685,6 @@ "additionalProperties": { "type": "string" }, - "patternProperties": { - "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$": { - "type": "string" - } - }, "type": "object" } ], @@ -801,27 +758,6 @@ ], "title": "Expression" }, - "patternProperties": { - "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "number" - }, - { - "pattern": "^[\\w * ^ . : ; () {} = > < + \\- /]+$", - "type": "string" - } - ], - "examples": [ - "SIM1;OIL_PROD {+} 1000", - 5 - ], - "title": "Expression" - } - }, "type": "object" } ], @@ -874,27 +810,6 @@ ], "title": "Expression" }, - "patternProperties": { - "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "number" - }, - { - "pattern": "^[\\w * ^ . : ; () {} = > < + \\- /]+$", - "type": "string" - } - ], - "examples": [ - "SIM1;OIL_PROD {+} 1000", - 5 - ], - "title": "Expression" - } - }, "type": "object" } ], @@ -979,11 +894,6 @@ "additionalProperties": { "type": "string" }, - "patternProperties": { - "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$": { - "type": "string" - } - }, "type": "object" } ], @@ -1037,14 +947,6 @@ }, "type": "array" }, - "patternProperties": { - "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$": { - "items": { - "$ref": "#/definitions/YamlPumpSystemOperationalSettings" - }, - "type": "array" - } - }, "type": "object" } ], @@ -1291,6 +1193,7 @@ }, "YamlSingleVariable": { "additionalProperties": false, + "description": "A variable in YAML that can be set to a valid eCalc Expression", "properties": { "VALUE": { "anyOf": [ diff --git a/src/ecalc/libraries/libecalc/common/tests/input/validation/test_validation_json_schemas.py b/src/ecalc/libraries/libecalc/common/tests/input/validation/test_validation_json_schemas.py index 1bb87019b8..7018e7c966 100644 --- a/src/ecalc/libraries/libecalc/common/tests/input/validation/test_validation_json_schemas.py +++ b/src/ecalc/libraries/libecalc/common/tests/input/validation/test_validation_json_schemas.py @@ -47,11 +47,12 @@ def test_temporal_property_placeholder(self): }, { "type": "object", - "patternProperties": { - "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$": { - "$ref": "https://test.ecalc.equinor.com/api/v1/schema-validation/energy-usage-model.json#properties/ENERGY_USAGE_MODEL" - } - }, + # TODO: We need to make a trade off now, either make parsing fail, or that the Open API JSON Schema does not indicate the date pattern we require as key + # "patternProperties": { + # "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$": { + # "$ref": "https://test.ecalc.equinor.com/api/v1/schema-validation/energy-usage-model.json#properties/ENERGY_USAGE_MODEL" + # } + # }, "additionalProperties": { "$ref": "https://test.ecalc.equinor.com/api/v1/schema-validation/energy-usage-model.json#properties/ENERGY_USAGE_MODEL" }, diff --git a/src/ecalc/libraries/libecalc/common/tests/input/yaml_types/test_yaml_temporal_model.py b/src/ecalc/libraries/libecalc/common/tests/input/yaml_types/test_yaml_temporal_model.py index c59d893cbe..84ef8da968 100644 --- a/src/ecalc/libraries/libecalc/common/tests/input/yaml_types/test_yaml_temporal_model.py +++ b/src/ecalc/libraries/libecalc/common/tests/input/yaml_types/test_yaml_temporal_model.py @@ -16,9 +16,9 @@ def test_temporal_model_schema(self): }, { "type": "object", - "patternProperties": { - "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$": {"type": "string"} - }, + # "patternProperties": { + # "^\\d{4}\\-(0?[1-9]|1[012])\\-(0?[1-9]|[12][0-9]|3[01])$": {"type": "string"} + # }, "additionalProperties": {"type": "string"}, }, ],