From 5ef4947f7626e1942b002e4d2eab5f0837277c92 Mon Sep 17 00:00:00 2001 From: Ludwig Schneider Date: Mon, 26 Feb 2024 13:30:13 -0600 Subject: [PATCH 1/8] ensure that constructors only update (and accept) JSON attrs that are valid. --- src/cript/nodes/primary_nodes/collection.py | 5 ++--- src/cript/nodes/primary_nodes/computation.py | 5 ++--- src/cript/nodes/primary_nodes/computation_process.py | 5 ++--- src/cript/nodes/primary_nodes/data.py | 5 ++--- src/cript/nodes/primary_nodes/experiment.py | 5 ++--- src/cript/nodes/primary_nodes/inventory.py | 3 ++- src/cript/nodes/primary_nodes/material.py | 3 ++- src/cript/nodes/primary_nodes/project.py | 4 ++-- src/cript/nodes/primary_nodes/reference.py | 1 - src/cript/nodes/subobjects/algorithm.py | 4 ++-- src/cript/nodes/subobjects/citation.py | 4 ++-- src/cript/nodes/subobjects/computational_forcefield.py | 4 ++-- src/cript/nodes/subobjects/condition.py | 4 ++-- src/cript/nodes/subobjects/equipment.py | 4 ++-- src/cript/nodes/subobjects/ingredient.py | 4 ++-- src/cript/nodes/subobjects/parameter.py | 4 ++-- src/cript/nodes/subobjects/property.py | 4 ++-- src/cript/nodes/subobjects/quantity.py | 4 ++-- src/cript/nodes/subobjects/software.py | 4 ++-- src/cript/nodes/subobjects/software_configuration.py | 4 ++-- src/cript/nodes/supporting_nodes/file.py | 5 ++--- src/cript/nodes/supporting_nodes/user.py | 5 ++--- 22 files changed, 42 insertions(+), 48 deletions(-) diff --git a/src/cript/nodes/primary_nodes/collection.py b/src/cript/nodes/primary_nodes/collection.py index b7763d739..3beaf5c57 100644 --- a/src/cript/nodes/primary_nodes/collection.py +++ b/src/cript/nodes/primary_nodes/collection.py @@ -105,7 +105,7 @@ def __init__(self, name: str, experiment: Optional[List[Any]] = None, inventory: if citation is None: citation = [] - self._json_attrs = replace( + new_json_attrs = replace( self._json_attrs, name=name, experiment=experiment, @@ -113,8 +113,7 @@ def __init__(self, name: str, experiment: Optional[List[Any]] = None, inventory: doi=doi, citation=citation, ) - - self.validate() + self._update_json_attrs_if_valid(new_json_attrs) @property @beartype diff --git a/src/cript/nodes/primary_nodes/computation.py b/src/cript/nodes/primary_nodes/computation.py index b318e95a0..87d9b317a 100644 --- a/src/cript/nodes/primary_nodes/computation.py +++ b/src/cript/nodes/primary_nodes/computation.py @@ -141,7 +141,7 @@ def __init__( if citation is None: citation = [] - self._json_attrs = replace( + new_json_attrs = replace( self._json_attrs, type=type, input_data=input_data, @@ -151,8 +151,7 @@ def __init__( prerequisite_computation=prerequisite_computation, citation=citation, ) - - self.validate() + self._update_json_attrs_if_valid(new_json_attrs) # ------------------ Properties ------------------ diff --git a/src/cript/nodes/primary_nodes/computation_process.py b/src/cript/nodes/primary_nodes/computation_process.py index 5ab289dd1..576c69ede 100644 --- a/src/cript/nodes/primary_nodes/computation_process.py +++ b/src/cript/nodes/primary_nodes/computation_process.py @@ -220,7 +220,7 @@ def __init__( if citation is None: citation = [] - self._json_attrs = replace( + new_json_attrs = replace( self._json_attrs, type=type, input_data=input_data, @@ -231,8 +231,7 @@ def __init__( property=property, citation=citation, ) - - # self.validate() + self._update_json_attrs_if_valid(new_json_attrs) @property @beartype diff --git a/src/cript/nodes/primary_nodes/data.py b/src/cript/nodes/primary_nodes/data.py index 51edde3f5..586e292cb 100644 --- a/src/cript/nodes/primary_nodes/data.py +++ b/src/cript/nodes/primary_nodes/data.py @@ -167,7 +167,7 @@ def __init__( if citation is None: citation = [] - self._json_attrs = replace( + new_json_attrs = replace( self._json_attrs, type=type, file=file, @@ -178,8 +178,7 @@ def __init__( process=process, citation=citation, ) - - self.validate() + self._update_json_attrs_if_valid(new_json_attrs) @property @beartype diff --git a/src/cript/nodes/primary_nodes/experiment.py b/src/cript/nodes/primary_nodes/experiment.py index 17e683490..3d8433ebf 100644 --- a/src/cript/nodes/primary_nodes/experiment.py +++ b/src/cript/nodes/primary_nodes/experiment.py @@ -135,7 +135,7 @@ def __init__( super().__init__(name=name, notes=notes, **kwargs) - self._json_attrs = replace( + new_json_attrs = replace( self._json_attrs, name=name, process=process, @@ -147,8 +147,7 @@ def __init__( notes=notes, ) - # check if the code is still valid - self.validate() + self._update_json_attrs_if_valid(new_json_attrs) @property @beartype diff --git a/src/cript/nodes/primary_nodes/inventory.py b/src/cript/nodes/primary_nodes/inventory.py index 511d99d8e..d654e8902 100644 --- a/src/cript/nodes/primary_nodes/inventory.py +++ b/src/cript/nodes/primary_nodes/inventory.py @@ -99,7 +99,8 @@ def __init__(self, name: str, material: List[Material], notes: str = "", **kwarg super().__init__(name=name, notes=notes, **kwargs) - self._json_attrs = replace(self._json_attrs, material=material) + new_json_attrs = replace(self._json_attrs, material=material) + self._update_json_attrs_if_valid(new_json_attrs) @property @beartype diff --git a/src/cript/nodes/primary_nodes/material.py b/src/cript/nodes/primary_nodes/material.py index 4e7fffa4a..110ee47e5 100644 --- a/src/cript/nodes/primary_nodes/material.py +++ b/src/cript/nodes/primary_nodes/material.py @@ -126,7 +126,7 @@ def __init__( if keyword is None: keyword = [] - self._json_attrs = replace( + new_json_attrs = replace( self._json_attrs, name=name, identifier=identifier, @@ -137,6 +137,7 @@ def __init__( computational_forcefield=computational_forcefield, keyword=keyword, ) + self._update_json_attrs_if_valid(new_json_attrs) @property @beartype diff --git a/src/cript/nodes/primary_nodes/project.py b/src/cript/nodes/primary_nodes/project.py index 4da8b427b..bab62cbfa 100644 --- a/src/cript/nodes/primary_nodes/project.py +++ b/src/cript/nodes/primary_nodes/project.py @@ -101,8 +101,8 @@ def __init__(self, name: str, collection: Optional[List[Collection]] = None, mat if material is None: material = [] - self._json_attrs = replace(self._json_attrs, name=name, collection=collection, material=material) - self.validate() + new_json_attrs = replace(self._json_attrs, name=name, collection=collection, material=material) + self._update_json_attrs_if_valid(new_json_attrs) def validate(self, api=None, is_patch=False, force_validation: bool = False): from cript.nodes.exceptions import ( diff --git a/src/cript/nodes/primary_nodes/reference.py b/src/cript/nodes/primary_nodes/reference.py index 996bf7e0d..4f23a5e70 100644 --- a/src/cript/nodes/primary_nodes/reference.py +++ b/src/cript/nodes/primary_nodes/reference.py @@ -168,7 +168,6 @@ def __init__( new_attrs = replace(self._json_attrs, type=type, title=title, author=author, journal=journal, publisher=publisher, year=year, volume=volume, issue=issue, pages=pages, doi=doi, issn=issn, arxiv_id=arxiv_id, pmid=pmid, website=website) self._update_json_attrs_if_valid(new_attrs) - self.validate() @property @beartype diff --git a/src/cript/nodes/subobjects/algorithm.py b/src/cript/nodes/subobjects/algorithm.py index b5cc59069..d45f5f894 100644 --- a/src/cript/nodes/subobjects/algorithm.py +++ b/src/cript/nodes/subobjects/algorithm.py @@ -106,8 +106,8 @@ def __init__(self, key: str, type: str, parameter: Optional[List[Parameter]] = N if citation is None: citation = [] super().__init__(**kwargs) - self._json_attrs = replace(self._json_attrs, key=key, type=type, parameter=parameter) - self.validate() + new_json_attrs = replace(self._json_attrs, key=key, type=type, parameter=parameter) + self._update_json_attrs_if_valid(new_json_attrs) @property def key(self) -> str: diff --git a/src/cript/nodes/subobjects/citation.py b/src/cript/nodes/subobjects/citation.py index abcba9df5..e4457a251 100644 --- a/src/cript/nodes/subobjects/citation.py +++ b/src/cript/nodes/subobjects/citation.py @@ -104,8 +104,8 @@ def __init__(self, type: str, reference: Reference, **kwargs): Instantiate citation subobject """ super().__init__(**kwargs) - self._json_attrs = replace(self._json_attrs, type=type, reference=reference) - self.validate() + new_json_attrs = replace(self._json_attrs, type=type, reference=reference) + self._update_json_attrs_if_valid(new_json_attrs) @property @beartype diff --git a/src/cript/nodes/subobjects/computational_forcefield.py b/src/cript/nodes/subobjects/computational_forcefield.py index e9176014b..d77d38b7e 100644 --- a/src/cript/nodes/subobjects/computational_forcefield.py +++ b/src/cript/nodes/subobjects/computational_forcefield.py @@ -140,7 +140,7 @@ def __init__(self, key: str, building_block: str, coarse_grained_mapping: str = if data is None: data = [] - self._json_attrs = replace( + new_json_attrs = replace( self._json_attrs, key=key, building_block=building_block, @@ -151,7 +151,7 @@ def __init__(self, key: str, building_block: str, coarse_grained_mapping: str = data=data, citation=citation, ) - self.validate() + self._update_json_attrs_if_valid(new_json_attrs) @property @beartype diff --git a/src/cript/nodes/subobjects/condition.py b/src/cript/nodes/subobjects/condition.py index acc17f3de..71903857b 100644 --- a/src/cript/nodes/subobjects/condition.py +++ b/src/cript/nodes/subobjects/condition.py @@ -151,7 +151,7 @@ def __init__( if data is None: data = [] - self._json_attrs = replace( + new_json_attrs = replace( self._json_attrs, key=key, type=type, @@ -164,7 +164,7 @@ def __init__( measurement_id=measurement_id, data=data, ) - self.validate() + self._update_json_attrs_if_valid(new_json_attrs) @property @beartype diff --git a/src/cript/nodes/subobjects/equipment.py b/src/cript/nodes/subobjects/equipment.py index ec6f6c7eb..99fa03e2c 100644 --- a/src/cript/nodes/subobjects/equipment.py +++ b/src/cript/nodes/subobjects/equipment.py @@ -94,8 +94,8 @@ def __init__(self, key: str, description: str = "", condition: Union[List[Condit if citation is None: citation = [] super().__init__(**kwargs) - self._json_attrs = replace(self._json_attrs, key=key, description=description, condition=condition, file=file, citation=citation) - self.validate() + new_json_attrs = replace(self._json_attrs, key=key, description=description, condition=condition, file=file, citation=citation) + self._update_json_attrs_if_valid(new_json_attrs) @property @beartype diff --git a/src/cript/nodes/subobjects/ingredient.py b/src/cript/nodes/subobjects/ingredient.py index a6ec82aa9..fc698173a 100644 --- a/src/cript/nodes/subobjects/ingredient.py +++ b/src/cript/nodes/subobjects/ingredient.py @@ -106,8 +106,8 @@ def __init__(self, material: Material, quantity: List[Quantity], keyword: Option super().__init__(**kwargs) if keyword is None: keyword = [] - self._json_attrs = replace(self._json_attrs, material=material, quantity=quantity, keyword=keyword) - self.validate() + new_json_attrs = replace(self._json_attrs, material=material, quantity=quantity, keyword=keyword) + self._update_json_attrs_if_valid(new_json_attrs) @classmethod def _from_json(cls, json_dict: dict): diff --git a/src/cript/nodes/subobjects/parameter.py b/src/cript/nodes/subobjects/parameter.py index 68031cb79..98e1d501e 100644 --- a/src/cript/nodes/subobjects/parameter.py +++ b/src/cript/nodes/subobjects/parameter.py @@ -92,8 +92,8 @@ def __init__(self, key: str, value: Number, unit: Optional[str] = None, **kwargs create Parameter sub-object """ super().__init__(**kwargs) - self._json_attrs = replace(self._json_attrs, key=key, value=value, unit=unit) - self.validate() + new_json_attrs = replace(self._json_attrs, key=key, value=value, unit=unit) + self._update_json_attrs_if_valid(new_json_attrs) @classmethod def _from_json(cls, json_dict: dict): diff --git a/src/cript/nodes/subobjects/property.py b/src/cript/nodes/subobjects/property.py index 831cfc5df..fb97e8f30 100644 --- a/src/cript/nodes/subobjects/property.py +++ b/src/cript/nodes/subobjects/property.py @@ -167,7 +167,7 @@ def __init__( citation = [] super().__init__(**kwargs) - self._json_attrs = replace( + new_json_attrs = replace( self._json_attrs, key=key, type=type, @@ -185,7 +185,7 @@ def __init__( citation=citation, notes=notes, ) - self.validate() + self._update_json_attrs_if_valid(new_json_attrs) @property @beartype diff --git a/src/cript/nodes/subobjects/quantity.py b/src/cript/nodes/subobjects/quantity.py index 392bed68f..dcd4b0d74 100644 --- a/src/cript/nodes/subobjects/quantity.py +++ b/src/cript/nodes/subobjects/quantity.py @@ -94,8 +94,8 @@ def __init__(self, key: str, value: Number, unit: str, uncertainty: Optional[Num create Quantity sub-object """ super().__init__(**kwargs) - self._json_attrs = replace(self._json_attrs, key=key, value=value, unit=unit, uncertainty=uncertainty, uncertainty_type=uncertainty_type) - self.validate() + new_json_attrs = replace(self._json_attrs, key=key, value=value, unit=unit, uncertainty=uncertainty, uncertainty_type=uncertainty_type) + self._update_json_attrs_if_valid(new_json_attrs) @classmethod def _from_json(cls, json_dict: dict): diff --git a/src/cript/nodes/subobjects/software.py b/src/cript/nodes/subobjects/software.py index e96ff60c2..e558e31e4 100644 --- a/src/cript/nodes/subobjects/software.py +++ b/src/cript/nodes/subobjects/software.py @@ -82,8 +82,8 @@ def __init__(self, name: str, version: str, source: str = "", **kwargs): """ super().__init__(**kwargs) - self._json_attrs = replace(self._json_attrs, name=name, version=version, source=source) - self.validate() + new_json_attrs = replace(self._json_attrs, name=name, version=version, source=source) + self._update_json_attrs_if_valid(new_json_attrs) @property @beartype diff --git a/src/cript/nodes/subobjects/software_configuration.py b/src/cript/nodes/subobjects/software_configuration.py index 8fba5a945..02dbfa9bd 100644 --- a/src/cript/nodes/subobjects/software_configuration.py +++ b/src/cript/nodes/subobjects/software_configuration.py @@ -97,8 +97,8 @@ def __init__(self, software: Software, algorithm: Optional[List[Algorithm]] = No if citation is None: citation = [] super().__init__(**kwargs) - self._json_attrs = replace(self._json_attrs, software=software, algorithm=algorithm, notes=notes, citation=citation) - self.validate() + new_json_attrs = replace(self._json_attrs, software=software, algorithm=algorithm, notes=notes, citation=citation) + self._update_json_attrs_if_valid(new_json_attrs) @property @beartype diff --git a/src/cript/nodes/supporting_nodes/file.py b/src/cript/nodes/supporting_nodes/file.py index c3e79b458..fcfb4ec54 100644 --- a/src/cript/nodes/supporting_nodes/file.py +++ b/src/cript/nodes/supporting_nodes/file.py @@ -178,7 +178,7 @@ def __init__(self, name: str, source: str, type: str, extension: str, data_dicti super().__init__(name=name, notes=notes, **kwargs) # setting every attribute except for source, which will be handled via setter - self._json_attrs = replace( + new_json_attrs = replace( self._json_attrs, type=type, # always giving the function the required str regardless if the input `Path` or `str` @@ -186,8 +186,7 @@ def __init__(self, name: str, source: str, type: str, extension: str, data_dicti extension=extension, data_dictionary=data_dictionary, ) - - self.validate() + self._update_json_attrs_if_valid(new_json_attrs) def ensure_uploaded(self, api=None): """ diff --git a/src/cript/nodes/supporting_nodes/user.py b/src/cript/nodes/supporting_nodes/user.py index c5374d0e6..9bfa97d8b 100644 --- a/src/cript/nodes/supporting_nodes/user.py +++ b/src/cript/nodes/supporting_nodes/user.py @@ -74,9 +74,8 @@ def __init__(self, username: str, email: Optional[str] = "", orcid: Optional[str user ORCID """ super().__init__(**kwargs) - self._json_attrs = replace(self._json_attrs, username=username, email=email, orcid=orcid) - - self.validate() + new_json_attrs = replace(self._json_attrs, username=username, email=email, orcid=orcid) + self._update_json_attrs_if_valid(new_json_attrs) @property @beartype From 7aa0be1b47e1f4bc6f166ff72e971a1c9144fed1 Mon Sep 17 00:00:00 2001 From: Ludwig Schneider Date: Mon, 26 Feb 2024 13:49:37 -0600 Subject: [PATCH 2/8] fix alternative name usage --- src/cript/nodes/primary_nodes/computation_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cript/nodes/primary_nodes/computation_process.py b/src/cript/nodes/primary_nodes/computation_process.py index 576c69ede..12502b738 100644 --- a/src/cript/nodes/primary_nodes/computation_process.py +++ b/src/cript/nodes/primary_nodes/computation_process.py @@ -154,7 +154,7 @@ def __init__( >>> input_data = cript.Data(name="my data name", type="afm_amp", file=[data_files]) >>> my_material = cript.Material( ... name="my material", - ... identifier=[{"alternative_names": "my material alternative name"}] + ... identifier=[{"names": ["my material alternative name"]}] ... ) >>> my_quantity = cript.Quantity(key="mass", value=1.23, unit="kg") >>> ingredient = cript.Ingredient( From beecaa45a0e93a447ad69203724f03fd437a106a Mon Sep 17 00:00:00 2001 From: Ludwig Schneider Date: Mon, 26 Feb 2024 13:58:15 -0600 Subject: [PATCH 3/8] fix toluene --- docs/examples/synthesis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/examples/synthesis.md b/docs/examples/synthesis.md index 27350256a..cd9c60638 100644 --- a/docs/examples/synthesis.md +++ b/docs/examples/synthesis.md @@ -132,7 +132,7 @@ These materials are simple, notice how we use the SMILES notation here as an ide Similarly, we can create more initial materials. ```python -toluene = cript.Material(name="toluene", identifier=[{"smiles": "Cc1ccccc1"}, {"pubchem_id": 1140}]) +toluene = cript.Material(name="toluene", identifier=[{"smiles": "Cc1ccccc1"}, {"pubchem_cid": 1140}]) styrene = cript.Material(name="styrene", identifier=[{"smiles": "c1ccccc1C=C"}, {"inchi": "InChI=1S/C8H8/c1-2-8-6-4-3-5-7-8/h2-7H,1H2"}]) butanol = cript.Material(name="1-butanol", identifier=[{"smiles": "OCCCC"}, {"inchi_key": "InChIKey=LRHPLDYGYMQRHN-UHFFFAOYSA-N"}]) methanol = cript.Material(name="methanol", identifier=[{"smiles": "CO"}, {"names": ["Butan-1-ol", "Butyric alcohol", "Methylolpropane", "n-Butan-1-ol", "methanol"]}]) From 27d96d0e1f45ee50e86e6b1ab27ec0fc1f323a3b Mon Sep 17 00:00:00 2001 From: Ludwig Schneider Date: Mon, 26 Feb 2024 16:37:08 -0600 Subject: [PATCH 4/8] fix test and make validation message more helpful --- src/cript/api/data_schema.py | 9 ++++++++- tests/nodes/primary_nodes/test_material.py | 4 ++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/cript/api/data_schema.py b/src/cript/api/data_schema.py index 87c51c12d..938ac8249 100644 --- a/src/cript/api/data_schema.py +++ b/src/cript/api/data_schema.py @@ -211,7 +211,14 @@ def is_node_schema_valid(self, node_json: str, is_patch: bool = False, force_val # logging out info to the terminal for the user feedback # (improve UX because the program is currently slow) - log_message = f"Validating {node_type} graph..." + log_message = f"Validating {node_type} graph" + try: + log_message += " '" + str(node_dict["name"]) + "'" + except KeyError: + log_message += " '" + str(node_dict["uid"]) + "'" + + log_message += " ... " + if force_validation: log_message = "Forced: " + log_message + " if error occur, try setting `cript.API.skip_validation = False` for debugging." else: diff --git a/tests/nodes/primary_nodes/test_material.py b/tests/nodes/primary_nodes/test_material.py index e37aba38c..627eb3c04 100644 --- a/tests/nodes/primary_nodes/test_material.py +++ b/tests/nodes/primary_nodes/test_material.py @@ -9,7 +9,7 @@ from tests.utils.util import strip_uid_from_dict -def test_create_complex_material(simple_material_node, simple_computational_forcefield_node, simple_process_node) -> None: +def test_create_complex_material(cript_api, simple_material_node, simple_computational_forcefield_node, simple_process_node) -> None: """ tests that a simple material can be created with only the required arguments """ @@ -20,7 +20,7 @@ def test_create_complex_material(simple_material_node, simple_computational_forc material_notes = "my material notes" component = [simple_material_node] - forcefield = [simple_computational_forcefield_node] + forcefield = simple_computational_forcefield_node my_property = [cript.Property(key="modulus_shear", type="min", value=1.23, unit="gram")] From 83e4a3ce19c48478e7bcdf46d134ae646574f553 Mon Sep 17 00:00:00 2001 From: Ludwig Schneider Date: Mon, 26 Feb 2024 17:24:28 -0600 Subject: [PATCH 5/8] making it more robuts --- src/cript/api/data_schema.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/cript/api/data_schema.py b/src/cript/api/data_schema.py index 938ac8249..284cd685b 100644 --- a/src/cript/api/data_schema.py +++ b/src/cript/api/data_schema.py @@ -215,7 +215,10 @@ def is_node_schema_valid(self, node_json: str, is_patch: bool = False, force_val try: log_message += " '" + str(node_dict["name"]) + "'" except KeyError: - log_message += " '" + str(node_dict["uid"]) + "'" + try: + log_message += " '" + str(node_dict["uid"]) + "'" + except KeyError: + pass log_message += " ... " From b36bcff1c9df2b18ac9852f194aefb247feb669f Mon Sep 17 00:00:00 2001 From: Ludwig Schneider Date: Tue, 27 Feb 2024 08:25:29 -0600 Subject: [PATCH 6/8] Allow users to handle invalid nodes coming from a search against the API --- docs/faq.md | 53 ++++++++++++++++++++++++++++++++++++-- src/cript/api/api.py | 2 +- src/cript/api/paginator.py | 30 ++++++++++++++++----- tests/api/test_search.py | 17 +++++++++++- 4 files changed, 91 insertions(+), 11 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 8786dd986..867abc356 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -55,7 +55,7 @@ tab for developer documentation._ **Q:** Is there documentation detailing the internal workings of the code? -**A:** _Absolutely! For an in-depth look at the CRIPT Python SDK code, +**A:** _Absolutely! For an in-depth look at the CRIPT Python SDK code, consult the [GitHub repository wiki internal documentation](https://github.com/C-Accel-CRIPT/Python-SDK/wiki)._ --- @@ -84,7 +84,7 @@ A GitHub account is required._ **Q:** Where can I find the release notes for each SDK version? -**A:** _The release notes can be found on our +**A:** _The release notes can be found on our [CRIPT Python SDK repository releases section](https://github.com/C-Accel-CRIPT/Python-SDK/releases)_ --- @@ -97,6 +97,55 @@ the code is written to get a better grasp of it? There you will find documentation on everything from how our code is structure, how we aim to write our documentation, CI/CD, and more._ +--- + +**Q:** What can I do, when my `api.search(...)` fails with a `cript.nodes.exception.CRIPTJsonDeserializationError` or similar? + +**A:** _There is a solution for you. Sometimes CRIPT can contain nodes formatted in a way that the Python SDK does not understand. We can disable the automatic conversion from the API response into SDK nodes. Here is an example of how to achieve this: +```python +# Create API object in with statement, here it assumes host, token, and storage token are in your environment variables +with cript.API() as api: + # Find the paginator object, which is a python iterator over the search results. + materials_paginator = cript_api.search(node_type=cript.Material, search_mode=cript.SearchModes.NODE_TYPE) + # Usually you would do + # `materials_list = list(materials_paginator)` + # or + # for node in materials_paginator: + # #do node stuff + # But now we want more control over the iteration to ignore failing node decoding. + # And store the result in a list of valid nodes + materials_list = [] + # We use a while True loop to iterate over the results + while True: + # This first try catches, when we reach the end of the search results. + # The `next()` function raises a StopIteration exception in that case + try: + # First we try to convert the current response into a node directly + try: + material_node = next(materials_paginator) + # But if that fails, we catch the exception from CRIPT + except cript.CRIPTException as exc: + # In case of failure, we disable the auto_load_function temporarily + materials_paginator.auto_load_nodes = False + # And only obtain the unloaded node JSON instead + material_json = next(materials_paginator) + # Here you can inspect and manually handle the problem. + # In the example, we just print it and ignore it otherwise + print(exc, material_json) + else: + # After a valid node is loaded (try block didn't fail) + # we store the valid node in the list + materials_list += [material_node] + finally: + # No matter what happened, for the next iteration we want to try to obtain + # an auto loaded node again, so we reset the paginator state. + materials_paginator.auto_load_nodes = True + except StopIteration: + # If next() of the paginator indicates an end of the search results, break the loop + break +``` + + _We try to also have type hinting, comments, and docstrings for all the code that we work on so it is clear and easy for anyone reading it to easily understand._ _if all else fails, contact us on our [GitHub Repository](https://github.com/C-Accel-CRIPT/Python-SDK)._ diff --git a/src/cript/api/api.py b/src/cript/api/api.py index 980c08876..3081399f8 100644 --- a/src/cript/api/api.py +++ b/src/cript/api/api.py @@ -988,7 +988,7 @@ def _capsule_request(self, url_path: str, method: str, api_request: bool = True, response: requests.Response = requests.request(url=url, method=method, headers=headers, timeout=timeout, **kwargs) post_log_message: str = f"Request return with {response.status_code}" if self.extra_api_log_debug_info: - post_log_message += f" {response.raw}" + post_log_message += f" {response.text}" self.logger.debug(post_log_message) return response diff --git a/src/cript/api/paginator.py b/src/cript/api/paginator.py index 929954727..0625138a1 100644 --- a/src/cript/api/paginator.py +++ b/src/cript/api/paginator.py @@ -1,4 +1,4 @@ -from json import JSONDecodeError +import json from typing import Dict, Union from urllib.parse import quote @@ -33,6 +33,7 @@ class Paginator: _current_position: int _fetched_nodes: list _number_fetched_pages: int = 0 + auto_load_nodes: bool = True @beartype def __init__( @@ -119,8 +120,11 @@ def _fetch_next_page(self) -> None: # if converting API response to JSON gives an error # then there must have been an API error, so raise the requests error # this is to avoid bad indirect errors and make the errors more direct for users - except JSONDecodeError: - response.raise_for_status() + except json.JSONDecodeError as json_exc: + try: + response.raise_for_status() + except Exception as exc: + raise exc from json_exc # handling both cases in case there is result inside of data or just data try: @@ -137,8 +141,10 @@ def _fetch_next_page(self) -> None: if api_response["code"] != 200: raise APIError(api_error=str(response.json()), http_method="GET", api_url=temp_url_path) - node_list = load_nodes_from_json(current_page_results) - self._fetched_nodes += node_list + # Here we only load the JSON into the temporary results. + # This delays error checking, and allows users to disable auto node conversion + json_list = current_page_results + self._fetched_nodes += json_list def __next__(self): if self._current_position >= len(self._fetched_nodes): @@ -147,14 +153,24 @@ def __next__(self): raise StopIteration self._fetch_next_page() - self._current_position += 1 try: - return self._fetched_nodes[self._current_position - 1] + next_node_json = self._fetched_nodes[self._current_position - 1] except IndexError: # This is not a random access iteration. # So if fetching a next page wasn't enough to get the index inbound, # The iteration stops raise StopIteration + if self.auto_load_nodes: + return_data = load_nodes_from_json(next_node_json) + else: + return_data = next_node_json + + # Advance position last, so if an exception occurs, for example when + # node decoding fails, we do not advance, and users can try again without decoding + self._current_position += 1 + + return return_data + def __iter__(self): self._current_position = 0 return self diff --git a/tests/api/test_search.py b/tests/api/test_search.py index 860ab43ea..47d3de802 100644 --- a/tests/api/test_search.py +++ b/tests/api/test_search.py @@ -24,7 +24,22 @@ def test_api_search_node_type(cript_api: cript.API) -> None: # test search results assert isinstance(materials_paginator, Paginator) - materials_list = list(materials_paginator) + materials_list = [] + while True: + try: + try: + material_node = next(materials_paginator) + except cript.CRIPTException as exc: + materials_paginator.auto_load_nodes = False + material_json = next(materials_paginator) + print(exc, material_json) + else: + materials_list += [material_node] + finally: + materials_paginator.auto_load_nodes = True + except StopIteration: + break + # Assure that we paginated more then one page assert materials_paginator._current_page_number > 0 assert len(materials_list) > 5 From 39a0862b720cc17210ae5fb5fffd3542fc16ad85 Mon Sep 17 00:00:00 2001 From: Ludwig Schneider Date: Tue, 27 Feb 2024 08:35:55 -0600 Subject: [PATCH 7/8] fix issues and shorten test --- tests/api/test_search.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/api/test_search.py b/tests/api/test_search.py index 47d3de802..7030e434a 100644 --- a/tests/api/test_search.py +++ b/tests/api/test_search.py @@ -39,11 +39,14 @@ def test_api_search_node_type(cript_api: cript.API) -> None: materials_paginator.auto_load_nodes = True except StopIteration: break + # We don't need to search for a million pages here. + if materials_paginator._number_fetched_pages > 6: + break # Assure that we paginated more then one page - assert materials_paginator._current_page_number > 0 + assert materials_paginator._number_fetched_pages > 0 assert len(materials_list) > 5 - first_page_first_result = materials_list[0]["name"] + first_page_first_result = materials_list[0].name # just checking that the word has a few characters in it assert len(first_page_first_result) > 3 From c596cd7086bd3965e1a4c52dbac598fb27211219 Mon Sep 17 00:00:00 2001 From: Ludwig Schneider Date: Fri, 8 Mar 2024 14:53:54 -0600 Subject: [PATCH 8/8] fix log message for logging validation message correctly. --- src/cript/api/data_schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cript/api/data_schema.py b/src/cript/api/data_schema.py index 284cd685b..8d5337633 100644 --- a/src/cript/api/data_schema.py +++ b/src/cript/api/data_schema.py @@ -223,9 +223,9 @@ def is_node_schema_valid(self, node_json: str, is_patch: bool = False, force_val log_message += " ... " if force_validation: - log_message = "Forced: " + log_message + " if error occur, try setting `cript.API.skip_validation = False` for debugging." + log_message = "Forced: " + log_message + " if error occur, try setting `cript_api.schema.skip_validation = False` for debugging." else: - log_message += " (Can be disabled by setting `cript.API.skip_validation = True`.)" + log_message += " (Can be disabled by setting `cript_api.schema.skip_validation = True`.)" self._api.logger.info(log_message)