From c593c92e33a1ae546af4b01e46a0c6340ce86d10 Mon Sep 17 00:00:00 2001 From: Rubel Date: Tue, 28 Nov 2023 17:31:22 +0100 Subject: [PATCH] removing x-ray utils in from xrd reader. --- dev-requirements.txt | 12 +-- pynxtools/dataconverter/readers/xrd/config.py | 36 +++++-- .../dataconverter/readers/xrd/xrd_helper.py | 33 ++++++- .../dataconverter/readers/xrd/xrd_parser.py | 98 +++++++++++++++---- pyproject.toml | 1 - 5 files changed, 138 insertions(+), 42 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 71dbce60a..2ae067b07 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile --extra=dev --output-file=dev-requirements.txt pyproject.toml @@ -104,7 +104,6 @@ h5py==3.7.0 # pyfai # pynxtools (pyproject.toml) # silx - # xrayutilities hyperspy==1.7.4 # via # kikuchipy @@ -167,9 +166,7 @@ lazy-object-proxy==1.9.0 llvmlite==0.39.1 # via numba lmfit==1.2.0 - # via - # pyxem - # xrayutilities + # via pyxem locket==1.0.0 # via partd markupsafe==2.1.1 @@ -279,7 +276,6 @@ numpy==1.21.6 # sparse # tifffile # xarray - # xrayutilities # zarr numpy-quaternion==2022.4.3 # via orix @@ -436,7 +432,6 @@ scipy==1.7.3 # scikit-image # scikit-learn # sparse - # xrayutilities silx==1.1.2 # via pyfai six==1.16.0 @@ -515,6 +510,7 @@ typing-extensions==4.3.0 # astroid # mypy # numcodecs + # pylint tzdata==2023.3 # via pytz-deprecation-shim tzlocal==4.3 @@ -541,8 +537,6 @@ wrapt==1.14.1 # via astroid xarray==0.20.2 # via pynxtools (pyproject.toml) -xrayutilities==1.7.6 - # via pynxtools (pyproject.toml) zarr==2.12.0 # via hyperspy zipfile37==0.1.3 diff --git a/pynxtools/dataconverter/readers/xrd/config.py b/pynxtools/dataconverter/readers/xrd/config.py index cdb11005a..fc5513419 100644 --- a/pynxtools/dataconverter/readers/xrd/config.py +++ b/pynxtools/dataconverter/readers/xrd/config.py @@ -6,27 +6,45 @@ "@units": "", "@chi_indices": 0}, }, - "/ENTRY[entry]/2theta_plot/intensity": {"xrdml_1.5": {"value": "/detector", - "@units": ""} + "/ENTRY[entry]/2theta_plot/intensity": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/intensities", + "@units": "counts/s"} }, - "/ENTRY[entry]/2theta_plot/omega": {"xrdml_1.5": {"value": "/Omega", + "/ENTRY[entry]/2theta_plot/omega": {"xrdml_1.5": {"value": "", "@units": "", "@omega_indices": 1}, }, "/ENTRY[entry]/2theta_plot/title": "Intensity Vs. Two Theta (deg.)", - "/ENTRY[entry]/2theta_plot/phi": {"xrdml_1.5": {"value": "/Omega", + "/ENTRY[entry]/2theta_plot/phi": {"xrdml_1.5": {"value": "", "@units": "", "@phi_indices": 0}, }, - "/ENTRY[entry]/2theta_plot/two_theta": {"xrdml_1.5": {"value": "/2Theta", + "/ENTRY[entry]/2theta_plot/two_theta": {"xrdml_1.5": {"value": "", "@units": "deg", "@two_theta_indices": 0}, }, "/ENTRY[entry]/COLLECTION[collection]/beam_attenuation_factors": {"xrdml_1.5": {"value": "/beamAttenuationFactors", "@units": ""}, }, - "/ENTRY[entry]/COLLECTION[collection]/count_time": {"xrdml_1.5": {"value": "/countTime", - "@units": ""}, + "/ENTRY[entry]/COLLECTION[collection]/omega/start": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_2/startPosition", + "@units": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_2/unit"}, + }, + "/ENTRY[entry]/COLLECTION[collection]/omega/end": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_2/endPosition", + "@units": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_2/unit"}, + }, + "/ENTRY[entry]/COLLECTION[collection]/omega/step": {"xrdml_1.5": {"value": "/xrdMeasurements/comment/entry_2/MinimumstepsizeOmega", + "@units": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_2/unit"}, + }, + "/ENTRY[entry]/COLLECTION[collection]/2theta/start": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_1/startPosition", + "@units": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_1/unit"}, + }, + "/ENTRY[entry]/COLLECTION[collection]/2theta/end": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_1/endPosition", + "@units": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_1/unit"}, + }, + "/ENTRY[entry]/COLLECTION[collection]/2theta/step": {"xrdml_1.5": {"value": "/xrdMeasurements/comment/entry_2/Minimumstepsize2Theta", + "@units": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/positions_1/unit"}, + }, + "/ENTRY[entry]/COLLECTION[collection]/count_time": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/commonCountingTime", + "@units": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/commonCountingTime/unit"}, }, "/ENTRY[entry]/COLLECTION[collection]/data_file": {"xrdml_1.5": {"value": ""} }, @@ -88,8 +106,8 @@ }, "/ENTRY[entry]/definition": "NXxrd_pan", "/ENTRY[entry]/method": "X-Ray Diffraction (XRD)", - "/ENTRY[entry]/q_plot/intensity": {"xrdml_1.5": {"value": "/detector", - "@units": ""}, + "/ENTRY[entry]/q_plot/intensity": {"xrdml_1.5": {"value": "/xrdMeasurements/xrdMeasurement/scan/dataPoints/intensities", + "@units": "counts/s"}, }, "/ENTRY[entry]/q_plot/q": {"xrdml_1.5": {"value": "", "@units": ""}, diff --git a/pynxtools/dataconverter/readers/xrd/xrd_helper.py b/pynxtools/dataconverter/readers/xrd/xrd_helper.py index 6696ddb0c..b7112d5e6 100644 --- a/pynxtools/dataconverter/readers/xrd/xrd_helper.py +++ b/pynxtools/dataconverter/readers/xrd/xrd_helper.py @@ -101,9 +101,33 @@ def fill_template_from_config_data(config_dict: dict, template: Template, template[nx_key] = val def two_theta_plot(): - two_theta_gr = "/ENTRY[entry]/2theta_plot" + + intesity = transform_to_intended_dt(template.get("/ENTRY[entry]/2theta_plot/intensity", + None)) + if intesity is not None: + intsity_len = np.shape(intesity)[0] + else: + raise ValueError("No intensity is found") + + two_theta_gr = "/ENTRY[entry]/2theta_plot/" + if template.get(f"{two_theta_gr}omega", None) is None: + omega_start = template.get("/ENTRY[entry]/COLLECTION[collection]/omega/start", None) + omega_end = template.get("/ENTRY[entry]/COLLECTION[collection]/omega/end", None) + + template["/ENTRY[entry]/2theta_plot/omega"] = np.linspace(float(omega_start), + float(omega_end), + intsity_len) + + if template.get(f"{two_theta_gr}two_theta", None) is None: + tw_theta_start = template.get("/ENTRY[entry]/COLLECTION[collection]/2theta/start", + None) + tw_theta_end = template.get("/ENTRY[entry]/COLLECTION[collection]/2theta/end", None) + template[f"{two_theta_gr}two_theta"] = np.linspace(float(tw_theta_start), + float(tw_theta_end), + intsity_len) template[two_theta_gr + "/" + "@axes"] = ["two_theta"] template[two_theta_gr + "/" + "@signal"] = "intensity" + template[two_theta_gr + "/" + "@auxiliary_signals"] = "omega" def q_plot(): q_plot_gr = "/ENTRY[entry]/q_plot" @@ -112,13 +136,14 @@ def q_plot(): alpha_1 = template.get("/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/k_alpha_one", None) two_theta: np.ndarray = template.get("/ENTRY[entry]/2theta_plot/two_theta", None) + if two_theta is None: + raise ValueError("Two-theta data is not found") if isinstance(two_theta, np.ndarray): theta: np.ndarray = two_theta / 2 - # pylint: disable=line-too-long - ratio_key = "/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/ratio_k_alphatwo_k_alphaone" + ratio_k = "/ENTRY[entry]/INSTRUMENT[instrument]/SOURCE[source]/ratio_k_alphatwo_k_alphaone" if alpha_1 and alpha_2: ratio = alpha_2 / alpha_1 - template[ratio_key] = ratio + template[ratio_k] = ratio lamda = ratio * alpha_1 + (1 - ratio) * alpha_2 q_vec = (4 * np.pi / lamda) * np.sin(np.deg2rad(theta)) template[q_plot_gr + "/" + "q_vec"] = q_vec diff --git a/pynxtools/dataconverter/readers/xrd/xrd_parser.py b/pynxtools/dataconverter/readers/xrd/xrd_parser.py index ed5dd0f75..c0fb8d8ea 100644 --- a/pynxtools/dataconverter/readers/xrd/xrd_parser.py +++ b/pynxtools/dataconverter/readers/xrd/xrd_parser.py @@ -18,6 +18,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Any, Dict, Tuple, Optional, List + from pathlib import Path import warnings import xml.etree.ElementTree as ET # for XML parsing @@ -107,11 +109,11 @@ def handle_with_panalytical_module(self): comes with xml file. """ self.parse_each_elm(parent_path='/', xml_node=self.xml_root) - + nested_data_dict: Dict[str, any] = {} # Extract other numerical data e.g. 'hkl', 'Omega', '2Theta', CountTime etc # using panalytical_xml module - parsed_data = XRDMLFile(self.__file_path) - nested_data_dict = parsed_data.scan.ddict + # parsed_data = XRDMLFile(self.__file_path) + # nested_data_dict = parsed_data.scan.ddict fill_slash_sep_dict_from_nested_dict('/', nested_data_dict, self.__xrd_dict) def process_node_text(self, parent_path, node_txt) -> None: @@ -152,7 +154,9 @@ def process_node_text(self, parent_path, node_txt) -> None: warnings.warn(f'Element text {node_txt} is ignored from parseing!', IgnoreNodeTextWarning) - def parse_each_elm(self, parent_path, xml_node): + def parse_each_elm(self, parent_path, xml_node, + multi_childs_tag: str = '', + tag_extensions: Optional[List[int]] = None): """Check each xml element and send the element to intended function. Parameters @@ -161,40 +165,86 @@ def parse_each_elm(self, parent_path, xml_node): Path to be in the starting of the key composing from element e.g. '/'. xml_node : XML.Element Any element except process instruction nodes. + multi_childs_tag : str + Tag that is available on several child nodes. + tag_extension : List[int] + List of extension of the child tag if there are several childs having the same + tag. Returns ------ None """ + tag = remove_namespace_from_tag(xml_node.tag) # Take care of special node of 'entry' tag if tag == 'entry': - parent_path = self.parse_entry_elm(parent_path, xml_node) + parent_path = self.parse_entry_elm(parent_path, xml_node, + multi_childs_tag, tag_extensions) else: - parent_path = self.parse_general_elm(parent_path, xml_node) + parent_path = self.parse_general_elm(parent_path, xml_node, + multi_childs_tag, tag_extensions) + _, multi_childs_tag = self.has_multi_childs_with_same_tag(xml_node) + # List of tag extensions for child nodes which have the same tag. + tag_extensions: List[int] = [0] for child in iter(xml_node): if child is not None: - self.parse_each_elm(parent_path, child) + self.parse_each_elm(parent_path, child, + multi_childs_tag, tag_extensions) - def parse_general_elm(self, parent_path, xml_node): - """Handle general element except entry element. + def has_multi_childs_with_same_tag(self, parent_node: ET.Element) -> Tuple[str, bool]: + """Check for multiple childs that have the same tag. + Parameter: + ---------- + parent_node : ET.Element + Parent node that might has multiple childs with the same tag. + + Returns: + -------- + Tuple[bool, str] + (true if multiple childs with the same tag, tag). + """ + tag: str = None + for child in iter(parent_node): + temp_tag = remove_namespace_from_tag(child.tag) + if tag is None: + tag = temp_tag + else: + if tag == temp_tag: + return (True, tag) + + return (False, '') + + def parse_general_elm(self, parent_path, xml_node, + multi_childs_tag, tag_extensions: List[int]): + """Handle general element except entry element. Parameters ---------- parent_path : str Path to be in the starting of the key composing from element e.g. '/'. xml_node : XML.Element Any element except process instruction and entry nodes. + multi_childs_tag : str + Tag that is available on several siblings. + tag_extension : List[int] + List of extension of the shiblings tag if there are several shiblings having + the same tag. Returns ------- None """ + tag = remove_namespace_from_tag(xml_node.tag) + if tag == multi_childs_tag: + new_ext = tag_extensions[-1] + 1 + tag = tag + '_' + str(new_ext) + tag_extensions.append(new_ext) if parent_path == '/': - parent_path = '/' + tag + parent_path = parent_path + tag else: # New parent path ends with element tag parent_path = '/'.join([parent_path, tag]) @@ -214,7 +264,8 @@ def parse_general_elm(self, parent_path, xml_node): return parent_path - def parse_entry_elm(self, parent_path, xml_node): + def parse_entry_elm(self, parent_path: str, xml_node: ET.Element, + multi_childs_tag: str, tag_extensions: List[int]): """Handle entry element. Parameters @@ -223,14 +274,25 @@ def parse_entry_elm(self, parent_path, xml_node): Path to be in the starting of the key composing from element e.g. '/'. xml_node : XML.Element Any entry node. + multi_childs_tag : str + Tag that is available on several siblings. + tag_extension : List[int] + List of extension of the shiblings tag if there are several shiblings having + the same tag. Returns ------- str: Parent path. """ + tag = remove_namespace_from_tag(xml_node.tag) + if tag == multi_childs_tag: + new_ext = tag_extensions[-1] + 1 + tag_extensions.append(new_ext) + tag = tag + '_' + str(new_ext) + if parent_path == '/': parent_path = '/' + tag else: @@ -335,14 +397,12 @@ def parse_and_populate_template(self, template, config_dict, eln_dict): None """ - file_format = self.get_file_format() - if file_format == ".xrdml": - xrd_dict = self.parse() - if len(config_dict) == 0 and self.file_parser.xrdml_version == '1.5': - from pynxtools.dataconverter.readers.xrd.config import xrdml - config_dict = xrdml - feed_xrdml_to_template(template, xrd_dict, eln_dict, - file_term=self.file_term, config_dict=config_dict) + xrd_dict = self.parse() + if len(config_dict) == 0 and self.file_parser.xrdml_version == '1.5': + from pynxtools.dataconverter.readers.xrd.config import xrdml + config_dict = xrdml + feed_xrdml_to_template(template, xrd_dict, eln_dict, + file_term=self.file_term, config_dict=config_dict) def parse(self): '''Parses the file based on its format. diff --git a/pyproject.toml b/pyproject.toml index 2fbec6688..8b1af5f0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,6 @@ dependencies = [ "tzlocal<=4.3", "scipy>=1.7.1", "lark>=1.1.5", - "xrayutilities>=1.7.4", "requests", "requests_cache", "nanonispy@git+https://github.com/ramav87/nanonispy.git@a0da87c58482d29624a2bf5deecb763dd1274212",