From 71fddf30b0ecd18e83578d9548b93fc3a0b11679 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 30 Jan 2024 16:29:30 +0100 Subject: [PATCH] complete reading of .spe metadata --- .../dataconverter/readers/xps/spe/spe_phi.py | 1022 ++++++++++++----- 1 file changed, 765 insertions(+), 257 deletions(-) diff --git a/pynxtools/dataconverter/readers/xps/spe/spe_phi.py b/pynxtools/dataconverter/readers/xps/spe/spe_phi.py index 9da5fc107..2653d8e83 100644 --- a/pynxtools/dataconverter/readers/xps/spe/spe_phi.py +++ b/pynxtools/dataconverter/readers/xps/spe/spe_phi.py @@ -27,7 +27,6 @@ import copy import datetime import xarray as xr -from dataclasses import fields import numpy as np from pynxtools.dataconverter.readers.xps.reader_utils import ( @@ -84,44 +83,379 @@ def construct_data(self): self._xps_dict["data"]: dict = {} key_map = { - "user": [], + "user": [ + "user_name", + ], "instrument": [ - "work_function", + "bias_box_mode", + "instrument_model", + "vendor", + "sca_multiplier_voltage", + "sca_multiplier_voltage/@units", + "source_analyser_angle", + "source_analyser_angle/@units", ], "source": [ - "source_label", + "scan_deflection_offset_x", + "scan_deflection_offset_y", + "scan_deflection_span_x", + "scan_deflection_span_y", + "xray_anode_material", + "xray_anode_position", + "xray_beam_diameter", + "xray_beam_diameter/@units", + "xray_beam_voltage", + "xray_beam_voltage/@units", + "xray_blanking_voltage", + "xray_blanking_voltage/@units", + "xray_condenser_lens_voltage", + "xray_condenser_lens_voltage/@units", + "xray_delay_factor_x", + "xray_delay_factor_y", + "xray_emission_control", + "xray_emission_current", + "xray_emission_current/@units", + "xray_energy", + "xray_energy/@units", + "xray_filament_current", + "xray_filament_current/@units", + "xray_high_power", + "xray_high_voltage", + "xray_high_voltage/@units", + "xray_interlace_interval", + "xray_magnification_factor_x", + "xray_magnification_factor_y", + "xray_max_filament_current", + "xray_max_filament_current/@units", + "xray_monochromatized", + "xray_objective_coil_current", + "xray_objective_coil_current/@units", + "xray_offset", + "xray_offset/@units", + "xray_power", + "xray_power/@units", + "xray_rotation", + "xray_rotation/@units", + "xray_spot_size", + "xray_spot_size/@units", + "xray_step_delay_read_beam", + "xray_steps_per_diameter", + "xray_stigmator_x", + "xray_stigmator_y", + ], + "beam": [ + "xray_beam_diameter", + "xray_beam_diameter/@units", + ], + "analyser": [ + "analyser_retardation_gain", + "analyser_solid_angle", + "analyser_solid_angle/@units", + "analyser_work_function", + "analyser_work_function/@units", + "sxi_auto_contrast", + "sxi_auto_contrast_high", + "sxi_auto_contrast_low", + "sxi_binding_energy", + "sxi_binding_energy/@units", + "sxi_display_mode", + "sxi_filename", + "sxi_lens2_voltage", + "sxi_lens2_voltage/@units", + "sxi_lens3_voltage", + "sxi_lens3_voltage/@units", + "sxi_lens4_voltage", + "sxi_lens4_voltage/@units", + "sxi_lens5_voltage", + "sxi_lens5_voltage/@units", + "sxi_lens_bias_voltage", + "sxi_lens_bias_voltage/@units", + "sxi_pass_energy", + "sxi_pass_energy/@units", + "sxi_persistence", + "sxi_rotator", + "sxi_rotator/@units", + "sxi_sec_per_display", + "sxi_shutter_bias", + "sxi_shutter_bias_voltage", + "sxi_shutter_bias_voltage/@units", + ], + "collectioncolumn": [ + "narrow_acceptance_angle", ], - "beam": ["excitation_energy"], - "analyser": ["analyser_name"], - "collectioncolumn": ["lens_mode"], "energydispersion": [ - "scan_mode", - "pass_energy", + "energy_scan_mode", ], "detector": [ - "detector_voltage", + "channel_1_info", + "channel_2_info", + "channel_3_info", + "channel_4_info", + "channel_5_info", + "channel_6_info", + "channel_7_info", + "channel_8_info", + "channel_9_info", + "channel_10_info", + "channel_11_info", + "channel_12_info", + "channel_13_info", + "channel_14_info", + "channel_15_info", + "channel_16_info", + "channel_17_info", + "channel_18_info", + "channel_19_info", + "channel_20_info", + "channel_21_info", + "channel_22_info", + "channel_23_info", + "channel_24_info", + "channel_25_info", + "channel_26_info", + "channel_27_info", + "channel_28_info", + "channel_29_info", + "channel_30_info", + "channel_31_info", + "channel_32_info", + "delay_before_acquire", + "delay_before_acquire/@units", + "detector_acquisition_time", + "detector_acquisition_time/@units", + "number_of_channels", + "refresh_persistence", + "survey_dwell_time", + "survey_dwell_time/@units", + ], + "manipulator": [ + "stage_x", + "stage_x/@units", + "stage_y", + "stage_y/@units", + "stage_z", + "stage_z/@units", + "stage_azimuth", + "stage_azimuth/@units", + "stage_polar", + "stage_polar/@units", + "stage_current_rotation_speed", + "stage_current_rotation_speed/@units", + ], + "defect_positioner": [ + "defect_positioner_alignment", + "defect_positioner_comment", + "defect_positioner_id", + "defect_positioner_reference_image", + "defect_positioner_rotation", + "defect_positioner_rotation/@units", + "defect_positioner_tilt", + "defect_positioner_tilt/@units", + "defect_positioner_u", + "defect_positioner_u/@units", + "defect_positioner_v", + "defect_positioner_v/@units", + "defect_positioner_x", + "defect_positioner_x/@units", + "defect_positioner_y", + "defect_positioner_y/@units", + "defect_positioner_z", + "defect_positioner_z/@units", + ], + "c60_ion_gun": [ + "c60_ion_gun", ], - "manipulator": [], - "sample": ["target_bias"], - "calibration": [], - "data": [ - "x_units", - "y_units", - "n_values", - "start_energy", - "step_size", - "dwell_time", + "flood_gun": [ + "auto_flood_gun", + "flood_gun_current", + "flood_gun_current/@units", + "flood_gun_energy", + "flood_gun_energy/@units", + "flood_gun_extractor", + "flood_gun_extractor/@units", + "flood_gun_filament_current", + "flood_gun_filament_current/@units", + "flood_gun_gain", + "flood_gun_mode", + "flood_gun_pulse_frequency", + "flood_gun_pulse_length", + "flood_gun_pulse_length/@units", + "flood_gun_ramp_rate", + "flood_gun_ramp_rate/@units", + "flood_gun_time_per_step", + "flood_gun_time_per_step/@units", + "flood_gun_x_steering", + "flood_gun_y_steering", + ], + "gcib": [ + "gcib_bend_voltage", + "gcib_bend_voltage/@units", + "gcib_cluster_size", + "gcib_cluster_size/@units", + "gcib_emission", + "gcib_emission/@units", + "gcib_energy_per_atom", + "gcib_energy_per_atom/@units", + "gcib_extractor", + "gcib_extractor/@units", + "gcib_focus", + "gcib_focus/@units", + "gcib_gas_pressure", + "gcib_gas_pressure/@units", + "gcib_high_voltage", + "gcib_high_voltage/@units", + "gcib_ionization", + "gcib_ionization/@units", + "gcib_magnet_current", + "gcib_magnet_current/@units", + "gcib_objective", + "gcib_objective/@units", + "gcib_raster_offset_x", + "gcib_raster_offset_x/@units", + "gcib_raster_offset_y", + "gcib_raster_offset_y/@units", + "gcib_raster_size_x", + "gcib_raster_size_x/@units", + "gcib_raster_size_y", + "gcib_raster_size_y/@units", + "gcib_sputter_rate", + "gcib_sputter_rate/@units", + "gcib_wien_filter_voltage", + "gcib_wien_filter_voltage/@units", + ], + "neutral_ion_gun": [ + "auto_neutral_ion_source", + "neutral_bend_voltage", + "neutral_bend_voltage/@units", + "neutral_condenser_lens_voltage", + "neutral_condenser_lens_voltage/@units", + "neutral_current", + "neutral_current/@units", + "neutral_deflection_bias", + "neutral_deflection_bias/@units", + "neutral_emission", + "neutral_emission/@units", + "neutral_energy", + "neutral_energy/@units", + "neutral_float_enabled", + "neutral_float_voltage", + "neutral_float_voltage/@units", + "neutral_grid_voltage", + "neutral_grid_voltage/@units", + "neutral_ion", + "neutral_ion_gun_gas_pressure", + "neutral_ion_gun_gas_pressure/@units", + "neutral_objective_lens_voltage", + "neutral_objective_lens_voltage/@units", + "neutral_raster_offset_x", + "neutral_raster_offset_x/@units", + "neutral_raster_offset_y", + "neutral_raster_offset_y/@units", + "neutral_raster_x", + "neutral_raster_x/@units", + "neutral_raster_y", + "neutral_raster_y/@units", + "neutral_rate", + "neutral_rate/@units", + "neutral_target_timed_on_time", + "neutral_target_timed_on_time/@units", + ], + "sputter_gun": [ + "bend_voltage", + "bend_voltage/@units", + "condenser_lens_voltage", + "condenser_lens_voltage/@units", + "deflection_bias", + "deflection_bias/@units", + "float_enabled", + "float_voltage", + "float_voltage/@units", + "grid_voltage", + "grid_voltage/@units", + "ion_gun_gas_pressure", + "ion_gun_gas_pressure/@units", + "ion_gun_mode", + "objective_lens_voltage", + "objective_lens_voltage/@units", + "sputter_current", + "sputter_current/@units", + "sputter_emission", + "sputter_emission/@units", + "sputter_energy", + "sputter_energy/@units", + "sputter_ion", + "sputter_raster_offset_x", + "sputter_raster_offset_x/@units", + "sputter_raster_offset_y", + "sputter_raster_offset_y/@units", + "sputter_raster_x", + "sputter_raster_x/@units", + "sputter_raster_y", + "sputter_raster_y/@units", + "sputter_rate", + "sputter_rate/@units", + "target_sputter_time", + "target_sputter_time/@units", + ], + "sample": [], + "process": [ + "deconvolution", + "deconvolution_pass_energy", + "deconvolution_pass_energy/@units", + "energy_recalibration", + "energy_reference_energy", + "energy_reference_energy/@units", + "energy_reference_peak", + "intensity_calibration_coefficients", + "intensity_recalibration", + ], + "data": [], + "file": [ + "photo_filename", + "software_version", ], "region": [ - "analysis_method", - "spectrum_type", - "dwell_time", - "comments", - "spectrum_id", - "time_stamp", - "scans", + "acquisition_file_date", + "acquisition_filename", + "experiment_id", + "file_date", + "file_description", + "file_type", + "image_size_x", + "image_size_y", + "no_spatial_areas", + "no_spectral_regions", + "no_spectral_regions_full", + "sem_field_of_view", + "spatial_area_definition", + "spatial_area_description", + "spatial_hr_photo_correction", + "spectral_region_background", + "spectral_region_background_full", + "spectral_region_definition", + "spectral_region_definition2", + "spectral_region_definition2_full", + "spectral_region_definition_full", + "spectral_region_hero", + "spectral_region_hero_full", + "spectral_region_ir", + "spectral_region_ir_full", "spectrum_id", - "cycle_no", + "survey_num_of_cycles", + "technique", + "technique_ex", + "tfc_parameters", + "xps_scan_mode", + ], + "unused": [ + "peak_to_noise_ratio_state", + "platform", + "platen_id", + "presputter", + "register_image", + "register_image_interval", + "register_image_last", + "register_image_mode", ], } @@ -151,7 +485,11 @@ def _update_xps_dict_with_spectrum(self, spectrum, key_map): "energydispersion": f"{analyser_parent}/energydispersion", "detector": f"{analyser_parent}/detector", "manipulator": f"{instrument_parent}/manipulator", - "calibration": f"{instrument_parent}/calibration", + "defect_positioner": f"{instrument_parent}/defect_positioner", + "flood_gun": f"{instrument_parent}/flood_gun", + "gcib": f"{instrument_parent}/gcib", + "sputter_gun": f"{instrument_parent}/sputter_gun", + "process": f"{instrument_parent}/process", "sample": f"{region_parent}/sample", "data": f"{region_parent}/data", "region": f"{region_parent}", @@ -177,67 +515,62 @@ def _update_xps_dict_with_spectrum(self, spectrum, key_map): detector_data_key_child = construct_detector_data_key(spectrum) detector_data_key = f'{path_map["detector"]}/{detector_data_key_child}/counts' - x_units = spectrum["x_units"] - energy = np.array(spectrum["data"]["x"]) - intensity = np.array(spectrum["data"]["y"]) - - if entry not in self._xps_dict["data"]: - self._xps_dict["data"][entry] = xr.Dataset() - - # Write raw data to detector. - self._xps_dict[detector_data_key] = intensity - - if not self.parser.export_settings["Separate Channel Data"]: - averaged_channels = intensity - else: - all_channel_data = [ - value - for key, value in self._xps_dict.items() - if detector_data_key.split("Channel_")[0] in key - ] - averaged_channels = np.mean(all_channel_data, axis=0) - - if not self.parser.export_settings["Separate Scan Data"]: - averaged_scans = intensity - else: - all_scan_data = [ - value - for key, value in self._xps_dict.items() - if detector_data_key.split("Scan_")[0] in key - ] - averaged_scans = np.mean(all_scan_data, axis=0) - - # Write to data in order: scan, cycle, channel - - # Write averaged cycle data to 'data'. - self._xps_dict["data"][entry][scan_key.split("_")[0]] = xr.DataArray( - data=averaged_scans, - coords={x_units: energy}, - ) - if self.parser.export_settings["Separate Scan Data"]: - # Write average cycle data to 'data'. - self._xps_dict["data"][entry][scan_key] = xr.DataArray( - data=averaged_channels, - coords={x_units: energy}, - ) - - if ( - self.parser.export_settings["Separate Channel Data"] - and self.write_channels_to_data - ): - # Write channel data to 'data'. - channel_no = spectrum["channel_no"] - self._xps_dict["data"][entry][ - f"{scan_key}_chan{channel_no}" - ] = xr.DataArray(data=intensity, coords={x_units: energy}) - -# %% - - -def convert_pascal_to_snake(str_value): - pattern = re.compile(r"(? analyser_work_function: 4.506, - analyser_work_function_units: eV, + Extract units for the metadata containing unit information. + + Example: + analyser_work_function: 4.506 eV + -> analyser_work_function: 4.506, + analyser_work_function_units: eV, + + Parameters + ---------- + key : str + Key of the associated value. + value : str + Combined unit and value information. + + Returns + ------- + value : + value with units. + unit : str + Associated unit. + + """ unit_map = {"seconds": "s", "uA": "micro-A", "(min)": "min"} special_cases = { @@ -575,9 +955,14 @@ def extract_unit(self, key, value): } unit_missing = { "grid_voltage": "V", - "condensor_lens_voltage": "V", + "condenser_lens_voltage": "V", "objective_lens_voltage": "V", "bend_voltage": "V", + "neutral_grid_voltage": "V", + "neutral_condenser_lens_voltage": "V", + "neutral_objective_lens_voltage": "V", + "stage_current_rotation_speed": "degree/s", + "neutral_bend_voltage": "V", "defect_positioner_u": "mm", "defect_positioner_v": "mm", "defect_positioner_x": "mm", @@ -600,17 +985,38 @@ def extract_unit(self, key, value): return value, unit - def map_values(self, key, value): + def map_values(self, key, value, field_type): + """ + Map values to corresponding structure and field type. + + Parameters + ---------- + key : str + PhiMetadata dataclass key. + value : str + Original value. + field_type : class + Class to be mapped onto. + + Returns + ------- + value + Value of correct type and internal structure. + + """ value_function_map = { - # "file_date": _parse_datetime, - # "acquisition_file_date": _parse_datetime, + "technique": _convert_experimental_technique, + "technique_ex": _convert_experimental_technique, + "file_type": _map_file_type, + "file_date": _parse_datetime, + "acquisition_file_date": _parse_datetime, "energy_reference": _convert_energy_referencing, "intensity_calibration_coefficients": _map_to_list, "energy_recalibration": _convert_bool, - "scan_deflection_span": _map_to_list, - "scan_deflection_offset": _map_to_list, + "scan_deflection_span": _map_to_xy, + "scan_deflection_offset": _map_to_xy, "tfc_parameters": _map_to_list, - "image_size_xy": _map_to_xy, + "image_size": _map_to_xy, "float_enabled": _convert_bool, "sputter_raster": _map_to_xy_with_units, "sputter_raster_offset": _map_to_xy_with_units, @@ -620,7 +1026,7 @@ def map_values(self, key, value): "xray_source": _convert_xray_source_params, "xray_stigmator": _map_to_xy, "xray_magnification_factor": _map_to_xy, - "x_ray_delay_factor": _map_to_xy, + "xray_delay_factor": _map_to_xy, "xray_high_power": _convert_bool, "xray_emission_control": _convert_bool, "xray_settings": _convert_xray_source_settings, @@ -628,7 +1034,7 @@ def map_values(self, key, value): "sxi_shutter_bias": _convert_bool, "stage_positions": _convert_stage_positions, "gcib_raster_size": _map_to_xy_with_units, - "gcib_raster_size_offset": _map_to_xy_with_units, + "gcib_raster_offset": _map_to_xy_with_units, "auto_flood_gun": _convert_bool, "auto_neutral_ion_source": _convert_bool, "presputter": _convert_bool, @@ -636,38 +1042,105 @@ def map_values(self, key, value): if key in value_function_map: map_fn = value_function_map[key] - new_value = map_fn(value) - return new_value - return value + value = map_fn(value) + return field_type(value) + + def add_metadata_to_each_spectrum(self): + """ + Add flattened dict with PhiMetadata fields to each spectrum. + + """ + flattened_metadata = self._flatten_dict(self.metadata.dict()) + + for spectrum in self.spectra: + spectrum.update(flattened_metadata) + + def _flatten_dict(self, metadata_dict): + """ + + + Parameters + ---------- + metadata_dict : dict + Metadata dict with PhiMetadata fields as keys. + + Returns + ------- + flattened_dict : dict + Flatted metadata_dict without any nested substructure. + + """ + + def shorten_supkey(supkey): + shortened_key_map = { + "xray_source": "xray", + "xray_settings": "xray", + "stage_positions": "stage", + } + if supkey in shortened_key_map: + return shortened_key_map[supkey] + return supkey + + flattened_dict = {} + + for key, value in metadata_dict.items(): + if isinstance(value, dict): + for subkey, subvalue in value.items(): + supkey = shorten_supkey(key) + flattened_dict[f"{supkey}_{subkey}"] = subvalue + else: + flattened_dict[key] = value + + for key in flattened_dict.copy().keys(): + if "_units" in key: + new_key = key.replace("_units", "/@units") + flattened_dict[new_key] = flattened_dict.pop(key) + + return flattened_dict -def _parse_datetime(date): +def convert_pascal_to_snake(str_value): + pattern = re.compile(r"(?