From ad768bceed8c232d39e1036d59e833237f091790 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Thu, 11 Jan 2024 11:20:03 +0100 Subject: [PATCH] Reorganize location of tests, etc. --- debug/apex.dev.ipynb | 81 ++ axon.batch.sh => debug/axon.batch.sh | 0 .../concept_mapper.ipynb | 0 imgs.batch.sh => debug/imgs.batch.sh | 0 debug/imgs.dev.ipynb | 812 +++++++++++++++++ debug/metadata_tiff.ipynb | 210 +++++ nion.batch.sh => debug/nion.batch.sh | 0 debug/nion.dev.ipynb | 57 ++ pyxem.batch.sh => debug/pyxem.batch.sh | 0 pyxem.dev.ipynb => debug/pyxem.dev.ipynb | 0 .../spctrscpy.batch.sh | 4 +- debug/spctrscpy.dev.ipynb | 838 ++++++++++++++++++ delete/test.ebsd.sh | 12 + 13 files changed, 2012 insertions(+), 2 deletions(-) create mode 100644 debug/apex.dev.ipynb rename axon.batch.sh => debug/axon.batch.sh (100%) rename concept_mapper.ipynb => debug/concept_mapper.ipynb (100%) rename imgs.batch.sh => debug/imgs.batch.sh (100%) create mode 100644 debug/imgs.dev.ipynb create mode 100644 debug/metadata_tiff.ipynb rename nion.batch.sh => debug/nion.batch.sh (100%) create mode 100644 debug/nion.dev.ipynb rename pyxem.batch.sh => debug/pyxem.batch.sh (100%) rename pyxem.dev.ipynb => debug/pyxem.dev.ipynb (100%) rename spctrscpy.batch.sh => debug/spctrscpy.batch.sh (86%) create mode 100644 debug/spctrscpy.dev.ipynb create mode 100755 delete/test.ebsd.sh diff --git a/debug/apex.dev.ipynb b/debug/apex.dev.ipynb new file mode 100644 index 000000000..8d5a55da3 --- /dev/null +++ b/debug/apex.dev.ipynb @@ -0,0 +1,81 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "f39f5e57-4926-4ccd-b187-9c1aba568a95", + "metadata": {}, + "outputs": [], + "source": [ + "from pynxtools.dataconverter.readers.em.subparsers.hfive_base import HdfFiveBaseParser\n", + "from jupyterlab_h5web import H5Web\n", + "! which python" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1617083-f6ab-4bad-ad6d-aaa6beffec89", + "metadata": {}, + "outputs": [], + "source": [ + "fpath = f\"/home/kaiobach/Research/paper_paper_paper/scidat_nomad_ebsd/bb_analysis/\" \\\n", + " f\"data/development_spctrscpy/pdi/APEX-single-spectrum/InGaN_nanowires_spectra.edaxh5\"\n", + "# H5Web(fpath)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a7191dce-db83-4252-bfe3-ffc36efb5e71", + "metadata": {}, + "outputs": [], + "source": [ + "hdf = HdfFiveBaseParser(file_path=fpath)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "944c0809-4ec6-406a-be61-1dd56dcf6d00", + "metadata": {}, + "outputs": [], + "source": [ + "hdf.get_content()\n", + "hdf.report_content()\n", + "hdf.store_report(store_instances=True,\n", + " store_instances_templatized=True,\n", + " store_templates=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac77fbda-ae54-4c1d-9441-e4d771605052", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/axon.batch.sh b/debug/axon.batch.sh similarity index 100% rename from axon.batch.sh rename to debug/axon.batch.sh diff --git a/concept_mapper.ipynb b/debug/concept_mapper.ipynb similarity index 100% rename from concept_mapper.ipynb rename to debug/concept_mapper.ipynb diff --git a/imgs.batch.sh b/debug/imgs.batch.sh similarity index 100% rename from imgs.batch.sh rename to debug/imgs.batch.sh diff --git a/debug/imgs.dev.ipynb b/debug/imgs.dev.ipynb new file mode 100644 index 000000000..93b767356 --- /dev/null +++ b/debug/imgs.dev.ipynb @@ -0,0 +1,812 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "6da1aea0-545b-446b-a3d1-1574af72f6c6", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from PIL import Image\n", + "from PIL.TiffTags import TAGS\n", + "# print(TAGS)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3889c47f-11c4-4bf3-97de-04fc52f0798d", + "metadata": {}, + "outputs": [], + "source": [ + "src = \"/home/kaiobach/Research/paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_imgs\"\n", + "fnms = [(\"kit\", \"kit/FeMoOx_AntiA_04_1k5x_CN.tif\"),\n", + " (\"ikz_changming\", \"ikz_changming/STO252b Camera Ceta 1818 790 kx.png\"),\n", + " (\"axon\", \"axon/20210426T224437.049Raw0.png\"),\n", + " (\"ikz_robert_t1\", \"ikz_robert/0c8nA_3deg_003_AplusB_test.tif\"),\n", + " (\"ikz_martin_t2\", \"ikz_robert/ALN_baoh_021.tif\"),\n", + " (\"ikz_robert_t3\", \"ikz_robert/T3_image.tif\"),\n", + " (\"ikz_robert_etd\", \"ikz_robert/ETD_image.tif\"),\n", + " (\"ikz_robert_navcam\", \"ikz_robert/NavCam_normal_vis_light_ccd.tif\")\n", + " ]\n", + "fnm = f\"{src}/{fnms[2][1]}\"\n", + "print(fnm)" + ] + }, + { + "cell_type": "markdown", + "id": "4f4bf73d-66b7-414b-abb1-db99b2bf370a", + "metadata": {}, + "source": [ + "***" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9ef2a35-a260-4a54-9b83-eae1d588966f", + "metadata": {}, + "outputs": [], + "source": [ + "all_parent_concepts = set()\n", + "for idx in np.arange(3, len(fnms)):\n", + " fnm = f\"{src}/{fnms[idx][1]}\"\n", + " # print(f\"{fnm}\")\n", + " with Image.open(fnm, mode=\"r\") as fp:\n", + " if True is False:\n", + " czi_keys = [34118, 34119]\n", + " for czi_key in czi_keys:\n", + " if czi_key in fp.tag_v2:\n", + " print(f\"Found czi_key {tfs_key}...\")\n", + " utf = fp.tag[czi_key]\n", + " print(type(utf))\n", + " if len(utf) == 1:\n", + " print(utf[0])\n", + " # exit(1)\n", + " tfs_keys = [34682]\n", + " for tfs_key in tfs_keys:\n", + " if tfs_key in fp.tag_v2:\n", + " # print(f\"Found tfs_key {tfs_key}...\")\n", + " utf = fp.tag[tfs_key]\n", + " # print(type(utf))\n", + " if len(utf) == 1:\n", + " print(utf[0])\n", + " for line in utf[0].splitlines():\n", + " if line.startswith(\"[\") is True:\n", + " all_parent_concepts.add(line)\n", + "\n", + "for concept in all_parent_concepts:\n", + " print(concept)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b280857b-df16-4f66-b81a-618fc34c69e5", + "metadata": {}, + "outputs": [], + "source": [ + "# fish all concepts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a7bfb57a-583d-47af-97d8-4f450b36a351", + "metadata": {}, + "outputs": [], + "source": [ + "def sort_ascendingly_by_second_argument(tup):\n", + " # convert the list of tuples to a numpy array with data type (object, int)\n", + " arr = np.array(tup, dtype=[('col1', object), ('col2', int)])\n", + " # get the indices that would sort the array based on the second column\n", + " indices = np.argsort(arr['col2'])\n", + " # use the resulting indices to sort the array\n", + " sorted_arr = arr[indices]\n", + " # convert the sorted numpy array back to a list of tuples\n", + " sorted_tup = [(row['col1'], row['col2']) for row in sorted_arr]\n", + " return sorted_tup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "05dd5af4-b32c-45be-bc80-da5a1166494e", + "metadata": {}, + "outputs": [], + "source": [ + "import mmap\n", + "TiffTfsParentConcepts = [\"Accessories\", \n", + " \"EasyLift\",\n", + " \"ETD\",\n", + " \"Specimen\",\n", + " \"T3\",\n", + " \"User\",\n", + " \"Scan\",\n", + " \"T1\",\n", + " \"Beam\",\n", + " \"Image\",\n", + " \"PrivateFei\",\n", + " \"EBeamDeceleration\",\n", + " \"EBeam\",\n", + " \"T2\",\n", + " \"CompoundLensFilter\",\n", + " \"Stage\",\n", + " \"Nav-Cam\",\n", + " \"EScan\",\n", + " \"HotStageMEMS\",\n", + " \"ColdStage\",\n", + " \"HotStage\",\n", + " \"System\",\n", + " \"GIS\",\n", + " \"Vacuum\",\n", + " \"IRBeam\",\n", + " \"HotStageHVHS\",\n", + " \"HiResIllumination\",\n", + " \"Detectors\"]\n", + "\n", + "all_concepts = set()\n", + "for idx in np.arange(3, len(fnms)):\n", + " fnm = f\"{src}/{fnms[idx][1]}\"\n", + " # print(f\"{fnm}\")\n", + " with open(fnm, 'rb', 0) as fp:\n", + " s = mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ)\n", + " s.seek(0, 2)\n", + " fsize = s.tell()\n", + " print(fsize)\n", + " s.seek(0, 0)\n", + " tfs_parent_concepts_byte_offset = {}\n", + " for concept in TiffTfsParentConcepts:\n", + " pos = s.find(bytes(f\"[{concept}]\", \"utf8\")) # != -1\n", + " if pos != -1:\n", + " tfs_parent_concepts_byte_offset[concept] = pos\n", + " # else:\n", + " # print(f\"Instance of concept [{concept}] was not found !\")\n", + " # print(tfs_parent_concepts_byte_offset)\n", + "\n", + " sequence = [] # decide I/O order in which metadata for childs of parent concepts will be read\n", + " for key, value in tfs_parent_concepts_byte_offset.items():\n", + " if value is not None:\n", + " sequence.append((key, value))\n", + " # tuple of parent_concept name and byte offset\n", + " sequence = sort_ascendingly_by_second_argument(sequence)\n", + " # print(sequence)\n", + "\n", + " idx = 0\n", + " for parent, byte_offset in sequence:\n", + " pos_s = byte_offset\n", + " pos_e = None\n", + " if idx < len(sequence) - 1:\n", + " pos_e = sequence[idx + 1][1]\n", + " else:\n", + " pos_e = fsize # np.iinfo(np.uint64).max\n", + " # TODO::better use official convention to not read beyond the end of file\n", + " idx += 1\n", + " if pos_s is None or pos_e is None:\n", + " raise ValueError(f\"Definition of byte boundaries for reading childs of [{parent}] was unsuccessful !\")\n", + " print(f\"Search for [{parent}] in between byte offsets {pos_s} and {pos_e}\")\n", + " s.seek(pos_s, 0)\n", + " payload = s.read(pos_e - pos_s).decode(\"utf8\")\n", + " for line in payload.split(\"\\n\"):\n", + " # print(line)\n", + " if line.count(\"=\") == 1:\n", + " all_concepts.add(f\"{parent}/{line.split('=')[0]}\")\n", + " # print(payload)\n", + "\n", + "for concept in all_concepts:\n", + " print(concept)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1963afb6-6e48-4628-a0e8-d2da0874701e", + "metadata": {}, + "outputs": [], + "source": [ + "# handle TIFF\n", + "with Image.open(fnm, mode=\"r\") as fp:\n", + " for key in fp.tag_v2:\n", + " if key in [34118, 34119]:\n", + " print(type(fp.tag[key]))\n", + " print(len(fp.tag[key])) \n", + " # print(f\"{key}, {fp.tag[key]}\")\n", + " if key not in TAGS.keys():\n", + " print(f\"--->tag {key}, is not in PIL.TiffTAGS !\")\n", + " # self.tags = {TAGS[key] : fp.tag[key] for key in fp.tag_v2}\n", + " # for key, val in self.tags.items():\n", + " # print(f\"{key}, {val}\")\n", + " nparr = np.array(fp)\n", + " print(f\"{type(nparr)}\")\n", + " print(f\"{nparr.dtype}\")\n", + " print(f\"{np.shape(nparr)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "28687c0e-6f14-484c-b511-3a4906d9672e", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a8ada062-e308-4288-8f00-b3e620f3c890", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "# https://www.geeksforgeeks.org/python-program-to-sort-a-list-of-tuples-by-second-item/\n", + "def sort_tuple(tup):\n", + " # convert the list of tuples to a numpy array with data type (object, int)\n", + " arr = np.array(tup, dtype=[('col1', object), ('col2', int)])\n", + " # get the indices that would sort the array based on the second column\n", + " indices = np.argsort(arr['col2'])\n", + " # use the resulting indices to sort the array\n", + " sorted_arr = arr[indices]\n", + " # convert the sorted numpy array back to a list of tuples\n", + " sorted_tup = [(row['col1'], row['col2']) for row in sorted_arr]\n", + " return sorted_tup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d27df293-626c-4d37-80df-96c182d4f401", + "metadata": {}, + "outputs": [], + "source": [ + "def if_str_represents_float(s):\n", + " try:\n", + " return isinstance(float(s), float)\n", + " # return str(float(s)) == s\n", + " except ValueError:\n", + " return False" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f647fa79-330b-48b2-8360-f92fc5ead187", + "metadata": {}, + "outputs": [], + "source": [ + "\"10\".isdigit()\n", + "# isinstance(float(\"8.99306e-010\"), float)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a2f0864-f8b3-4d53-bf9d-08a5787c32fb", + "metadata": {}, + "outputs": [], + "source": [ + "# TFS sections based on IKZ ALN_baoh_021.tif example\n", + "import mmap\n", + "\n", + "tfs_section_names = [\"[User]\",\n", + " \"[System]\",\n", + " \"[Beam]\",\n", + " \"[EBeam]\", \n", + " \"[GIS]\",\n", + " \"[Scan]\",\n", + " \"[EScan]\",\n", + " \"[Stage]\",\n", + " \"[Image]\",\n", + " \"[Vacuum]\",\n", + " \"[Specimen]\",\n", + " \"[Detectors]\",\n", + " \"[T2]\",\n", + " \"[Accessories]\",\n", + " \"[EBeamDeceleration]\",\n", + " \"[CompoundLensFilter]\",\n", + " \"[PrivateFei]\",\n", + " \"[HiResIllumination]\",\n", + " \"[EasyLift]\",\n", + " \"[HotStageMEMS]\",\n", + " \"[HotStage]\",\n", + " \"[HotStageHVHS]\",\n", + " \"[ColdStage]\"]\n", + "\n", + "tfs_section_details = {\"[System]\": [\"Type\", \"Dnumber\", \"Software\", \"BuildNr\", \"Source\", \"Column\", \"FinalLens\", \"Chamber\", \"Stage\", \"Pump\",\n", + " \"ESEM\", \"Aperture\", \"Scan\", \"Acq\", \"EucWD\", \"SystemType\", \"DisplayWidth\", \"DisplayHeight\"]}\n", + "tfs_section_offsets = {}\n", + "\n", + "with open(fnm, 'rb', 0) as file:\n", + " s = mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ)\n", + " for section_name in tfs_section_names:\n", + " pos = s.find(bytes(section_name, \"utf8\")) # != -1\n", + " tfs_section_offsets[section_name] = pos\n", + " print(tfs_section_offsets)\n", + "\n", + " # define search offsets\n", + " tpl = []\n", + " for key, value in tfs_section_offsets.items():\n", + " tpl.append((key, value))\n", + " # print(tpl)\n", + " tpl = sort_tuple(tpl)\n", + " print(tpl)\n", + " # if section_name == \"[System]\":\n", + " pos_s = None\n", + " pos_e = None\n", + " for idx in np.arange(0, len(tpl)):\n", + " if tpl[idx][0] != \"[System]\":\n", + " continue\n", + " else:\n", + " pos_s = tpl[idx][1]\n", + " if idx <= len(tpl) - 1:\n", + " pos_e = tpl[idx + 1][1]\n", + " break\n", + " print(f\"Search in between byte offsets {pos_s} and {pos_e}\")\n", + " # fish metadata of e.g. the system section\n", + " section_metadata = {}\n", + " for term in tfs_section_details[\"[System]\"]:\n", + " \n", + " s.seek(pos_s, 0)\n", + " pos = s.find(bytes(term, \"utf8\"))\n", + " if pos < pos_e: # check if pos_e is None\n", + " s.seek(pos, 0)\n", + " section_metadata[f\"{term}\"] = f\"{s.readline().strip().decode('utf8').replace(f'{term}=', '')}\"\n", + " if if_str_represents_float(section_metadata[f\"{term}\"]) is True:\n", + " section_metadata[f\"{term}\"] = np.float64(section_metadata[f\"{term}\"])\n", + " elif section_metadata[f\"{term}\"].isdigit() is True:\n", + " section_metadata[f\"{term}\"] = np.int64(section_metadata[f\"{term}\"])\n", + " else:\n", + " pass\n", + " # print(f\"{term}, {pos}, {pos + len(term) + 1}\")\n", + " # tfs_section_offswr\n", + " # file.seek(pos, 0) #\n", + " print(section_metadata)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2f3eb287-8f55-424c-a016-a07fc59f068a", + "metadata": {}, + "outputs": [], + "source": [ + "'2'.isdigit()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1341e30-fcce-4a3d-a099-d342b8bbe318", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "48f31e6c-1554-4476-8688-5f5323d513c8", + "metadata": {}, + "outputs": [], + "source": [ + "# https://codereview.stackexchange.com/a/21035\n", + "# https://stackoverflow.com/questions/38852822/how-to-flatten-xml-file-in-python\n", + "import xmltodict\n", + "from collections import OrderedDict\n", + "\n", + "def flatten_dict(d):\n", + " def items():\n", + " for key, value in d.items():\n", + " # nested subtree\n", + " if isinstance(value, dict):\n", + " for subkey, subvalue in flatten_dict(value).items():\n", + " yield '{}.{}'.format(key, subkey), subvalue\n", + " # nested list\n", + " elif isinstance(value, list):\n", + " for num, elem in enumerate(value):\n", + " for subkey, subvalue in flatten_dict(elem).items():\n", + " yield '{}.[{}].{}'.format(key, num, subkey), subvalue\n", + " # everything else (only leafs should remain)\n", + " else:\n", + " yield key, value\n", + " return OrderedDict(items())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9358182-2cde-45c2-bfd5-07266b142601", + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "def get_protochip_variadic_concept(token):\n", + " # return val is a tuple\n", + " # zeroth entry specifies the prefix of the instance concept\n", + " # first entry specifies associated parent concept\n", + " # second entry is True is token resolves an AXON parent concept\n", + " if token.count(\" = \") == 1:\n", + " tmp = token.split(\"=\")\n", + " concept = tmp[0].strip()\n", + " value = tmp[1].strip()\n", + " # print(f\"{concept}, {value}\")\n", + " idxs = re.finditer(\".\\[[0-9]+\\].\", concept)\n", + " is_variadic = False\n", + " if (sum(1 for _ in idxs) > 0): # and (\"DataValues\" in concept):\n", + " is_variadic = True\n", + " variadic = concept\n", + " concept_id = []\n", + " for idx in re.finditer(\".\\[[0-9]+\\].\", token):\n", + " variadic = variadic.replace(concept[idx.start(0):idx.end(0)], \".[*].\")\n", + " concept_id.append(f\"{concept[idx.start(0):idx.end(0)]}\".replace(\".[\", \"\").replace(\"].\", \"\"))\n", + " if (\"DataValues\" in concept) and concept.count(\".Name = \") == 1:\n", + " return (variadic, f\"{variadic}.{'_'.join(concept_id)}\", True) # parent concept\n", + " elif concept.count(\"PositionerName\") == 1:\n", + " return (variadic, f\"{variadic}.{'_'.join(concept_id)}\", True) # parent concept\n", + " else:\n", + " return (variadic, f\"{variadic}.{'_'.join(concept_id)}\", False) # child concept\n", + " else:\n", + " return (concept, concept, True)\n", + " else:\n", + " return (None, None, None)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "93b116cf-0ead-4bb9-9496-774221feb70f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3caa30a0-e050-4898-b553-9d71e149160c", + "metadata": {}, + "outputs": [], + "source": [ + "def get_protochips_variadic_concept(key, value):\n", + " # return val is a tuple\n", + " # zeroth entry specifies the prefix of the instance concept\n", + " # first entry specifies associated parent concept\n", + " # second entry is True is token resolves an AXON parent concept\n", + " if isinstance(key, str) and key != \"\":\n", + " concept = key.strip()\n", + " idxs = re.finditer(\".\\[[0-9]+\\].\", concept)\n", + " is_variadic = False\n", + " if (sum(1 for _ in idxs) > 0):\n", + " is_variadic = True\n", + " variadic = concept\n", + " concept_id = []\n", + " for idx in re.finditer(\".\\[[0-9]+\\].\", concept):\n", + " variadic = variadic.replace(concept[idx.start(0):idx.end(0)], \".[*].\")\n", + " concept_id.append(f\"{concept[idx.start(0):idx.end(0)]}\".replace(\".[\", \"\").replace(\"].\", \"\"))\n", + " print(f\"concept_id: {concept_id}\")\n", + " if variadic.endswith(\".Name\"):\n", + " variadic = variadic.replace(\".Name\", f\".{value}\")\n", + " return variadic\n", + " else:\n", + " return concept\n", + " return None" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "501ddd47-46d5-4191-be82-bac7ed007749", + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "key = \"MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[0].DataValues.AuxiliaryDataValue.[20].HeatingPower\"\n", + "def specific_to_variadic(token):\n", + " if isinstance(token, str) and key != \"\":\n", + " concept = token.strip()\n", + " idxs = re.finditer(r\".\\[[0-9]+\\].\", concept)\n", + " if (sum(1 for _ in idxs) > 0):\n", + " variadic = concept\n", + " for idx in re.finditer(\".\\[[0-9]+\\].\", concept):\n", + " variadic = variadic.replace(concept[idx.start(0):idx.end(0)], \".[*].\")\n", + " return variadic\n", + " else:\n", + " return concept\n", + " return None\n", + "print(specific_to_variadic(key))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6feb02fa-d179-4d7e-b1cc-09d8af396315", + "metadata": {}, + "outputs": [], + "source": [ + "key = \"MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[0].DataValues.AuxiliaryDataValue.[20].Name\"\n", + "value = \"HeatingPower\"\n", + "key = \"MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[0].DataValues.AuxiliaryDataValue.[20].Value\"\n", + "value = 0.077978329305\n", + "key = \"MicroscopeControlImageMetadata.@xmlns:i\"\n", + "value = \"http://www.w3.org/2001/XMLSchema-instance\"\n", + "key = \"MicroscopeControlImageMetadata.ImagerSettings.ImagePhysicalSize.X\"\n", + "value = 0.99\n", + "key = \"MicroscopeControlImageMetadata.ActivePositionerSettings.PositionerSettings.[0].X\"\n", + "value = -369.84159375\n", + "\n", + "# first phase analyse the collection of Protochips metadata concept instance symbols and reduce to unique concepts\n", + "concepts = {}\n", + "for concept, value in flattened_xml.items():\n", + " # not every key is allowed to define a concept\n", + " # print(f\"{concept}: {value}\")\n", + " idxs = re.finditer(\".\\[[0-9]+\\].\", concept)\n", + " if (sum(1 for _ in idxs) > 0): # is_variadic (allowed to define eventually a key!)\n", + " markers = [\".Name\", \".PositionerName\"]\n", + " for marker in markers:\n", + " if concept.endswith(marker):\n", + " concepts[f\"{concept[0:len(concept)-len(marker)]}\"] = value # .{value}\"] = None\n", + " else:\n", + " # not variadic, i.e. defines a concept\n", + " concepts[concept] = value\n", + "#for c, v in concepts.items():\n", + "# print(f\"{c}: {v}\")\n", + "print(\"1. done\")\n", + "# second phase, evaluate each concept instance symbol wrt to its prefix coming from the unique concept\n", + "for k, v in flattened_xml.items():\n", + " # print(f\"-->{k}: {v}\")\n", + " grpnms = None\n", + " idxs = re.finditer(\".\\[[0-9]+\\].\", k)\n", + " if (sum(1 for _ in idxs) > 0): # is variadic\n", + " search_argument = k[0:k.rfind(\"].\")+1]\n", + " for parent_grpnm, child_grpnm in concepts.items():\n", + " if parent_grpnm.startswith(search_argument):\n", + " grpnms = (parent_grpnm, child_grpnm)\n", + " break\n", + " if grpnms is not None:\n", + " if len(grpnms) == 2:\n", + " if \"PositionerSettings\" in k and k.endswith(\".PositionerName\") is False:\n", + " print(f\"vv: {grpnms[0]}.{grpnms[1]}{k[k.rfind('.') + 1:]}: {v}\")\n", + " if k.endswith(\".Value\"):\n", + " print(f\"vv: {grpnms[0]}.{grpnms[1]}: {v}\")\n", + " else:\n", + " print(f\"nv: {k}: {v}\")\n", + "# token = \"MicroscopeControlImageMetadata.ActivePositionerSettings.PositionerSettings.[0].PositionerName = Stage\"\n", + "# token = \"\"\n", + "# idxs = re.finditer(\".\\[[0-9]+\\].\", token)\n", + "# print(sum(1 for _ in idxs))\n", + "# a, b, c = instance_to_concept(token)\n", + "# print(f\"{a}, {b}, {c}\")\n", + "# print(get_protochips_variadic_concept(key, value))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "67dc0c90-7b37-4d9c-8bbe-3db50040094d", + "metadata": {}, + "outputs": [], + "source": [ + "def get_axon_concept_prefix(glossary, what):\n", + " # return val is a tuple\n", + " # zeroth entry specifies the prefix of the instance concept\n", + " # first entry specifies associated parent concept\n", + " # second entry is True is token resolves an AXON parent concept\n", + " for term in glossary:\n", + " if token.count(\" = \") == 1:\n", + " concept = term.split(\"=\")[0].strip()\n", + " value = term.split(\"=\")[1].strip()\n", + " if concept.endswith(\"Name\") and what in value:\n", + " return concept[0:len(concept)-len(\"Name\")-1]\n", + " return None" + ] + }, + { + "cell_type": "markdown", + "id": "1300ffd9-06c3-44b2-87bf-a9f8e50db713", + "metadata": {}, + "source": [ + "Human understand that the following metadata structure from AXON suggests that:\n", + "MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[0].DataValues.AuxiliaryDataValue.[0].Name = SystemStatus\n", + "MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[0].DataValues.AuxiliaryDataValue.[0].Value = Ready\n", + "MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[0].DataValues.AuxiliaryDataValue.[0].ValueType = String\n", + "MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[0].DataValues.AuxiliaryDataValue.[0].ValueUnits = None\n", + "MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[0].DataValues.AuxiliaryDataValue.[0].DisplayUnits = None\n", + "MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[0].DataValues.AuxiliaryDataValue.[0].Description = None\n", + "MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[0].DataValues.AuxiliaryDataValue.[0].MeasurementTypeName = None\n", + "\n", + "Is equivalent to\n", + "*MicroscopeControlImageMetadata.SystemStatus* being a group # NeXus concept name\n", + "MicroscopeControlImageMetadata.SystemStatus.Value = Ready # instance data\n", + "MicroscopeControlImageMetadata.SystemStatus.ValueType = String # NeXus value category\n", + "MicroscopeControlImageMetadata.SystemStatus.ValueUnits = None # unit\n", + "MicroscopeControlImageMetadata.SystemStatus.DisplayUnits = None # essentially NOMAD MetaInfo display unit relevant for the GUI\n", + "MicroscopeControlImageMetadata.SystemStatus.Description = None # essentially the NeXus docstring\n", + "MicroscopeControlImageMetadata.SystemStatus.MeasurementTypeName = None # no equivalent in NeXus" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2cf76281-756f-4794-8406-0a4df1acb682", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "import os\n", + "import mmap\n", + "for file_name in os.listdir(f\"{src}/axon\"):\n", + " with open(f\"{src}/axon/{file_name}\", 'rb', 0) as fp:\n", + " s = mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ)\n", + " magic = s.read(8)\n", + " if magic == b'\\x89PNG\\r\\n\\x1a\\n': # https://en.wikipedia.org/wiki/List_of_file_signatures\n", + " print(f\"{file_name} is PNG\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5d5cbb4-c5a2-44a1-b6a4-277167582869", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import mmap\n", + "import PIL\n", + "import datetime\n", + "events = []\n", + "for file_name in os.listdir(f\"{src}/axon\"):\n", + " # if we try to open non-image files with PIL directly we get an error so one should hunt first on the magic number\n", + " with open(f\"{src}/axon/{file_name}\", 'rb', 0) as file:\n", + " s = mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ)\n", + " magic = s.read(8)\n", + " if magic != b'\\x89PNG\\r\\n\\x1a\\n':\n", + " continue\n", + " \n", + " with Image.open(f\"{src}/axon/{file_name}\", mode=\"r\") as fp:\n", + " # if f\"{file_name}\" != \"20210426T225128.453Raw382.png\": # \"20210426T224913.001Raw257.png\":\n", + " # continue\n", + " if isinstance(fp, PIL.PngImagePlugin.PngImageFile) is True:\n", + " print(f\"Parsing {src}/axon/{file_name}...\")\n", + " fp.load() # Needed only for .png EXIF data (see citation above)\n", + " # print(fp.info.keys())\n", + "\n", + " if \"MicroscopeControlImage\" in fp.info.keys():\n", + " # print(fp.info[\"MicroscopeControlImage\"])\n", + " xml_content = xmltodict.parse(fp.info[\"MicroscopeControlImage\"])\n", + " flattened_xml = flatten_dict(xml_content)\n", + " # first display all instance metadata in their unmodified AXON-specific representation\n", + " axon_concepts = []\n", + " for k,v in flattened_xml.items():\n", + " # print('{} = {}'.format(k,v))\n", + " print(f\"{k}: {v}\")\n", + " continue\n", + " if \"AuxiliaryDataValue\" in f\"{k}\":\n", + " axon_concepts.append(f\"{k}.{v}\")\n", + " else:\n", + " axon_concepts.append(f\"{k}\")\n", + " # if f\"{k}\".endswith(\".Name\"):\n", + " # axon_concepts.append(f\"{k[0:len(k)-len('Name')-1]}\")\n", + " # else:\n", + " # axon_concepts.append(f\"{k}\")\n", + " # for concept in axon_concepts:\n", + " # print(concept)\n", + " # print(axon_concepts)\n", + " reqs = [\"MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[0].DataValues.AuxiliaryDataValue.[0].Name.SystemStatus\",\n", + " \"MicroscopeControlImageMetadata.MicroscopeDateTime\",\n", + " \"MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[0].DataValues.AuxiliaryDataValue.[7].Name.HolderTemperature\"]\n", + " for req in reqs:\n", + " if req not in axon_concepts:\n", + " raise ValueError(f\"{src}/axon/{file_name} does not contain concept instance {req} !\")\n", + " # all required concepts have instance data so parse\n", + " # print(f\"{flattened_xml['MicroscopeControlImageMetadata.MicroscopeDateTime']}\")\n", + " if f\"{flattened_xml['MicroscopeControlImageMetadata.MicroscopeDateTime']}\".count(\".\") == 1:\n", + " datetime_obj = datetime.datetime.strptime(f\"{flattened_xml['MicroscopeControlImageMetadata.MicroscopeDateTime']}\", '%Y-%m-%dT%H:%M:%S.%f%z')\n", + " else:\n", + " datetime_obj = datetime.datetime.strptime(f\"{flattened_xml['MicroscopeControlImageMetadata.MicroscopeDateTime']}\", '%Y-%m-%dT%H:%M:%S%z')\n", + " events.append((f\"{file_name}\", datetime_obj))\n", + " # print(f\"converted to datetime: {datetime_obj}\")\n", + " print(f\"{flattened_xml['MicroscopeControlImageMetadata.AuxiliaryData.AuxiliaryDataCategory.[0].DataValues.AuxiliaryDataValue.[7].Value']}\")\n", + " fp.seek(0)\n", + " nparr = np.array(fp)\n", + " print(f\"{np.shape(nparr)}, {nparr.dtype}\")\n", + " continue\n", + "\n", + " if True is False:\n", + " # identify all unique axon_concepts\n", + " axon_concepts = []\n", + " for k,v in flattened_xml.items():\n", + " prefix, parent, is_parent = instance_to_concept('{} = {}'.format(k,v))\n", + " if is_parent is True:\n", + " if parent not in axon_concepts:\n", + " axon_concepts.append(parent)\n", + " else:\n", + " raise ValueError(f\"Found duplicated axon conceptid {conceptid} !\")\n", + " for entry in axon_concepts:\n", + " print(entry)\n", + "\n", + " # make members of the same axon concept childs\n", + " for k,v in flattened_xml.items():\n", + " varname, conceptid, is_parent = instance_to_concept('{} = {}'.format(k,v))\n", + " if conceptid is not None: \n", + " if conceptid in axon_concepts.keys():\n", + " if is_parent is True:\n", + " print(f\"{axon_concepts[conceptid]}\")\n", + " else:\n", + " print(f\"{axon_concepts[conceptid]}.{varname[varname.rfind('.') + 1:]}\")\n", + " else:\n", + " raise ValueError(f\"Unable to find matching axon parent conceptid {conceptid} !\")\n", + " else:\n", + " print('{} = {}'.format(k,v))\n", + " # elif fnm.lower().endswith(\".png\") is True: # check for mime type instead\n", + " # print(f\"There is no iTXt chunk in {fnm} which has embedded XML within the AXON namespace MicroscopeControlImage!\")\n", + " else:\n", + " print(f\"There is nothing to harvest here!\")\n", + " # print(axon_concepts)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60da8d37-6e03-4c04-b47c-b24224de6a02", + "metadata": {}, + "outputs": [], + "source": [ + "def sort_ascendingly_by_second_argument_iso8601(tup):\n", + " # convert the list of tuples to a numpy array with data type (object, int)\n", + " arr = np.array(tup, dtype=[('col1', object), ('col2', datetime.datetime)])\n", + " # get the indices that would sort the array based on the second column\n", + " indices = np.argsort(arr['col2'])\n", + " # use the resulting indices to sort the array\n", + " sorted_arr = arr[indices]\n", + " # convert the sorted numpy array back to a list of tuples\n", + " sorted_tup = [(row['col1'], row['col2']) for row in sorted_arr]\n", + " return sorted_tup\n", + "\n", + "new_lst = sort_ascendingly_by_second_argument_iso8601(events)\n", + "# events = sorted(events, key=lambda x: x if isinstance(x, datetime.datetime) else datetime.datetime(x.year, x.month, x.day))\n", + "time_series_start = new_lst[0][1]\n", + "print(f\"Time series start: {time_series_start}\")\n", + "for file_name, iso8601 in new_lst:\n", + " continue\n", + " print(f\"{file_name}, {iso8601}, {(iso8601 - time_series_start).total_seconds()} s\")\n", + "print(f\"Time series end: {new_lst[-1][1]}, {(new_lst[-1][1] - time_series_start).total_seconds()} s\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3c18541f-43d2-433d-8bbc-0ef3ec3bf9f7", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01d55278-ba89-4c02-9728-108da1c598ec", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/debug/metadata_tiff.ipynb b/debug/metadata_tiff.ipynb new file mode 100644 index 000000000..8bed68164 --- /dev/null +++ b/debug/metadata_tiff.ipynb @@ -0,0 +1,210 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "5eafd031", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f655f67", + "metadata": {}, + "outputs": [], + "source": [ + "src = \"/home/kaiobach/Research/paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_imgs/ikz_robert\"\n", + "filepath=f\"{src}/0c8nA_3deg_003_AplusB_test.tif\"\n", + "lines=[]\n", + "with open(filepath, 'r', encoding=\"utf8\",errors=\"ignore\") as fp:\n", + " contents=fp.read()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "268eba7c", + "metadata": {}, + "outputs": [], + "source": [ + "text=contents[contents.find(\"Date\"):] #metadata at the end of the file starts with the date" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a7f988b3", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "print(text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3b69b8ea", + "metadata": {}, + "outputs": [], + "source": [ + "def get_metadata_tif(filepath):\n", + " with open(filepath, 'r', encoding=\"utf8\",errors=\"ignore\") as fp:\n", + " contents=fp.read() \n", + " text=contents[contents.find(\"Date\"):]\n", + " # qtu <-- quantitiy_type_unit\n", + " #Beam\n", + " qtu=[[\"\\nSystemType=\",\"str\",None],\n", + " [\"\\nPixelWidth=\",\"np.float64\",\"m\"],\n", + " [\"\\nPixelHeight=\",\"np.float64\",\"m\"], \n", + " \n", + " [\"\\nHV=\",\"np.float64\",\"V\"],\n", + " [\"\\nBeamCurrent=\",\"np.float64\",\"A\"],\n", + " [\"\\nWorkingDistance=\",\"np.float64\",\"m\"],\n", + " [\"\\nDwelltime=\",\"np.float64\",\"s\"],\n", + " [\"\\nSpot=\",\"np.float64\",\"nm\"],\n", + " [\"\\nStigmatorX=\",\"np.float64\",None],\n", + " [\"\\nStigmatorY=\",\"np.float64\",None],\n", + " [\"\\nBeamShiftX=\",\"np.float64\",None],\n", + " [\"\\nBeamShiftY=\",\"np.float64\",None],\n", + " [\"\\nSourceTiltX=\",\"np.float64\",None],\n", + " [\"\\nSourceTiltY=\",\"np.float64\",None],\n", + " [\"\\nEmissionCurrent=\",\"np.float64\",\"A\"],\n", + " [\"\\nSpecimenCurrent=\",\"np.float64\",\"A\"],\n", + " [\"\\nApertureDiameter=\",\"np.float64\",\"m\"],\n", + " [\"\\nATubeVoltage=\",\"np.float64\",\"V\"],\n", + " #Scan \n", + " [\"\\nScanRotation=\",\"np.float64\",\"deg\"],\n", + " [\"\\nTiltCorrectionIsOn=\",\"str\",None],#yes,no \n", + " [\"\\nUseCase=\",\"str\",None],\n", + "\n", + " #CompoundLens\n", + " [\"\\nIsOn=\",\"str\",None], #On,Off\n", + " [\"\\nThresholdEnergy=\",\"np.float64\",\"eV\"],\n", + " #Stage \n", + " [\"\\nStageX=\",\"np.float64\",\"m\"],\n", + " [\"\\nStageY=\",\"np.float64\",\"m\"],\n", + " [\"\\nStageZ=\",\"np.float64\",\"m\"],\n", + " [\"\\nStageR=\",\"np.float64\",\"deg\"],\n", + " [\"\\nStageTa=\",\"np.float64\",\"deg\"],\n", + " [\"\\nStageTb=\",\"np.float64\",\"deg\"],\n", + " [\"\\nStageBias=\",\"np.float64\",\"V\"],\n", + " [\"\\nChPressure=\",\"np.float64\",\"Pa\"],\n", + "\n", + " #Detecor\n", + " [\"\\nName=\",\"str\",None],\n", + " [\"\\nMode=\",\"str\",None],\n", + " [\"\\nContrast=\",\"np.float64\",None],\n", + " [\"\\nBrightness=\",\"np.float64\",None],\n", + " [\"\\nSignal=\",\"str\",None],\n", + " [\"\\nContrastDB=\",\"np.float64\",\"dB\"],\n", + " [\"\\nBrightnessDB=\",\"np.float64\",\"dB\"],\n", + " [\"\\nAverage=\",\"int\",None],\n", + " [\"\\nIntegrate=\",\"int\",None],\n", + "\n", + "\n", + " [\"\\nResolutionX=\",\"int\",None],\n", + " [\"\\nResolutionY=\",\"int\",None],\n", + " [\"\\nHorFieldsize=\",\"np.float64\",\"m\"],\n", + " [\"\\nVerFieldsize=\",\"np.float64\",\"m\"],\n", + " [\"\\nFrameTime=\",\"np.float64\",\"s\"],\n", + " #Digital\n", + " [\"\\nDigitalContrast=\",\"np.float64\",None],\n", + " [\"\\nDigitalBrightness=\",\"np.float64\",None],\n", + " [\"\\nDigitalGamma=\",\"np.float64\",None]]\n", + " \n", + " res=[]\n", + " typechanges=[]\n", + " counter=0\n", + " for i in qtu:\n", + " kw=i[0]\n", + " start=text.find(kw)+len(kw)\n", + " end=text[start:].find(\"\\n\")\n", + " if i[1]==\"int\": \n", + " res.append(int(text[start:start+end]))\n", + " elif i[1]==\"np.float64\":\n", + " res.append(np.double(text[start:start+end]))\n", + " elif i[1]==\"str\" or i[1]==\"string\":\n", + " if text[start:start+end] in [\"no\",\"No\",\"off\",\"Off\",\"false\",\"False\"]:\n", + " res.append(False)\n", + " typechanges.append(counter)\n", + " elif text[start:start+end] in [\"yes\",\"Yes\",\"on\",\"On\",\"true\",\"True\"]:\n", + " res.append(True)\n", + " typechanges.append(counter)\n", + " else:\n", + " res.append(text[start:start+end])\n", + " counter +=1\n", + "\n", + " for j in typechanges:\n", + " qtu[j][1]=\"bool\"\n", + " \n", + " return res,qtu" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "74365e99", + "metadata": {}, + "outputs": [], + "source": [ + "filepath=f\"{src}/0c8nA_3deg_003_AplusB_test.tif\"\n", + "res,qtu=get_metadata_tif(filepath)\n", + "res" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0000d17e", + "metadata": {}, + "outputs": [], + "source": [ + "qtu" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3435cdbc-9ffa-4035-8b8d-ee4a648cc597", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "584e4103-2e86-43f7-ab58-fb5d3e7dc36b", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/nion.batch.sh b/debug/nion.batch.sh similarity index 100% rename from nion.batch.sh rename to debug/nion.batch.sh diff --git a/debug/nion.dev.ipynb b/debug/nion.dev.ipynb new file mode 100644 index 000000000..24750c59b --- /dev/null +++ b/debug/nion.dev.ipynb @@ -0,0 +1,57 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "f6fd745e-93c4-4754-ade9-0f4aa4b36213", + "metadata": {}, + "outputs": [], + "source": [ + "from jupyterlab_h5web import H5Web\n", + "src = \"/home/kaiobach/Research/paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_nion/2022-02-18_Metadata_Kuehbach.zip/2022-02-18_Metadata_Kuehbach Data/2022/02/18/20220218-140947/\"\n", + "H5Web(f\"{src}data_7EPPSHNUFKH6F6A4JCR45J03G.h5\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8469fb59-4e2b-4f7e-9012-9059192f210a", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "a = np.asarray([1, 1, 1], np.float64)\n", + "print(isinstance(a, np.ndarray))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "207488e4-fa3c-4757-96bb-fa5f37bd14db", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyxem.batch.sh b/debug/pyxem.batch.sh similarity index 100% rename from pyxem.batch.sh rename to debug/pyxem.batch.sh diff --git a/pyxem.dev.ipynb b/debug/pyxem.dev.ipynb similarity index 100% rename from pyxem.dev.ipynb rename to debug/pyxem.dev.ipynb diff --git a/spctrscpy.batch.sh b/debug/spctrscpy.batch.sh similarity index 86% rename from spctrscpy.batch.sh rename to debug/spctrscpy.batch.sh index ea5b091af..0d2a219cc 100755 --- a/spctrscpy.batch.sh +++ b/debug/spctrscpy.batch.sh @@ -1,11 +1,11 @@ #!/bin/bash -datasource="../../../../paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_spctrscpy/" +datasource="../../../../paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_spctrscpy/pdi/" # apex examples ikz, pdi # examples="ikz/VInP_108_L2.h5 ikz/GeSn_13.h5 pynx/46_ES-LP_L1_brg.bcf pynx/1613_Si_HAADF_610_kx.emd pynx/EELS_map_2_ROI_1_location_4.dm3 pynx/H5OINA_examples_Specimen_1_Map_EDS_+_EBSD_Map_Data_2.h5oina" examples="ikz/VInP_108_L2.h5" -examples="pdi/InGaN_nanowires_spectra.edaxh5" +examples="InGaN_nanowires_spectra.edaxh5" for example in $examples; do echo $example diff --git a/debug/spctrscpy.dev.ipynb b/debug/spctrscpy.dev.ipynb new file mode 100644 index 000000000..c6b5871e2 --- /dev/null +++ b/debug/spctrscpy.dev.ipynb @@ -0,0 +1,838 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 15, + "id": "6da1aea0-545b-446b-a3d1-1574af72f6c6", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from rsciio import bruker, emd, digitalmicrograph\n", + "from jupyterlab_h5web import H5Web\n", + "import h5py\n", + "from matplotlib import pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "3889c47f-11c4-4bf3-97de-04fc52f0798d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/home/kaiobach/Research/paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_spctrscpy/ikz/VInP_108_L2.h5\n" + ] + } + ], + "source": [ + "src = \"/home/kaiobach/Research/paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_spctrscpy\"\n", + "fnms = [(\"apex\", \"ikz/VInP_108_L2.h5\"),\n", + " (\"apex\", \"ikz/GeSn_13.h5\"),\n", + " (\"bruker\", \"pynx/46_ES-LP_L1_brg.bcf\"),\n", + " (\"emd\", \"pynx/1613_Si_HAADF_610_kx.emd\"),\n", + " (\"digitalmicrograph\", \"pynx/EELS_map_2_ROI_1_location_4.dm3\"),\n", + " (\"oxfordinstruments\", \"pynx/H5OINA_examples_Specimen_1_Map_EDS_+_EBSD_Map_Data_2.h5oina\")]\n", + "# pyUSID, HSMA\n", + "fnm = f\"{src}/{fnms[0][1]}\"\n", + "print(fnm)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "a4d9ef96-3c70-4c12-80ba-ea4a7d716d47", + "metadata": {}, + "outputs": [ + { + "data": { + "application/x-hdf5": "/home/kaiobach/Research/paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/development_spctrscpy/ikz/VInP_108_L2.h5", + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "H5Web(fnm)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "f0a7f9ac-1ade-43d7-aedd-b2572d163b34", + "metadata": { + "jupyter": { + "source_hidden": true + } + }, + "outputs": [], + "source": [ + "\n", + "from typing import Dict\n", + "\n", + "\n", + "class NxObject:\n", + " \"\"\"An object in a graph e.g. an attribute, dataset, or group in NeXus.\"\"\"\n", + "\n", + " def __init__(self,\n", + " name: str = None,\n", + " unit: str = None,\n", + " dtype=str,\n", + " value=None,\n", + " **kwargs):\n", + " if (name is not None) and (name == \"\"):\n", + " raise ValueError(f\"Value for argument name needs to be a non-empty string !\")\n", + " if (unit is not None) and (unit == \"\"):\n", + " raise ValueError(f\"Value for argument unit needs to be a non-empty string !\")\n", + " if (dtype is not None) and isinstance(dtype, type) is False:\n", + " raise ValueError(f\"Value of argument dtype must not be None \" \\\n", + " f\" and a valid, ideally a numpy datatype !\")\n", + " # self.doc = None # docstring\n", + " self.name = name # name of the field\n", + " self.unit = unit # not unit category but actual unit\n", + " # use special values \"unitless\" for NX_UNITLESS (e.g. 1) and\n", + " # \"dimensionless\" for NX_DIMENSIONLESS (e.g. 1m / 1m)\n", + " self.dtype = dtype # use np.dtype if possible\n", + " if value is None or dtype is str:\n", + " self.unit = \"unitless\"\n", + " if value is not None:\n", + " self.value = value\n", + " # value should be a numpy scalar, tensor, or string if possible\n", + " self.eqv_hdf = None\n", + " if \"eqv_hdf\" in kwargs:\n", + " if kwargs[\"eqv_hdf\"] in [\"group\", \"dataset\", \"attribute\"]:\n", + " self.eqv_hdf = kwargs[\"eqv_hdf\"]\n", + " else:\n", + " raise ValueError(f\"Value of keyword argument eqv_hdf needs to be one of grp, dset, attr !\")\n", + "\n", + " def __repr__(self):\n", + " \"\"\"Report values.\"\"\"\n", + " return f\"Name: {self.name}, unit: {self.unit}, dtype: {self.dtype}, eqv_hdf: {self.eqv_hdf}\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "97c3a10f-903a-4d7e-883b-779c6c34f4a0", + "metadata": {}, + "outputs": [], + "source": [ + "NX_IMAGE_REAL_SPACE_SET_HDF_PATH = [\"image_oned/axis_x-field\",\n", + "\"image_oned/axis_x@long_name-attribute\",\n", + "\"image_oned/intensity-field\",\n", + "\"image_threed/axis_x-field\",\n", + "\"image_threed/axis_x@long_name-attribute\",\n", + "\"image_threed/axis_y-field\",\n", + "\"image_threed/axis_y@long_name-attribute\",\n", + "\"image_threed/axis_z-field\",\n", + "\"image_threed/axis_z@long_name-attribute\",\n", + "\"image_threed/intensity-field\",\n", + "\"image_twod/axis_x-field\",\n", + "\"image_twod/axis_x@long_name-attribute\",\n", + "\"image_twod/axis_y-field\",\n", + "\"image_twod/axis_y@long_name-attribute\",\n", + "\"image_twod/intensity-field\",\n", + "\"stack_oned/axis_image_identifier-field\",\n", + "\"stack_oned/axis_image_identifier@long_name-attribute\",\n", + "\"stack_oned/axis_x-field\",\n", + "\"stack_oned/axis_x@long_name-attribute\",\n", + "\"stack_oned/intensity-field\",\n", + "\"stack_threed/axis_image_identifier-field\",\n", + "\"stack_threed/axis_image_identifier@long_name-attribute\",\n", + "\"stack_threed/axis_x-field\",\n", + "\"stack_threed/axis_x@long_name-attribute\",\n", + "\"stack_threed/axis_y-field\",\n", + "\"stack_threed/axis_y@long_name-attribute\",\n", + "\"stack_threed/axis_z-field\",\n", + "\"stack_threed/axis_z@long_name-attribute\",\n", + "\"stack_threed/intensity-field\",\n", + "\"stack_twod/axis_image_identifier-field\",\n", + "\"stack_twod/axis_image_identifier@long_name-attribute\",\n", + "\"stack_twod/axis_x-field\",\n", + "\"stack_twod/axis_x@long_name-attribute\",\n", + "\"stack_twod/axis_y-field\",\n", + "\"stack_twod/axis_y@long_name-attribute\",\n", + "\"stack_twod/intensity-field\"]\n", + "\n", + "class NxEmImageRealSpaceSet():\n", + " def __init__(self):\n", + " self.tmp: Dict = {}\n", + " for entry in NX_IMAGE_REAL_SPACE_SET_HDF_PATH:\n", + " if entry.endswith(\"-field\") is True:\n", + " self.tmp[entry[0:len(entry)-len(\"-field\")]] = NxObject(eqv_hdf=\"dataset\")\n", + " elif entry.endswith(\"-attribute\") is True:\n", + " self.tmp[entry[0:len(entry)-len(\"-attribute\")]] = NxObject(eqv_hdf=\"attribute\")\n", + " else:\n", + " self.tmp[entry[0:len(entry)-len(\"-group\")]] = NxObject(eqv_hdf=\"group\")" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "8bbbaa03-0aac-43fb-941a-f63910496fa3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'image_oned/axis_x': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'image_oned/axis_x@long_name': Name: None, unit: unitless, dtype: , eqv_hdf: attribute, 'image_oned/intensity': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'image_threed/axis_x': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'image_threed/axis_x@long_name': Name: None, unit: unitless, dtype: , eqv_hdf: attribute, 'image_threed/axis_y': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'image_threed/axis_y@long_name': Name: None, unit: unitless, dtype: , eqv_hdf: attribute, 'image_threed/axis_z': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'image_threed/axis_z@long_name': Name: None, unit: unitless, dtype: , eqv_hdf: attribute, 'image_threed/intensity': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'image_twod/axis_x': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'image_twod/axis_x@long_name': Name: None, unit: unitless, dtype: , eqv_hdf: attribute, 'image_twod/axis_y': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'image_twod/axis_y@long_name': Name: None, unit: unitless, dtype: , eqv_hdf: attribute, 'image_twod/intensity': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'stack_oned/axis_image_identifier': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'stack_oned/axis_image_identifier@long_name': Name: None, unit: unitless, dtype: , eqv_hdf: attribute, 'stack_oned/axis_x': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'stack_oned/axis_x@long_name': Name: None, unit: unitless, dtype: , eqv_hdf: attribute, 'stack_oned/intensity': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'stack_threed/axis_image_identifier': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'stack_threed/axis_image_identifier@long_name': Name: None, unit: unitless, dtype: , eqv_hdf: attribute, 'stack_threed/axis_x': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'stack_threed/axis_x@long_name': Name: None, unit: unitless, dtype: , eqv_hdf: attribute, 'stack_threed/axis_y': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'stack_threed/axis_y@long_name': Name: None, unit: unitless, dtype: , eqv_hdf: attribute, 'stack_threed/axis_z': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'stack_threed/axis_z@long_name': Name: None, unit: unitless, dtype: , eqv_hdf: attribute, 'stack_threed/intensity': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'stack_twod/axis_image_identifier': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'stack_twod/axis_image_identifier@long_name': Name: None, unit: unitless, dtype: , eqv_hdf: attribute, 'stack_twod/axis_x': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'stack_twod/axis_x@long_name': Name: None, unit: unitless, dtype: , eqv_hdf: attribute, 'stack_twod/axis_y': Name: None, unit: unitless, dtype: , eqv_hdf: dataset, 'stack_twod/axis_y@long_name': Name: None, unit: unitless, dtype: , eqv_hdf: attribute, 'stack_twod/intensity': Name: None, unit: unitless, dtype: , eqv_hdf: dataset}\n" + ] + } + ], + "source": [ + "tmp = NxEmImageRealSpaceSet()\n", + "# print(tmp.tmp)" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "58052fb7-723f-476d-a8ca-df99efffcc05", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ", (1,), {'names': ['Version', 'ImageType', 'Label', 'SMin', 'SMax', 'Par_Color', 'EdsPresetMode', 'EdsPresetTime', 'EdsMapDataType', 'TiltCorrected', 'RasterMode', 'ChannelStart', 'ChannelEnd', 'IntDummy1', 'IAdc', 'ISize', 'IBits', 'NReads', 'NFrames', 'FDwell', 'KVolt', 'Tilt', 'TakeOff', 'Magnification', 'WorkingDistance', 'MicronsPerPixelX', 'MicronsPerPixelY', 'NumberOfCommentLines', 'TextLines', 'Fpar1', 'NOverlayElements', 'OverlayColors', 'XmpEdiTimeCnst', 'Fpar'], 'formats': [', (' 38\u001b[0m spd_naive[one, two, :] \u001b[38;5;241m=\u001b[39m \u001b[43mspd_edax\u001b[49m\u001b[43m[\u001b[49m\u001b[43mone\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtwo\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 39\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNaive done\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 41\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m: \n", + "File \u001b[0;32mh5py/_objects.pyx:54\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n", + "File \u001b[0;32mh5py/_objects.pyx:55\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n", + "File \u001b[0;32m~/miniconda3/envs/py3.10.13/lib/python3.10/site-packages/h5py/_hl/dataset.py:841\u001b[0m, in \u001b[0;36mDataset.__getitem__\u001b[0;34m(self, args, new_dtype)\u001b[0m\n\u001b[1;32m 839\u001b[0m mspace \u001b[38;5;241m=\u001b[39m h5s\u001b[38;5;241m.\u001b[39mcreate_simple(selection\u001b[38;5;241m.\u001b[39mmshape)\n\u001b[1;32m 840\u001b[0m fspace \u001b[38;5;241m=\u001b[39m selection\u001b[38;5;241m.\u001b[39mid\n\u001b[0;32m--> 841\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mid\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmspace\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfspace\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43marr\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmtype\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdxpl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dxpl\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 843\u001b[0m \u001b[38;5;66;03m# Patch up the output for NumPy\u001b[39;00m\n\u001b[1;32m 844\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m arr\u001b[38;5;241m.\u001b[39mshape \u001b[38;5;241m==\u001b[39m ():\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "with h5py.File(fnm, \"r\") as h5r:\n", + " src = \"/VInP/VInP_108_L2/Area 10/Live Map 1\"\n", + " trg = \"SPD\"\n", + " reqs = [\"MicronPerPixelX\", \"MicronPerPixelY\", \"NumberOfLines\", \"NumberOfPoints\", \"NumberofChannels\"]\n", + " for req in reqs:\n", + " if req not in h5r[f\"{src}/{trg}\"].attrs.keys():\n", + " # also check for shape\n", + " raise ValueError(f\"Required attribute named {req} not found in {src}/{trg} !\")\n", + " nyxe = {\"y\": h5r[f\"{src}/{trg}\"].attrs[\"NumberOfLines\"][0],\n", + " \"x\": h5r[f\"{src}/{trg}\"].attrs[\"NumberOfPoints\"][0],\n", + " \"e\": h5r[f\"{src}/{trg}\"].attrs[\"NumberofChannels\"][0]}\n", + " print(nyxe)\n", + " # the native APEX SPD concept instance is a two-dimensional array of arrays of length e (n_energy_bins)\n", + " # likely EDAX has in their C(++) code a vector of vector or something equivalent either way we faced\n", + " # nested C arrays of the base data type (here (u)int 16\n", + " # even worse, chunked in HDF5 thus the e-long arrays are just some payload inside the compressed\n", + " # chunk without some extra logic to resolve the third (energy) dimension:\n", + " # how to reshape this efficiently without creating unnecessary copies\n", + " # the following code is ugly as it needs a maximum large copy of the dataset\n", + " spd_edax = h5r[f\"{src}/{trg}\"]\n", + " print(f\"edax: {np.shape(spd_edax)}, {type(spd_edax)}, {spd_edax.dtype}\")\n", + " spd_naive = np.zeros((nyxe[\"y\"], nyxe[\"x\"], nyxe[\"e\"]), \"\n", + "axes\n", + "[{'name': 'height', 'offset': 0, 'scale': 0.00351909256747931, 'units': 'µm', 'size': 512}, {'name': 'width', 'offset': 0, 'scale': 0.00351909256747931, 'units': 'µm', 'size': 512}]\n", + "metadata\n", + "{'Acquisition_instrument': {'TEM': {'beam_energy': 200, 'magnification': 56000}}, 'Sample': {'name': 'map 4'}, 'Signal': {}, 'General': {'title': 'HAADF', 'original_filename': '46_ES-LP_L1_brg.bcf'}}\n", + "original_metadata\n", + "{'Microscope': {'HV': 200, 'WD': -1, 'Mag': 56000, 'DX': 0.00351909256747931, 'DY': 0.00351909256747931, 'Flags': 16776960, 'XmlClassType': 'TRTSEMData'}, 'DSP Configuration': {'ImageWidth': 512, 'ImageHeight': 512, 'PixelAverage': 60, 'LineAverage': 1, 'SEBitCount': 16, 'ChannelCount': 4, 'ChannelName0': 'BF', 'ChannelName1': 'DF', 'ChannelName2': 'DF4', 'Channel3': 1, 'ChannelName3': 'HAADF', 'CounterIndex': 0, 'CounterChannelUsed': 0, 'TiltAngle': 0, 'CounterMode': 0, 'PixelTime': 1, 'XmlClassType': 'TRTDSPConfiguration'}, 'Stage': {'State': 7936, 'XmlClassType': 'TRTSEMStageData'}}\n", + "mapping\n", + "{'Stage.Rotation': ('Acquisition_instrument.TEM.Stage.rotation', None), 'Stage.Tilt': ('Acquisition_instrument.TEM.Stage.tilt_alpha', None), 'Stage.X': ('Acquisition_instrument.TEM.Stage.x', None), 'Stage.Y': ('Acquisition_instrument.TEM.Stage.y', None), 'Stage.Z': ('Acquisition_instrument.TEM.Stage.z', None)}\n", + "data\n", + "[[19447 20033 16242 ... 48594 49493 50088]\n", + " [21447 21375 21792 ... 47285 48770 49734]\n", + " [22395 22443 22459 ... 48080 48883 49461]\n", + " ...\n", + " [18395 18379 18853 ... 17190 17801 17768]\n", + " [17785 19704 18772 ... 15905 16178 17511]\n", + " [19471 20226 20355 ... 17142 17254 17793]]\n", + "Loading type(entry) \n", + "data\n", + "[[[0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " ...\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + "\n", + " [[0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " ...\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + "\n", + " [[0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " ...\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + "\n", + " ...\n", + "\n", + " [[0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " ...\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + "\n", + " [[0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " ...\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]\n", + "\n", + " [[0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " ...\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]\n", + " [0 0 0 ... 0 0 0]]]\n", + "axes\n", + "[{'name': 'height', 'size': 512, 'offset': 0, 'scale': 0.00351909256747931, 'units': 'µm', 'navigate': True}, {'name': 'width', 'size': 512, 'offset': 0, 'scale': 0.00351909256747931, 'units': 'µm', 'navigate': True}, {'name': 'Energy', 'size': 2048, 'offset': -0.4798465772, 'scale': 0.01000934711, 'units': 'keV', 'navigate': False}]\n", + "metadata\n", + "{'Acquisition_instrument': {'TEM': {'beam_energy': 200, 'magnification': 56000, 'Detector': {'EDS': {'elevation_angle': 22.0, 'detector_type': 'Custom type', 'azimuth_angle': 45.0, 'real_time': 723.7632, 'live_time': 13.678}}}}, 'General': {'original_filename': '46_ES-LP_L1_brg.bcf', 'title': 'EDX', 'date': '2020-07-22', 'time': '14:18:32'}, 'Sample': {'name': 'map 4', 'elements': ['Al', 'Ca', 'Fe', 'Hf', 'Lu', 'Mg', 'Nd', 'O', 'Si', 'Sm', 'U'], 'xray_lines': ['Al_Ka', 'Ca_Ka', 'Fe_Ka', 'Hf_La', 'Lu_La', 'Mg_Ka', 'Nd_La', 'O_Ka', 'Si_Ka', 'Sm_La', 'U_Ma']}, 'Signal': {'signal_type': 'EDS_TEM', 'quantity': 'X-rays (Counts)'}}\n", + "original_metadata\n", + "{'Hardware': {'TRTKnownHeader': {'Type': 'RTHardware', 'Size': 137}, 'RealTime': 16331, 'LifeTime': 13678, 'DeadTime': 16.0, 'ZeroPeakPosition': 95, 'ZeroPeakFrequency': 800, 'PulseDensity': 89486, 'Amplification': 20000.0, 'ShapingTime': 60000, 'XmlClassType': 'TRTSpectrumHardwareHeader'}, 'Detector': {'TRTKnownHeader': {'Type': 'RTDetector', 'Version': 5, 'Size': 9932}, 'Technology': 'SDD', 'Type': 'Custom type', 'DetectorThickness': 0.45, 'SiDeadLayerThickness': 0.01, 'DetLayers': {}, 'WindowType': 'Custom type', 'WindowLayers': None, 'Corrections': {'Escape': None, 'Tail': {'FormulaType': 'Internal', 'MainCorrection': 1}, 'Shelf': {'FormulaType': 'Internal', 'RangeStart': 0.08, 'RangeEnd': 10.0, 'MainCorrection': 1, 'Coefficient0': 1}, 'Shift': {'FormulaType': 'Internal', 'RangeStart': 0.08, 'RangeEnd': 0.555, 'MainCorrection': 1}, 'FWHMShift': None}, 'CorrectionType': 2, 'ResponseFunctionCount': 21, 'SampleCount': 5, 'SampleOffset': -3, 'PulsePairResTimeCount': 0, 'PileUpMinEnergy': 1, 'PileUpWithBG': False, 'TailFactor': 0, 'ShelfFactor': 0, 'ShiftFactor': 0, 'ShiftFactor2': 0, 'ShiftData': (0.079, 0, 0.08, 0.01, 0.555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0), 'ResponseFunction': [(0, 3.3, 0.000801, 3.3, 0.00298, 3.3, 0.008902, 3.3, 0.025, 3.300046, 0.041098, 3.303475, 0.04702, 3.307302, 0.049199, 3.309237, 0.05, 3.31), (0, 3.3, 0.00444, 3.3, 0.01651, 3.3, 0.049318, 3.3, 0.1385, 3.300046, 0.227682, 3.303475, 0.26049, 3.307302, 0.27256, 3.309237, 0.277, 3.31), (0, 1.1, 0.006283, 1.1, 0.023364, 1.1, 0.069793, 1.1, 0.196, 1.102513, 0.322207, 1.291145, 0.368636, 1.50163, 0.385717, 1.608042, 0.392, 1.65), (0, 0.4, 0.008415, 0.4, 0.031291, 0.4, 0.093473, 0.4, 0.2625, 0.40457, 0.431527, 0.747537, 0.493709, 1.13024, 0.516585, 1.323712, 0.525, 1.4), (0, 0.2, 0.010836, 0.2, 0.040291, 0.2, 0.120357, 0.2, 0.338, 0.202513, 0.555643, 0.391145, 0.635709, 0.601632, 0.665164, 0.708042, 0.676, 0.75), (0, 0.03, 0.016687, 0.03, 0.062045, 0.03, 0.185343, 0.03, 0.5205, 0.032513, 0.855657, 0.221145, 0.978955, 0.43163, 1.024313, 0.538043, 1.041, 0.58), (0, 0.055, 0.020101, 0.055, 0.07474, 0.055, 0.223266, 0.055, 0.627, 0.057057, 1.030734, 0.21139, 1.17926, 0.383607, 1.233899, 0.470671, 1.254, 0.505), (0, 0.05, 0.023836, 0.05, 0.088627, 0.05, 0.26475, 0.05, 0.7435, 0.050732, 1.22225, 0.105607, 1.398373, 0.166839, 1.463164, 0.197794, 1.487, 0.21), (0, 0.03, 0.027891, 0.03, 0.103707, 0.03, 0.309795, 0.03, 0.87, 0.030594, 1.430205, 0.075181, 1.636293, 0.124932, 1.712109, 0.150082, 1.74, 0.16), (0, 0.15, 0.030776, 0.15, 0.114435, 0.15, 0.341842, 0.15, 0.96, 0.152377, 1.578158, 0.330719, 1.805565, 0.529724, 1.889224, 0.63033, 1.92, 0.67), (0, 0.15, 0.032283, 0.15, 0.120037, 0.15, 0.358578, 0.15, 1.007, 0.152377, 1.655422, 0.330719, 1.893963, 0.529725, 1.981717, 0.63033, 2.014, 0.67), (0, 0.085, 0.036996, 0.085, 0.13756, 0.085, 0.410923, 0.085, 1.154, 0.087055, 1.897077, 0.241391, 2.17044, 0.413607, 2.271004, 0.50067, 2.308, 0.535), (0, 0.085, 0.05918, 0.085, 0.220049, 0.085, 0.657334, 0.085, 1.846, 0.08589, 3.034665, 0.152769, 3.471952, 0.227397, 3.63282, 0.265124, 3.692, 0.28), (0, 0.035, 0.079378, 0.035, 0.295146, 0.035, 0.881668, 0.035, 2.476, 0.035549, 4.070332, 0.076705, 4.656854, 0.122629, 4.872623, 0.145845, 4.952, 0.155), (0, 0.035, 0.119867, 0.035, 0.445699, 0.035, 1.331404, 0.035, 3.739, 0.035457, 6.146595, 0.069754, 7.0323, 0.108024, 7.358133, 0.127371, 7.478, 0.135), (0, 0.035, 0.148303, 0.035, 0.551433, 0.035, 1.647253, 0.035, 4.626, 0.035457, 7.604747, 0.069754, 8.700567, 0.108024, 9.103698, 0.127371, 9.252, 0.135), (0, 0.023571, 0.176322, 0.023571, 0.655616, 0.023571, 1.958472, 0.023571, 5.5, 0.023597, 9.041529, 0.025499, 10.344384, 0.02762, 10.823678, 0.028693, 11, 0.029116), (0, 0.009286, 0.208381, 0.009286, 0.774819, 0.009286, 2.314557, 0.009286, 6.5, 0.009297, 10.685443, 0.010101, 12.225181, 0.010999, 12.791619, 0.011453, 13, 0.011632), (0, 0.010714, 0.24044, 0.010714, 0.894022, 0.010714, 2.670643, 0.010714, 7.5, 0.010714, 12.329357, 0.010714, 14.105978, 0.010714, 14.759561, 0.010714, 15, 0.010714), (0, 0, 0.320586, 0, 1.192029, 0, 3.560857, 0, 10, 0, 16.439142, 0, 18.80797, 0, 19.679415, 0, 20, 0), (0, 0, 1.60293, 0, 5.960146, 0, 17.804287, 0, 50, 0, 82.195709, 0, 94.039856, 0, 98.397072, 0, 100, 0)], 'XmlClassType': 'TRTDetectorHeader'}, 'Analysis': {'TRTKnownHeader': {'Type': 'RTESMA', 'Size': 662}, 'PrimaryEnergy': 200.0, 'ReferenceFactor': -1, 'ReferenceStdDev': -1, 'BaseRefStdDev': 0.002000100008, 'ElevationAngle': 22.0, 'AzimutAngle': 45.0, 'DetectorAngle': 15.0, 'CoatCorrection': None, 'XmlClassType': 'TRTESMAHeader'}, 'Spectrum': {'Size': 82, 'Date': '22.7.2020', 'Time': '14:18:32', 'ChannelCount': 2048, 'CalibAbs': -0.4798465772, 'CalibLin': 0.01000934711, 'SigmaAbs': 0.0004952410698, 'SigmaLin': 0.0004825546962, 'XmlClassType': 'TRTSpectrumHeader'}, 'DSP Configuration': {'ImageWidth': 512, 'ImageHeight': 512, 'PixelAverage': 60, 'LineAverage': 1, 'SEBitCount': 16, 'ChannelCount': 4, 'ChannelName0': 'BF', 'ChannelName1': 'DF', 'ChannelName2': 'DF4', 'Channel3': 1, 'ChannelName3': 'HAADF', 'CounterIndex': 0, 'CounterChannelUsed': 0, 'TiltAngle': 0, 'CounterMode': 0, 'PixelTime': 1, 'XmlClassType': 'TRTDSPConfiguration'}, 'Line counter': (47, 47, 47, 47, 47, 47, 47, 47, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46, 46), 'Stage': {'State': 7936, 'XmlClassType': 'TRTSEMStageData'}, 'Microscope': {'HV': 200, 'WD': -1, 'Mag': 56000, 'DX': 0.00351909256747931, 'DY': 0.00351909256747931, 'Flags': 16776960, 'XmlClassType': 'TRTSEMData'}}\n", + "mapping\n", + "{'Stage.Rotation': ('Acquisition_instrument.TEM.Stage.rotation', None), 'Stage.Tilt': ('Acquisition_instrument.TEM.Stage.tilt_alpha', None), 'Stage.X': ('Acquisition_instrument.TEM.Stage.x', None), 'Stage.Y': ('Acquisition_instrument.TEM.Stage.y', None), 'Stage.Z': ('Acquisition_instrument.TEM.Stage.z', None)}\n" + ] + } + ], + "source": [ + "objs = bruker.file_reader(f\"{src}/{fnms[2][1]}\")\n", + "# objs = emd.file_reader(f\"{src}/{fnms[3][1]}\")\n", + "# objs = digitalmicrograph.file_reader(f\"{src}/{fnms[4][1]}\")\n", + "if isinstance(objs, list) is True:\n", + " for entry in objs:\n", + " print(f\"Loading type(entry) {type(entry)}\")\n", + " if isinstance(entry, dict) is True:\n", + " for key, val in entry.items():\n", + " print(key)\n", + " print(val) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c92b0a71-e9d8-460e-99b5-b12208b56258", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e04b275f-bc59-4fbc-8c56-ae4d6e964d14", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "48f31e6c-1554-4476-8688-5f5323d513c8", + "metadata": {}, + "outputs": [], + "source": [ + "# https://codereview.stackexchange.com/a/21035\n", + "# https://stackoverflow.com/questions/38852822/how-to-flatten-xml-file-in-python\n", + "from collections import OrderedDict\n", + "\n", + "def flatten_dict(d):\n", + " def items():\n", + " for key, value in d.items():\n", + " # nested subtree\n", + " if isinstance(value, dict):\n", + " for subkey, subvalue in flatten_dict(value).items():\n", + " yield '{}.{}'.format(key, subkey), subvalue\n", + " # nested list\n", + " elif isinstance(value, list):\n", + " for num, elem in enumerate(value):\n", + " for subkey, subvalue in flatten_dict(elem).items():\n", + " yield '{}.[{}].{}'.format(key, num, subkey), subvalue\n", + " # everything else (only leafs should remain)\n", + " else:\n", + " yield key, value\n", + " return OrderedDict(items())\n", + "\n", + "import xmltodict" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5d5cbb4-c5a2-44a1-b6a4-277167582869", + "metadata": {}, + "outputs": [], + "source": [ + "with Image.open(fnm, mode=\"r\") as fp:\n", + " fp.load() # Needed only for .png EXIF data (see citation above)\n", + " if \"MicroscopeControlImage\" in fp.info.keys():\n", + " # print(fp.info[\"MicroscopeControlImage\"])\n", + " xml_content = xmltodict.parse(fp.info[\"MicroscopeControlImage\"])\n", + " flattened_xml = flatten_dict(xml_content)\n", + " for k,v in flattened_xml.items():\n", + " print('{} = {}'.format(k,v))\n", + " elif fnm.lower().endswith(\".png\") is True: # check for mime type instead\n", + " print(f\"There is no iTXt chunk in {fnm} which has embedded XML within the AXON namespace MicroscopeControlImage!\")\n", + " else:\n", + " print(f\"There is nothing to harvest here!\")" + ] + }, + { + "cell_type": "markdown", + "id": "4f4bf73d-66b7-414b-abb1-db99b2bf370a", + "metadata": {}, + "source": [ + "***" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1963afb6-6e48-4628-a0e8-d2da0874701e", + "metadata": {}, + "outputs": [], + "source": [ + "# handle TIFF\n", + "with Image.open(fnm, mode=\"r\") as fp:\n", + " for key in fp.tag_v2:\n", + " if key in [34118, 34119]:\n", + " print(type(fp.tag[key]))\n", + " print(len(fp.tag[key])) \n", + " # print(f\"{key}, {fp.tag[key]}\")\n", + " if key not in TAGS.keys():\n", + " print(f\"--->tag {key}, is not in PIL.TiffTAGS !\")\n", + " # self.tags = {TAGS[key] : fp.tag[key] for key in fp.tag_v2}\n", + " # for key, val in self.tags.items():\n", + " # print(f\"{key}, {val}\")\n", + " nparr = np.array(fp)\n", + " print(f\"{type(nparr)}\")\n", + " print(f\"{nparr.dtype}\")\n", + " print(f\"{np.shape(nparr)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9ef2a35-a260-4a54-9b83-eae1d588966f", + "metadata": {}, + "outputs": [], + "source": [ + "with Image.open(fnm, mode=\"r\") as fp:\n", + " if True is False:\n", + " czi_keys = [34118, 34119]\n", + " for czi_key in czi_keys:\n", + " if czi_key in fp.tag_v2:\n", + " print(f\"Found czi_key {tfs_key}...\")\n", + " utf = fp.tag[czi_key]\n", + " print(type(utf))\n", + " if len(utf) == 1:\n", + " print(utf[0])\n", + " # exit(1)\n", + " tfs_keys = [34682]\n", + " for tfs_key in tfs_keys:\n", + " if tfs_key in fp.tag_v2:\n", + " print(f\"Found tfs_key {tfs_key}...\")\n", + " utf = fp.tag[tfs_key]\n", + " print(type(utf))\n", + " if len(utf) == 1:\n", + " print(utf[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "28687c0e-6f14-484c-b511-3a4906d9672e", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a8ada062-e308-4288-8f00-b3e620f3c890", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "# https://www.geeksforgeeks.org/python-program-to-sort-a-list-of-tuples-by-second-item/\n", + "def sort_tuple(tup):\n", + " # convert the list of tuples to a numpy array with data type (object, int)\n", + " arr = np.array(tup, dtype=[('col1', object), ('col2', int)])\n", + " # get the indices that would sort the array based on the second column\n", + " indices = np.argsort(arr['col2'])\n", + " # use the resulting indices to sort the array\n", + " sorted_arr = arr[indices]\n", + " # convert the sorted numpy array back to a list of tuples\n", + " sorted_tup = [(row['col1'], row['col2']) for row in sorted_arr]\n", + " return sorted_tup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d27df293-626c-4d37-80df-96c182d4f401", + "metadata": {}, + "outputs": [], + "source": [ + "def if_str_represents_float(s):\n", + " try:\n", + " return isinstance(float(s), float)\n", + " # return str(float(s)) == s\n", + " except ValueError:\n", + " return False" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f647fa79-330b-48b2-8360-f92fc5ead187", + "metadata": {}, + "outputs": [], + "source": [ + "\"10\".isdigit()\n", + "# isinstance(float(\"8.99306e-010\"), float)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a2f0864-f8b3-4d53-bf9d-08a5787c32fb", + "metadata": {}, + "outputs": [], + "source": [ + "# TFS sections based on IKZ ALN_baoh_021.tif example\n", + "import mmap\n", + "\n", + "tfs_section_names = [\"[User]\",\n", + " \"[System]\",\n", + " \"[Beam]\",\n", + " \"[EBeam]\", \n", + " \"[GIS]\",\n", + " \"[Scan]\",\n", + " \"[EScan]\",\n", + " \"[Stage]\",\n", + " \"[Image]\",\n", + " \"[Vacuum]\",\n", + " \"[Specimen]\",\n", + " \"[Detectors]\",\n", + " \"[T2]\",\n", + " \"[Accessories]\",\n", + " \"[EBeamDeceleration]\",\n", + " \"[CompoundLensFilter]\",\n", + " \"[PrivateFei]\",\n", + " \"[HiResIllumination]\",\n", + " \"[EasyLift]\",\n", + " \"[HotStageMEMS]\",\n", + " \"[HotStage]\",\n", + " \"[HotStageHVHS]\",\n", + " \"[ColdStage]\"]\n", + "\n", + "tfs_section_details = {\"[System]\": [\"Type\", \"Dnumber\", \"Software\", \"BuildNr\", \"Source\", \"Column\", \"FinalLens\", \"Chamber\", \"Stage\", \"Pump\",\n", + " \"ESEM\", \"Aperture\", \"Scan\", \"Acq\", \"EucWD\", \"SystemType\", \"DisplayWidth\", \"DisplayHeight\"]}\n", + "tfs_section_offsets = {}\n", + "\n", + "with open(fnm, 'rb', 0) as file:\n", + " s = mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ)\n", + " for section_name in tfs_section_names:\n", + " pos = s.find(bytes(section_name, \"utf8\")) # != -1\n", + " tfs_section_offsets[section_name] = pos\n", + " print(tfs_section_offsets)\n", + "\n", + " # define search offsets\n", + " tpl = []\n", + " for key, value in tfs_section_offsets.items():\n", + " tpl.append((key, value))\n", + " # print(tpl)\n", + " tpl = sort_tuple(tpl)\n", + " print(tpl)\n", + " # if section_name == \"[System]\":\n", + " pos_s = None\n", + " pos_e = None\n", + " for idx in np.arange(0, len(tpl)):\n", + " if tpl[idx][0] != \"[System]\":\n", + " continue\n", + " else:\n", + " pos_s = tpl[idx][1]\n", + " if idx <= len(tpl) - 1:\n", + " pos_e = tpl[idx + 1][1]\n", + " break\n", + " print(f\"Search in between byte offsets {pos_s} and {pos_e}\")\n", + " # fish metadata of e.g. the system section\n", + " section_metadata = {}\n", + " for term in tfs_section_details[\"[System]\"]:\n", + " \n", + " s.seek(pos_s, 0)\n", + " pos = s.find(bytes(term, \"utf8\"))\n", + " if pos < pos_e: # check if pos_e is None\n", + " s.seek(pos, 0)\n", + " section_metadata[f\"{term}\"] = f\"{s.readline().strip().decode('utf8').replace(f'{term}=', '')}\"\n", + " if if_str_represents_float(section_metadata[f\"{term}\"]) is True:\n", + " section_metadata[f\"{term}\"] = np.float64(section_metadata[f\"{term}\"])\n", + " elif section_metadata[f\"{term}\"].isdigit() is True:\n", + " section_metadata[f\"{term}\"] = np.int64(section_metadata[f\"{term}\"])\n", + " else:\n", + " pass\n", + " # print(f\"{term}, {pos}, {pos + len(term) + 1}\")\n", + " # tfs_section_offswr\n", + " # file.seek(pos, 0) #\n", + " print(section_metadata)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2f3eb287-8f55-424c-a016-a07fc59f068a", + "metadata": {}, + "outputs": [], + "source": [ + "'2'.isdigit()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1341e30-fcce-4a3d-a099-d342b8bbe318", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/delete/test.ebsd.sh b/delete/test.ebsd.sh new file mode 100755 index 000000000..4730b1d5b --- /dev/null +++ b/delete/test.ebsd.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +src_prefix="~/Research/paper_paper_paper/scidat_nomad_ebsd/bb_analysis/data/production_ebsd" +src_pyxem="${src_prefix}_pyxem" +src_mtex="${src_prefix}_mtex" +echo "$src_prefix" +echo "$src_pyxem" +echo "$src_mtex" +ohfiles="${src_pyxem}/*.oh5" +cmd="ls ${ohfiles}" +$cmd +