From f34712e29b080861977b437a88ae5b231e7b584d Mon Sep 17 00:00:00 2001
From: Steven Zeltmann <steven.zeltmann@cornell.edu>
Date: Mon, 12 Feb 2024 16:17:17 -0500
Subject: [PATCH 01/13] EMPAD2 load progress in status bar

---
 src/py4D_browser/empad2_reader.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/src/py4D_browser/empad2_reader.py b/src/py4D_browser/empad2_reader.py
index 3fedfe8..81fb961 100644
--- a/src/py4D_browser/empad2_reader.py
+++ b/src/py4D_browser/empad2_reader.py
@@ -1,8 +1,21 @@
 import empad2
-from PyQt5.QtWidgets import QFileDialog, QMessageBox
+from PyQt5.QtWidgets import QFileDialog, QMessageBox, QApplication
 import numpy as np
 
 
+class StatusBarWriter:
+    def __init__(self, statusBar):
+        self.statusBar = statusBar
+        self.app = app = QApplication.instance()
+
+    def write(self, message):
+        self.statusBar.showMessage(message, 1_000)
+        self.app.processEvents()
+
+    def flush(self):
+        pass
+
+
 def set_empad2_sensor(self, sensor_name):
     self.empad2_calibrations = empad2.load_calibration_data(sensor=sensor_name)
     self.statusBar().showMessage(f"{sensor_name} calibrations loaded", 5_000)
@@ -41,7 +54,14 @@ def load_empad2_dataset(self):
 
         filename = raw_file_dialog(self)
         self.datacube = empad2.load_dataset(
-            filename, self.empad2_background, self.empad2_calibrations
+            filename,
+            self.empad2_background,
+            self.empad2_calibrations,
+            _tqdm_args={
+                "desc": "Loading",
+                "file": StatusBarWriter(self.statusBar()),
+                "mininterval": 1.0,
+            },
         )
 
         if dummy_data:

From 91603d8481bd7e6807eb8f3508b6e175655b5ec2 Mon Sep 17 00:00:00 2001
From: Steven Zeltmann <steven.zeltmann@cornell.edu>
Date: Sat, 17 Feb 2024 11:57:57 -0500
Subject: [PATCH 02/13] add arina reader

---
 src/py4D_browser/main_window.py  |  5 +++++
 src/py4D_browser/menu_actions.py | 30 ++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/src/py4D_browser/main_window.py b/src/py4D_browser/main_window.py
index 0b4ef66..fa76c77 100644
--- a/src/py4D_browser/main_window.py
+++ b/src/py4D_browser/main_window.py
@@ -34,6 +34,7 @@ class DataViewer(QMainWindow):
 
     from py4D_browser.menu_actions import (
         load_file,
+        load_data_arina,
         load_data_auto,
         load_data_bin,
         load_data_mmap,
@@ -115,6 +116,10 @@ def setup_menus(self):
         self.load_binned_action.triggered.connect(self.load_data_bin)
         self.file_menu.addAction(self.load_binned_action)
 
+        self.load_arina_action = QAction("Load &Arina Data...", self)
+        self.load_arina_action.triggered.connect(self.load_data_arina)
+        self.file_menu.addAction(self.load_arina_action)
+
         self.file_menu.addSeparator()
 
         export_label = QAction("Export", self)
diff --git a/src/py4D_browser/menu_actions.py b/src/py4D_browser/menu_actions.py
index 94ff7c1..ef8efac 100644
--- a/src/py4D_browser/menu_actions.py
+++ b/src/py4D_browser/menu_actions.py
@@ -5,6 +5,7 @@
 import numpy as np
 import matplotlib.pyplot as plt
 from py4D_browser.help_menu import KeyboardMapMenu
+from py4DSTEM.io.filereaders import read_arina
 
 
 def load_data_auto(self):
@@ -22,6 +23,35 @@ def load_data_bin(self):
     filename = self.show_file_dialog()
     self.load_file(filename, mmap=False, binning=4)
 
+def load_data_arina(self):
+    filename = self.show_file_dialog()
+    dataset = read_arina(filename)
+
+    # Try to reshape the data to be square
+    N_patterns = dataset.data.shape[1]
+    Nxy = np.sqrt(N_patterns)
+    if np.abs(Nxy - np.round(Nxy)) <= 1e-10:
+        Nxy = int(Nxy)
+        dataset.data = dataset.data.reshape(Nxy, Nxy, dataset.data.shape[2], dataset.data.shape[3])
+    else:
+        self.statusBar().showMessage(f"The scan appears to not be square! Found {N_patterns} patterns", 5_000)
+
+    self.datacube = dataset
+    self.diffraction_scale_bar.pixel_size = self.datacube.calibration.get_Q_pixel_size()
+    self.diffraction_scale_bar.units = self.datacube.calibration.get_Q_pixel_units()
+
+    self.real_space_scale_bar.pixel_size = self.datacube.calibration.get_R_pixel_size()
+    self.real_space_scale_bar.units = self.datacube.calibration.get_R_pixel_units()
+
+    self.fft_scale_bar.pixel_size = (
+        1.0 / self.datacube.calibration.get_R_pixel_size() / self.datacube.R_Ny
+    )
+    self.fft_scale_bar.units = f"1/{self.datacube.calibration.get_R_pixel_units()}"
+
+    self.update_diffraction_space_view(reset=True)
+    self.update_real_space_view(reset=True)
+
+    self.setWindowTitle(filename)
 
 def load_file(self, filepath, mmap=False, binning=1):
     print(f"Loading file {filepath}")

From 244bdc18c3429b8aa8be7a20bd1f28a82a260f45 Mon Sep 17 00:00:00 2001
From: Steven Zeltmann <steven.zeltmann@cornell.edu>
Date: Sat, 17 Feb 2024 12:04:16 -0500
Subject: [PATCH 03/13] fix incorrect positioning of realspace detector

---
 src/py4D_browser/update_views.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/py4D_browser/update_views.py b/src/py4D_browser/update_views.py
index 7b7c332..36c47db 100644
--- a/src/py4D_browser/update_views.py
+++ b/src/py4D_browser/update_views.py
@@ -279,7 +279,7 @@ def update_realspace_detector(self):
     if self.datacube is None:
         return
 
-    x, y = self.datacube.shape[2:]
+    x, y = self.datacube.data.shape[:2]
     x0, y0 = x / 2, y / 2
     xr, yr = x / 10, y / 10
 
@@ -322,7 +322,7 @@ def update_diffraction_detector(self):
     if self.datacube is None:
         return
 
-    x, y = self.datacube.shape[2:]
+    x, y = self.datacube.data.shape[2:]
     x0, y0 = x / 2, y / 2
     xr, yr = x / 10, y / 10
 

From d165ed46658157040d099d26215d49044e29ac47 Mon Sep 17 00:00:00 2001
From: Steven Zeltmann <steven.zeltmann@cornell.edu>
Date: Sat, 17 Feb 2024 12:15:33 -0500
Subject: [PATCH 04/13] use clipped autoscaling

---
 src/py4D_browser/menu_actions.py | 10 ++++++++--
 src/py4D_browser/update_views.py | 18 ++++++++++++++++--
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/src/py4D_browser/menu_actions.py b/src/py4D_browser/menu_actions.py
index ef8efac..fbe3e9e 100644
--- a/src/py4D_browser/menu_actions.py
+++ b/src/py4D_browser/menu_actions.py
@@ -23,6 +23,7 @@ def load_data_bin(self):
     filename = self.show_file_dialog()
     self.load_file(filename, mmap=False, binning=4)
 
+
 def load_data_arina(self):
     filename = self.show_file_dialog()
     dataset = read_arina(filename)
@@ -32,9 +33,13 @@ def load_data_arina(self):
     Nxy = np.sqrt(N_patterns)
     if np.abs(Nxy - np.round(Nxy)) <= 1e-10:
         Nxy = int(Nxy)
-        dataset.data = dataset.data.reshape(Nxy, Nxy, dataset.data.shape[2], dataset.data.shape[3])
+        dataset.data = dataset.data.reshape(
+            Nxy, Nxy, dataset.data.shape[2], dataset.data.shape[3]
+        )
     else:
-        self.statusBar().showMessage(f"The scan appears to not be square! Found {N_patterns} patterns", 5_000)
+        self.statusBar().showMessage(
+            f"The scan appears to not be square! Found {N_patterns} patterns", 5_000
+        )
 
     self.datacube = dataset
     self.diffraction_scale_bar.pixel_size = self.datacube.calibration.get_Q_pixel_size()
@@ -53,6 +58,7 @@ def load_data_arina(self):
 
     self.setWindowTitle(filename)
 
+
 def load_file(self, filepath, mmap=False, binning=1):
     print(f"Loading file {filepath}")
     extension = os.path.splitext(filepath)[-1].lower()
diff --git a/src/py4D_browser/update_views.py b/src/py4D_browser/update_views.py
index 36c47db..ecb7b90 100644
--- a/src/py4D_browser/update_views.py
+++ b/src/py4D_browser/update_views.py
@@ -174,9 +174,16 @@ def update_real_space_view(self, reset=False):
 
     self.unscaled_realspace_image = vimg
 
+    auto_level = reset or self.realspace_rescale_button.latched
+
     self.real_space_widget.setImage(
         new_view.T,
-        autoLevels=reset or self.realspace_rescale_button.latched,
+        autoLevels=False,
+        levels=(
+            (np.percentile(new_view, 2), np.percentile(new_view, 98))
+            if auto_level
+            else None
+        ),
         autoRange=reset,
     )
 
@@ -251,9 +258,16 @@ def update_diffraction_space_view(self, reset=False):
     else:
         raise ValueError("Mode not recognized")
 
+    auto_level = reset or self.diffraction_rescale_button.latched
+
     self.diffraction_space_widget.setImage(
         new_view.T,
-        autoLevels=reset or self.diffraction_rescale_button.latched,
+        autoLevels=False,
+        levels=(
+            (np.percentile(new_view, 2), np.percentile(new_view, 98))
+            if auto_level
+            else None
+        ),
         autoRange=reset,
     )
 

From 17650994aec67009eefea6a5b6cfcfb3cf2dcb9f Mon Sep 17 00:00:00 2001
From: Steven Zeltmann <steven.zeltmann@cornell.edu>
Date: Wed, 6 Mar 2024 09:19:47 -0500
Subject: [PATCH 05/13] add complex FFT view

---
 src/py4D_browser/main_window.py  |  9 ++++++++-
 src/py4D_browser/update_views.py | 22 +++++++++++++++++++++-
 src/py4D_browser/utils.py        | 25 +++++++++++++++++++++++++
 3 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/src/py4D_browser/main_window.py b/src/py4D_browser/main_window.py
index fa76c77..875ee07 100644
--- a/src/py4D_browser/main_window.py
+++ b/src/py4D_browser/main_window.py
@@ -365,13 +365,20 @@ def setup_menus(self):
         img_fft_action = QAction("Virtual Image FFT", self)
         img_fft_action.setCheckable(True)
         img_fft_action.setChecked(True)
+        img_fft_action.triggered.connect(partial(self.update_real_space_view, False))
         self.fft_menu.addAction(img_fft_action)
         self.fft_source_action_group.addAction(img_fft_action)
+
+        img_complex_fft_action = QAction("Virtual Image FFT (complex)", self)
+        img_complex_fft_action.setCheckable(True)
+        self.fft_menu.addAction(img_complex_fft_action)
+        self.fft_source_action_group.addAction(img_complex_fft_action)
+        img_complex_fft_action.triggered.connect(partial(self.update_real_space_view, False))
+
         img_ewpc_action = QAction("EWPC", self)
         img_ewpc_action.setCheckable(True)
         self.fft_menu.addAction(img_ewpc_action)
         self.fft_source_action_group.addAction(img_ewpc_action)
-        img_fft_action.triggered.connect(partial(self.update_real_space_view, False))
         img_ewpc_action.triggered.connect(
             partial(self.update_diffraction_space_view, False)
         )
diff --git a/src/py4D_browser/update_views.py b/src/py4D_browser/update_views.py
index ecb7b90..e2853ba 100644
--- a/src/py4D_browser/update_views.py
+++ b/src/py4D_browser/update_views.py
@@ -3,7 +3,7 @@
 import py4DSTEM
 from functools import partial
 
-from py4D_browser.utils import pg_point_roi, make_detector
+from py4D_browser.utils import pg_point_roi, make_detector, complex_to_Lab
 
 
 def update_real_space_view(self, reset=False):
@@ -200,6 +200,26 @@ def update_real_space_view(self, reset=False):
         if mode_switch:
             # Need to autorange after setRect
             self.fft_widget.autoRange()
+    elif self.fft_source_action_group.checkedAction().text() == "Virtual Image FFT (complex)":
+        fft = np.fft.fftshift(np.fft.fft2(new_view))
+        levels = (np.min(np.abs(fft)), np.percentile(np.abs(fft), 99.9))
+        mode_switch = self.fft_widget_text.textItem.toPlainText() != "Virtual Image FFT"
+        self.fft_widget_text.setText("Virtual Image FFT")
+        fft_img = complex_to_Lab(
+            fft.T,
+            amin=levels[0],
+            amax=levels[1],
+            ab_scale=128,
+            gamma=0.5,
+        )
+        self.fft_widget.setImage(
+            fft_img, autoLevels=False, autoRange=mode_switch, levels=(0,1),
+        )
+
+        self.fft_widget.getImageItem().setRect(0, 0, fft.shape[1], fft.shape[1])
+        if mode_switch:
+            # Need to autorange after setRect
+            self.fft_widget.autoRange()
 
 
 def update_diffraction_space_view(self, reset=False):
diff --git a/src/py4D_browser/utils.py b/src/py4D_browser/utils.py
index fb7be24..e773096 100644
--- a/src/py4D_browser/utils.py
+++ b/src/py4D_browser/utils.py
@@ -108,3 +108,28 @@ def make_detector(shape: tuple, mode: str, geometry) -> np.ndarray:
             raise ValueError(f"mode and geometry not understood: {unknown}")
 
     return mask
+
+def complex_to_Lab(im,amin=None,amax=None,gamma=1,L_scale=100,ab_scale=64,uniform_L=None):
+    from skimage.color import lab2rgb
+    from matplotlib.colors import Normalize
+    import warnings
+    
+    Lab = np.zeros(im.shape + (3,),dtype=np.float64)
+    angle = np.angle(im)
+    
+    L = Normalize(vmin=amin, vmax=amax, clip=True)(np.abs(im)) ** gamma
+    L = Normalize()(L)
+    
+    # attempt at polynomial saturation
+    # ab_prescale = 4*L - 4*L*L
+    ab_prescale = 0.5
+    
+    Lab[...,0] = uniform_L or L * L_scale
+    Lab[...,1] = np.cos(angle) * ab_scale * ab_prescale
+    Lab[...,2] = np.sin(angle) * ab_scale * ab_prescale
+    
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        rgb = lab2rgb(Lab)
+    
+    return rgb
\ No newline at end of file

From a29f6a9c0fca16e2aaddf78093fe6f0f476f19bd Mon Sep 17 00:00:00 2001
From: Steven Zeltmann <steven.zeltmann@cornell.edu>
Date: Wed, 6 Mar 2024 09:23:10 -0500
Subject: [PATCH 06/13] format with black

---
 src/py4D_browser/main_window.py  |  4 +++-
 src/py4D_browser/update_views.py | 10 ++++++++--
 src/py4D_browser/utils.py        | 27 +++++++++++++++------------
 3 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/src/py4D_browser/main_window.py b/src/py4D_browser/main_window.py
index 875ee07..ccb32bd 100644
--- a/src/py4D_browser/main_window.py
+++ b/src/py4D_browser/main_window.py
@@ -373,7 +373,9 @@ def setup_menus(self):
         img_complex_fft_action.setCheckable(True)
         self.fft_menu.addAction(img_complex_fft_action)
         self.fft_source_action_group.addAction(img_complex_fft_action)
-        img_complex_fft_action.triggered.connect(partial(self.update_real_space_view, False))
+        img_complex_fft_action.triggered.connect(
+            partial(self.update_real_space_view, False)
+        )
 
         img_ewpc_action = QAction("EWPC", self)
         img_ewpc_action.setCheckable(True)
diff --git a/src/py4D_browser/update_views.py b/src/py4D_browser/update_views.py
index e2853ba..acb5e2d 100644
--- a/src/py4D_browser/update_views.py
+++ b/src/py4D_browser/update_views.py
@@ -200,7 +200,10 @@ def update_real_space_view(self, reset=False):
         if mode_switch:
             # Need to autorange after setRect
             self.fft_widget.autoRange()
-    elif self.fft_source_action_group.checkedAction().text() == "Virtual Image FFT (complex)":
+    elif (
+        self.fft_source_action_group.checkedAction().text()
+        == "Virtual Image FFT (complex)"
+    ):
         fft = np.fft.fftshift(np.fft.fft2(new_view))
         levels = (np.min(np.abs(fft)), np.percentile(np.abs(fft), 99.9))
         mode_switch = self.fft_widget_text.textItem.toPlainText() != "Virtual Image FFT"
@@ -213,7 +216,10 @@ def update_real_space_view(self, reset=False):
             gamma=0.5,
         )
         self.fft_widget.setImage(
-            fft_img, autoLevels=False, autoRange=mode_switch, levels=(0,1),
+            fft_img,
+            autoLevels=False,
+            autoRange=mode_switch,
+            levels=(0, 1),
         )
 
         self.fft_widget.getImageItem().setRect(0, 0, fft.shape[1], fft.shape[1])
diff --git a/src/py4D_browser/utils.py b/src/py4D_browser/utils.py
index e773096..e1ec718 100644
--- a/src/py4D_browser/utils.py
+++ b/src/py4D_browser/utils.py
@@ -109,27 +109,30 @@ def make_detector(shape: tuple, mode: str, geometry) -> np.ndarray:
 
     return mask
 
-def complex_to_Lab(im,amin=None,amax=None,gamma=1,L_scale=100,ab_scale=64,uniform_L=None):
+
+def complex_to_Lab(
+    im, amin=None, amax=None, gamma=1, L_scale=100, ab_scale=64, uniform_L=None
+):
     from skimage.color import lab2rgb
     from matplotlib.colors import Normalize
     import warnings
-    
-    Lab = np.zeros(im.shape + (3,),dtype=np.float64)
+
+    Lab = np.zeros(im.shape + (3,), dtype=np.float64)
     angle = np.angle(im)
-    
+
     L = Normalize(vmin=amin, vmax=amax, clip=True)(np.abs(im)) ** gamma
     L = Normalize()(L)
-    
+
     # attempt at polynomial saturation
     # ab_prescale = 4*L - 4*L*L
     ab_prescale = 0.5
-    
-    Lab[...,0] = uniform_L or L * L_scale
-    Lab[...,1] = np.cos(angle) * ab_scale * ab_prescale
-    Lab[...,2] = np.sin(angle) * ab_scale * ab_prescale
-    
+
+    Lab[..., 0] = uniform_L or L * L_scale
+    Lab[..., 1] = np.cos(angle) * ab_scale * ab_prescale
+    Lab[..., 2] = np.sin(angle) * ab_scale * ab_prescale
+
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
         rgb = lab2rgb(Lab)
-    
-    return rgb
\ No newline at end of file
+
+    return rgb

From 36e4591a7cf7ee7911096e53c4b2d0f59fedb554 Mon Sep 17 00:00:00 2001
From: Steven Zeltmann <steven.zeltmann@cornell.edu>
Date: Wed, 6 Mar 2024 13:36:48 -0500
Subject: [PATCH 07/13] add statistics for displays

---
 src/py4D_browser/main_window.py  | 6 ++++++
 src/py4D_browser/update_views.py | 8 ++++++++
 2 files changed, 14 insertions(+)

diff --git a/src/py4D_browser/main_window.py b/src/py4D_browser/main_window.py
index ccb32bd..4a1d33b 100644
--- a/src/py4D_browser/main_window.py
+++ b/src/py4D_browser/main_window.py
@@ -480,6 +480,12 @@ def setup_views(self):
         self.fft_widget.getView().setMenuEnabled(False)
 
         # Setup Status Bar
+        self.realspace_statistics_text = QLabel("Image Stats")
+        self.diffraction_statistics_text = QLabel("Diffraction Stats")
+        self.statusBar().addPermanentWidget(VLine())
+        self.statusBar().addPermanentWidget(self.realspace_statistics_text)
+        self.statusBar().addPermanentWidget(VLine())
+        self.statusBar().addPermanentWidget(self.diffraction_statistics_text)
         self.statusBar().addPermanentWidget(VLine())
         self.statusBar().addPermanentWidget(self.diffraction_space_view_text)
         self.statusBar().addPermanentWidget(VLine())
diff --git a/src/py4D_browser/update_views.py b/src/py4D_browser/update_views.py
index acb5e2d..48948f4 100644
--- a/src/py4D_browser/update_views.py
+++ b/src/py4D_browser/update_views.py
@@ -174,6 +174,10 @@ def update_real_space_view(self, reset=False):
 
     self.unscaled_realspace_image = vimg
 
+    self.realspace_statistics_text.setToolTip(
+        f"min\t{vimg.min():.5g}\nmax\t{vimg.max():.5g}\nmean\t{vimg.mean():.5g}\nsum\t{vimg.sum():.5g}\nstd\t{np.std(vimg):.5g}"
+    )
+
     auto_level = reset or self.realspace_rescale_button.latched
 
     self.real_space_widget.setImage(
@@ -284,6 +288,10 @@ def update_diffraction_space_view(self, reset=False):
     else:
         raise ValueError("Mode not recognized")
 
+    self.diffraction_statistics_text.setToolTip(
+        f"min\t{DP.min():.5g}\nmax\t{DP.max():.5g}\nmean\t{DP.mean():.5g}\nsum\t{DP.sum():.5g}\nstd\t{np.std(DP):.5g}"
+    )
+
     auto_level = reset or self.diffraction_rescale_button.latched
 
     self.diffraction_space_widget.setImage(

From 4548c9885989ef3e816a6c086f7242dccdb68a1a Mon Sep 17 00:00:00 2001
From: Steven Zeltmann <steven.zeltmann@cornell.edu>
Date: Wed, 6 Mar 2024 13:47:07 -0500
Subject: [PATCH 08/13] add pre-commit config

---
 .github/workflows/black.yml |  2 +-
 .pre-commit-config.yaml     | 14 ++++++++++++++
 CITATION.cff                |  2 +-
 LICENSE.txt                 |  1 -
 README.md                   | 16 ++++++++--------
 py4DGUI-keymap.html         | 16 ++++++++--------
 pyproject.toml              |  2 +-
 7 files changed, 33 insertions(+), 20 deletions(-)
 create mode 100644 .pre-commit-config.yaml

diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml
index 09b2a0f..f7afd36 100644
--- a/.github/workflows/black.yml
+++ b/.github/workflows/black.yml
@@ -11,4 +11,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: psf/black@stable
\ No newline at end of file
+      - uses: psf/black@stable
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 0000000..503f4f9
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,14 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+-   repo: https://github.com/psf/black
+    rev: 24.2.0
+    hooks:
+        -   id: black
diff --git a/CITATION.cff b/CITATION.cff
index 89d614c..1892afa 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -1,7 +1,7 @@
 cff-version: 1.1.0
 message: "If you use this software, please cite the accompanying paper."
 abstract: "Scanning transmission electron microscopy (STEM) allows for imaging, diffraction, and spectroscopy of materials on length scales ranging from microns to atoms. By using a high-speed, direct electron detector, it is now possible to record a full two-dimensional (2D) image of the diffracted electron beam at each probe position, typically a 2D grid of probe positions. These 4D-STEM datasets are rich in information, including signatures of the local structure, orientation, deformation, electromagnetic fields, and other sample-dependent properties. However, extracting this information requires complex analysis pipelines that include data wrangling, calibration, analysis, and visualization, all while maintaining robustness against imaging distortions and artifacts. In this paper, we present py4DSTEM, an analysis toolkit for measuring material properties from 4D-STEM datasets, written in the Python language and released with an open-source license. We describe the algorithmic steps for dataset calibration and various 4D-STEM property measurements in detail and present results from several experimental datasets. We also implement a simple and universal file format appropriate for electron microscopy data in py4DSTEM, which uses the open-source HDF5 standard. We hope this tool will benefit the research community and help improve the standards for data and computational methods in electron microscopy, and we invite the community to contribute to this ongoing project."
-authors: 
+authors:
   -
     affiliation: "National Center for Electron Microscopy, Molecular Foundry, Lawrence Berkeley National Laboratory, 1 Cyclotron Road, Berkeley, CA 94720, USA"
     family-names: Savitzky
diff --git a/LICENSE.txt b/LICENSE.txt
index 72f3b9d..77f1435 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -672,4 +672,3 @@ may consider it more useful to permit linking proprietary applications with
 the library.  If this is what you want to do, use the GNU Lesser General
 Public License instead of this License.  But first, please read
 <https://www.gnu.org/licenses/why-not-lgpl.html>.
-
diff --git a/README.md b/README.md
index e68e9d3..c7132b6 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,10 @@
 
 # The `py4DSTEM` GUI
 
-This repository hosts the `pyqt` based graphical 4D--STEM data browser that was originally part of **py4DSTEM** until version 0.13.11. 
+This repository hosts the `pyqt` based graphical 4D--STEM data browser that was originally part of **py4DSTEM** until version 0.13.11.
 
-## Installation 
-The GUI is available on PyPI and conda-forge: 
+## Installation
+The GUI is available on PyPI and conda-forge:
 
 `pip install py4D-browser`
 
@@ -15,19 +15,19 @@ The GUI is available on PyPI and conda-forge:
 Run `py4DGUI` in your terminal to open the GUI. Then just drag and drop a 4D-STEM dataset into the window!
 
 ### Controls
-* Move the virtual detector and the real-space selector using the mouse or using the keyboard shortcuts: WASD moves the detector and IJKL moves the selector, and holding down shift moves 5 pixels at a time. 
-* Auto scaling of both views is on by default. Press the "Autoscale" buttons in the bottom right to disable. Press either button to apply automatic scaling once, or Shift + click to lock autoscaling back on. 
+* Move the virtual detector and the real-space selector using the mouse or using the keyboard shortcuts: WASD moves the detector and IJKL moves the selector, and holding down shift moves 5 pixels at a time.
+* Auto scaling of both views is on by default. Press the "Autoscale" buttons in the bottom right to disable. Press either button to apply automatic scaling once, or Shift + click to lock autoscaling back on.
 * Different shapes of virtual detector are available in the "Detector Shape" menu, and different detector responses are available in the "Detector Response" menu.
-* The information in the bottom bar contains the details of the virtual detector used to generate the images, and can be entered into py4DSTEM to generate the same image. 
+* The information in the bottom bar contains the details of the virtual detector used to generate the images, and can be entered into py4DSTEM to generate the same image.
 * The FFT pane can be switched between displaying the FFT of the virtual image and displaying the [exit wave power cepstrum](https://doi.org/10.1016/j.ultramic.2020.112994).
 * Virtual images can be exported either as the scaled and clipped displays shown in the GUI or as raw data. The exact datatype stored in the raw TIFF image depends on both the datatype of the dataset and the type of virtual image being displayed (in particular, integer datatypes are converted internally to floating point to prevent overflows when generating any synthesized virtual images).
-* If the [EMPAD-G2 Raw Reader](https://github.com/sezelt/empad2) is installed in the same environment, an extra menu will appear that allows the concatenated binary format data to be background subtracted and calibrated in the GUI. You can also save the calibrated data as an HDF5 file for later analysis. 
+* If the [EMPAD-G2 Raw Reader](https://github.com/sezelt/empad2) is installed in the same environment, an extra menu will appear that allows the concatenated binary format data to be background subtracted and calibrated in the GUI. You can also save the calibrated data as an HDF5 file for later analysis.
 
 ![Demonstration](/images/demo.gif)
 
 The keyboard map in the Help menu was made using [this tool](https://archie-adams.github.io/keyboard-shortcut-map-maker/) and the map file is in the top level of this repo.
 
-## About 
+## About
 
 ![py4DSTEM logo](/images/py4DSTEM_logo.png)
 
diff --git a/py4DGUI-keymap.html b/py4DGUI-keymap.html
index 95996c2..b701bf8 100644
--- a/py4DGUI-keymap.html
+++ b/py4DGUI-keymap.html
@@ -21,8 +21,8 @@
 
 .ulButtons ul {
   margin: 0;
-  list-style-type: none; 
-  text-align: center; 
+  list-style-type: none;
+  text-align: center;
   background-color: #b4b3bd;
 }
 .ulButtons ul li {
@@ -51,7 +51,7 @@
 }
 
 .bodyStyle {
-  width:1109px; 
+  width:1109px;
   margin:0 auto;
 }
 
@@ -68,8 +68,8 @@
 
 /* Footer styling. */
 footer {
-  background-color: #b4b3bd; 
-  margin-top: 20px; 
+  background-color: #b4b3bd;
+  margin-top: 20px;
   margin-bottom: 0;
 }
 
@@ -79,12 +79,12 @@
 
 .footer-div {
   width:960px;
-  margin:0 auto; 
+  margin:0 auto;
   background-color: #b4b3bd;
 }
 
 #to-top-button{
-  margin-left:426px; 
+  margin-left:426px;
   padding-top: 14px;
 }
 #to-top-input {
@@ -689,4 +689,4 @@ <h3 style="float: left;" class="editable">Keyboard Shortcuts</h3>
         </div>
 
       </div> <!-- End of keyboard. -->
-    </li></ol></div></body></html>
\ No newline at end of file
+    </li></ol></div></body></html>
diff --git a/pyproject.toml b/pyproject.toml
index 3426704..229a8ea 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -43,4 +43,4 @@ include-package-data = true
 where = ["src"]
 
 [tool.setuptools.package-data]
-py4D_browser = ["*.png"]
\ No newline at end of file
+py4D_browser = ["*.png"]

From 664fb20ece7e4a0094dc7530f90a9242c5c8fbac Mon Sep 17 00:00:00 2001
From: Steven Zeltmann <steven.zeltmann@cornell.edu>
Date: Wed, 6 Mar 2024 13:47:46 -0500
Subject: [PATCH 09/13] disable some pre-commit hooks

---
 .pre-commit-config.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 503f4f9..196c01b 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -4,9 +4,9 @@ repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v3.2.0
     hooks:
-    -   id: trailing-whitespace
-    -   id: end-of-file-fixer
-    -   id: check-yaml
+    # -   id: trailing-whitespace
+    # -   id: end-of-file-fixer
+    # -   id: check-yaml
     -   id: check-added-large-files
 -   repo: https://github.com/psf/black
     rev: 24.2.0

From 604f3e5d5d3e3083a3838421d474cef4ea1c73b1 Mon Sep 17 00:00:00 2001
From: Steven Zeltmann <steven.zeltmann@cornell.edu>
Date: Wed, 6 Mar 2024 14:08:01 -0500
Subject: [PATCH 10/13] VERSION ONE POINT OH

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 229a8ea..a715a7e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "py4D_browser"
-version = "0.999999"
+version = "1.0.0"
 authors = [
   { name="Steven Zeltmann", email="steven.zeltmann@lbl.gov" },
 ]

From 80ae86562aacf95d81b323d050cab4c2151c1dcf Mon Sep 17 00:00:00 2001
From: Steven Zeltmann <steven.zeltmann@cornell.edu>
Date: Sun, 17 Mar 2024 10:48:35 -0400
Subject: [PATCH 11/13] add ResizeDialog for picking appropriate data shapes
 (not yet used)

---
 src/py4D_browser/menu_actions.py |   7 ++-
 src/py4D_browser/utils.py        | 101 ++++++++++++++++++++++++++++++-
 2 files changed, 104 insertions(+), 4 deletions(-)

diff --git a/src/py4D_browser/menu_actions.py b/src/py4D_browser/menu_actions.py
index fbe3e9e..27f95aa 100644
--- a/src/py4D_browser/menu_actions.py
+++ b/src/py4D_browser/menu_actions.py
@@ -64,7 +64,7 @@ def load_file(self, filepath, mmap=False, binning=1):
     extension = os.path.splitext(filepath)[-1].lower()
     print(f"Type: {extension}")
     if extension in (".h5", ".hdf5", ".py4dstem", ".emd"):
-        datacubes = get_4D(h5py.File(filepath, "r"))
+        datacubes = get_ND(h5py.File(filepath, "r"))
         print(f"Found {len(datacubes)} 4D datasets inside the HDF5 file...")
         if len(datacubes) >= 1:
             # Read the first datacube in the HDF5 file into RAM
@@ -243,7 +243,8 @@ def get_savefile_name(self, file_format) -> str:
         raise ValueError("Could get save file")
 
 
-def get_4D(f, datacubes=None):
+def get_ND(f, datacubes=None, N=4):
+    # Traverse an h5py.File and look for Datasets with N dimensions
     if datacubes is None:
         datacubes = []
     for k in f.keys():
@@ -252,7 +253,7 @@ def get_4D(f, datacubes=None):
             if len(f[k].shape) == 4:
                 datacubes.append(f[k])
         elif isinstance(f[k], h5py.Group):
-            get_4D(f[k], datacubes)
+            get_ND(f[k], datacubes)
     return datacubes
 
 
diff --git a/src/py4D_browser/utils.py b/src/py4D_browser/utils.py
index e1ec718..94a979c 100644
--- a/src/py4D_browser/utils.py
+++ b/src/py4D_browser/utils.py
@@ -1,8 +1,9 @@
 import pyqtgraph as pg
 import numpy as np
-from PyQt5.QtWidgets import QFrame, QPushButton, QApplication
+from PyQt5.QtWidgets import QFrame, QPushButton, QApplication, QLabel
 from PyQt5.QtCore import pyqtSignal
 from PyQt5.QtCore import Qt
+from PyQt5.QtWidgets import QDialog, QHBoxLayout, QVBoxLayout, QSpinBox
 
 
 class VLine(QFrame):
@@ -52,6 +53,104 @@ def on_click(self, *args):
                     self.status_bar.showMessage("Shift+click to keep on", 5_000)
 
 
+class ResizeDialog(QDialog):
+    def __init__(self, size, parent=None):
+        super().__init__(parent=parent)
+
+        self.new_size = size
+        Nmax = size[0] * size[1]
+
+        layout = QVBoxLayout(self)
+
+        layout.addWidget(QLabel("Dataset size unknown. Please enter the shape:"))
+
+        box_layout = QHBoxLayout()
+        box_layout.addWidget(QLabel("X:"))
+
+        xbox = QSpinBox()
+        xbox.setRange(1, Nmax)
+        xbox.setSingleStep(1)
+        xbox.setKeyboardTracking(False)
+        xbox.valueChanged.connect(self.x_box_changed)
+        box_layout.addWidget(xbox)
+
+        box_layout.addStretch()
+        box_layout.addWidget(QLabel("Y:"))
+
+        ybox = QSpinBox()
+        ybox.setRange(1, Nmax)
+        ybox.setSingleStep(1)
+        ybox.setValue(Nmax)
+        ybox.setKeyboardTracking(False)
+        ybox.valueChanged.connect(self.y_box_changed)
+        box_layout.addWidget(ybox)
+
+        layout.addLayout(box_layout)
+
+        button_layout = QHBoxLayout()
+        button_layout.addStretch()
+        done_button = QPushButton("Done")
+        done_button.pressed.connect(self.close)
+        button_layout.addWidget(done_button)
+        layout.addLayout(button_layout)
+
+        self.x_box = xbox
+        self.y_box = ybox
+        self.x_box_last = xbox.value()
+        self.y_box_last = ybox.value()
+        self.N = Nmax
+
+        self.resize(600, 400)
+
+    @classmethod
+    def get_new_size(cls, size, parent=None):
+        dialog = cls(size=size, parent=parent)
+        dialog.exec_()
+        return dialog.new_size
+
+    def x_box_changed(self, new_value):
+        if new_value == self.x_box_last:
+            return
+        x_new, y_new = self.get_next_rect(
+            new_value, "down" if new_value < self.x_box_last else "up"
+        )
+
+        self.x_box_last = x_new
+        self.y_box_last = y_new
+
+        self.x_box.setValue(x_new)
+        self.y_box.setValue(y_new)
+
+        self.new_size = [x_new, y_new]
+
+    def y_box_changed(self, new_value):
+        if new_value == self.y_box_last:
+            return
+        y_new, x_new = self.get_next_rect(
+            new_value, "down" if new_value < self.y_box_last else "up"
+        )
+
+        self.x_box_last = x_new
+        self.y_box_last = y_new
+
+        self.x_box.setValue(x_new)
+        self.y_box.setValue(y_new)
+
+        self.new_size = [x_new, y_new]
+
+    def get_next_rect(self, current, direction):
+        # get the next perfect rectangle
+        iterator = (
+            range(current, 0, -1) if direction == "down" else range(current, self.N + 1)
+        )
+
+        for i in iterator:
+            if self.N % i == 0:
+                return i, self.N // i
+
+        raise ValueError("Factor finding failed, frustratingly.")
+
+
 def pg_point_roi(view_box):
     """
     Point selection.  Based in pyqtgraph, and returns a pyqtgraph CircleROI object.

From 1465a76e4b049fff25fe2257464d50b24847107d Mon Sep 17 00:00:00 2001
From: Steven Zeltmann <steven.zeltmann@cornell.edu>
Date: Sun, 17 Mar 2024 11:04:52 -0400
Subject: [PATCH 12/13] show reshape dialog when a H5 file has 3D data

---
 src/py4D_browser/menu_actions.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/py4D_browser/menu_actions.py b/src/py4D_browser/menu_actions.py
index 27f95aa..b3bc7fc 100644
--- a/src/py4D_browser/menu_actions.py
+++ b/src/py4D_browser/menu_actions.py
@@ -1,3 +1,4 @@
+from numbers import Real
 import py4DSTEM
 from PyQt5.QtWidgets import QFileDialog, QMessageBox
 import h5py
@@ -5,6 +6,7 @@
 import numpy as np
 import matplotlib.pyplot as plt
 from py4D_browser.help_menu import KeyboardMapMenu
+from py4D_browser.utils import ResizeDialog
 from py4DSTEM.io.filereaders import read_arina
 
 
@@ -63,8 +65,9 @@ def load_file(self, filepath, mmap=False, binning=1):
     print(f"Loading file {filepath}")
     extension = os.path.splitext(filepath)[-1].lower()
     print(f"Type: {extension}")
-    if extension in (".h5", ".hdf5", ".py4dstem", ".emd"):
-        datacubes = get_ND(h5py.File(filepath, "r"))
+    if extension in (".h5", ".hdf5", ".py4dstem", ".emd", ".mat"):
+        file = h5py.File(filepath, "r")
+        datacubes = get_ND(file)
         print(f"Found {len(datacubes)} 4D datasets inside the HDF5 file...")
         if len(datacubes) >= 1:
             # Read the first datacube in the HDF5 file into RAM
@@ -81,7 +84,17 @@ def load_file(self, filepath, mmap=False, binning=1):
             self.datacube.calibration.set_Q_pixel_units(Q_units)
 
         else:
-            raise ValueError("No 4D data detected in the H5 file!")
+            # if no 4D data was found, look for 3D data
+            datacubes = get_ND(file, N=3)
+            print(f"Found {len(datacubes)} 3D datasets inside the HDF5 file...")
+            if len(datacubes) >= 1:
+                array = datacubes[0] if mmap else datacubes[0][()]
+                new_shape = ResizeDialog.get_new_size([1, array.shape[0]], parent=self)
+                self.datacube = py4DSTEM.DataCube(
+                    array.reshape(*new_shape, *array.shape[1:])
+                )
+            else:
+                raise ValueError("No 4D (or even 3D) data detected in the H5 file!")
     elif extension in [".npy"]:
         self.datacube = py4DSTEM.DataCube(np.load(filepath))
     else:
@@ -192,7 +205,7 @@ def show_file_dialog(self) -> str:
         self,
         "Open 4D-STEM Data",
         "",
-        "4D-STEM Data (*.dm3 *.dm4 *.raw *.mib *.gtg *.h5 *.hdf5 *.emd *.py4dstem *.npy *.npz);;Any file (*)",
+        "4D-STEM Data (*.dm3 *.dm4 *.raw *.mib *.gtg *.h5 *.hdf5 *.emd *.py4dstem *.npy *.npz *.mat);;Any file (*)",
     )
     if filename is not None and len(filename[0]) > 0:
         return filename[0]
@@ -250,7 +263,7 @@ def get_ND(f, datacubes=None, N=4):
     for k in f.keys():
         if isinstance(f[k], h5py.Dataset):
             # we found data
-            if len(f[k].shape) == 4:
+            if len(f[k].shape) == N:
                 datacubes.append(f[k])
         elif isinstance(f[k], h5py.Group):
             get_ND(f[k], datacubes)

From 97b84a872ad036eb92b4ed6e86a2d5e981a2c2ad Mon Sep 17 00:00:00 2001
From: Steven Zeltmann <steven.zeltmann@cornell.edu>
Date: Sun, 17 Mar 2024 11:09:22 -0400
Subject: [PATCH 13/13] add reshape menu to adjust shape any time

---
 src/py4D_browser/main_window.py  |  5 +++++
 src/py4D_browser/menu_actions.py | 12 ++++++++++++
 2 files changed, 17 insertions(+)

diff --git a/src/py4D_browser/main_window.py b/src/py4D_browser/main_window.py
index 4a1d33b..81f7391 100644
--- a/src/py4D_browser/main_window.py
+++ b/src/py4D_browser/main_window.py
@@ -43,6 +43,7 @@ class DataViewer(QMainWindow):
         export_datacube,
         export_virtual_image,
         show_keyboard_map,
+        reshape_data,
     )
 
     from py4D_browser.update_views import (
@@ -120,6 +121,10 @@ def setup_menus(self):
         self.load_arina_action.triggered.connect(self.load_data_arina)
         self.file_menu.addAction(self.load_arina_action)
 
+        self.reshape_data_action = QAction("&Reshape Data...", self)
+        self.reshape_data_action.triggered.connect(self.reshape_data)
+        self.file_menu.addAction(self.reshape_data_action)
+
         self.file_menu.addSeparator()
 
         export_label = QAction("Export", self)
diff --git a/src/py4D_browser/menu_actions.py b/src/py4D_browser/menu_actions.py
index b3bc7fc..1b09ad3 100644
--- a/src/py4D_browser/menu_actions.py
+++ b/src/py4D_browser/menu_actions.py
@@ -121,6 +121,18 @@ def load_file(self, filepath, mmap=False, binning=1):
     self.setWindowTitle(filepath)
 
 
+def reshape_data(self):
+    new_shape = ResizeDialog.get_new_size(self.datacube.shape[:2], parent=self)
+    self.datacube.data = self.datacube.data.reshape(
+        *new_shape, *self.datacube.data.shape[2:]
+    )
+
+    print(f"Reshaping data to {new_shape}")
+
+    self.update_diffraction_space_view(reset=True)
+    self.update_real_space_view(reset=True)
+
+
 def export_datacube(self, save_format: str):
     assert save_format in [
         "Raw float32",