Cms giwaxs support version 1

.github/workflows/main.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ${{ matrix.os}}
     strategy:
       matrix:
-        python-version: [ '3.8', '3.9', '3.10','3.11']
+        python-version: ['3.10','3.11']
         os: [ubuntu-latest, macOS-latest, windows-latest]
 
     steps:
@@ -58,6 +58,8 @@ jobs:
         unzip PSS300nm_C_ccd100.zip
         wget https://github.com/usnistgov/PyHyperScattering/releases/download/0.0.0-example-data/smi_example.zip
         unzip smi_example.zip
+        wget https://github.com/usnistgov/PyHyperScattering/releases/download/0.0.0-example-data/CMS_giwaxs_series.zip
+        unzip CMS_giwaxs_series.zip
     - name: Fetch and unzip example data (Windows)
       if: ${{ matrix.os == 'windows-latest' }}
       run: |
@@ -71,6 +73,8 @@ jobs:
         unzip PSS300nm_C_ccd100.zip
         C:\msys64\usr\bin\wget.exe https://github.com/usnistgov/PyHyperScattering/releases/download/0.0.0-example-data/smi_example.zip
         unzip smi_example.zip
+        C:\msys64\usr\bin\wget.exe https://github.com/usnistgov/PyHyperScattering/releases/download/0.0.0-example-data/CMS_giwaxs_series.zip
+        unzip CMS_giwaxs_series.zip
     - name: Test with pytest
       run: |
         #pytest -v

.gitignore

@@ -12,18 +12,18 @@ venv/*
 OrientationRunDec2019/**
 **/*.p
 **/.*
-**/*.ipynb
 *.png
 *.h5
 *.nxs
+.ipynb_checkpoints
 Example/**
 example-data/**
 src/__pycache__
 *.egg-info
 src/PyHyperScattering/__pycache__
 src/PyHyperScattering/.ipynb_checkpoints/*
 dist
-docs/_build/*
+dcs/_build/*
 docs/_build/html/*
 *.pyc
 example_data/11012/Dark_56367-AI.txt

requirements.txt

@@ -5,7 +5,10 @@ h5py
 nodejs
 numpy
 pandas
+# pygix fails to improt if silx > 2.0.0 
+silx==2.0.0
 pyfai
+pygix
 scikit-image
 scipy
 pillow

src/PyHyperScattering/FileIO.py

@@ -34,7 +34,25 @@ def __init__(self,xr_obj):
     def savePickle(self,filename):
         with open(filename, 'wb') as file:
             pickle.dump(self._obj, file)
-
+
+
+    # - This was copied from the Toney group contribution for GIWAXS.
+    def saveZarr(self,  filename, mode: str = 'w'):
+         """
+         Save the DataArray as a .zarr file in a specific path, with a file name constructed from a prefix and suffix.
+
+         Parameters:
+             da (xr.DataArray): The DataArray to be saved.
+             base_path (Union[str, pathlib.Path]): The base path to save the .zarr file.
+             prefix (str): The prefix to use for the file name.
+             suffix (str): The suffix to use for the file name.
+             mode (str): The mode to use when saving the file. Default is 'w'.
+         """
+         da = self._obj
+         ds = da.to_dataset(name='DA')
+         file_path = pathlib.Path(filename)
+         ds.to_zarr(file_path, mode=mode)
+
     def saveNexus(self,fileName,compression=5):
         data = self._obj
         timestamp = datetime.datetime.now()

src/PyHyperScattering/FileLoader.py

@@ -14,13 +14,12 @@ class FileLoader():
     Abstract class defining a generic scattering file loader.  Input is a (or multiple) filename/s and output is a
     xarray I(pix_x,pix_y,dims,coords) where dims and coords are loaded by user request.
 
-
-     Difference: all coords are dims but not all dims are coords.  Dims can also be auto-hinted using the following   
-     standard names: energy,exposure,pos_x,pos_y,pos_z,theta.
+    Difference: all coords are dims but not all dims are coords.  Dims can also be auto-hinted using the following   
+    standard names: energy,exposure,pos_x,pos_y,pos_z,theta.
 
-     Individual loaders can try searching metadata for other dim names but this is not guaranteed.
-     Coords can be used to provide a list of values for a dimension when that dimension cannot be hinted, e.g. where vals
-     come from external data.
+    Individual loaders can try searching metadata for other dim names but this is not guaranteed.
+    Coords can be used to provide a list of values for a dimension when that dimension cannot be hinted, e.g. where vals
+    come from external data.
     '''
     file_ext = ''  # file extension to be used to filter files from this instrument
     md_loading_is_quick = False
@@ -32,20 +31,35 @@ def loadSingleImage(self,filepath,coords=None,return_q=None,**kwargs):
 
     def peekAtMd(self,filepath):
         return self.loadSingleImage(filepath,{})
-
 
-
-    def loadFileSeries(self,basepath,dims,coords={},file_filter=None,file_filter_regex=None,file_skip=None,md_filter={},quiet=True,output_qxy=False,dest_qx=None,dest_qy=None,output_raw=False,image_slice=None):
+    def loadFileSeries(self, 
+                       basepath, 
+                       dims, 
+                       coords={},
+                       file_filter=None,
+                       file_filter_regex=None,
+                       file_skip=None,
+                       md_filter={},
+                       quiet=True,
+                       output_qxy=False,
+                       dest_qx=None,
+                       dest_qy=None,
+                       output_raw=False,
+                       image_slice=None):
         '''
         Load a series into a single xarray.
 
+        * = not implemented yet
+
         Args:
             basepath (str or Path): path to the directory to load
-            dims (list): dimensions of the resulting xarray, as list of str
+            dims (list): dimensions of the resulting xarray (excluding 'pix_x' and 'pix_y'), as list of str
             coords (dict): dictionary of any dims that are *not* present in metadata
             file_filter (str): string that must be in each file name
-            file_filer_regex(str): regex string that must match in each file name
-            file_skip (str): string that, if present in file name, means file should be skipped.
+
+            *file_filer_regex(str): regex string that must match in each file name
+            *file_skip (str): string that, if present in file name, means file should be skipped.
+
             md_filter (dict): dict of *required* metadata values; points without these metadata values will be dropped
             md_filter_regex (dict): dict of *required* metadata regex; points without these metadata values will be dropped
             quiet (bool): skip printing most intermediate output if true.
@@ -61,14 +75,20 @@ def loadFileSeries(self,basepath,dims,coords={},file_filter=None,file_filter_reg
         nfiles = len(os.listdir(basepath))
         nprocessed = 0
         nloaded = 0
-        print(f'Found {str(nfiles)} files.')
+        print(f'Found {str(nfiles)} total files.')
         data_rows = []
         qnew = None
         dest_coords = defaultdict(list)
         if file_filter_regex is not None:
             file_filter_regex = re.compile(file_filter_regex)
-
-        for file in tqdm(sorted(os.listdir(basepath))):
+        if file_filter is not None:
+            filepaths = sorted(basepath.glob(f'*{file_filter}*'))
+            files = [f.name for f in filepaths]
+        else:
+            files = sorted(os.listdir(basepath))
+        print(f"Found {str(len(files))} files after applying 'file_filter'.")
+
+        for file in tqdm(files):
             nprocessed += 1
 
             if re.match(self.file_ext,file) is None:
@@ -102,7 +122,7 @@ def loadFileSeries(self,basepath,dims,coords={},file_filter=None,file_filter_reg
                     if not quiet:
                         print(f'Not loading {file}, expected {key} to be {val} but it was {md[key]}')
             if load_this_image:
-                if img == None:
+                if img is None:
                     if not quiet:
                         print(f'Loading {file}')
                     img = self.loadSingleImage(basepath/file,coords=local_coords, return_q = output_qxy,image_slice=image_slice,use_cached_md=False)

src/PyHyperScattering/IntegrationUtils.py

@@ -2,6 +2,7 @@
 import xarray as xr
 import numpy as np
 import math
+from tqdm.auto import tqdm
 
 try:
     import holoviews as hv
@@ -98,6 +99,8 @@ def checkAll(integrator,img,img_min=1,img_max=10000,img_scaling='log',alpha=1,d_
         ax.add_patch(guide1)
         ax.add_patch(guide2)
         ax.imshow(integrator.mask,origin='lower',alpha=alpha)
+
+
 class DrawMask:
     '''
     Utility class for interactively drawing a mask in a Jupyter notebook.
@@ -114,21 +117,22 @@ class DrawMask:
 
     '''
 
-    def __init__(self,frame):
+    def __init__(self,frame, cmap='viridis', clim=(5e0, 5e3), width=800, height=700):
         '''
         Construct a DrawMask object
 
         Args:
             frame (xarray): a single data frame with pix_x and pix_y axes
 
-
         '''
+
         if len(frame.shape) > 2:
             warnings.warn('This tool needs a single frame, not a stack!  .sel down to a single frame before starting!',stacklevel=2)
 
-        self.frame=frame
+        self.frame = frame
 
-        self.fig = frame.hvplot(cmap='terrain',clim=(5,5000),logz=True,data_aspect=1)
+        self.fig = frame.hvplot(cmap=cmap, clim=clim, logz=True, data_aspect=1, 
+                                width=width, height=height)
 
         self.poly = hv.Polygons([])
         self.path_annotator = hv.annotate.instance()
@@ -140,17 +144,13 @@ def ui(self):
 
         Returns: the holoviews object
 
-
-
         '''
         print('Usage: click the "PolyAnnotator" tool at top right.  DOUBLE CLICK to start drawing a masked object, SINGLE CLICK to add a vertex, then DOUBLE CLICK to finish.  Click/drag individual vertex to adjust.')
-        return self.path_annotator(
-                self.fig * self.poly.opts(
-                            width=self.frame.shape[0], 
-                            height=self.frame.shape[1], 
-                            responsive=False), 
-                annotations=['Label'], 
-            vertex_annotations=['Value'])
+        annotator_plot = self.path_annotator(
+                                    self.fig * self.poly.opts(responsive=False), 
+                                    annotations=['Label'], 
+                                    vertex_annotations=['Value'])
+        return annotator_plot.opts(toolbar='left')
 
 
     def save(self,fname):
@@ -178,11 +178,11 @@ def load(self,fname):
         '''
         with open(fname,'r') as f:
             strlist = json.load(f)
-        print(strlist)
+        # print(strlist)
         dflist = []
         for item in strlist:
             dflist.append(pd.read_json(item))
-        print(dflist)
+        # print(dflist)
         self.poly = hv.Polygons(dflist)
 
         self.path_annotator(
@@ -204,3 +204,58 @@ def mask(self):
             mask |= skimage.draw.polygon2mask(self.frame.shape,self.path_annotator.annotated.iloc[i].dframe(['x','y']))
 
         return mask
+
+
+class CMSGIWAXS:
+    """For streamlined loading for CMS data"""
+    def __init__(self, files, loader, integrator):
+        """
+        Inputs: files: indexable object str or pathlib.Path filepaths to 
+                       raw GIWAXS data
+                loader: custom PyHyperScattering CMSGIWAXSLoader object, must 
+                        return DataArray with attributes metadata
+                integrator: instance of PGGeneralIntegrator object
+        """
+        self.files = files
+        self.loader = loader
+        self.integrator = integrator
+
+    def single_images_to_dataset(self):
+        """
+        Method that takes a subscriptable object of filepaths corresponding to raw GIWAXS
+        beamline data, loads the raw data into an xarray DataArray, generates pygix-transformed 
+        cartesian and polar DataArrays, and creates 3 corresponding xarray Datasets 
+        containing a DataArray per sample. 
+        The raw dataarrays must contain the attributes 'scan_id' and 'incident_angle'
+
+        Outputs: 2 Datasets: raw & reciprocal space (cartesian or polar based on integrator object)
+        """
+        # Select the first element of the sorted set outside of the for loop to initialize the xr.DataSet
+        DA = self.loader.loadSingleImage(self.files[0])
+        assert 'scan_id' in DA.attrs.keys(), "'scan_id' is a required attribute to use this function"
+
+        # Update incident angle per sample:
+        assert 'incident_angle' in DA.attrs.keys(), "'incident_angle' is a required attribute to use this function"
+        self.integrator.incident_angle = float(DA.incident_angle[2:])
+
+        # Integrate single image
+        integ_DA = self.integrator.integrateSingleImage(DA)
+
+        # Save coordinates for interpolating other dataarrays 
+        integ_coords = integ_DA.coords
+
+        # Create a DataSet, each DataArray will be named according to it's scan id
+        raw_DS = DA.to_dataset(name=DA.scan_id)
+        integ_DS = integ_DA.to_dataset(name=DA.scan_id)
+
+        # Populate the DataSet with 
+        for filepath in tqdm(self.files[1:], desc=f'Transforming Raw Data'):
+            DA = self.loader.loadSingleImage(filepath)
+            integ_DA = self.integrator.integrateSingleImage(DA)
+
+            integ_DA = integ_DA.interp(integ_coords)
+
+            raw_DS[f'{DA.scan_id}'] = DA
+            integ_DS[f'{DA.scan_id}'] = integ_DA
+
+        return raw_DS, integ_DS