Merge pull request #102 from leewujung/master

Fix provenance group netCDF4 error

README.rst

@@ -76,7 +76,8 @@ Contributors
 `Kavin Nguyen <https://github.com/ngkavin>`_ (@ngkavin)
 are primary developers of this project.
 `Valentina Staneva <https://escience.washington.edu/people/valentina-staneva/>`_ (@valentina-s)
-provides consultation and also contributes to development. Other contributors are listed `here <echopype documentation>`_.
+provides consultation and also contributes to development.
+Other contributors are listed `here <echopype documentation>`_.
 
 We thank Dave Billenness of ASL Environmental Sciences for
 providing the AZFP Matlab Toolbox as reference for our

echopype/convert/ek60.py

@@ -462,7 +462,7 @@ def copyfiles():
 
             # Check if nc file already exists and deletes it if overwrite is true
             if os.path.exists(out_file) and overwrite:
-                print("          overwriting: " + out_file)
+                print("          overwriting: " + out_file)  # TODO: this should be printed after 'converting...'
                 os.remove(out_file)
             # Check if nc file already exists
             # ... if yes, abort conversion and issue warning

echopype/convert/utils/set_groups_base.py

@@ -41,30 +41,23 @@ def set_provenance(self, src_file_names, prov_dict):
                           prov_dict['conversion_software_version']
                           prov_dict['conversion_time']
         """
-        # create group
-        files = ", ".join([os.path.basename(file) for file in src_file_names])
-        if self.format == '.nc':
-            file = netCDF4.Dataset(self.file_path, "a", format="NETCDF4")
-            pr = file.createGroup("Provenance")
-            # dimensions
-            pr.createDimension("filenames", None)
-            # variables
-            pr_src_fnames = pr.createVariable(files, str, "filenames")
-            pr_src_fnames.long_name = "Source filenames"
+        # Save the source filenames as a data variable
+        ds = xr.Dataset(
+            {
+                'filenames': ('file_num', src_file_names, {'long_name': 'Source filenames'})
+            },
+            coords={'file_num': np.arange(len(src_file_names))},
+        )
 
-            # set group attributes
-            for k, v in prov_dict.items():
-                pr.setncattr(k, v)
-            # close nc file
-            file.close()
+        # Save all attributes
+        for k, v in prov_dict.items():
+            ds.attrs[k] = v
 
+        # save to file
+        if self.format == '.nc':
+            ds.to_netcdf(path=self.file_path, mode='a', group='Provenance')
         elif self.format == '.zarr':
-            file = zarr.open(self.file_path, 'a')
-            pr = file.create_group('Provenance')
-            pr_src_fnames = pr.create_dataset('filenames', data=files)
-            pr_src_fnames.attrs['long_name'] = "Source filenames"
-            for k, v in prov_dict.items():
-                pr[k] = v
+            ds.to_zarr(path=self.file_path, mode='a', group='Provenance')
 
     def set_sonar(self, sonar_dict):
         """Set the Sonar group in the nc file.

echopype/model/azfp.py

@@ -139,6 +139,8 @@ def calibrate(self, save=False, save_postfix='_Sv', save_path=None):
         save_path : str
             Full filename to save to, overwriting the RAWFILE_Sv.nc default
         """
+        # Print raw data nc file
+        print('%s  calibrating data in %s' % (dt.datetime.now().strftime('%H:%M:%S'), self.file_path))
 
         # Open data set for Environment and Beam groups
         ds_beam = xr.open_dataset(self.file_path, group="Beam")

echopype/model/ek60.py

@@ -76,6 +76,9 @@ def calibrate(self, save=False, save_postfix='_Sv', save_path=None):
         save_path : str
             Full filename to save to, overwriting the RAWFILENAME_Sv.nc default
         """
+        # Print raw data nc file
+        print('%s  calibrating data in %s' % (dt.datetime.now().strftime('%H:%M:%S'), self.file_path))
+
         # Open data set for Environment and Beam groups
         ds_beam = xr.open_dataset(self.file_path, group="Beam")
 

echopype/model/modelbase.py

@@ -220,6 +220,9 @@ def _assemble_path():
                 file_out = _assemble_path()
 
         # Create folder if not already exists
+        if save_dir == '':
+            # TODO: should we use '.' instead of os.getcwd()?
+            save_dir = os.getcwd()  # explicit about path to current directory
         if not os.path.exists(save_dir):
             os.mkdir(save_dir)
 
@@ -557,7 +560,7 @@ def get_MVBS(self, source_postfix='_Sv', source_path=None,
 
         # Get Sv by validating path and calibrate if not already done
         if self.Sv is not None:
-            print('%s  Use Sv stored in memory to calculate MVBS.' % dt.datetime.now().strftime('%H:%M:%S'))
+            print('%s  use Sv stored in memory to calculate MVBS.' % dt.datetime.now().strftime('%H:%M:%S'))
             print_src = False
         else:
             print_src = True

notebooks/EK60_demo_OOI.ipynb

@@ -670,7 +670,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.3"
+   "version": "3.8.0"
   }
  },
  "nbformat": 4,