Merge pull request #977 from OSOceanAcoustics/dev

Release/v0.6.4

.ci_helpers/docker/http.dockerfile

@@ -1,3 +1,4 @@
 FROM httpd:2.4
+ARG TARGETPLATFORM
 
 COPY echopype/test_data /usr/local/apache2/htdocs/data

.ci_helpers/docker/minioci.dockerfile

@@ -1,4 +1,5 @@
 FROM minio/minio
+ARG TARGETPLATFORM
 
 # Install git
 RUN microdnf install git

.ci_helpers/py3.10.yaml

@@ -14,4 +14,3 @@ dependencies:
   - s3fs==2022.5.0
   - matplotlib-base
   - cmocean
-  - mamba=0.20.0

.ci_helpers/py3.8.yaml

@@ -14,4 +14,3 @@ dependencies:
   - s3fs==2022.5.0
   - matplotlib-base
   - cmocean
-  - mamba=0.20.0

.ci_helpers/py3.9.yaml

@@ -14,4 +14,3 @@ dependencies:
   - s3fs==2022.5.0
   - matplotlib-base
   - cmocean
-  - mamba=0.20.0

.ci_helpers/run-test.py

@@ -32,6 +32,8 @@
     "utils": {},
     "visualize": {},
     "metrics": {},
+    "mask": {},
+    "consolidate": {},
 }
 
 if __name__ == "__main__":
@@ -57,7 +59,7 @@
     )
     args = parser.parse_args()
     if args.local:
-        temp_path = Path("temp_echopype_output")
+        temp_path = Path("~/.echopype/temp_output")
         dump_path = Path("echopype/test_data/dump")
         if temp_path.exists():
             shutil.rmtree(temp_path)

.github/workflows/build.yaml

@@ -65,7 +65,7 @@ jobs:
       - name: Install dev tools
         shell: bash -l {0}
         run: |
-          mamba install -c conda-forge --yes --file requirements-dev.txt
+          micromamba install -c conda-forge -n ${{ env.CONDA_ENV }} --yes --file requirements-dev.txt
       - name: Install echopype
         shell: bash -l {0}
         run: |

.github/workflows/docker.yaml

@@ -47,6 +47,7 @@ jobs:
           context: ./
           file: ./.ci_helpers/docker/${{ matrix.image_name }}.dockerfile
           push: true
+          platforms: linux/amd64,linux/arm64
           tags: |
             ${{ env.IMAGE_SPEC }}:${{ env.DATE_TAG }}
             ${{ env.IMAGE_SPEC }}:latest

.github/workflows/pr.yaml

@@ -61,7 +61,7 @@ jobs:
       - name: Install dev tools
         shell: bash -l {0}
         run: |
-          mamba install -c conda-forge --yes --file requirements-dev.txt
+          micromamba install -c conda-forge -n ${{ env.CONDA_ENV }} --yes --file requirements-dev.txt
       - name: Install echopype
         shell: bash -l {0}
         run: |

.github/workflows/windows-utils.yaml

@@ -0,0 +1,69 @@
+name: windows-utils-test
+
+on:
+  push:
+    paths-ignore: [".ci_helpers/docker/**", "**/docker.yaml"]
+  pull_request:
+    paths-ignore: [".ci_helpers/docker/**", "**/docker.yaml"]
+  pull_request_target:
+    paths-ignore: [".ci_helpers/docker/**", "**/docker.yaml"]
+  workflow_dispatch:
+
+env:
+  CONDA_ENV: echopype
+
+jobs:
+  windows-test:
+      name: windows-${{ matrix.python-version }}-build
+      runs-on: "windows-latest"
+      continue-on-error: ${{ matrix.experimental }}
+      strategy:
+        fail-fast: false
+        matrix:
+          include:
+          - python-version: 3.9
+            experimental: false
+      defaults:
+        run:
+          shell: powershell
+      steps:
+        - name: Checkout repo
+          uses: actions/checkout@v3
+          with:
+            fetch-depth: 0 # Fetch all history for all branches and tags.
+        - name: Set environment variables
+          run: |
+            echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
+        - name: Setup miniconda w/mamba
+          uses: conda-incubator/setup-miniconda@v2
+          with:
+            python-version: ${{ matrix.python-version }}
+            mamba-version: "*"
+            channels: conda-forge,defaults
+            channel-priority: true
+            activate-environment: ${{ env.CONDA_ENV }}
+            environment-file: .ci_helpers/py${{ matrix.python-version }}.yaml
+        # Currently problematic! See https://github.com/mamba-org/mamba/issues/2157
+        # - name: Setup micromamba
+        #   uses: mamba-org/provision-with-micromamba@v15
+        #   with:
+        #     environment-file: .ci_helpers/py${{ matrix.python-version }}.yaml
+        #     environment-name: ${{ env.CONDA_ENV }}
+        #     cache-env: true
+        #     cache-env-key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles(format('.ci_helpers/py{0}.yaml', matrix.python-version)) }}
+        - name: Print conda info
+          run: |
+            conda info
+            conda list
+            conda config --show-sources
+            conda config --show
+            printenv | sort
+        - name: Install dev tools
+          run: |
+            mamba install -c conda-forge -n ${{ env.CONDA_ENV }} --yes --file requirements-dev.txt
+        - name: Install echopype
+          run: |
+            python -m pip install -e .
+        - name: Running all Tests
+          run: |
+            pytest -vvv -rx --cov=echopype --cov-report=xml --log-cli-level=WARNING --disable-warnings echopype/tests/utils |& tee ci_${{ matrix.python-version }}_test_log.log

.gitignore

@@ -29,7 +29,6 @@ toolbox
 *.mat
 ek60/
 azfp/
-temp_echopype_output/
 notebooks/*_dev/
 _echopype_version.py
 !echopype/test_data

.pre-commit-config.yaml

@@ -33,14 +33,10 @@ repos:
     hooks:
       - id: flake8
 
-  - repo: https://github.com/executablebooks/jupyter-book
-    rev: v0.12.3
-    hooks:
-    - id: jb-to-sphinx
-      args: ["docs/source"]
-
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.2
     hooks:
       - id: codespell
-        args: ["--skip=*.ipynb", "-w", "docs/source", "echopype"]
+        # Checks spelling in `docs/source` and `echopype` dirs ONLY
+        # Ignores `.ipynb` files and `_build` folders
+        args: ["--skip=*.ipynb,docs/source/_build", "-w", "docs/source", "echopype"]

.readthedocs.yml

@@ -5,6 +5,16 @@
 # Required
 version: 2
 
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.8"
+  jobs:
+    pre_build:
+      # Generate the Sphinx configuration for this Jupyter Book so it builds.
+      - "jupyter-book config sphinx docs/source"
+
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
    configuration: docs/source/conf.py
@@ -18,7 +28,6 @@ formats: []
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
-   version: "3.8"
    install:
       - requirements: docs/requirements.txt
       - method: pip

CITATION.cff

@@ -13,6 +13,9 @@ authors:
 - family-names: "Mayorga"
   given-names: "Emilio"
   orcid: "https://orcid.org/0000-0003-2574-4623"
+- family-names: "Reyes"
+  given-names: "Brandon"
+  orcid: "https://orcid.org/0000-0003-3496-7490"
 - family-names: "Majeed"
   given-names: "Imran"
   orcid: "https://orcid.org/0000-0002-2032-6476"
@@ -23,9 +26,10 @@ title: "echopype"
 doi: 10.5281/zenodo.3906999
 # but pointing to a specific release/version will
 # require having to update this file with each release
-version: 0.5.4
-date-released: 2021-10-01
-url: "https://github.com/OSOceanAcoustics/echopype"
+version: 0.6.4
+date-released: 2023-01-03
+url: "https://echopype.readthedocs.io"
+repository-code: "https://github.com/OSOceanAcoustics/echopype"
 preferred-citation:
   type: article
   authors:
@@ -48,6 +52,7 @@ preferred-citation:
     given-names: "Valentina"
     orcid: "https://orcid.org/0000-0002-3412-0364"
   journal: "arXiv preprint arXiv:2111.00187"
+  doi: https://doi.org/10.48550/arXiv.2111.00187
   month: 10
   title: "Echopype: A Python library for interoperable and scalable processing of water column sonar data for biological information"
   year: 2021

docs/source/api.rst

@@ -72,6 +72,13 @@ qc
    :no-inheritance-diagram:
    :no-heading:
 
+mask
+^^^^
+
+.. automodapi:: echopype.mask
+   :no-inheritance-diagram:
+   :no-heading:
+
 Utilities
 ---------
 

docs/source/convert.rst

@@ -86,7 +86,7 @@ all files from the same deployment.
     by allowing users to directly write variables that may consume a large amount of memory
     into a temporary zarr store (see `#774 <https://github.com/OSOceanAcoustics/echopype/pull/774>`_).
 
-    This feature is accessible through ``open_raw`` via arguments ``offload_to_zarr`` and ``max_zarr_mb``
+    This feature is accessible through ``open_raw`` via arguments ``use_swap`` and ``max_mb``
     and is only available for the following echosounders: EK60, ES70, EK80, ES80, EA640.
     See :ref:`API reference <api-open_raw>` for usage.
     This is currently a beta feature that will benefit from user feedback.
@@ -179,8 +179,8 @@ The examples below apply equally to both methods, except as noted.
 A destination folder or file path should be specified with the ``save_path``
 argument in these methods in order to control the location of the converted files.
 If the argument is not specified, the converted ``.nc`` and ``.zarr``
-files are saved into a folder called ``temp_echopype_output`` under the
-current execution folder. This folder will be created if it doesn't already exists.
+files are saved into the directory ``~/.echopype/temp_output``.
+This folder will be created if it doesn't already exists.
 
 
 Specify metadata attributes
@@ -227,7 +227,7 @@ the platform code from the
 .. ``combine`` argument (the default is ``combine=False``). In that case,
 .. ``save_path`` must be specified explicitly. If ``save_path`` is only a filename
 .. rather than a full file path, the combined output file will be saved to the
-.. default ``temp_echopype_output`` folder.
+.. default ``~/.echopype/temp_output`` folder.
 
 .. .. code-block:: python
 

docs/source/index.md

@@ -35,6 +35,13 @@ of the NOAA Alaska Fisheries Science Center
 for providing low-level file parsing routines for
 Simrad EK60 and EK80 echosounders.
 
+## Citing echopype
+
+Please cite echopype as follows:
+
+Lee, W., Mayorga, E., Setiawan, L., Majeed, I., Nguyen, K., & Staneva, V. (2021). Echopype: A Python library for interoperable and scalable processing of water column sonar data for biological information. arXiv preprint arXiv:2111.00187
+
+Citation information and project metadata are stored in `CITATION.cff`, which uses the [Citation File Format](https://citation-file-format.github.io/).
 
 ## License
 

docs/source/open-converted.ipynb

@@ -101,11 +101,11 @@
     "    ed_list.append(ep.open_converted(converted_file))  # already converted files are lazy-loaded\n",
     "```\n",
     "\n",
-    "Finally, we apply `combine_echodata` on this list to combine all the data into a single `EchoData` object. Here, we will store the final combined form in the Zarr path `path_to/combined_echodatas.zarr` and use the client we established above: \n",
+    "Finally, we apply `combine_echodata` on this list to combine all the data into a single `EchoData` object. Here, we will store the final combined form in the Zarr path `path_to/combined_echodata.zarr` and use the client we established above: \n",
     "```python\n",
     "combined_ed = ep.combine_echodata(\n",
     "    ed_list, \n",
-    "    zarr_path='path_to/combined_echodatas.zarr', \n",
+    "    zarr_path='path_to/combined_echodata.zarr', \n",
     "    client=client\n",
     ")\n",
     "```\n",

docs/source/whats-new.md

@@ -4,11 +4,65 @@ What's new
 See [GitHub releases page](https://github.com/OSOceanAcoustics/echopype/releases) for the complete history.
 
 
+# v0.6.4 (2023 March 13)
+
+## Overview
+
+This is a release that includes important performance enhancements that allow user-provided calibration parameters for broadband and narrowband calibration, new functionalities to compute and attach split-beam angles to calibrated Sv dataset, perform frequency-differencing and masking, as well as a number of bug fixes and other improvements.
+
+## New features
+- Allow passing in `cal_params` as a dictionary for narrowband and broadband calibration (#955)
+- Add default chunk encoding for zarr output (#939)
+- Add `add_splitbeam_angle` function to the `consolidate` subpackage (#916, #971)
+- Add a new `mask` subpackage
+  - Add `apply_mask` function to the `mask` subpackage (#912)
+  - Add `frequency-differencing` function to the `mask` subpackage (#901)
+- Allow selection of a subset of channels when combining multiple echodata objects (#892)
+  - This is done via the added `channel_selection` input argument to `combine_echodata`
+- Add default consolidated flag for `echodata.to_zarr` (#855)
+- Generalize and improve efficiency of `compute_MVBS` (#878)
+  - Allow `echo_range` that vary with `ping_time`
+  - Allow `Sv` data that are dask arrays
+- Add `.nbytes` to obtain that data size of an `echodata` object (#874)
+- Add new data variables from the raw data files to facilitate EK80 calibration (#944)
+  - transceiver type, transmit impedance, receive impedance, receiver sampling frequency
+
+## Enhancements and other changes
+- Unify the order of dimensions of `echo_range` across all sonar models (#968)
+- Create and use the default echopype home directory (#896, #954)
+  - This default directory is at `~/.echopype`
+- Refactor the `calibrate` subpackage (#904)
+  - routines for `env_params` and `cal_params` intake and range computation are now in different modules rather than methods in each classes
+  - Tidy up `cal_params` related routines (#953)
+  - Revise `env_params` related routines (#952)
+- Improving pulse compressed broadband Sv computation and echo range computation (#944)
+  - The resulting values are tested against pyEcholab and Echoview outputs
+  - Note there is unresolved discrepancy between pyEcholab/Echopype outputs with Echoview outputs for the first section of Sv values
+- Change input argument names for `open_raw` (#962)
+  - Change `offload_to_zarr` to `use_swap`
+  - Change `max_zarr_mb` to `max_mb`
+
+## Bug fixes
+- Handling of provenance attributes in apply-mask and add-depth, especially for testing (#930)
+- Fix bugs in combining multiple echodata objects (the `combine_echodata` function)
+  - Fix meta_source_filenames bug and enable (meta)source_filenames appending of path and list (#913)
+- Fix bugs in `env_params` intake for calibration (#952)
+
+## Infrastructure
+- Update docker build for arm64 mac silicon chips (#964)
+- Fix windows github actions workflow for utils module (#947)
+- Remove mamba dependency (#946)
+- Fix pre-commit ci and update RTD config for jupyter-book (#934)
+
+
+
+
+
 # v0.6.3 (2022 October 15)
 
 ## Overview
 
-This is a minor release that includes an important performance enhancement for combining large volumes of data residing in individual files into a single entityr, a number of bug fixes, and other smaller improvements.
+This is a minor release that includes an important performance enhancement for combining large volumes of data residing in individual files into a single entity, a number of bug fixes, and other smaller improvements.
 
 ## New features
 - Overhaul `combine_echodata` function

echopype/__init__.py

@@ -2,20 +2,25 @@
 
 from _echopype_version import version as __version__  # noqa
 
-from . import calibrate, consolidate, preprocess, utils
+from . import calibrate, consolidate, mask, preprocess, utils
 from .convert.api import open_raw
 from .echodata.api import open_converted
 from .echodata.combine import combine_echodata
+from .utils.io import init_ep_dir
 from .utils.log import verbose
 
+# Turn off verbosity for echopype
 verbose(override=True)
 
+init_ep_dir()
+
 __all__ = [
     "open_raw",
     "open_converted",
     "combine_echodata",
     "calibrate",
     "consolidate",
+    "mask",
     "preprocess",
     "utils",
     "verbose",