From 94ec11066d70f6990161778dc1cf1dd4757bb3ab Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Fri, 25 Oct 2024 15:33:18 -0400 Subject: [PATCH 01/23] Added sonar model checkers for EK60, EK80 Future releases of echopype may accommodate cases where the user is not required to provide sonar model numbers. We have here a simple checker to do that. -Added model checkers for EK60 and EK80 sonar models. --- echopype/convert/parse_ek60.py | 17 ++++++++++++++++- echopype/convert/parse_ek80.py | 17 ++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/echopype/convert/parse_ek60.py b/echopype/convert/parse_ek60.py index 88d749ea7..1ba8c2d9b 100644 --- a/echopype/convert/parse_ek60.py +++ b/echopype/convert/parse_ek60.py @@ -1,5 +1,6 @@ from .parse_base import ParseEK - +from .utils.ek_raw_io import RawSimradFile +import numpy as np class ParseEK60(ParseEK): """Class for converting data from Simrad EK60 echosounders.""" @@ -14,3 +15,17 @@ def __init__( **kwargs, ): super().__init__(file, bot_file, idx_file, storage_options, sonar_model) + +def _is_EK60(raw_file, storage_options): + """Parse raw data to check if it is from Simrad EK80 echosounder.""" + with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid: + config_datagram = fid.read(1) + config_datagram["timestamp"] = np.datetime64( + config_datagram["timestamp"].replace(tzinfo=None), "[ns]" + ) + + # Only EK80 files have configuration in self.config_datagram + if config_datagram["sounder_name"] == "ER60" or config_datagram["sounder_name"] == "EK60": + return True + else: + return False diff --git a/echopype/convert/parse_ek80.py b/echopype/convert/parse_ek80.py index 9d3932b3a..79368e755 100644 --- a/echopype/convert/parse_ek80.py +++ b/echopype/convert/parse_ek80.py @@ -1,5 +1,6 @@ from .parse_base import ParseEK - +from .utils.ek_raw_io import RawSimradFile +import numpy as np class ParseEK80(ParseEK): """Class for converting data from Simrad EK80 echosounders.""" @@ -15,3 +16,17 @@ def __init__( ): super().__init__(file, bot_file, idx_file, storage_options, sonar_model) self.environment = {} # dictionary to store environment data + +def is_EK80(raw_file, storage_options): + """Parse raw data to check if it is from Simrad EK80 echosounder.""" + with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid: + config_datagram = fid.read(1) + config_datagram["timestamp"] = np.datetime64( + config_datagram["timestamp"].replace(tzinfo=None), "[ns]" + ) + + # Only EK80 files have configuration in self.config_datagram + if "configuration" in config_datagram: + return True + else: + return False \ No newline at end of file From cae865e4383918c63ab44c97ddcedd6fdb199d5d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 25 Oct 2024 19:47:31 +0000 Subject: [PATCH 02/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- echopype/convert/parse_ek60.py | 7 +++++-- echopype/convert/parse_ek80.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/echopype/convert/parse_ek60.py b/echopype/convert/parse_ek60.py index 1ba8c2d9b..67a0451e5 100644 --- a/echopype/convert/parse_ek60.py +++ b/echopype/convert/parse_ek60.py @@ -1,6 +1,8 @@ +import numpy as np + from .parse_base import ParseEK from .utils.ek_raw_io import RawSimradFile -import numpy as np + class ParseEK60(ParseEK): """Class for converting data from Simrad EK60 echosounders.""" @@ -15,7 +17,8 @@ def __init__( **kwargs, ): super().__init__(file, bot_file, idx_file, storage_options, sonar_model) - + + def _is_EK60(raw_file, storage_options): """Parse raw data to check if it is from Simrad EK80 echosounder.""" with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid: diff --git a/echopype/convert/parse_ek80.py b/echopype/convert/parse_ek80.py index 79368e755..7f3e8c2bf 100644 --- a/echopype/convert/parse_ek80.py +++ b/echopype/convert/parse_ek80.py @@ -1,6 +1,8 @@ +import numpy as np + from .parse_base import ParseEK from .utils.ek_raw_io import RawSimradFile -import numpy as np + class ParseEK80(ParseEK): """Class for converting data from Simrad EK80 echosounders.""" @@ -17,6 +19,7 @@ def __init__( super().__init__(file, bot_file, idx_file, storage_options, sonar_model) self.environment = {} # dictionary to store environment data + def is_EK80(raw_file, storage_options): """Parse raw data to check if it is from Simrad EK80 echosounder.""" with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid: @@ -29,4 +32,4 @@ def is_EK80(raw_file, storage_options): if "configuration" in config_datagram: return True else: - return False \ No newline at end of file + return False From 2cb262542adf1121721311c841b980540e33b5bc Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Fri, 25 Oct 2024 15:53:46 -0400 Subject: [PATCH 03/23] Update parse_ek60.py --- echopype/convert/parse_ek60.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/echopype/convert/parse_ek60.py b/echopype/convert/parse_ek60.py index 67a0451e5..d29dea2bc 100644 --- a/echopype/convert/parse_ek60.py +++ b/echopype/convert/parse_ek60.py @@ -19,7 +19,7 @@ def __init__( super().__init__(file, bot_file, idx_file, storage_options, sonar_model) -def _is_EK60(raw_file, storage_options): +def is_EK60(raw_file, storage_options): """Parse raw data to check if it is from Simrad EK80 echosounder.""" with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid: config_datagram = fid.read(1) From 7bf89a1d56135fb23c7b0a61851ad4ba18381324 Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Mon, 28 Oct 2024 10:19:05 -0400 Subject: [PATCH 04/23] Added test cases for EK60 and EK80 model validation Added test cases for EK60 and EK80 raw parser functions to validate their identification of echosounder types through raw data alone. --- echopype/convert/parse_ek60.py | 9 +++------ echopype/convert/parse_ek80.py | 9 +++------ echopype/tests/convert/test_convert_ek60.py | 18 +++++++++++++++++- echopype/tests/convert/test_convert_ek80.py | 12 +++++++++++- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/echopype/convert/parse_ek60.py b/echopype/convert/parse_ek60.py index d29dea2bc..ee09f876f 100644 --- a/echopype/convert/parse_ek60.py +++ b/echopype/convert/parse_ek60.py @@ -20,15 +20,12 @@ def __init__( def is_EK60(raw_file, storage_options): - """Parse raw data to check if it is from Simrad EK80 echosounder.""" + """Check if a raw data file is from Simrad EK60 echosounder.""" with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid: config_datagram = fid.read(1) config_datagram["timestamp"] = np.datetime64( config_datagram["timestamp"].replace(tzinfo=None), "[ns]" ) - # Only EK80 files have configuration in self.config_datagram - if config_datagram["sounder_name"] == "ER60" or config_datagram["sounder_name"] == "EK60": - return True - else: - return False + # Return True if the sounder name matches "ER60" or "EK60" + return config_datagram["sounder_name"] in {"ER60", "EK60"} diff --git a/echopype/convert/parse_ek80.py b/echopype/convert/parse_ek80.py index 7f3e8c2bf..66a1695df 100644 --- a/echopype/convert/parse_ek80.py +++ b/echopype/convert/parse_ek80.py @@ -21,15 +21,12 @@ def __init__( def is_EK80(raw_file, storage_options): - """Parse raw data to check if it is from Simrad EK80 echosounder.""" + """Check if a raw data file is from Simrad EK80 echosounder.""" with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid: config_datagram = fid.read(1) config_datagram["timestamp"] = np.datetime64( config_datagram["timestamp"].replace(tzinfo=None), "[ns]" ) - # Only EK80 files have configuration in self.config_datagram - if "configuration" in config_datagram: - return True - else: - return False + # Return True if "configuration" exists in config_datagram + return "configuration" in config_datagram diff --git a/echopype/tests/convert/test_convert_ek60.py b/echopype/tests/convert/test_convert_ek60.py index a77481380..5ad780634 100644 --- a/echopype/tests/convert/test_convert_ek60.py +++ b/echopype/tests/convert/test_convert_ek60.py @@ -6,7 +6,7 @@ import pytest from echopype import open_raw -from echopype.convert import ParseEK60 +from echopype.convert import ParseEK60, is_EK60 @pytest.fixture @@ -264,3 +264,19 @@ def test_converting_ek60_raw_with_missing_channel_power(): # Check that all empty power channels do not exist in the EchoData Beam group for _, empty_power_channel_name in empty_power_chs.items(): assert empty_power_channel_name not in ed["Sonar/Beam_group1"]["channel"] + + +def test_is_EK60_ek60_file(): + # Replace with the path to a valid EK60 test file + test_file_path = "path/to/ek60_test_file.raw" + assert is_EK60(test_file_path, storage_options={}) == True + +def test_is_EK60_er60_file(): + # Replace with the path to a valid ER60 test file + test_file_path = "path/to/er60_test_file.raw" + assert is_EK60(test_file_path, storage_options={}) == True + +def test_is_EK60_non_ek60_file(): + # Replace with the path to a non-EK60/ER60 test file + test_file_path = "path/to/non_ek60_test_file.raw" + assert is_EK60(test_file_path, storage_options={}) == False \ No newline at end of file diff --git a/echopype/tests/convert/test_convert_ek80.py b/echopype/tests/convert/test_convert_ek80.py index c84116c68..bf8b8fa3a 100644 --- a/echopype/tests/convert/test_convert_ek80.py +++ b/echopype/tests/convert/test_convert_ek80.py @@ -7,7 +7,7 @@ from echopype import open_raw, open_converted from echopype.testing import TEST_DATA_FOLDER -from echopype.convert.parse_ek80 import ParseEK80 +from echopype.convert.parse_ek80 import ParseEK80, is_EK80 from echopype.convert.set_groups_ek80 import WIDE_BAND_TRANS, PULSE_COMPRESS, FILTER_IMAG, FILTER_REAL, DECIMATION @@ -531,3 +531,13 @@ def test_parse_ek80_with_invalid_env_datagrams(): env_var = ed["Environment"][var] assert env_var.notnull().all() and env_var.dtype == np.float64 + +def test_is_EK80_ek80_file(): + # Replace with the path to a valid EK80 test file that includes "configuration" + test_file_path = "path/to/ek80_test_file.raw" + assert is_EK80(test_file_path, storage_options={}) == True + +def test_is_EK80_non_ek80_file(): + # Replace with the path to a test file without "configuration" key + test_file_path = "path/to/non_ek80_test_file.raw" + assert is_EK80(test_file_path, storage_options={}) == False \ No newline at end of file From 1affd2d5c221a1f552f3d83c2f0fae4c4a82c36b Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Mon, 28 Oct 2024 10:48:06 -0400 Subject: [PATCH 05/23] Fixed the tests. The test failed because a proper path wasn't provided to them. This commit should fix that. --- echopype/tests/convert/test_convert_ek60.py | 4 ++-- echopype/tests/convert/test_convert_ek80.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/echopype/tests/convert/test_convert_ek60.py b/echopype/tests/convert/test_convert_ek60.py index 5ad780634..78b3ff30f 100644 --- a/echopype/tests/convert/test_convert_ek60.py +++ b/echopype/tests/convert/test_convert_ek60.py @@ -273,10 +273,10 @@ def test_is_EK60_ek60_file(): def test_is_EK60_er60_file(): # Replace with the path to a valid ER60 test file - test_file_path = "path/to/er60_test_file.raw" + test_file_path = "echopype/test_data/ek60/L0003-D20040909-T161906-EK60.raw" assert is_EK60(test_file_path, storage_options={}) == True def test_is_EK60_non_ek60_file(): # Replace with the path to a non-EK60/ER60 test file - test_file_path = "path/to/non_ek60_test_file.raw" + test_file_path = "echopype/test_data/ek80/D20170912-T234910.raw" assert is_EK60(test_file_path, storage_options={}) == False \ No newline at end of file diff --git a/echopype/tests/convert/test_convert_ek80.py b/echopype/tests/convert/test_convert_ek80.py index bf8b8fa3a..fe9001401 100644 --- a/echopype/tests/convert/test_convert_ek80.py +++ b/echopype/tests/convert/test_convert_ek80.py @@ -534,10 +534,10 @@ def test_parse_ek80_with_invalid_env_datagrams(): def test_is_EK80_ek80_file(): # Replace with the path to a valid EK80 test file that includes "configuration" - test_file_path = "path/to/ek80_test_file.raw" + test_file_path = "echopype/test_data/ek80/D20170912-T234910.raw" assert is_EK80(test_file_path, storage_options={}) == True def test_is_EK80_non_ek80_file(): # Replace with the path to a test file without "configuration" key - test_file_path = "path/to/non_ek80_test_file.raw" + test_file_path = "echopype/test_data/ek60/L0003-D20040909-T161906-EK60.raw" assert is_EK80(test_file_path, storage_options={}) == False \ No newline at end of file From 2abe8be1d4658cc005a801220e7a8f2529443251 Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Mon, 28 Oct 2024 10:58:38 -0400 Subject: [PATCH 06/23] Fixed an import error I forgot to import the outside-of-class functions in the __init__. Consider this fixed. --- echopype/convert/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/echopype/convert/__init__.py b/echopype/convert/__init__.py index 78c4e2859..b8f6b6520 100644 --- a/echopype/convert/__init__.py +++ b/echopype/convert/__init__.py @@ -14,8 +14,8 @@ from .parse_azfp import ParseAZFP from .parse_azfp6 import ParseAZFP6 from .parse_base import ParseBase -from .parse_ek60 import ParseEK60 -from .parse_ek80 import ParseEK80 +from .parse_ek60 import ParseEK60, is_EK60 +from .parse_ek80 import ParseEK80, is_EK80 from .set_groups_ad2cp import SetGroupsAd2cp from .set_groups_azfp import SetGroupsAZFP from .set_groups_azfp6 import SetGroupsAZFP6 From fa7dc1c9615429bfd29b965ea32e38abff436dd2 Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Mon, 28 Oct 2024 11:34:16 -0400 Subject: [PATCH 07/23] Test fix + Update to documentation for Docker and sudo privileges - Added notes about the potential need to remove sudo privileges for some users. - Improved tests by adding an exception handler for key errors, returning False when encountered. --- docs/source/contributing.md | 18 +++++++++++++++++- echopype/convert/parse_ek60.py | 7 +++++-- echopype/tests/convert/test_convert_ek60.py | 2 +- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/source/contributing.md b/docs/source/contributing.md index 7fca175f4..dd0d72d6e 100644 --- a/docs/source/contributing.md +++ b/docs/source/contributing.md @@ -81,7 +81,23 @@ One can replace `conda` with `mamba` in the above commands when creating the env Currently, test data are stored in a private Google Drive folder and made available via the [`cormorack/http`](https://hub.docker.com/r/cormorack/http) -Docker image on Docker hub. +Docker image on Docker hub. There’s no need to pull directly from Docker Hub, as the scripts below will handle that for you. For Linux Users: If you run into an issue where accessing the Docker daemon requires sudo privileges, which may not suit your environment, you can adjust your settings to allow non-root access to Docker: + +```shell +sudo groupadd docker +``` + +Add the current user to the Docker group +```shell +sudo gpasswd -a $USER docker +``` + +Restart the docker daemon +```shell +sudo service docker restart +``` + +You might also need to restart your computer if the steps above don't resolve the issue. The image is rebuilt daily when new test data are added. If your tests require adding new test data, ping the maintainers (@leewujung, @ctuguinay) to get them added to the the Google Drive. diff --git a/echopype/convert/parse_ek60.py b/echopype/convert/parse_ek60.py index ee09f876f..2fb0089fe 100644 --- a/echopype/convert/parse_ek60.py +++ b/echopype/convert/parse_ek60.py @@ -27,5 +27,8 @@ def is_EK60(raw_file, storage_options): config_datagram["timestamp"].replace(tzinfo=None), "[ns]" ) - # Return True if the sounder name matches "ER60" or "EK60" - return config_datagram["sounder_name"] in {"ER60", "EK60"} + try: + # Return True if the sounder name matches "ER60" or "EK60" + return config_datagram["sounder_name"] in {"ER60", "EK60"} + except KeyError as e: + False diff --git a/echopype/tests/convert/test_convert_ek60.py b/echopype/tests/convert/test_convert_ek60.py index 78b3ff30f..6336134bf 100644 --- a/echopype/tests/convert/test_convert_ek60.py +++ b/echopype/tests/convert/test_convert_ek60.py @@ -268,7 +268,7 @@ def test_converting_ek60_raw_with_missing_channel_power(): def test_is_EK60_ek60_file(): # Replace with the path to a valid EK60 test file - test_file_path = "path/to/ek60_test_file.raw" + test_file_path = "echopype/test_data/ek60/L0003-D20040909-T161906-EK60.raw" assert is_EK60(test_file_path, storage_options={}) == True def test_is_EK60_er60_file(): From adb51112696e22c5fddcdbb3e2b5855332fdf471 Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Mon, 28 Oct 2024 11:40:42 -0400 Subject: [PATCH 08/23] Update parse_ek60.py Syntax error fixed. --- echopype/convert/parse_ek60.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/echopype/convert/parse_ek60.py b/echopype/convert/parse_ek60.py index 2fb0089fe..44a06fa08 100644 --- a/echopype/convert/parse_ek60.py +++ b/echopype/convert/parse_ek60.py @@ -31,4 +31,4 @@ def is_EK60(raw_file, storage_options): # Return True if the sounder name matches "ER60" or "EK60" return config_datagram["sounder_name"] in {"ER60", "EK60"} except KeyError as e: - False + return False From 8967c3e45f9f173b2aaa12c92caeb01b937a9a29 Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Mon, 28 Oct 2024 11:52:36 -0400 Subject: [PATCH 09/23] Refactor error handling: Removed unnecessary 'e' variable The 'e' variable commonly used in error handling was not utilized, so I removed it. This change caused tests to fail, prompting its removal. --- echopype/convert/parse_ek60.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/echopype/convert/parse_ek60.py b/echopype/convert/parse_ek60.py index 44a06fa08..eb97b79be 100644 --- a/echopype/convert/parse_ek60.py +++ b/echopype/convert/parse_ek60.py @@ -30,5 +30,5 @@ def is_EK60(raw_file, storage_options): try: # Return True if the sounder name matches "ER60" or "EK60" return config_datagram["sounder_name"] in {"ER60", "EK60"} - except KeyError as e: + except KeyError: return False From 53367cde6d8ba7e370278ac8dc2e9433da252f4d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:49:29 -0700 Subject: [PATCH 10/23] chore(deps): bump actions/cache from 4.1.1 to 4.1.2 (#1400) Bumps [actions/cache](https://github.com/actions/cache) from 4.1.1 to 4.1.2. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v4.1.1...v4.1.2) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/windows.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml index 233ddb835..290579389 100644 --- a/.github/workflows/windows.yaml +++ b/.github/workflows/windows.yaml @@ -51,7 +51,7 @@ jobs: python-version: ${{ matrix.python-version }} architecture: x64 - name: Cache conda - uses: actions/cache@v4.1.1 + uses: actions/cache@v4.1.2 env: # Increase this value to reset cache if '.ci_helpers/py{0}.yaml' has not changed CACHE_NUMBER: 0 From ca2adf9afb9aff17959001e620be8a8425b5a95c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:49:39 -0700 Subject: [PATCH 11/23] chore(deps): bump actions/setup-python from 5.2.0 to 5.3.0 (#1401) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.2.0 to 5.3.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.2.0...v5.3.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yaml | 2 +- .github/workflows/packit.yaml | 4 ++-- .github/workflows/pr.yaml | 2 +- .github/workflows/pypi.yaml | 4 ++-- .github/workflows/windows.yaml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 819176595..1591ae05d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -42,7 +42,7 @@ jobs: run: | echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV - name: Set up Python - uses: actions/setup-python@v5.2.0 + uses: actions/setup-python@v5.3.0 with: python-version: ${{ matrix.python-version }} - name: Upgrade pip diff --git a/.github/workflows/packit.yaml b/.github/workflows/packit.yaml index 472d9bf30..fb120efef 100644 --- a/.github/workflows/packit.yaml +++ b/.github/workflows/packit.yaml @@ -20,7 +20,7 @@ jobs: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5.2.0 + uses: actions/setup-python@v5.3.0 with: python-version: 3.9 @@ -52,7 +52,7 @@ jobs: needs: build-artifact runs-on: ubuntu-20.04 steps: - - uses: actions/setup-python@v5.2.0 + - uses: actions/setup-python@v5.3.0 name: Install Python with: python-version: 3.9 diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 155a8c72f..1afceeaf2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -35,7 +35,7 @@ jobs: with: fetch-depth: 0 # Fetch all history for all branches and tags. - name: Set up Python - uses: actions/setup-python@v5.2.0 + uses: actions/setup-python@v5.3.0 with: python-version: ${{ matrix.python-version }} - name: Upgrade pip diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml index 8cdf870e1..d01fbcfbf 100644 --- a/.github/workflows/pypi.yaml +++ b/.github/workflows/pypi.yaml @@ -24,7 +24,7 @@ jobs: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5.2.0 + uses: actions/setup-python@v5.3.0 with: python-version: 3.9 @@ -56,7 +56,7 @@ jobs: needs: build-artifact runs-on: ubuntu-20.04 steps: - - uses: actions/setup-python@v5.2.0 + - uses: actions/setup-python@v5.3.0 name: Install Python with: python-version: 3.9 diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml index 290579389..04b126036 100644 --- a/.github/workflows/windows.yaml +++ b/.github/workflows/windows.yaml @@ -46,7 +46,7 @@ jobs: # Check data endpoint curl http://localhost:8080/data/ - name: Setup Python - uses: actions/setup-python@v5.2.0 + uses: actions/setup-python@v5.3.0 with: python-version: ${{ matrix.python-version }} architecture: x64 From 907424daba6255afe9fcf6a65c838e989817cd3f Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Thu, 31 Oct 2024 09:36:45 -0400 Subject: [PATCH 12/23] Made tests better so that they run through all test files in the test file directory. For EK60 only. --- echopype/tests/convert/test_convert_ek60.py | 35 +++++++++++++-------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/echopype/tests/convert/test_convert_ek60.py b/echopype/tests/convert/test_convert_ek60.py index 6336134bf..7909aa456 100644 --- a/echopype/tests/convert/test_convert_ek60.py +++ b/echopype/tests/convert/test_convert_ek60.py @@ -1,5 +1,5 @@ import warnings - +import glob import numpy as np import pandas as pd from scipy.io import loadmat @@ -266,17 +266,26 @@ def test_converting_ek60_raw_with_missing_channel_power(): assert empty_power_channel_name not in ed["Sonar/Beam_group1"]["channel"] -def test_is_EK60_ek60_file(): - # Replace with the path to a valid EK60 test file - test_file_path = "echopype/test_data/ek60/L0003-D20040909-T161906-EK60.raw" - assert is_EK60(test_file_path, storage_options={}) == True +def test_is_EK60_ek60_files(): + # Collect all .raw files in the ek60 directory + ek60_files = glob.glob("echopype/test_data/ek60/*.raw") + + # Check that each file in ek60 is identified as EK60 + for test_file_path in ek60_files: + assert is_EK60(test_file_path, storage_options={}) == True -def test_is_EK60_er60_file(): - # Replace with the path to a valid ER60 test file - test_file_path = "echopype/test_data/ek60/L0003-D20040909-T161906-EK60.raw" - assert is_EK60(test_file_path, storage_options={}) == True +def test_is_EK60_er60_files(): + # Collect all .raw files in the ek60 directory (assuming ER60 files are also here) + er60_files = glob.glob("echopype/test_data/ek60/*.raw") + + # Check that each file in ek60 is identified as EK60 + for test_file_path in er60_files: + assert is_EK60(test_file_path, storage_options={}) == True -def test_is_EK60_non_ek60_file(): - # Replace with the path to a non-EK60/ER60 test file - test_file_path = "echopype/test_data/ek80/D20170912-T234910.raw" - assert is_EK60(test_file_path, storage_options={}) == False \ No newline at end of file +def test_is_EK60_non_ek60_files(): + # Collect all .raw files in the ek80 directory (non-EK60 files) + ek80_files = glob.glob("echopype/test_data/ek80/*.raw") + + # Check that each file in ek80 is not identified as EK60 + for test_file_path in ek80_files: + assert is_EK60(test_file_path, storage_options={}) == False \ No newline at end of file From 8a2517d52d1190cf58436fb43fcc9e533fa03368 Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Thu, 31 Oct 2024 09:51:51 -0400 Subject: [PATCH 13/23] Added better tests for ek80 checker functions. Added better tests for ek80 checker functions. --- echopype/tests/convert/test_convert_ek60.py | 3 +++ echopype/tests/convert/test_convert_ek80.py | 28 +++++++++++++-------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/echopype/tests/convert/test_convert_ek60.py b/echopype/tests/convert/test_convert_ek60.py index 7909aa456..1a4c2f681 100644 --- a/echopype/tests/convert/test_convert_ek60.py +++ b/echopype/tests/convert/test_convert_ek60.py @@ -267,6 +267,7 @@ def test_converting_ek60_raw_with_missing_channel_power(): def test_is_EK60_ek60_files(): + """Check that EK60 files are identified as EK60""" # Collect all .raw files in the ek60 directory ek60_files = glob.glob("echopype/test_data/ek60/*.raw") @@ -275,6 +276,7 @@ def test_is_EK60_ek60_files(): assert is_EK60(test_file_path, storage_options={}) == True def test_is_EK60_er60_files(): + """Check that ER60 files are identified as EK60""" # Collect all .raw files in the ek60 directory (assuming ER60 files are also here) er60_files = glob.glob("echopype/test_data/ek60/*.raw") @@ -283,6 +285,7 @@ def test_is_EK60_er60_files(): assert is_EK60(test_file_path, storage_options={}) == True def test_is_EK60_non_ek60_files(): + """Check that non-EK60 files are not identified as EK60""" # Collect all .raw files in the ek80 directory (non-EK60 files) ek80_files = glob.glob("echopype/test_data/ek80/*.raw") diff --git a/echopype/tests/convert/test_convert_ek80.py b/echopype/tests/convert/test_convert_ek80.py index fe9001401..c76862c52 100644 --- a/echopype/tests/convert/test_convert_ek80.py +++ b/echopype/tests/convert/test_convert_ek80.py @@ -1,5 +1,5 @@ import shutil - +import glob import pytest import numpy as np import pandas as pd @@ -532,12 +532,20 @@ def test_parse_ek80_with_invalid_env_datagrams(): assert env_var.notnull().all() and env_var.dtype == np.float64 -def test_is_EK80_ek80_file(): - # Replace with the path to a valid EK80 test file that includes "configuration" - test_file_path = "echopype/test_data/ek80/D20170912-T234910.raw" - assert is_EK80(test_file_path, storage_options={}) == True - -def test_is_EK80_non_ek80_file(): - # Replace with the path to a test file without "configuration" key - test_file_path = "echopype/test_data/ek60/L0003-D20040909-T161906-EK60.raw" - assert is_EK80(test_file_path, storage_options={}) == False \ No newline at end of file +def test_is_EK80_ek80_files(): + """Test that EK80 files are identified as EK80.""" + # Collect all .raw files in the ek80 directory + ek80_files = glob.glob("echopype/test_data/ek80/*.raw") + + # Check that each file in ek80 is identified as EK80 + for test_file_path in ek80_files: + assert is_EK80(test_file_path, storage_options={}) == True + +def test_is_EK80_non_ek80_files(): + """Test that non-EK80 files are not identified as EK80.""" + # Collect all .raw files in the ek60 directory (non-EK80 files) + ek60_files = glob.glob("echopype/test_data/ek60/*.raw") + + # Check that each file in ek60 is not identified as EK80 + for test_file_path in ek60_files: + assert is_EK80(test_file_path, storage_options={}) == False \ No newline at end of file From 01ceb52b0685077218a5a31cff5adba670f60f62 Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Thu, 31 Oct 2024 15:31:59 -0400 Subject: [PATCH 14/23] Added the rest of the sonar checkers. Added the rest of the sonar checkers. --- echopype/convert/parse_ad2cp.py | 22 +++++++++++++++++++++ echopype/convert/parse_azfp.py | 34 +++++++++++++++++++++++++++++++++ echopype/convert/parse_azfp6.py | 22 +++++++++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/echopype/convert/parse_ad2cp.py b/echopype/convert/parse_ad2cp.py index 51ebc9848..27fac68e6 100644 --- a/echopype/convert/parse_ad2cp.py +++ b/echopype/convert/parse_ad2cp.py @@ -1853,3 +1853,25 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec DataRecordType.ECHOSOUNDER_RAW_TRANSMIT: ECHOSOUNDER_RAW_DATA_RECORD_FORMAT, DataRecordType.STRING: STRING_DATA_RECORD_FORMAT, } + + +def is_AD2CP(raw_file): + """ + Check if the provided file has a .ad2cp extension. + + Parameters: + raw_file (str): The name of the file to check. + + Returns: + bool: True if the file has a .ad2cp extension, False otherwise. + """ + + # Check if the input is a string + if not isinstance(raw_file, str): + return False # Return False if the input is not a string + + # Use the str.lower() method to check for the .ad2cp extension + has_ad2cp_extension = raw_file.lower().endswith('.ad2cp') + + # Return the result of the check + return has_ad2cp_extension \ No newline at end of file diff --git a/echopype/convert/parse_azfp.py b/echopype/convert/parse_azfp.py index 1e307ab9e..b5f9254b5 100644 --- a/echopype/convert/parse_azfp.py +++ b/echopype/convert/parse_azfp.py @@ -570,3 +570,37 @@ def _calc_Sv_offset(freq, pulse_len): ) return SV_OFFSET[freq][pulse_len] + +def is_AZFP(raw_file): + """ + Check if the specified XML file contains an with string="AZFP". + + Parameters: + raw_file (str): The base name of the XML file (with or without extension). + + Returns: + bool: True if with string="AZFP" is found, False otherwise. + """ + + + # Check if the filename ends with .xml or .XML, and strip the extension if it does + base_filename = raw_file.rstrip('.xml').rstrip('.XML') + + # Create a list of possible filenames with both extensions + possible_files = [f"{base_filename}.xml", f"{base_filename}.XML"] + + for full_filename in possible_files: + if os.path.isfile(full_filename): + try: + # Parse the XML file + tree = ET.parse(full_filename) + root = tree.getroot() + + # Check for elements + for instrument in root.findall('.//InstrumentType'): + if instrument.get('string') == 'AZFP': + return True + except ET.ParseError: + print(f"Error parsing the XML file: {full_filename}.") + + return False \ No newline at end of file diff --git a/echopype/convert/parse_azfp6.py b/echopype/convert/parse_azfp6.py index aa98eca55..a7313aad7 100644 --- a/echopype/convert/parse_azfp6.py +++ b/echopype/convert/parse_azfp6.py @@ -689,3 +689,25 @@ def _calc_Sv_offset(freq, pulse_len): ) return SV_OFFSET[freq][pulse_len] + + +def is_AZFP6(raw_file): + """ + Check if the provided file has a .azfp extension. + + Parameters: + raw_file (str): The name of the file to check. + + Returns: + bool: True if the file has a .azfp extension, False otherwise. + """ + + # Check if the input is a string + if not isinstance(raw_file, str): + return False # Return False if the input is not a string + + # Use the str.lower() method to check for the .azfp extension + has_azfp_extension = raw_file.lower().endswith('.azfp') + + # Return the result of the check + return has_azfp_extension \ No newline at end of file From 05345893aedc582c39f485bdc3a87f6a97274254 Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Fri, 1 Nov 2024 09:44:06 -0400 Subject: [PATCH 15/23] Update test_convert_azfp6.py --- echopype/tests/convert/test_convert_azfp6.py | 24 ++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/echopype/tests/convert/test_convert_azfp6.py b/echopype/tests/convert/test_convert_azfp6.py index 29122a0bb..4ec705ca5 100644 --- a/echopype/tests/convert/test_convert_azfp6.py +++ b/echopype/tests/convert/test_convert_azfp6.py @@ -4,14 +4,14 @@ - verify echopype converted files against those from AZFP Matlab scripts and EchoView - convert AZFP file with different range settings across frequency """ - +import glob import numpy as np import pandas as pd from datetime import datetime, timedelta from scipy.io import loadmat from echopype import open_raw import pytest -from echopype.convert.parse_azfp6 import ParseAZFP6 +from echopype.convert.parse_azfp6 import ParseAZFP6, is_AZFP6 @pytest.fixture @@ -166,3 +166,23 @@ def test_convert_azfp_02a_gps_lat_long(azfp_path): ) check_platform_required_scalar_vars(echodata) + + + +def test_is_AZFP6_valid_files(): + """Test that .azfp files are identified as valid AZFP files.""" + # Collect all .azfp files in the test directory + azfp_files = glob.glob("echopype/test_data/azfp6/*.azfp") + + # Check that each file in azfp is identified as valid AZFP + for test_file_path in azfp_files: + assert is_AZFP6(test_file_path) == True + +def test_is_AZFP6_invalid_files(): + """Test that non-.azfp files are not identified as valid AZFP files.""" + # Collect all non-.azfp files in the test directory + non_azfp_files = glob.glob("echopype/test_data/azfp/*") + + # Check that each file in non_azfp is not identified as valid AZFP + for test_file_path in non_azfp_files: + assert is_AZFP6(test_file_path) == False \ No newline at end of file From 853d01b5caede2db4b22a3b110e1f71ad85e57fe Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Fri, 1 Nov 2024 09:48:48 -0400 Subject: [PATCH 16/23] Added more tests Added more tests for azfp and ad2cp. --- echopype/tests/convert/test_convert_ad2cp.py | 21 ++++++++++++++++++++ echopype/tests/convert/test_convert_azfp.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/echopype/tests/convert/test_convert_ad2cp.py b/echopype/tests/convert/test_convert_ad2cp.py index bf7656f16..a92b49425 100644 --- a/echopype/tests/convert/test_convert_ad2cp.py +++ b/echopype/tests/convert/test_convert_ad2cp.py @@ -12,10 +12,12 @@ import pytest from tempfile import TemporaryDirectory from pathlib import Path +import glob from echopype import open_raw, open_converted from echopype.testing import TEST_DATA_FOLDER +from echopype.convert.parse_ad2cp import is_AD2CP @pytest.fixture def ocean_contour_export_dir(test_path): @@ -260,3 +262,22 @@ def _check_raw_output( atol=absolute_tolerance, ) base.close() + + +def test_is_AD2CP_valid_files(): + """Test that .ad2cp files are identified as valid AD2CP files.""" + # Collect all .ad2cp files in the test directory + ad2cp_files = glob.glob("test_data/ad2cp/*.ad2cp") + + # Check that each file in ad2cp is identified as valid AD2CP + for test_file_path in ad2cp_files: + assert is_AD2CP(test_file_path) == True + +def test_is_AD2CP_invalid_files(): + """Test that non-.ad2cp files are not identified as valid AD2CP files.""" + # Collect all non-.ad2cp files in the test directory + non_ad2cp_files = glob.glob("echopype/test_data/azfp6/*") + + # Check that each file in non_ad2cp is not identified as valid AD2CP + for test_file_path in non_ad2cp_files: + assert is_AD2CP(test_file_path) == False \ No newline at end of file diff --git a/echopype/tests/convert/test_convert_azfp.py b/echopype/tests/convert/test_convert_azfp.py index f0a0451b9..6961539ac 100644 --- a/echopype/tests/convert/test_convert_azfp.py +++ b/echopype/tests/convert/test_convert_azfp.py @@ -10,7 +10,7 @@ from scipy.io import loadmat from echopype import open_raw import pytest -from echopype.convert.parse_azfp import ParseAZFP +from echopype.convert.parse_azfp import ParseAZFP, is_AZFP @pytest.fixture From d3467f5c57f6b171c1ee215088f8b54476af8fc2 Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Fri, 1 Nov 2024 09:55:31 -0400 Subject: [PATCH 17/23] Update test_convert_azfp.py --- echopype/tests/convert/test_convert_azfp.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/echopype/tests/convert/test_convert_azfp.py b/echopype/tests/convert/test_convert_azfp.py index 6961539ac..6767de13e 100644 --- a/echopype/tests/convert/test_convert_azfp.py +++ b/echopype/tests/convert/test_convert_azfp.py @@ -4,6 +4,8 @@ - verify echopype converted files against those from AZFP Matlab scripts and EchoView - convert AZFP file with different range settings across frequency """ +import xml.etree.ElementTree as ET +import glob import numpy as np import pandas as pd @@ -263,3 +265,21 @@ def test_load_parse_azfp_xml(azfp_path): assert parseAZFP.parameters['pulse_len_phase2'] == [0, 0, 0, 0] assert parseAZFP.parameters['range_samples_phase1'] == [8273, 8273, 8273, 8273] assert parseAZFP.parameters['range_samples_phase2'] == [2750, 2750, 2750, 2750] + + + +def test_is_AZFP_valid_files(): + """Test that XML files with are identified as valid AZFP files.""" + # Collect all valid XML files in the test directory + valid_files = glob.glob("echopype/test_data/azfp/*.xml") + glob.glob("test_data/azfp/*.XML") + + for test_file_path in valid_files: + assert is_AZFP(test_file_path) == True + +def test_is_AZFP_invalid_files(): + """Test that XML files without are not identified as valid AZFP files.""" + # Collect all invalid XML files in the test directory + invalid_files = glob.glob("echopype/test_data/azfp6/*") + + for test_file_path in invalid_files: + assert is_AZFP(test_file_path) == False \ No newline at end of file From bf73faacd8c72ff8165bd2bcf01fb37a5c868da9 Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Fri, 1 Nov 2024 10:25:22 -0400 Subject: [PATCH 18/23] Corrected some tests. Updated tests. --- echopype/convert/parse_ek60.py | 19 ++++++++++++++++++- echopype/tests/convert/test_convert_ad2cp.py | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/echopype/convert/parse_ek60.py b/echopype/convert/parse_ek60.py index 88d749ea7..0758033bd 100644 --- a/echopype/convert/parse_ek60.py +++ b/echopype/convert/parse_ek60.py @@ -1,5 +1,7 @@ -from .parse_base import ParseEK +import numpy as np +from .parse_base import ParseEK +from .utils.ek_raw_io import RawSimradFile class ParseEK60(ParseEK): """Class for converting data from Simrad EK60 echosounders.""" @@ -14,3 +16,18 @@ def __init__( **kwargs, ): super().__init__(file, bot_file, idx_file, storage_options, sonar_model) + + +def is_ER60(raw_file, storage_options): + """Check if a raw data file is from Simrad EK60 echosounder.""" + with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid: + config_datagram = fid.read(1) + config_datagram["timestamp"] = np.datetime64( + config_datagram["timestamp"].replace(tzinfo=None), "[ns]" + ) + + try: + # Return True if the sounder name matches "ER60" or "EK60" + return config_datagram["sounder_name"] in {"ER60"} + except KeyError: + return False \ No newline at end of file diff --git a/echopype/tests/convert/test_convert_ad2cp.py b/echopype/tests/convert/test_convert_ad2cp.py index a92b49425..1a737e9d7 100644 --- a/echopype/tests/convert/test_convert_ad2cp.py +++ b/echopype/tests/convert/test_convert_ad2cp.py @@ -267,7 +267,7 @@ def _check_raw_output( def test_is_AD2CP_valid_files(): """Test that .ad2cp files are identified as valid AD2CP files.""" # Collect all .ad2cp files in the test directory - ad2cp_files = glob.glob("test_data/ad2cp/*.ad2cp") + ad2cp_files = glob.glob("echopype/test_data/ad2cp/normal/*.ad2cp") # Check that each file in ad2cp is identified as valid AD2CP for test_file_path in ad2cp_files: From 4db7dd50254cb0d224f73464b40792dc97d525e9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2024 14:31:04 +0000 Subject: [PATCH 19/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- echopype/convert/parse_ad2cp.py | 8 ++++---- echopype/convert/parse_azfp.py | 16 ++++++++-------- echopype/convert/parse_azfp6.py | 8 ++++---- echopype/convert/parse_ek60.py | 6 ------ 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/echopype/convert/parse_ad2cp.py b/echopype/convert/parse_ad2cp.py index 27fac68e6..b7815f8de 100644 --- a/echopype/convert/parse_ad2cp.py +++ b/echopype/convert/parse_ad2cp.py @@ -1865,13 +1865,13 @@ def is_AD2CP(raw_file): Returns: bool: True if the file has a .ad2cp extension, False otherwise. """ - + # Check if the input is a string if not isinstance(raw_file, str): return False # Return False if the input is not a string - + # Use the str.lower() method to check for the .ad2cp extension - has_ad2cp_extension = raw_file.lower().endswith('.ad2cp') + has_ad2cp_extension = raw_file.lower().endswith(".ad2cp") # Return the result of the check - return has_ad2cp_extension \ No newline at end of file + return has_ad2cp_extension diff --git a/echopype/convert/parse_azfp.py b/echopype/convert/parse_azfp.py index b5f9254b5..445866c18 100644 --- a/echopype/convert/parse_azfp.py +++ b/echopype/convert/parse_azfp.py @@ -571,6 +571,7 @@ def _calc_Sv_offset(freq, pulse_len): return SV_OFFSET[freq][pulse_len] + def is_AZFP(raw_file): """ Check if the specified XML file contains an with string="AZFP". @@ -582,25 +583,24 @@ def is_AZFP(raw_file): bool: True if with string="AZFP" is found, False otherwise. """ - # Check if the filename ends with .xml or .XML, and strip the extension if it does - base_filename = raw_file.rstrip('.xml').rstrip('.XML') + base_filename = raw_file.rstrip(".xml").rstrip(".XML") # Create a list of possible filenames with both extensions possible_files = [f"{base_filename}.xml", f"{base_filename}.XML"] - + for full_filename in possible_files: if os.path.isfile(full_filename): try: # Parse the XML file tree = ET.parse(full_filename) root = tree.getroot() - + # Check for elements - for instrument in root.findall('.//InstrumentType'): - if instrument.get('string') == 'AZFP': + for instrument in root.findall(".//InstrumentType"): + if instrument.get("string") == "AZFP": return True except ET.ParseError: print(f"Error parsing the XML file: {full_filename}.") - - return False \ No newline at end of file + + return False diff --git a/echopype/convert/parse_azfp6.py b/echopype/convert/parse_azfp6.py index a7313aad7..9a6af7fdf 100644 --- a/echopype/convert/parse_azfp6.py +++ b/echopype/convert/parse_azfp6.py @@ -701,13 +701,13 @@ def is_AZFP6(raw_file): Returns: bool: True if the file has a .azfp extension, False otherwise. """ - + # Check if the input is a string if not isinstance(raw_file, str): return False # Return False if the input is not a string - + # Use the str.lower() method to check for the .azfp extension - has_azfp_extension = raw_file.lower().endswith('.azfp') + has_azfp_extension = raw_file.lower().endswith(".azfp") # Return the result of the check - return has_azfp_extension \ No newline at end of file + return has_azfp_extension diff --git a/echopype/convert/parse_ek60.py b/echopype/convert/parse_ek60.py index ccb954335..adf06468a 100644 --- a/echopype/convert/parse_ek60.py +++ b/echopype/convert/parse_ek60.py @@ -1,11 +1,9 @@ import numpy as np - from .parse_base import ParseEK from .utils.ek_raw_io import RawSimradFile - class ParseEK60(ParseEK): """Class for converting data from Simrad EK60 echosounders.""" @@ -21,7 +19,6 @@ def __init__( super().__init__(file, bot_file, idx_file, storage_options, sonar_model) - def is_ER60(raw_file, storage_options): """Check if a raw data file is from Simrad EK60 echosounder.""" with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid: @@ -36,9 +33,7 @@ def is_ER60(raw_file, storage_options): return False - def is_EK60(raw_file, storage_options): - """Check if a raw data file is from Simrad EK60 echosounder.""" with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid: config_datagram = fid.read(1) @@ -51,4 +46,3 @@ def is_EK60(raw_file, storage_options): return config_datagram["sounder_name"] in {"EK60"} except KeyError: return False - From 9929523aab0ecfd8369300548ffb7f73d7ac1cbd Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Fri, 1 Nov 2024 10:36:15 -0400 Subject: [PATCH 20/23] Needed some fixes for the merge. Sorry about that. --- echopype/convert/parse_ad2cp.py | 1 + echopype/convert/parse_azfp6.py | 22 ++++++++++++++++++++ echopype/tests/convert/test_convert_ad2cp.py | 3 ++- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/echopype/convert/parse_ad2cp.py b/echopype/convert/parse_ad2cp.py index 27fac68e6..062a856f8 100644 --- a/echopype/convert/parse_ad2cp.py +++ b/echopype/convert/parse_ad2cp.py @@ -8,6 +8,7 @@ from .parse_base import ParseBase + @unique class BurstAverageDataRecordVersion(Enum): """ diff --git a/echopype/convert/parse_azfp6.py b/echopype/convert/parse_azfp6.py index a7313aad7..0008bc1af 100644 --- a/echopype/convert/parse_azfp6.py +++ b/echopype/convert/parse_azfp6.py @@ -691,6 +691,28 @@ def _calc_Sv_offset(freq, pulse_len): return SV_OFFSET[freq][pulse_len] +def is_AZFP6(raw_file): + """ + Check if the provided file has a .azfp extension. + + Parameters: + raw_file (str): The name of the file to check. + + Returns: + bool: True if the file has a .azfp extension, False otherwise. + """ + + # Check if the input is a string + if not isinstance(raw_file, str): + return False # Return False if the input is not a string + + # Use the str.lower() method to check for the .azfp extension + has_azfp_extension = raw_file.lower().endswith('.azfp') + + # Return the result of the check + return has_azfp_extension + + def is_AZFP6(raw_file): """ Check if the provided file has a .azfp extension. diff --git a/echopype/tests/convert/test_convert_ad2cp.py b/echopype/tests/convert/test_convert_ad2cp.py index 1a737e9d7..262e95c05 100644 --- a/echopype/tests/convert/test_convert_ad2cp.py +++ b/echopype/tests/convert/test_convert_ad2cp.py @@ -4,7 +4,8 @@ Files under "normal" contain default data variables, whereas files under "raw" additionally contain the IQ samples. """ - +import glob +from echopype.convert import is_AD2CP import xarray as xr import numpy as np From d128ff70e01f99646a7ea72fd70b68e7992cc051 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 1 Nov 2024 14:39:10 +0000 Subject: [PATCH 21/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- echopype/convert/parse_ad2cp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/echopype/convert/parse_ad2cp.py b/echopype/convert/parse_ad2cp.py index 715e2eca4..b7815f8de 100644 --- a/echopype/convert/parse_ad2cp.py +++ b/echopype/convert/parse_ad2cp.py @@ -8,7 +8,6 @@ from .parse_base import ParseBase - @unique class BurstAverageDataRecordVersion(Enum): """ From 48d9d6b41b8220035956ad7d661c548ea6f07410 Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Fri, 1 Nov 2024 10:47:55 -0400 Subject: [PATCH 22/23] More merge fixes. More merge fixes. --- echopype/convert/parse_ek60.py | 4 +-- echopype/tests/convert/test_convert_azfp.py | 2 +- echopype/tests/convert/test_convert_ek60.py | 29 ++++++++++++++------- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/echopype/convert/parse_ek60.py b/echopype/convert/parse_ek60.py index adf06468a..6a397256d 100644 --- a/echopype/convert/parse_ek60.py +++ b/echopype/convert/parse_ek60.py @@ -28,7 +28,7 @@ def is_ER60(raw_file, storage_options): ) # Return True if the sounder name matches "ER60" try: - return config_datagram["sounder_name"] in {"ER60"} + return config_datagram["sounder_name"] in {"ER60", "EK60"} except KeyError: return False @@ -43,6 +43,6 @@ def is_EK60(raw_file, storage_options): try: # Return True if the sounder name matches "EK60" - return config_datagram["sounder_name"] in {"EK60"} + return config_datagram["sounder_name"] in {"ER60", "EK60"} except KeyError: return False diff --git a/echopype/tests/convert/test_convert_azfp.py b/echopype/tests/convert/test_convert_azfp.py index 6767de13e..b2f4d43d8 100644 --- a/echopype/tests/convert/test_convert_azfp.py +++ b/echopype/tests/convert/test_convert_azfp.py @@ -271,7 +271,7 @@ def test_load_parse_azfp_xml(azfp_path): def test_is_AZFP_valid_files(): """Test that XML files with are identified as valid AZFP files.""" # Collect all valid XML files in the test directory - valid_files = glob.glob("echopype/test_data/azfp/*.xml") + glob.glob("test_data/azfp/*.XML") + valid_files = glob.glob("echopype/test_data/azfp/*.xml") + glob.glob("echopype/test_data/azfp/*.XML") for test_file_path in valid_files: assert is_AZFP(test_file_path) == True diff --git a/echopype/tests/convert/test_convert_ek60.py b/echopype/tests/convert/test_convert_ek60.py index 1a4c2f681..9b318b00a 100644 --- a/echopype/tests/convert/test_convert_ek60.py +++ b/echopype/tests/convert/test_convert_ek60.py @@ -6,7 +6,7 @@ import pytest from echopype import open_raw -from echopype.convert import ParseEK60, is_EK60 +from echopype.convert import ParseEK60, is_EK60, is_ER60 @pytest.fixture @@ -269,26 +269,35 @@ def test_converting_ek60_raw_with_missing_channel_power(): def test_is_EK60_ek60_files(): """Check that EK60 files are identified as EK60""" # Collect all .raw files in the ek60 directory - ek60_files = glob.glob("echopype/test_data/ek60/*.raw") + ek60_files = glob.glob("echopype/test_data/ek60/from_echopy/*.raw") # Check that each file in ek60 is identified as EK60 for test_file_path in ek60_files: assert is_EK60(test_file_path, storage_options={}) == True -def test_is_EK60_er60_files(): - """Check that ER60 files are identified as EK60""" - # Collect all .raw files in the ek60 directory (assuming ER60 files are also here) - er60_files = glob.glob("echopype/test_data/ek60/*.raw") +def test_is_EK60_non_ek60_files(): + """Check that non-EK60 files are not identified as EK60""" + # Collect all .raw files in the ek80 directory (non-EK60 files) + ek80_files = glob.glob("echopype/test_data/ek80/*.raw") + + # Check that each file in ek80 is not identified as EK60 + for test_file_path in ek80_files: + assert is_EK60(test_file_path, storage_options={}) == False + +def test_is_ER60_er60_files(): + """Check that EK60 files are identified as EK60""" + # Collect all .raw files in the ek60 directory + ek60_files = glob.glob("echopype/test_data/ek60/from_echopy/*.raw") # Check that each file in ek60 is identified as EK60 - for test_file_path in er60_files: - assert is_EK60(test_file_path, storage_options={}) == True + for test_file_path in ek60_files: + assert is_ER60(test_file_path, storage_options={}) == True -def test_is_EK60_non_ek60_files(): +def test_is_ER60_non_er60_files(): """Check that non-EK60 files are not identified as EK60""" # Collect all .raw files in the ek80 directory (non-EK60 files) ek80_files = glob.glob("echopype/test_data/ek80/*.raw") # Check that each file in ek80 is not identified as EK60 for test_file_path in ek80_files: - assert is_EK60(test_file_path, storage_options={}) == False \ No newline at end of file + assert is_ER60(test_file_path, storage_options={}) == False \ No newline at end of file From 80b665609a6483a7742af041bf4a438f62fc4f2e Mon Sep 17 00:00:00 2001 From: Michael C Ryan Date: Fri, 1 Nov 2024 11:15:26 -0400 Subject: [PATCH 23/23] Update __init__.py Need to import these functions. --- echopype/convert/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/echopype/convert/__init__.py b/echopype/convert/__init__.py index b8f6b6520..d4fe935a9 100644 --- a/echopype/convert/__init__.py +++ b/echopype/convert/__init__.py @@ -10,11 +10,11 @@ """ # flake8: noqa -from .parse_ad2cp import ParseAd2cp -from .parse_azfp import ParseAZFP -from .parse_azfp6 import ParseAZFP6 +from .parse_ad2cp import ParseAd2cp, is_AD2CP +from .parse_azfp import ParseAZFP, is_AZFP +from .parse_azfp6 import ParseAZFP6, is_AZFP6 from .parse_base import ParseBase -from .parse_ek60 import ParseEK60, is_EK60 +from .parse_ek60 import ParseEK60, is_EK60, is_ER60 from .parse_ek80 import ParseEK80, is_EK80 from .set_groups_ad2cp import SetGroupsAd2cp from .set_groups_azfp import SetGroupsAZFP