Added sonar model checkers for EK60, EK80, ER60, AZFP6, AZFP, AD2CP and tests

.github/workflows/build.yaml

@@ -42,7 +42,7 @@ jobs:
         run: |
           echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
       - name: Set up Python
-        uses: actions/setup-python@v5.2.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: ${{ matrix.python-version }}
       - name: Upgrade pip

.github/workflows/packit.yaml

@@ -20,7 +20,7 @@ jobs:
         fetch-depth: 0
 
     - name: Set up Python
-      uses: actions/setup-python@v5.2.0
+      uses: actions/setup-python@v5.3.0
       with:
         python-version: 3.9
 
@@ -52,7 +52,7 @@ jobs:
     needs: build-artifact
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/setup-python@v5.2.0
+    - uses: actions/setup-python@v5.3.0
       name: Install Python
       with:
         python-version: 3.9

.github/workflows/pr.yaml

@@ -35,7 +35,7 @@ jobs:
         with:
           fetch-depth: 0 # Fetch all history for all branches and tags.
       - name: Set up Python
-        uses: actions/setup-python@v5.2.0
+        uses: actions/setup-python@v5.3.0
         with:
           python-version: ${{ matrix.python-version }}
       - name: Upgrade pip

.github/workflows/pypi.yaml

@@ -24,7 +24,7 @@ jobs:
         fetch-depth: 0
 
     - name: Set up Python
-      uses: actions/setup-python@v5.2.0
+      uses: actions/setup-python@v5.3.0
       with:
         python-version: 3.9
 
@@ -56,7 +56,7 @@ jobs:
     needs: build-artifact
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/setup-python@v5.2.0
+    - uses: actions/setup-python@v5.3.0
       name: Install Python
       with:
         python-version: 3.9

.github/workflows/windows.yaml

@@ -46,12 +46,12 @@ jobs:
             # Check data endpoint
             curl http://localhost:8080/data/
         - name: Setup Python
-          uses: actions/setup-python@v5.2.0
+          uses: actions/setup-python@v5.3.0
           with:
             python-version: ${{ matrix.python-version }}
             architecture: x64
         - name: Cache conda
-          uses: actions/[email protected].1
+          uses: actions/[email protected].2
           env:
             # Increase this value to reset cache if '.ci_helpers/py{0}.yaml' has not changed
             CACHE_NUMBER: 0

docs/source/contributing.md

@@ -81,7 +81,23 @@ One can replace `conda` with `mamba` in the above commands when creating the env
 
 Currently, test data are stored in a private Google Drive folder and
 made available via the [`cormorack/http`](https://hub.docker.com/r/cormorack/http)
-Docker image on Docker hub.
+Docker image on Docker hub. There’s no need to pull directly from Docker Hub, as the scripts below will handle that for you. For Linux Users: If you run into an issue where accessing the Docker daemon requires sudo privileges, which may not suit your environment, you can adjust your settings to allow non-root access to Docker:
+
+```shell
+sudo groupadd docker
+```
+
+Add the current user to the Docker group
+```shell
+sudo gpasswd -a $USER docker
+```
+
+Restart the docker daemon
+```shell
+sudo service docker restart
+```
+
+You might also need to restart your computer if the steps above don't resolve the issue.
 The image is rebuilt daily when new test data are added.
 If your tests require adding new test data, ping the maintainers (@leewujung, @ctuguinay)
 to get them added to the the Google Drive.

echopype/convert/__init__.py

@@ -10,12 +10,12 @@
 """
 
 # flake8: noqa
-from .parse_ad2cp import ParseAd2cp
-from .parse_azfp import ParseAZFP
-from .parse_azfp6 import ParseAZFP6
+from .parse_ad2cp import ParseAd2cp, is_AD2CP
+from .parse_azfp import ParseAZFP, is_AZFP
+from .parse_azfp6 import ParseAZFP6, is_AZFP6
 from .parse_base import ParseBase
-from .parse_ek60 import ParseEK60
-from .parse_ek80 import ParseEK80
+from .parse_ek60 import ParseEK60, is_EK60, is_ER60
+from .parse_ek80 import ParseEK80, is_EK80
 from .set_groups_ad2cp import SetGroupsAd2cp
 from .set_groups_azfp import SetGroupsAZFP
 from .set_groups_azfp6 import SetGroupsAZFP6

echopype/convert/parse_ad2cp.py

@@ -1853,3 +1853,25 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec
         DataRecordType.ECHOSOUNDER_RAW_TRANSMIT: ECHOSOUNDER_RAW_DATA_RECORD_FORMAT,
         DataRecordType.STRING: STRING_DATA_RECORD_FORMAT,
     }
+
+
+def is_AD2CP(raw_file):
+    """
+    Check if the provided file has a .ad2cp extension.
+
+    Parameters:
+    raw_file (str): The name of the file to check.
+
+    Returns:
+    bool: True if the file has a .ad2cp extension, False otherwise.
+    """
+
+    # Check if the input is a string
+    if not isinstance(raw_file, str):
+        return False  # Return False if the input is not a string
+
+    # Use the str.lower() method to check for the .ad2cp extension
+    has_ad2cp_extension = raw_file.lower().endswith(".ad2cp")
+
+    # Return the result of the check
+    return has_ad2cp_extension

echopype/convert/parse_azfp.py

@@ -570,3 +570,37 @@ def _calc_Sv_offset(freq, pulse_len):
             )
 
         return SV_OFFSET[freq][pulse_len]
+
+
+def is_AZFP(raw_file):
+    """
+    Check if the specified XML file contains an <InstrumentType> with string="AZFP".
+
+    Parameters:
+    raw_file (str): The base name of the XML file (with or without extension).
+
+    Returns:
+    bool: True if <InstrumentType> with string="AZFP" is found, False otherwise.
+    """
+
+    # Check if the filename ends with .xml or .XML, and strip the extension if it does
+    base_filename = raw_file.rstrip(".xml").rstrip(".XML")
+
+    # Create a list of possible filenames with both extensions
+    possible_files = [f"{base_filename}.xml", f"{base_filename}.XML"]
+
+    for full_filename in possible_files:
+        if os.path.isfile(full_filename):
+            try:
+                # Parse the XML file
+                tree = ET.parse(full_filename)
+                root = tree.getroot()
+
+                # Check for <InstrumentType> elements
+                for instrument in root.findall(".//InstrumentType"):
+                    if instrument.get("string") == "AZFP":
+                        return True
+            except ET.ParseError:
+                print(f"Error parsing the XML file: {full_filename}.")
+
+    return False

echopype/convert/parse_azfp6.py

@@ -689,3 +689,25 @@ def _calc_Sv_offset(freq, pulse_len):
             )
 
         return SV_OFFSET[freq][pulse_len]
+
+
+def is_AZFP6(raw_file):
+    """
+    Check if the provided file has a .azfp extension.
+
+    Parameters:
+    raw_file (str): The name of the file to check.
+
+    Returns:
+    bool: True if the file has a .azfp extension, False otherwise.
+    """
+
+    # Check if the input is a string
+    if not isinstance(raw_file, str):
+        return False  # Return False if the input is not a string
+
+    # Use the str.lower() method to check for the .azfp extension
+    has_azfp_extension = raw_file.lower().endswith(".azfp")
+
+    # Return the result of the check
+    return has_azfp_extension

echopype/convert/parse_ek60.py

@@ -1,4 +1,7 @@
+import numpy as np
+
 from .parse_base import ParseEK
+from .utils.ek_raw_io import RawSimradFile
 
 
 class ParseEK60(ParseEK):
@@ -14,3 +17,32 @@ def __init__(
         **kwargs,
     ):
         super().__init__(file, bot_file, idx_file, storage_options, sonar_model)
+
+
+def is_ER60(raw_file, storage_options):
+    """Check if a raw data file is from Simrad EK60 echosounder."""
+    with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid:
+        config_datagram = fid.read(1)
+        config_datagram["timestamp"] = np.datetime64(
+            config_datagram["timestamp"].replace(tzinfo=None), "[ns]"
+        )
+        # Return True if the sounder name matches "ER60"
+        try:
+            return config_datagram["sounder_name"] in {"ER60", "EK60"}
+        except KeyError:
+            return False
+
+
+def is_EK60(raw_file, storage_options):
+    """Check if a raw data file is from Simrad EK60 echosounder."""
+    with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid:
+        config_datagram = fid.read(1)
+        config_datagram["timestamp"] = np.datetime64(
+            config_datagram["timestamp"].replace(tzinfo=None), "[ns]"
+        )
+
+        try:
+            # Return True if the sounder name matches "EK60"
+            return config_datagram["sounder_name"] in {"ER60", "EK60"}
+        except KeyError:
+            return False

echopype/convert/parse_ek80.py

@@ -1,4 +1,7 @@
+import numpy as np
+
 from .parse_base import ParseEK
+from .utils.ek_raw_io import RawSimradFile
 
 
 class ParseEK80(ParseEK):
@@ -15,3 +18,15 @@ def __init__(
     ):
         super().__init__(file, bot_file, idx_file, storage_options, sonar_model)
         self.environment = {}  # dictionary to store environment data
+
+
+def is_EK80(raw_file, storage_options):
+    """Check if a raw data file is from Simrad EK80 echosounder."""
+    with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid:
+        config_datagram = fid.read(1)
+        config_datagram["timestamp"] = np.datetime64(
+            config_datagram["timestamp"].replace(tzinfo=None), "[ns]"
+        )
+
+        # Return True if "configuration" exists in config_datagram
+        return "configuration" in config_datagram

echopype/tests/convert/test_convert_ad2cp.py

@@ -4,18 +4,21 @@
 Files under "normal" contain default data variables,
 whereas files under "raw" additionally contain the IQ samples.
 """
-
+import glob
+from echopype.convert import is_AD2CP
 
 import xarray as xr
 import numpy as np
 import netCDF4
 import pytest
 from tempfile import TemporaryDirectory
 from pathlib import Path
+import glob
 
 from echopype import open_raw, open_converted
 from echopype.testing import TEST_DATA_FOLDER
 
+from echopype.convert.parse_ad2cp import is_AD2CP
 
 @pytest.fixture
 def ocean_contour_export_dir(test_path):
@@ -260,3 +263,22 @@ def _check_raw_output(
                         atol=absolute_tolerance,
                     )
         base.close()
+
+
+def test_is_AD2CP_valid_files():
+    """Test that .ad2cp files are identified as valid AD2CP files."""
+    # Collect all .ad2cp files in the test directory
+    ad2cp_files = glob.glob("echopype/test_data/ad2cp/normal/*.ad2cp")
+
+    # Check that each file in ad2cp is identified as valid AD2CP
+    for test_file_path in ad2cp_files:
+        assert is_AD2CP(test_file_path) == True
+
+def test_is_AD2CP_invalid_files():
+    """Test that non-.ad2cp files are not identified as valid AD2CP files."""
+    # Collect all non-.ad2cp files in the test directory
+    non_ad2cp_files = glob.glob("echopype/test_data/azfp6/*")
+
+    # Check that each file in non_ad2cp is not identified as valid AD2CP
+    for test_file_path in non_ad2cp_files:
+        assert is_AD2CP(test_file_path) == False

echopype/tests/convert/test_convert_azfp.py

@@ -4,13 +4,15 @@
 - verify echopype converted files against those from AZFP Matlab scripts and EchoView
 - convert AZFP file with different range settings across frequency
 """
+import xml.etree.ElementTree as ET
+import glob
 
 import numpy as np
 import pandas as pd
 from scipy.io import loadmat
 from echopype import open_raw
 import pytest
-from echopype.convert.parse_azfp import ParseAZFP
+from echopype.convert.parse_azfp import ParseAZFP, is_AZFP
 
 
 @pytest.fixture
@@ -263,3 +265,21 @@ def test_load_parse_azfp_xml(azfp_path):
     assert parseAZFP.parameters['pulse_len_phase2'] == [0, 0, 0, 0]
     assert parseAZFP.parameters['range_samples_phase1'] == [8273, 8273, 8273, 8273]
     assert parseAZFP.parameters['range_samples_phase2'] == [2750, 2750, 2750, 2750]
+
+
+
+def test_is_AZFP_valid_files():
+    """Test that XML files with <InstrumentType string="AZFP"> are identified as valid AZFP files."""
+    # Collect all valid XML files in the test directory
+    valid_files = glob.glob("echopype/test_data/azfp/*.xml") + glob.glob("echopype/test_data/azfp/*.XML")
+
+    for test_file_path in valid_files:
+        assert is_AZFP(test_file_path) == True
+
+def test_is_AZFP_invalid_files():
+    """Test that XML files without <InstrumentType string="AZFP"> are not identified as valid AZFP files."""
+    # Collect all invalid XML files in the test directory
+    invalid_files = glob.glob("echopype/test_data/azfp6/*")
+
+    for test_file_path in invalid_files:
+        assert is_AZFP(test_file_path) == False

echopype/tests/convert/test_convert_azfp6.py

@@ -4,14 +4,14 @@
 - verify echopype converted files against those from AZFP Matlab scripts and EchoView
 - convert AZFP file with different range settings across frequency
 """
-
+import glob
 import numpy as np
 import pandas as pd
 from datetime import datetime, timedelta
 from scipy.io import loadmat
 from echopype import open_raw
 import pytest
-from echopype.convert.parse_azfp6 import ParseAZFP6
+from echopype.convert.parse_azfp6 import ParseAZFP6, is_AZFP6
 
 
 @pytest.fixture
@@ -166,3 +166,23 @@ def test_convert_azfp_02a_gps_lat_long(azfp_path):
     )
 
     check_platform_required_scalar_vars(echodata)
+
+
+
+def test_is_AZFP6_valid_files():
+    """Test that .azfp files are identified as valid AZFP files."""
+    # Collect all .azfp files in the test directory
+    azfp_files = glob.glob("echopype/test_data/azfp6/*.azfp")
+
+    # Check that each file in azfp is identified as valid AZFP
+    for test_file_path in azfp_files:
+        assert is_AZFP6(test_file_path) == True
+
+def test_is_AZFP6_invalid_files():
+    """Test that non-.azfp files are not identified as valid AZFP files."""
+    # Collect all non-.azfp files in the test directory
+    non_azfp_files = glob.glob("echopype/test_data/azfp/*")
+
+    # Check that each file in non_azfp is not identified as valid AZFP
+    for test_file_path in non_azfp_files:
+        assert is_AZFP6(test_file_path) == False