diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 819176595..1591ae05d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -42,7 +42,7 @@ jobs: run: | echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV - name: Set up Python - uses: actions/setup-python@v5.2.0 + uses: actions/setup-python@v5.3.0 with: python-version: ${{ matrix.python-version }} - name: Upgrade pip diff --git a/.github/workflows/packit.yaml b/.github/workflows/packit.yaml index 472d9bf30..fb120efef 100644 --- a/.github/workflows/packit.yaml +++ b/.github/workflows/packit.yaml @@ -20,7 +20,7 @@ jobs: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5.2.0 + uses: actions/setup-python@v5.3.0 with: python-version: 3.9 @@ -52,7 +52,7 @@ jobs: needs: build-artifact runs-on: ubuntu-20.04 steps: - - uses: actions/setup-python@v5.2.0 + - uses: actions/setup-python@v5.3.0 name: Install Python with: python-version: 3.9 diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 155a8c72f..1afceeaf2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -35,7 +35,7 @@ jobs: with: fetch-depth: 0 # Fetch all history for all branches and tags. - name: Set up Python - uses: actions/setup-python@v5.2.0 + uses: actions/setup-python@v5.3.0 with: python-version: ${{ matrix.python-version }} - name: Upgrade pip diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml index 8cdf870e1..d01fbcfbf 100644 --- a/.github/workflows/pypi.yaml +++ b/.github/workflows/pypi.yaml @@ -24,7 +24,7 @@ jobs: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v5.2.0 + uses: actions/setup-python@v5.3.0 with: python-version: 3.9 @@ -56,7 +56,7 @@ jobs: needs: build-artifact runs-on: ubuntu-20.04 steps: - - uses: actions/setup-python@v5.2.0 + - uses: actions/setup-python@v5.3.0 name: Install Python with: python-version: 3.9 diff --git a/.github/workflows/windows.yaml b/.github/workflows/windows.yaml index 233ddb835..04b126036 100644 --- a/.github/workflows/windows.yaml +++ b/.github/workflows/windows.yaml @@ -46,12 +46,12 @@ jobs: # Check data endpoint curl http://localhost:8080/data/ - name: Setup Python - uses: actions/setup-python@v5.2.0 + uses: actions/setup-python@v5.3.0 with: python-version: ${{ matrix.python-version }} architecture: x64 - name: Cache conda - uses: actions/cache@v4.1.1 + uses: actions/cache@v4.1.2 env: # Increase this value to reset cache if '.ci_helpers/py{0}.yaml' has not changed CACHE_NUMBER: 0 diff --git a/docs/source/contributing.md b/docs/source/contributing.md index 7fca175f4..dd0d72d6e 100644 --- a/docs/source/contributing.md +++ b/docs/source/contributing.md @@ -81,7 +81,23 @@ One can replace `conda` with `mamba` in the above commands when creating the env Currently, test data are stored in a private Google Drive folder and made available via the [`cormorack/http`](https://hub.docker.com/r/cormorack/http) -Docker image on Docker hub. +Docker image on Docker hub. There’s no need to pull directly from Docker Hub, as the scripts below will handle that for you. For Linux Users: If you run into an issue where accessing the Docker daemon requires sudo privileges, which may not suit your environment, you can adjust your settings to allow non-root access to Docker: + +```shell +sudo groupadd docker +``` + +Add the current user to the Docker group +```shell +sudo gpasswd -a $USER docker +``` + +Restart the docker daemon +```shell +sudo service docker restart +``` + +You might also need to restart your computer if the steps above don't resolve the issue. The image is rebuilt daily when new test data are added. If your tests require adding new test data, ping the maintainers (@leewujung, @ctuguinay) to get them added to the the Google Drive. diff --git a/echopype/convert/__init__.py b/echopype/convert/__init__.py index 78c4e2859..d4fe935a9 100644 --- a/echopype/convert/__init__.py +++ b/echopype/convert/__init__.py @@ -10,12 +10,12 @@ """ # flake8: noqa -from .parse_ad2cp import ParseAd2cp -from .parse_azfp import ParseAZFP -from .parse_azfp6 import ParseAZFP6 +from .parse_ad2cp import ParseAd2cp, is_AD2CP +from .parse_azfp import ParseAZFP, is_AZFP +from .parse_azfp6 import ParseAZFP6, is_AZFP6 from .parse_base import ParseBase -from .parse_ek60 import ParseEK60 -from .parse_ek80 import ParseEK80 +from .parse_ek60 import ParseEK60, is_EK60, is_ER60 +from .parse_ek80 import ParseEK80, is_EK80 from .set_groups_ad2cp import SetGroupsAd2cp from .set_groups_azfp import SetGroupsAZFP from .set_groups_azfp6 import SetGroupsAZFP6 diff --git a/echopype/convert/parse_ad2cp.py b/echopype/convert/parse_ad2cp.py index 51ebc9848..b7815f8de 100644 --- a/echopype/convert/parse_ad2cp.py +++ b/echopype/convert/parse_ad2cp.py @@ -1853,3 +1853,25 @@ def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRec DataRecordType.ECHOSOUNDER_RAW_TRANSMIT: ECHOSOUNDER_RAW_DATA_RECORD_FORMAT, DataRecordType.STRING: STRING_DATA_RECORD_FORMAT, } + + +def is_AD2CP(raw_file): + """ + Check if the provided file has a .ad2cp extension. + + Parameters: + raw_file (str): The name of the file to check. + + Returns: + bool: True if the file has a .ad2cp extension, False otherwise. + """ + + # Check if the input is a string + if not isinstance(raw_file, str): + return False # Return False if the input is not a string + + # Use the str.lower() method to check for the .ad2cp extension + has_ad2cp_extension = raw_file.lower().endswith(".ad2cp") + + # Return the result of the check + return has_ad2cp_extension diff --git a/echopype/convert/parse_azfp.py b/echopype/convert/parse_azfp.py index 1e307ab9e..445866c18 100644 --- a/echopype/convert/parse_azfp.py +++ b/echopype/convert/parse_azfp.py @@ -570,3 +570,37 @@ def _calc_Sv_offset(freq, pulse_len): ) return SV_OFFSET[freq][pulse_len] + + +def is_AZFP(raw_file): + """ + Check if the specified XML file contains an with string="AZFP". + + Parameters: + raw_file (str): The base name of the XML file (with or without extension). + + Returns: + bool: True if with string="AZFP" is found, False otherwise. + """ + + # Check if the filename ends with .xml or .XML, and strip the extension if it does + base_filename = raw_file.rstrip(".xml").rstrip(".XML") + + # Create a list of possible filenames with both extensions + possible_files = [f"{base_filename}.xml", f"{base_filename}.XML"] + + for full_filename in possible_files: + if os.path.isfile(full_filename): + try: + # Parse the XML file + tree = ET.parse(full_filename) + root = tree.getroot() + + # Check for elements + for instrument in root.findall(".//InstrumentType"): + if instrument.get("string") == "AZFP": + return True + except ET.ParseError: + print(f"Error parsing the XML file: {full_filename}.") + + return False diff --git a/echopype/convert/parse_azfp6.py b/echopype/convert/parse_azfp6.py index aa98eca55..9a6af7fdf 100644 --- a/echopype/convert/parse_azfp6.py +++ b/echopype/convert/parse_azfp6.py @@ -689,3 +689,25 @@ def _calc_Sv_offset(freq, pulse_len): ) return SV_OFFSET[freq][pulse_len] + + +def is_AZFP6(raw_file): + """ + Check if the provided file has a .azfp extension. + + Parameters: + raw_file (str): The name of the file to check. + + Returns: + bool: True if the file has a .azfp extension, False otherwise. + """ + + # Check if the input is a string + if not isinstance(raw_file, str): + return False # Return False if the input is not a string + + # Use the str.lower() method to check for the .azfp extension + has_azfp_extension = raw_file.lower().endswith(".azfp") + + # Return the result of the check + return has_azfp_extension diff --git a/echopype/convert/parse_ek60.py b/echopype/convert/parse_ek60.py index 88d749ea7..6a397256d 100644 --- a/echopype/convert/parse_ek60.py +++ b/echopype/convert/parse_ek60.py @@ -1,4 +1,7 @@ +import numpy as np + from .parse_base import ParseEK +from .utils.ek_raw_io import RawSimradFile class ParseEK60(ParseEK): @@ -14,3 +17,32 @@ def __init__( **kwargs, ): super().__init__(file, bot_file, idx_file, storage_options, sonar_model) + + +def is_ER60(raw_file, storage_options): + """Check if a raw data file is from Simrad EK60 echosounder.""" + with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid: + config_datagram = fid.read(1) + config_datagram["timestamp"] = np.datetime64( + config_datagram["timestamp"].replace(tzinfo=None), "[ns]" + ) + # Return True if the sounder name matches "ER60" + try: + return config_datagram["sounder_name"] in {"ER60", "EK60"} + except KeyError: + return False + + +def is_EK60(raw_file, storage_options): + """Check if a raw data file is from Simrad EK60 echosounder.""" + with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid: + config_datagram = fid.read(1) + config_datagram["timestamp"] = np.datetime64( + config_datagram["timestamp"].replace(tzinfo=None), "[ns]" + ) + + try: + # Return True if the sounder name matches "EK60" + return config_datagram["sounder_name"] in {"ER60", "EK60"} + except KeyError: + return False diff --git a/echopype/convert/parse_ek80.py b/echopype/convert/parse_ek80.py index 9d3932b3a..66a1695df 100644 --- a/echopype/convert/parse_ek80.py +++ b/echopype/convert/parse_ek80.py @@ -1,4 +1,7 @@ +import numpy as np + from .parse_base import ParseEK +from .utils.ek_raw_io import RawSimradFile class ParseEK80(ParseEK): @@ -15,3 +18,15 @@ def __init__( ): super().__init__(file, bot_file, idx_file, storage_options, sonar_model) self.environment = {} # dictionary to store environment data + + +def is_EK80(raw_file, storage_options): + """Check if a raw data file is from Simrad EK80 echosounder.""" + with RawSimradFile(raw_file, "r", storage_options=storage_options) as fid: + config_datagram = fid.read(1) + config_datagram["timestamp"] = np.datetime64( + config_datagram["timestamp"].replace(tzinfo=None), "[ns]" + ) + + # Return True if "configuration" exists in config_datagram + return "configuration" in config_datagram diff --git a/echopype/tests/convert/test_convert_ad2cp.py b/echopype/tests/convert/test_convert_ad2cp.py index bf7656f16..262e95c05 100644 --- a/echopype/tests/convert/test_convert_ad2cp.py +++ b/echopype/tests/convert/test_convert_ad2cp.py @@ -4,7 +4,8 @@ Files under "normal" contain default data variables, whereas files under "raw" additionally contain the IQ samples. """ - +import glob +from echopype.convert import is_AD2CP import xarray as xr import numpy as np @@ -12,10 +13,12 @@ import pytest from tempfile import TemporaryDirectory from pathlib import Path +import glob from echopype import open_raw, open_converted from echopype.testing import TEST_DATA_FOLDER +from echopype.convert.parse_ad2cp import is_AD2CP @pytest.fixture def ocean_contour_export_dir(test_path): @@ -260,3 +263,22 @@ def _check_raw_output( atol=absolute_tolerance, ) base.close() + + +def test_is_AD2CP_valid_files(): + """Test that .ad2cp files are identified as valid AD2CP files.""" + # Collect all .ad2cp files in the test directory + ad2cp_files = glob.glob("echopype/test_data/ad2cp/normal/*.ad2cp") + + # Check that each file in ad2cp is identified as valid AD2CP + for test_file_path in ad2cp_files: + assert is_AD2CP(test_file_path) == True + +def test_is_AD2CP_invalid_files(): + """Test that non-.ad2cp files are not identified as valid AD2CP files.""" + # Collect all non-.ad2cp files in the test directory + non_ad2cp_files = glob.glob("echopype/test_data/azfp6/*") + + # Check that each file in non_ad2cp is not identified as valid AD2CP + for test_file_path in non_ad2cp_files: + assert is_AD2CP(test_file_path) == False \ No newline at end of file diff --git a/echopype/tests/convert/test_convert_azfp.py b/echopype/tests/convert/test_convert_azfp.py index f0a0451b9..b2f4d43d8 100644 --- a/echopype/tests/convert/test_convert_azfp.py +++ b/echopype/tests/convert/test_convert_azfp.py @@ -4,13 +4,15 @@ - verify echopype converted files against those from AZFP Matlab scripts and EchoView - convert AZFP file with different range settings across frequency """ +import xml.etree.ElementTree as ET +import glob import numpy as np import pandas as pd from scipy.io import loadmat from echopype import open_raw import pytest -from echopype.convert.parse_azfp import ParseAZFP +from echopype.convert.parse_azfp import ParseAZFP, is_AZFP @pytest.fixture @@ -263,3 +265,21 @@ def test_load_parse_azfp_xml(azfp_path): assert parseAZFP.parameters['pulse_len_phase2'] == [0, 0, 0, 0] assert parseAZFP.parameters['range_samples_phase1'] == [8273, 8273, 8273, 8273] assert parseAZFP.parameters['range_samples_phase2'] == [2750, 2750, 2750, 2750] + + + +def test_is_AZFP_valid_files(): + """Test that XML files with are identified as valid AZFP files.""" + # Collect all valid XML files in the test directory + valid_files = glob.glob("echopype/test_data/azfp/*.xml") + glob.glob("echopype/test_data/azfp/*.XML") + + for test_file_path in valid_files: + assert is_AZFP(test_file_path) == True + +def test_is_AZFP_invalid_files(): + """Test that XML files without are not identified as valid AZFP files.""" + # Collect all invalid XML files in the test directory + invalid_files = glob.glob("echopype/test_data/azfp6/*") + + for test_file_path in invalid_files: + assert is_AZFP(test_file_path) == False \ No newline at end of file diff --git a/echopype/tests/convert/test_convert_azfp6.py b/echopype/tests/convert/test_convert_azfp6.py index 29122a0bb..4ec705ca5 100644 --- a/echopype/tests/convert/test_convert_azfp6.py +++ b/echopype/tests/convert/test_convert_azfp6.py @@ -4,14 +4,14 @@ - verify echopype converted files against those from AZFP Matlab scripts and EchoView - convert AZFP file with different range settings across frequency """ - +import glob import numpy as np import pandas as pd from datetime import datetime, timedelta from scipy.io import loadmat from echopype import open_raw import pytest -from echopype.convert.parse_azfp6 import ParseAZFP6 +from echopype.convert.parse_azfp6 import ParseAZFP6, is_AZFP6 @pytest.fixture @@ -166,3 +166,23 @@ def test_convert_azfp_02a_gps_lat_long(azfp_path): ) check_platform_required_scalar_vars(echodata) + + + +def test_is_AZFP6_valid_files(): + """Test that .azfp files are identified as valid AZFP files.""" + # Collect all .azfp files in the test directory + azfp_files = glob.glob("echopype/test_data/azfp6/*.azfp") + + # Check that each file in azfp is identified as valid AZFP + for test_file_path in azfp_files: + assert is_AZFP6(test_file_path) == True + +def test_is_AZFP6_invalid_files(): + """Test that non-.azfp files are not identified as valid AZFP files.""" + # Collect all non-.azfp files in the test directory + non_azfp_files = glob.glob("echopype/test_data/azfp/*") + + # Check that each file in non_azfp is not identified as valid AZFP + for test_file_path in non_azfp_files: + assert is_AZFP6(test_file_path) == False \ No newline at end of file diff --git a/echopype/tests/convert/test_convert_ek60.py b/echopype/tests/convert/test_convert_ek60.py index a77481380..9b318b00a 100644 --- a/echopype/tests/convert/test_convert_ek60.py +++ b/echopype/tests/convert/test_convert_ek60.py @@ -1,12 +1,12 @@ import warnings - +import glob import numpy as np import pandas as pd from scipy.io import loadmat import pytest from echopype import open_raw -from echopype.convert import ParseEK60 +from echopype.convert import ParseEK60, is_EK60, is_ER60 @pytest.fixture @@ -264,3 +264,40 @@ def test_converting_ek60_raw_with_missing_channel_power(): # Check that all empty power channels do not exist in the EchoData Beam group for _, empty_power_channel_name in empty_power_chs.items(): assert empty_power_channel_name not in ed["Sonar/Beam_group1"]["channel"] + + +def test_is_EK60_ek60_files(): + """Check that EK60 files are identified as EK60""" + # Collect all .raw files in the ek60 directory + ek60_files = glob.glob("echopype/test_data/ek60/from_echopy/*.raw") + + # Check that each file in ek60 is identified as EK60 + for test_file_path in ek60_files: + assert is_EK60(test_file_path, storage_options={}) == True + +def test_is_EK60_non_ek60_files(): + """Check that non-EK60 files are not identified as EK60""" + # Collect all .raw files in the ek80 directory (non-EK60 files) + ek80_files = glob.glob("echopype/test_data/ek80/*.raw") + + # Check that each file in ek80 is not identified as EK60 + for test_file_path in ek80_files: + assert is_EK60(test_file_path, storage_options={}) == False + +def test_is_ER60_er60_files(): + """Check that EK60 files are identified as EK60""" + # Collect all .raw files in the ek60 directory + ek60_files = glob.glob("echopype/test_data/ek60/from_echopy/*.raw") + + # Check that each file in ek60 is identified as EK60 + for test_file_path in ek60_files: + assert is_ER60(test_file_path, storage_options={}) == True + +def test_is_ER60_non_er60_files(): + """Check that non-EK60 files are not identified as EK60""" + # Collect all .raw files in the ek80 directory (non-EK60 files) + ek80_files = glob.glob("echopype/test_data/ek80/*.raw") + + # Check that each file in ek80 is not identified as EK60 + for test_file_path in ek80_files: + assert is_ER60(test_file_path, storage_options={}) == False \ No newline at end of file diff --git a/echopype/tests/convert/test_convert_ek80.py b/echopype/tests/convert/test_convert_ek80.py index c84116c68..c76862c52 100644 --- a/echopype/tests/convert/test_convert_ek80.py +++ b/echopype/tests/convert/test_convert_ek80.py @@ -1,5 +1,5 @@ import shutil - +import glob import pytest import numpy as np import pandas as pd @@ -7,7 +7,7 @@ from echopype import open_raw, open_converted from echopype.testing import TEST_DATA_FOLDER -from echopype.convert.parse_ek80 import ParseEK80 +from echopype.convert.parse_ek80 import ParseEK80, is_EK80 from echopype.convert.set_groups_ek80 import WIDE_BAND_TRANS, PULSE_COMPRESS, FILTER_IMAG, FILTER_REAL, DECIMATION @@ -531,3 +531,21 @@ def test_parse_ek80_with_invalid_env_datagrams(): env_var = ed["Environment"][var] assert env_var.notnull().all() and env_var.dtype == np.float64 + +def test_is_EK80_ek80_files(): + """Test that EK80 files are identified as EK80.""" + # Collect all .raw files in the ek80 directory + ek80_files = glob.glob("echopype/test_data/ek80/*.raw") + + # Check that each file in ek80 is identified as EK80 + for test_file_path in ek80_files: + assert is_EK80(test_file_path, storage_options={}) == True + +def test_is_EK80_non_ek80_files(): + """Test that non-EK80 files are not identified as EK80.""" + # Collect all .raw files in the ek60 directory (non-EK80 files) + ek60_files = glob.glob("echopype/test_data/ek60/*.raw") + + # Check that each file in ek60 is not identified as EK80 + for test_file_path in ek60_files: + assert is_EK80(test_file_path, storage_options={}) == False \ No newline at end of file