Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

hotfix: Temporarily skip datagrams generated by EC150 #1357

Merged
merged 6 commits into from
Jul 15, 2024
Merged

hotfix: Temporarily skip datagrams generated by EC150 #1357

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c27aa5f
change parse_ek to skip EC150 datagrams
leewujung Jul 13, 2024
c9f08dc
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 13, 2024
1677858
fix pre-commit errors, update test
leewujung Jul 13, 2024
2f00fd7
Merge branch 'skip_EC150' of https://github.com/leewujung/echopype in…
leewujung Jul 13, 2024
24151ec
fix bug to look for cconfiguration in self.config_datagram for ek60
leewujung Jul 13, 2024
b34f7bb
comment out print statement
ctuguinay Jul 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 64 additions & 31 deletions echopype/convert/parse_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,15 @@ def parse_raw(self):
self.config_datagram["timestamp"] = np.datetime64(
self.config_datagram["timestamp"].replace(tzinfo=None), "[ns]"
)

# Only EK80 files have configuration in self.config_datagram
if "configuration" in self.config_datagram:
# Remove EC150 (ADCP) from config
channel_id = list(self.config_datagram["configuration"].keys())
channel_id_rm = [ch for ch in channel_id if "EC150" in ch]
for ch in channel_id_rm:
_ = self.config_datagram["configuration"].pop(ch)

for v in self.config_datagram["configuration"].values():
if "pulse_duration" not in v and "pulse_length" in v:
# it seems like sometimes this field can appear with the name "pulse_length"
Expand Down Expand Up @@ -476,6 +484,12 @@ def _read_datagrams(self, fid):
new_datagram["timestamp"].replace(tzinfo=None), "[ns]"
)

# # For debugging EC150 datagrams
# if new_datagram["type"].startswith("XML") and "subtype" in new_datagram:
# print(f"{new_datagram['type']} - {new_datagram['subtype']}")
# else:
# print(new_datagram["type"])

num_datagrams_parsed += 1

# XML datagrams store environment or instrument parameters for EK80
Expand All @@ -485,7 +499,14 @@ def _read_datagrams(self, fid):
self.environment["xml"] = new_datagram["xml"]
self.environment["timestamp"] = new_datagram["timestamp"]
elif new_datagram["subtype"] == "parameter":
current_parameters = new_datagram["parameter"]
if "EC150" not in new_datagram["parameter"]["channel_id"]:
# print(
# f"{new_datagram['parameter']['channel_id']} from XML-parameter "
# "-- NOT SKIPPING"
# )
current_parameters = new_datagram["parameter"]
# else:
# print(f"{new_datagram['parameter']['channel_id']} from XML-parameter")

# RAW0 datagrams store raw acoustic data for a channel for EK60
elif new_datagram["type"].startswith("RAW0"):
Expand All @@ -502,33 +523,41 @@ def _read_datagrams(self, fid):
# - RAW3
# RAW3 datagrams store raw acoustic data for a channel for EK80
elif new_datagram["type"].startswith("RAW3"):
curr_ch_id = new_datagram["channel_id"]
# Check if the proceeding Parameter XML does not
# match with data in this RAW3 datagram
if current_parameters["channel_id"] != curr_ch_id:
raise ValueError("Parameter ID does not match RAW")

# Save channel-specific ping time
self.ping_time[curr_ch_id].append(new_datagram["timestamp"])

# Append ping by ping data
new_datagram.update(current_parameters)
self._append_channel_ping_data(new_datagram)
if "EC150" not in new_datagram["channel_id"]:
# print(f"{new_datagram['channel_id']} from RAW3 -- NOT SKIPPING")
curr_ch_id = new_datagram["channel_id"]
# Check if the proceeding Parameter XML does not
# match with data in this RAW3 datagram
if current_parameters["channel_id"] != curr_ch_id:
raise ValueError("Parameter ID does not match RAW")

# Save channel-specific ping time
self.ping_time[curr_ch_id].append(new_datagram["timestamp"])

# Append ping by ping data
new_datagram.update(current_parameters)
self._append_channel_ping_data(new_datagram)
# else:
# print(f"{new_datagram['channel_id']} from RAW3")

# RAW4 datagrams store raw transmit pulse for a channel for EK80
elif new_datagram["type"].startswith("RAW4"):
curr_ch_id = new_datagram["channel_id"]
# Check if the proceeding Parameter XML does not
# match with data in this RAW4 datagram
if current_parameters["channel_id"] != curr_ch_id:
raise ValueError("Parameter ID does not match RAW")

# Ping time is identical to the immediately following RAW3 datagram
# so does not need to be stored separately

# Append ping by ping data
new_datagram.update(current_parameters)
self._append_channel_ping_data(new_datagram, raw_type="transmit")
if "EC150" not in new_datagram["channel_id"]:
# print(f"{new_datagram['channel_id']} from RAW4 -- NOT SKIPPING")
curr_ch_id = new_datagram["channel_id"]
# Check if the proceeding Parameter XML does not
# match with data in this RAW4 datagram
if current_parameters["channel_id"] != curr_ch_id:
raise ValueError("Parameter ID does not match RAW")

# Ping time is identical to the immediately following RAW3 datagram
# so does not need to be stored separately

# Append ping by ping data
new_datagram.update(current_parameters)
self._append_channel_ping_data(new_datagram, raw_type="transmit")
# else:
# print(f"{new_datagram['channel_id']} from RAW4")

# NME datagrams store ancillary data as NMEA-0817 style ASCII data.
elif new_datagram["type"].startswith("NME"):
Expand All @@ -545,12 +574,16 @@ def _read_datagrams(self, fid):

# FIL datagrams contain filters for processing bascatter data for EK80
elif new_datagram["type"].startswith("FIL"):
self.fil_coeffs[new_datagram["channel_id"]][new_datagram["stage"]] = new_datagram[
"coefficients"
]
self.fil_df[new_datagram["channel_id"]][new_datagram["stage"]] = new_datagram[
"decimation_factor"
]
if "EC150" not in new_datagram["channel_id"]:
# print(f"{new_datagram['channel_id']} from FIL -- NOT SKIPPING")
self.fil_coeffs[new_datagram["channel_id"]][new_datagram["stage"]] = (
new_datagram["coefficients"]
)
self.fil_df[new_datagram["channel_id"]][new_datagram["stage"]] = new_datagram[
"decimation_factor"
]
# else:
# print(f"{new_datagram['channel_id']} from FIL")

# TAG datagrams contain time-stamped annotations inserted via the recording software
elif new_datagram["type"].startswith("TAG"):
Expand Down
13 changes: 13 additions & 0 deletions echopype/tests/convert/test_convert_ek80.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,3 +442,16 @@ def test_convert_ek80_mru1(ek80_path):
np.all(echodata["Platform"]["roll"].data == np.array(parser.mru["roll"]))
np.all(echodata["Platform"]["vertical_offset"].data == np.array(parser.mru["heave"]))
np.all(echodata["Platform"]["heading"].data == np.array(parser.mru["heading"]))


def test_skip_ec150(ek80_path):
"""Make sure we skip EC150 datagrams correctly."""
ek80_mru1_path = str(ek80_path.joinpath("RL2407_ADCP-D20240709-T150437.raw"))
echodata = open_raw(raw_file=ek80_mru1_path, sonar_model='EK80')

assert "EC150" not in echodata["Sonar/Beam_group1"]["channel"].values
assert "backscatter_i" in echodata["Sonar/Beam_group1"].data_vars
assert (
echodata["Sonar/Beam_group1"].dims
== {'channel': 1, 'ping_time': 2, 'range_sample': 115352, 'beam': 4}
)