From 3ee4917e08f4ded5ebc9e34394b09afcc0612298 Mon Sep 17 00:00:00 2001 From: Ronald Maj Date: Mon, 23 Dec 2024 06:34:18 +0000 Subject: [PATCH] add: New unittests for igslog.py functions --- gnssanalysis/gn_io/igslog.py | 105 ++++++-- tests/test_datasets/sitelog_test_data.py | 329 +++++++++++++++++++++++ tests/test_igslog.py | 73 +++++ 3 files changed, 481 insertions(+), 26 deletions(-) create mode 100644 tests/test_datasets/sitelog_test_data.py create mode 100644 tests/test_igslog.py diff --git a/gnssanalysis/gn_io/igslog.py b/gnssanalysis/gn_io/igslog.py index 12128c7..f1597fa 100644 --- a/gnssanalysis/gn_io/igslog.py +++ b/gnssanalysis/gn_io/igslog.py @@ -107,7 +107,7 @@ class LogVersionError(Exception): """ - Log file read does not conform to known IGS version standard + Log file does not conform to known IGS version standard """ pass @@ -148,7 +148,6 @@ def determine_log_version(data: bytes) -> str: :param bytes data: IGS log file bytes object to determine the version of :return str: Return the version number: "v1.0" or "v2.0" (or "Unknown" if file does not conform to standard) """ - # check for version: result_v1 = _REGEX_VERSION_1.search(data) if result_v1: @@ -161,54 +160,108 @@ def determine_log_version(data: bytes) -> str: raise LogVersionError("Log file does not conform to any known IGS version") -def parse_igs_log(filename_array: _np.ndarray) -> _np.ndarray: - """Parses igs log and outputs ndarray with parsed data +def extract_id_block(data: bytes, file_path: str, file_code: str, version: str = None) -> bytes: + """Extract the site ID block given the bytes object read from an IGS site log file - :param _np.ndarray filename_array: Metadata on input log file. Expects ndarray of the form [CODE DATE PATH] - :return _np.ndarray: Returns array with data from the IGS log file parsed + :param bytes data: The bytes object returned from an open() call on a IGS site log in "rb" mode + :param str file_path: The path to the file from which the "data" bytes object was obtained + :param str file_code: Code from the filename_array passed to the parse_igs_log() function + :param str version: Version number of log file (e.g. "v2.0") - determined if version=None, defaults to None + :raises LogVersionError: Raises an error if an unknown version string is passed in + :return bytes: The site ID block of the IGS site log """ - file_code, _, file_path = filename_array - - with open(file_path, "rb") as file: - data = file.read() - - try: + if version == None: version = determine_log_version(data) - except LogVersionError as e: - logger.warning(f"Error: {e}, skipping parsing the log file") if version == "v1.0": _REGEX_ID = _REGEX_ID_V1 - _REGEX_LOC = _REGEX_LOC_V1 elif version == "v2.0": _REGEX_ID = _REGEX_ID_V2 - _REGEX_LOC = _REGEX_LOC_V2 + else: + raise LogVersionError("Incorrect version string passed to the extract_id_block() function") - blk_id = _REGEX_ID.search(data) - if blk_id is None: + id_block = _REGEX_ID.search(data) + if id_block is None: logger.warning(f"ID rejected from {file_path}") return _np.array([]).reshape(0, 12) - blk_id = [blk_id[1].decode().upper(), blk_id[2].decode().upper()] # no .groups() thus 1 and 2 - code = blk_id[0] + id_block = [id_block[1].decode().upper(), id_block[2].decode().upper()] # no .groups() thus 1 and 2 + code = id_block[0] if code != file_code: logger.warning(f"{code}!={file_code} at {file_path}") return _np.array([]).reshape(0, 12) + return id_block + + +def extract_location_block(data: bytes, file_path: str, version: str = None) -> bytes: + """Extract the location block given the bytes object read from an IGS site log file + + :param bytes data: The bytes object returned from an open() call on a IGS site log in "rb" mode + :param str file_path: The path to the file from which the "data" bytes object was obtained + :param str version: Version number of log file (e.g. "v2.0") - determined if version=None, defaults to None + :raises LogVersionError: Raises an error if an unknown version string is passed in + :return bytes: The location block of the IGS site log + """ + if version == None: + version = determine_log_version(data) + + if version == "v1.0": + _REGEX_LOC = _REGEX_LOC_V1 + elif version == "v2.0": + _REGEX_LOC = _REGEX_LOC_V2 + else: + raise LogVersionError("Incorrect version string passed to extract_location_block() function") - blk_loc = _REGEX_LOC.search(data) - if blk_loc is None: + location_block = _REGEX_LOC.search(data) + if location_block is None: logger.warning(f"LOC rejected from {file_path}") return _np.array([]).reshape(0, 12) + return location_block + - blk_rec = _REGEX_REC.findall(data) - if blk_rec == []: +def extract_receiver_block(data: bytes, file_path: str) -> bytes: + """Extract the location block given the bytes object read from an IGS site log file + + :param bytes data: The bytes object returned from an open() call on a IGS site log in "rb" mode + :param str file_path: The path to the file from which the "data" bytes object was obtained + :return bytes: The receiver block of the IGS site log + """ + receiver_block = _REGEX_REC.findall(data) + if receiver_block == []: logger.warning(f"REC rejected from {file_path}") return _np.array([]).reshape(0, 12) + return receiver_block + - blk_ant = _REGEX_ANT.findall(data) - if blk_ant == []: +def extract_antenna_block(data: bytes, file_path: str) -> bytes: + antenna_block = _REGEX_ANT.findall(data) + if antenna_block == []: logger.warning(f"ANT rejected from {file_path}") return _np.array([]).reshape(0, 12) + return antenna_block + + +def parse_igs_log(filename_array: _np.ndarray) -> _np.ndarray: + """Parses igs log and outputs ndarray with parsed data + + :param _np.ndarray filename_array: Metadata on input log file. Expects ndarray of the form [CODE DATE PATH] + :return _np.ndarray: Returns array with data from the IGS log file parsed + """ + file_code, _, file_path = filename_array + + with open(file_path, "rb") as file: + data = file.read() + + try: + version = determine_log_version(data) + except LogVersionError as e: + logger.warning(f"Error: {e}, skipping parsing the log file") + return + + blk_id = extract_id_block(data, version, file_path, file_code) + blk_loc = extract_location_block(data, version, file_path) + blk_rec = extract_receiver_block(data, file_path) + blk_ant = extract_antenna_block(data, file_path) blk_loc = [group.decode(encoding="utf8", errors="ignore") for group in blk_loc.groups()] blk_rec = _np.asarray(blk_rec, dtype=str) diff --git a/tests/test_datasets/sitelog_test_data.py b/tests/test_datasets/sitelog_test_data.py new file mode 100644 index 0000000..42545ee --- /dev/null +++ b/tests/test_datasets/sitelog_test_data.py @@ -0,0 +1,329 @@ +# Central record of IGS site log test data sets to be shared across unit tests + +# first dataset is a truncated version of file abmf_20240710.log + +abmf_site_log_v1 = bytes( + """ + ABMF Site Information Form (site log) + International GNSS Service + See Instructions at: + https://files.igs.org/pub/station/general/sitelog_instr.txt + +0. Form + + Prepared by (full name) : RGP TEAM + Date Prepared : 2024-07-10 + Report Type : UPDATE + If Update: + Previous Site Log : (ssss_ccyymmdd.log) + Modified/Added Sections : (n.n,n.n,...) + + +1. Site Identification of the GNSS Monument + + Site Name : Aeroport du Raizet -LES ABYMES - Météo France + Four Character ID : ABMF + Monument Inscription : NONE + IERS DOMES Number : 97103M001 + CDP Number : NONE + Monument Description : INOX TRIANGULAR PLATE ON TOP OF METALLIC PILAR + Height of the Monument : 2.0 m + Monument Foundation : ROOF + Foundation Depth : 4.0 m + Marker Description : TOP AND CENTRE OF THE TRIANGULAR PLATE + Date Installed : 2008-07-15T00:00Z + Geologic Characteristic : + Bedrock Type : + Bedrock Condition : + Fracture Spacing : 11-50 cm + Fault zones nearby : + Distance/activity : + Additional Information : + + +2. Site Location Information + + City or Town : Les Abymes + State or Province : Guadeloupe (971) + Country : Guadeloupe + Tectonic Plate : CARIBBEAN + Approximate Position (ITRF) + X coordinate (m) : 2919786.0 + Y coordinate (m) : -5383745.0 + Z coordinate (m) : 1774604.0 + Latitude (N is +) : +161544.30 + Longitude (E is +) : -0613139.11 + Elevation (m,ellips.) : -25.0 + Additional Information : + + +3. GNSS Receiver Information + +3.1 Receiver Type : LEICA GR25 + Satellite System : GPS+GLO+GAL+BDS+SBAS + Serial Number : 1830399 + Firmware Version : 4.31 + Elevation Cutoff Setting : 3 deg + Date Installed : 2019-03-13T17:00Z + Date Removed : 2019-04-15T12:00Z + Temperature Stabiliz. : none + Additional Information : L2C disabled + +3.2 Receiver Type : SEPT POLARX5 + Satellite System : GPS+GLO+GAL+BDS+SBAS + Serial Number : 3013312 + Firmware Version : 5.2.0 + Elevation Cutoff Setting : 0 deg + Date Installed : 2019-04-15T12:00Z + Date Removed : 2019-10-01T16:00Z + Temperature Stabiliz. : none + Additional Information : L2C disabled + +3.3 Receiver Type : SEPT POLARX5 + Satellite System : GPS+GLO+GAL+BDS+SBAS + Serial Number : 3013312 + Firmware Version : 5.3.0 + Elevation Cutoff Setting : 0 deg + Date Installed : 2019-10-01T16:00Z + Date Removed : (CCYY-MM-DDThh:mmZ) + Temperature Stabiliz. : none + Additional Information : L2C disabled + +3.x Receiver Type : (A20, from rcvr_ant.tab; see instructions) + Satellite System : (GPS+GLO+GAL+BDS+QZSS+SBAS) + Serial Number : (A20, but note the first A5 is used in SINEX) + Firmware Version : (A11) + Elevation Cutoff Setting : (deg) + Date Installed : (CCYY-MM-DDThh:mmZ) + Date Removed : (CCYY-MM-DDThh:mmZ) + Temperature Stabiliz. : (none or tolerance in degrees C) + Additional Information : (multiple lines) + + +4. GNSS Antenna Information + +4.1 Antenna Type : AERAT2775_43 SPKE + Serial Number : 5546 + Antenna Reference Point : TOP + Marker->ARP Up Ecc. (m) : 000.0500 + Marker->ARP North Ecc(m) : 000.0000 + Marker->ARP East Ecc(m) : 000.0000 + Alignment from True N : 0 deg + Antenna Radome Type : SPKE + Radome Serial Number : NONE + Antenna Cable Type : + Antenna Cable Length : 30.0 m + Date Installed : 2008-07-15T00:00Z + Date Removed : 2009-10-15T20:00Z + Additional Information : + +4.2 Antenna Type : TRM55971.00 NONE + Serial Number : 1440911917 + Antenna Reference Point : BAM + Marker->ARP Up Ecc. (m) : 000.0000 + Marker->ARP North Ecc(m) : 000.0000 + Marker->ARP East Ecc(m) : 000.0000 + Alignment from True N : 0 deg + Antenna Radome Type : NONE + Radome Serial Number : + Antenna Cable Type : + Antenna Cable Length : 30.0 m + Date Installed : 2009-10-15T20:00Z + Date Removed : 2012-01-24T12:00Z + Additional Information : + +4.3 Antenna Type : TRM57971.00 NONE + Serial Number : 1441112501 + Antenna Reference Point : BAM + Marker->ARP Up Ecc. (m) : 000.0000 + Marker->ARP North Ecc(m) : 000.0000 + Marker->ARP East Ecc(m) : 000.0000 + Alignment from True N : 0 deg + Antenna Radome Type : NONE + Radome Serial Number : + Antenna Cable Type : + Antenna Cable Length : 30.0 m + Date Installed : 2012-01-24T12:00Z + Date Removed : (CCYY-MM-DDThh:mmZ) + Additional Information : + +4.x Antenna Type : (A20, from rcvr_ant.tab; see instructions) + Serial Number : (A*, but note the first A5 is used in SINEX) + Antenna Reference Point : (BPA/BCR/XXX from "antenna.gra"; see instr.) + Marker->ARP Up Ecc. (m) : (F8.4) + Marker->ARP North Ecc(m) : (F8.4) + Marker->ARP East Ecc(m) : (F8.4) + Alignment from True N : (deg; + is clockwise/east) + Antenna Radome Type : (A4 from rcvr_ant.tab; see instructions) + Radome Serial Number : + Antenna Cable Type : (vendor & type number) + Antenna Cable Length : (m) + Date Installed : (CCYY-MM-DDThh:mmZ) + Date Removed : (CCYY-MM-DDThh:mmZ) + Additional Information : (multiple lines) + """, + "utf-8", +) + +abmf_site_log_v2 = bytes( + """ + ABMF00GLP Site Information Form (site log v2.0) + International GNSS Service + See Instructions at: + https://files.igs.org/pub/station/general/sitelog_instr_v2.0.txt + +0. Form + + Prepared by (full name) : RGP TEAM + Date Prepared : 2024-07-10 + Report Type : UPDATE + If Update: + Previous Site Log : (ssssmrccc_ccyymmdd.log) + Modified/Added Sections : (n.n,n.n,...) + + +1. Site Identification of the GNSS Monument + + Site Name : Aeroport du Raizet -LES ABYMES - Météo France + Nine Character ID : ABMF00GLP + Monument Inscription : NONE + IERS DOMES Number : 97103M001 + CDP Number : NONE + Monument Description : INOX TRIANGULAR PLATE ON TOP OF METALLIC PILAR + Height of the Monument : 2.0 m + Monument Foundation : ROOF + Foundation Depth : 4.0 m + Marker Description : TOP AND CENTRE OF THE TRIANGULAR PLATE + Date Installed : 2008-07-15T00:00Z + Geologic Characteristic : + Bedrock Type : + Bedrock Condition : + Fracture Spacing : 11-50 cm + Fault zones nearby : + Distance/activity : + Additional Information : + + +2. Site Location Information + + City or Town : Les Abymes + State or Province : Guadeloupe (971) + Country or Region : GLP + Tectonic Plate : CARIBBEAN + Approximate Position (ITRF) + X coordinate (m) : 2919786.0 + Y coordinate (m) : -5383745.0 + Z coordinate (m) : 1774604.0 + Latitude (N is +) : +161544.30 + Longitude (E is +) : -0613139.11 + Elevation (m,ellips.) : -25.0 + Additional Information : + + +3. GNSS Receiver Information + +3.1 Receiver Type : LEICA GR25 + Satellite System : GPS+GLO+GAL+BDS+SBAS + Serial Number : 1830399 + Firmware Version : 4.31 + Elevation Cutoff Setting : 3 deg + Date Installed : 2019-03-13T17:00Z + Date Removed : 2019-04-15T12:00Z + Temperature Stabiliz. : none + Additional Information : L2C disabled + +3.2 Receiver Type : SEPT POLARX5 + Satellite System : GPS+GLO+GAL+BDS+SBAS + Serial Number : 3013312 + Firmware Version : 5.2.0 + Elevation Cutoff Setting : 0 deg + Date Installed : 2019-04-15T12:00Z + Date Removed : 2019-10-01T16:00Z + Temperature Stabiliz. : none + Additional Information : L2C disabled + +3.3 Receiver Type : SEPT POLARX5 + Satellite System : GPS+GLO+GAL+BDS+SBAS + Serial Number : 3013312 + Firmware Version : 5.3.0 + Elevation Cutoff Setting : 0 deg + Date Installed : 2019-10-01T16:00Z + Date Removed : (CCYY-MM-DDThh:mmZ) + Temperature Stabiliz. : none + Additional Information : L2C disabled + +3.x Receiver Type : (A20, from rcvr_ant.tab; see instructions) + Satellite System : (GPS+GLO+GAL+BDS+QZSS+SBAS) + Serial Number : (A20, but note the first A5 is used in SINEX) + Firmware Version : (A11) + Elevation Cutoff Setting : (deg) + Date Installed : (CCYY-MM-DDThh:mmZ) + Date Removed : (CCYY-MM-DDThh:mmZ) + Temperature Stabiliz. : (none or tolerance in degrees C) + Additional Information : (multiple lines) + + +4. GNSS Antenna Information + +4.1 Antenna Type : AERAT2775_43 SPKE + Serial Number : 5546 + Antenna Reference Point : TOP + Marker->ARP Up Ecc. (m) : 000.0500 + Marker->ARP North Ecc(m) : 000.0000 + Marker->ARP East Ecc(m) : 000.0000 + Alignment from True N : 0 deg + Antenna Radome Type : SPKE + Radome Serial Number : NONE + Antenna Cable Type : + Antenna Cable Length : 30.0 m + Date Installed : 2008-07-15T00:00Z + Date Removed : 2009-10-15T20:00Z + Additional Information : + +4.2 Antenna Type : TRM55971.00 NONE + Serial Number : 1440911917 + Antenna Reference Point : BAM + Marker->ARP Up Ecc. (m) : 000.0000 + Marker->ARP North Ecc(m) : 000.0000 + Marker->ARP East Ecc(m) : 000.0000 + Alignment from True N : 0 deg + Antenna Radome Type : NONE + Radome Serial Number : + Antenna Cable Type : + Antenna Cable Length : 30.0 m + Date Installed : 2009-10-15T20:00Z + Date Removed : 2012-01-24T12:00Z + Additional Information : + +4.3 Antenna Type : TRM57971.00 NONE + Serial Number : 1441112501 + Antenna Reference Point : BAM + Marker->ARP Up Ecc. (m) : 000.0000 + Marker->ARP North Ecc(m) : 000.0000 + Marker->ARP East Ecc(m) : 000.0000 + Alignment from True N : 0 deg + Antenna Radome Type : NONE + Radome Serial Number : + Antenna Cable Type : + Antenna Cable Length : 30.0 m + Date Installed : 2012-01-24T12:00Z + Date Removed : (CCYY-MM-DDThh:mmZ) + Additional Information : + +4.x Antenna Type : (A20, from rcvr_ant.tab; see instructions) + Serial Number : (A*, but note the first A5 is used in SINEX) + Antenna Reference Point : (BPA/BCR/XXX from "antenna.gra"; see instr.) + Marker->ARP Up Ecc. (m) : (F8.4) + Marker->ARP North Ecc(m) : (F8.4) + Marker->ARP East Ecc(m) : (F8.4) + Alignment from True N : (deg; + is clockwise/east) + Antenna Radome Type : (A4 from rcvr_ant.tab; see instructions) + Radome Serial Number : + Antenna Cable Type : (vendor & type number) + Antenna Cable Length : (m) + Date Installed : (CCYY-MM-DDThh:mmZ) + Date Removed : (CCYY-MM-DDThh:mmZ) + Additional Information : (multiple lines) + """, + "utf-8", +) diff --git a/tests/test_igslog.py b/tests/test_igslog.py new file mode 100644 index 0000000..aa5a368 --- /dev/null +++ b/tests/test_igslog.py @@ -0,0 +1,73 @@ +import unittest +import numpy as _np +import pandas as _pd + +from gnssanalysis.gn_io import igslog +from test_datasets.sitelog_test_data import abmf_site_log_v1 as v1_data, abmf_site_log_v2 as v2_data + + +class Testregex(unittest.TestCase): + def test_determine_log_version(self): + # Ensure version 1 and 2 strings are produced as expected + self.assertEqual(igslog.determine_log_version(v1_data), "v1.0") + self.assertEqual(igslog.determine_log_version(v2_data), "v2.0") + + def test_extract_id_block(self): + # Ensure the extract of ID information works and gives correct dome number: + self.assertEqual(igslog.extract_id_block(v1_data, "/example/path", "ABMF", "v1.0"), ["ABMF", "97103M001"]) + self.assertEqual(igslog.extract_id_block(v2_data, "/example/path", "ABMF", "v2.0"), ["ABMF", "97103M001"]) + + def test_extract_location_block(self): + + # Version 1 Location description results: + v1_location_block = igslog.extract_location_block(v1_data, "/example/path", "v1.0") + self.assertEqual(v1_location_block.group(1), b"Les Abymes") + self.assertEqual(v1_location_block.group(2), b"Guadeloupe") + + # Version 2 Location description results: + v2_location_block = igslog.extract_location_block(v2_data, "/example/path", "v2.0") + self.assertEqual(v2_location_block.group(1), b"Les Abymes") + self.assertEqual(v2_location_block.group(2), b"GLP") + + # Coordinate information remains the same: + self.assertEqual(v2_location_block.group(3), v1_location_block.group(3)) + + def test_extract_receiver_block(self): + + # Testing version 1: + v1_receiver_block = igslog.extract_receiver_block(v1_data, "/example/path") + self.assertEqual(v1_receiver_block[0][0], b"LEICA GR25") + self.assertEqual( + v1_receiver_block[1][0], v1_receiver_block[2][0] + ) # Testing that entries [1] and [2] are receiver: "SEPT POLARX5" + self.assertEqual(v1_receiver_block[1][3], b"5.2.0") # Difference between entries is a Firmware change + self.assertEqual(v1_receiver_block[2][3], b"5.3.0") # Difference between entries is a Firmware change + # Last receiver should not have an end date assigned (i.e. current): + self.assertEqual(v1_receiver_block[-1][-1], b"") + + # Same as above, but for version 2: + v2_receiver_block = igslog.extract_receiver_block(v2_data, "/example/path") + self.assertEqual(v2_receiver_block[0][0], b"LEICA GR25") + self.assertEqual( + v2_receiver_block[1][0], v2_receiver_block[2][0] + ) # Testing that entries 2 and 3 are "SEPT POLARX5" + self.assertEqual(v2_receiver_block[1][3], b"5.2.0") # Difference between entries 2 and 3 is in Firmware change + self.assertEqual(v2_receiver_block[2][3], b"5.3.0") + # Last receiver should not have an end date assigned (i.e. current): + self.assertEqual(v2_receiver_block[-1][-1], b"") + + def test_extract_antenna_block(self): + + # Testing version 1: + v1_antenna_block = igslog.extract_antenna_block(v1_data, "/example/path") + self.assertEqual(v1_antenna_block[0][0], b"AERAT2775_43") # Check antenna type of first entry + self.assertEqual(v1_antenna_block[0][8], b"2009-10-15T20:00Z") # Check end date of second entry + # Last antenna should not have an end date assigned (i.e. current): + self.assertEqual(v1_antenna_block[-1][-1], b"") + + # Testing version 2: + v2_antenna_block = igslog.extract_antenna_block(v2_data, "/example/path") + self.assertEqual(v2_antenna_block[0][0], b"AERAT2775_43") # Check antenna type of first entry + self.assertEqual(v2_antenna_block[0][8], b"2009-10-15T20:00Z") # Check end date of second entry + # Last antenna should not have an end date assigned (i.e. current): + self.assertEqual(v2_antenna_block[-1][-1], b"")