From ab0f3612fc5752d0eed2c77996b547acaccec940 Mon Sep 17 00:00:00 2001 From: Gavin Medley Date: Wed, 4 Sep 2024 17:38:47 -0600 Subject: [PATCH] Bump python requirement to 3.9 and remove 3.8 tests and github workflow Add support for MIL-1750A float parsing Add test cases pulled from MIL-1750A standard document --- .github/workflows/tests.yml | 2 +- docker-compose.yml | 7 --- docs/source/changelog.md | 1 + pyproject.toml | 3 +- space_packet_parser/xtcedef.py | 95 ++++++++++++++++++++++++++-------- tests/unit/test_xtcedef.py | 76 ++++++++++++++++++++++++--- 6 files changed, 146 insertions(+), 38 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c8a5bcc..948e448 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,7 +4,7 @@ jobs: python-version-matrix: strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] uses: ./.github/workflows/test-python-version.yml with: python-version: ${{ matrix.python-version }} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 15381c6..8857dfc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,13 +9,6 @@ services: build: target: style - 3.8-tests: - image: space-packets-3.8-test:latest - build: - target: test - args: - - BASE_IMAGE_PYTHON_VERSION=3.8 - 3.9-tests: image: space-packets-3.9-test:latest build: diff --git a/docs/source/changelog.md b/docs/source/changelog.md index 8eddfd0..a757334 100644 --- a/docs/source/changelog.md +++ b/docs/source/changelog.md @@ -16,6 +16,7 @@ Release notes for the `space_packet_parser` library ``f"{int.from_bytes(data, byteorder='big'):0{len(data)*8}b}"`` - Fix EnumeratedParameterType to handle duplicate labels - Add error reporting for unsupported and invalid parameter types +- Add support for MIL-1750A floats (32-bit only) ### v4.2.0 (released) - Parse short and long descriptions of parameters diff --git a/pyproject.toml b/pyproject.toml index acefa33..70b71a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ keywords = [ ] [tool.poetry.dependencies] -python = ">=3.8" +python = ">=3.9" lxml = ">=4.8.0" [tool.poetry.group.dev.dependencies] @@ -50,6 +50,7 @@ myst-parser = "*" sphinx-autoapi = "*" sphinx-rtd-theme = "*" coverage = "*" +numpy = "*" [tool.poetry.group.examples] optional = true diff --git a/space_packet_parser/xtcedef.py b/space_packet_parser/xtcedef.py index f8910e7..da99fb6 100644 --- a/space_packet_parser/xtcedef.py +++ b/space_packet_parser/xtcedef.py @@ -1,6 +1,6 @@ """Module for parsing XTCE xml files to specify packet format""" # Standard -from abc import ABCMeta +from abc import ABCMeta, abstractmethod from collections import namedtuple from dataclasses import dataclass, field import inspect @@ -111,6 +111,7 @@ class MatchCriteria(AttrComparable, metaclass=ABCMeta): } @classmethod + @abstractmethod def from_match_criteria_xml_element(cls, element: ElementTree.Element, ns: dict): """Abstract classmethod to create a match criteria object from an XML element. @@ -127,6 +128,7 @@ def from_match_criteria_xml_element(cls, element: ElementTree.Element, ns: dict) """ raise NotImplementedError() + @abstractmethod def evaluate(self, parsed_data: dict, current_parsed_value: Optional[Union[int, float]] = None) -> bool: """Evaluate match criteria down to a boolean. @@ -609,6 +611,7 @@ class Calibrator(AttrComparable, metaclass=ABCMeta): """Abstract base class for XTCE calibrators""" @classmethod + @abstractmethod def from_calibrator_xml_element(cls, element: ElementTree.Element, ns: dict) -> 'Calibrator': """Abstract classmethod to create a default_calibrator object from an XML element. @@ -625,6 +628,7 @@ def from_calibrator_xml_element(cls, element: ElementTree.Element, ns: dict) -> """ return NotImplemented + @abstractmethod def calibrate(self, uncalibrated_value: Union[int, float]) -> Union[int, float]: """Takes an integer-encoded or float-encoded value and returns a calibrated version. @@ -1035,6 +1039,7 @@ class DataEncoding(AttrComparable, metaclass=ABCMeta): """Abstract base class for XTCE data encodings""" @classmethod + @abstractmethod def from_data_encoding_xml_element(cls, element: ElementTree.Element, ns: dict) -> 'DataEncoding': """Abstract classmethod to create a data encoding object from an XML element. @@ -1456,6 +1461,7 @@ def __init__(self, size_in_bits: int, def _calculate_size(self, packet: Packet) -> int: return self.size_in_bits + @abstractmethod def _get_raw_value(self, packet: Packet) -> Union[int, float]: """Read the raw value from the packet data @@ -1472,6 +1478,16 @@ def _get_raw_value(self, packet: Packet) -> Union[int, float]: """ raise NotImplementedError() + @staticmethod + def _twos_complement(val: int, bit_width: int) -> int: + """Take the twos complement of val + + Used when parsing ints and some floats + """ + if (val & (1 << (bit_width - 1))) != 0: # if sign bit is set e.g., 8bit: 128-255 + return val - (1 << bit_width) # compute negative value + return val + def parse_value(self, packet: Packet, **kwargs) -> Tuple[Union[int, float], Union[int, float]]: @@ -1522,10 +1538,8 @@ def _get_raw_value(self, packet: Packet) -> int: ) if self.encoding == 'unsigned': return val - # It is a signed integer and we need to take into account the first bit - if (val & (1 << (self.size_in_bits - 1))) != 0: # if sign bit is set e.g., 8bit: 128-255 - return val - (1 << self.size_in_bits) # compute negative value - return val # return positive value as is + # It is a signed integer, and we need to take into account the first bit + return self._twos_complement(val, self.size_in_bits) @classmethod def from_data_encoding_xml_element(cls, element: ElementTree.Element, ns: dict) -> 'IntegerDataEncoding': @@ -1561,8 +1575,6 @@ def __init__(self, size_in_bits: int, encoding: str = 'IEEE-754', context_calibrators: Optional[List[ContextCalibrator]] = None): """Constructor - # TODO: Implement MIL-1650A encoding option - Parameters ---------- size_in_bits : int @@ -1581,33 +1593,72 @@ def __init__(self, size_in_bits: int, encoding: str = 'IEEE-754', if encoding not in self._supported_encodings: raise ValueError(f"Invalid encoding type {encoding} for float data. " f"Must be one of {self._supported_encodings}.") - if encoding == 'MIL-1750A': - raise NotImplementedError("MIL-1750A encoded floats are not supported by this library yet.") + if encoding == 'MIL-1750A' and size_in_bits != 32: + raise ValueError("MIL-1750A encoded floats must be 32 bits, per the MIL-1750A spec. See " + "https://www.xgc-tek.com/manuals/mil-std-1750a/c191.html#AEN324") if encoding == 'IEEE-754' and size_in_bits not in (16, 32, 64): raise ValueError(f"Invalid size_in_bits value for IEEE-754 FloatDataEncoding, {size_in_bits}. " "Must be 16, 32, or 64.") super().__init__(size_in_bits, encoding=encoding, byte_order=byte_order, default_calibrator=default_calibrator, context_calibrators=context_calibrators) - if self.byte_order == "leastSignificantByteFirst": - self._struct_format = "<" + if self.encoding == "MIL-1750A": + def _mil_parse_func(mil_bytes: bytes): + """Parsing function for MIL-1750A floats""" + # MIL 1750A floats are always 32 bit + # See: https://www.xgc-tek.com/manuals/mil-std-1750a/c191.html#AEN324 + # + # MSB LSB MSB LSB + # ------------------------------------------------------------------ + # | S| Mantissa | Exponent | + # ------------------------------------------------------------------ + # 0 1 23 24 31 + if self.byte_order == "mostSignificantByteFirst": + bytes_as_int = int.from_bytes(mil_bytes, byteorder='big') + else: + bytes_as_int = int.from_bytes(mil_bytes, byteorder='little') + exponent = bytes_as_int & 0xFF # last 8 bits + mantissa = (bytes_as_int >> 8) & 0xFFFFFF # bits 0 through 23 (24 bits) + # We include the sign bit with the mantissa because we can just take the twos complement + # of it directly and use it in the final calculation for the value + + # Both mantissa and exponent are stored as twos complement with no bias + exponent = self._twos_complement(exponent, 8) + mantissa = self._twos_complement(mantissa, 24) + + # Calculate float value using native Python floats, which are more precise + return mantissa * (2.0 ** (exponent - (24 - 1))) + + # Set up the parsing function just once, so we can use it repeatedly with _get_raw_value + self.parse_func = _mil_parse_func else: - # Big-endian is the default - self._struct_format = ">" + if self.byte_order == "leastSignificantByteFirst": + self._struct_format = "<" + else: + # Big-endian is the default + self._struct_format = ">" + + if self.size_in_bits == 16: + self._struct_format += "e" + elif self.size_in_bits == 32: + self._struct_format += "f" + elif self.size_in_bits == 64: + self._struct_format += "d" + + def ieee_parse_func(data: bytes): + """Parsing function for IEEE floats""" + # The packet data we got back is always extracted in big-endian order + # but the struct format code contains the endianness of the float data + return struct.unpack(self._struct_format, data)[0] - if self.size_in_bits == 16: - self._struct_format += "e" - elif self.size_in_bits == 32: - self._struct_format += "f" - elif self.size_in_bits == 64: - self._struct_format += "d" + # Set up the parsing function just once, so we can use it repeatedly with _get_raw_value + self.parse_func: callable = ieee_parse_func def _get_raw_value(self, packet): """Read the data in as bytes and return a float representation.""" data = packet.read_as_bytes(self.size_in_bits) - # The packet data we got back is always extracted in big-endian order - # but the struct format code contains the endianness of the float data - return struct.unpack(self._struct_format, data)[0] + # The parsing function is fully set during initialization to save time during parsing + return self.parse_func(data) @classmethod def from_data_encoding_xml_element(cls, element: ElementTree.Element, ns: dict) -> 'FloatDataEncoding': diff --git a/tests/unit/test_xtcedef.py b/tests/unit/test_xtcedef.py index 1239e85..ee1f768 100644 --- a/tests/unit/test_xtcedef.py +++ b/tests/unit/test_xtcedef.py @@ -1506,7 +1506,7 @@ def test_float_parameter_type(xml_string: str, expectation): # Test big endian 64-bit float (xtcedef.FloatParameterType('TEST_FLOAT', xtcedef.FloatDataEncoding(64)), xtcedef.Packet(b'\x3F\xF9\xE3\x77\x9B\x97\xF4\xA8'), # 64-bit IEEE 754 value of Phi - 1.61803), + 1.6180339), # Test float parameter type encoded as big endian 16-bit integer with contextual polynomial calibrator (xtcedef.FloatParameterType( 'TEST_FLOAT', @@ -1523,16 +1523,79 @@ def test_float_parameter_type(xml_string: str, expectation): xtcedef.Packet(0b1111111111010110.to_bytes(length=2, byteorder='big'), parsed_data={'PKT_APID': parser.ParsedDataItem('PKT_APID', 1101)}), -82.600000), + # Test MIL 1750A encoded floats. + # Test values taken from: https://www.xgc-tek.com/manuals/mil-std-1750a/c191.html#AEN324 + (xtcedef.FloatParameterType( + 'MIL_1750A_FLOAT', + xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")), + xtcedef.Packet(b'\x7f\xff\xff\x7f'), + 0.9999998 * (2 ** 127)), + (xtcedef.FloatParameterType( + 'MIL_1750A_FLOAT', + xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")), + xtcedef.Packet(b'\x40\x00\x00\x7f'), + 0.5 * (2 ** 127)), + (xtcedef.FloatParameterType( + 'MIL_1750A_FLOAT', + xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")), + xtcedef.Packet(b'\x50\x00\x00\x04'), + 0.625 * (2 ** 4)), + (xtcedef.FloatParameterType( + 'MIL_1750A_FLOAT', + xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")), + xtcedef.Packet(b'\x40\x00\x00\x01'), + 0.5 * (2 ** 1)), + (xtcedef.FloatParameterType( + 'MIL_1750A_FLOAT', + xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")), + xtcedef.Packet(b'\x40\x00\x00\x00'), + 0.5 * (2 ** 0)), + (xtcedef.FloatParameterType( + 'MIL_1750A_FLOAT', + xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")), + xtcedef.Packet(b'\x40\x00\x00\xff'), + 0.5 * (2 ** -1)), + (xtcedef.FloatParameterType( + 'MIL_1750A_FLOAT', + xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")), + xtcedef.Packet(b'\x40\x00\x00\x80'), + 0.5 * (2 ** -128)), + (xtcedef.FloatParameterType( + 'MIL_1750A_FLOAT', + xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")), + xtcedef.Packet(b'\x00\x00\x00\x00'), + 0.0 * (2 ** 0)), + (xtcedef.FloatParameterType( + 'MIL_1750A_FLOAT', + xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")), + xtcedef.Packet(b'\x80\x00\x00\x00'), + -1.0 * (2 ** 0)), + (xtcedef.FloatParameterType( + 'MIL_1750A_FLOAT', + xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")), + xtcedef.Packet(b'\xBF\xFF\xFF\x80'), + -0.5000001 * (2 ** -128)), + (xtcedef.FloatParameterType( + 'MIL_1750A_FLOAT', + xtcedef.FloatDataEncoding(32, encoding="MIL-1750A")), + xtcedef.Packet(b'\x9F\xFF\xFF\x04'), + -0.7500001 * (2 ** 4)), + # Little endian version of previous test + (xtcedef.FloatParameterType( + 'MIL_1750A_FLOAT', + xtcedef.FloatDataEncoding(32, encoding="MIL-1750A", byte_order="leastSignificantByteFirst")), + xtcedef.Packet(b'\x04\xFF\xFF\x9F'), + -0.7500001 * (2 ** 4)), ] ) def test_float_parameter_parsing(parameter_type, packet, expected): """Test parsing float parameters""" raw, derived = parameter_type.parse_value(packet) + # NOTE: These results are compared with a relative tolerance due to the imprecise storage of floats if derived: - # NOTE: These results are rounded due to the imprecise storage of floats - assert round(derived, 5) == expected + assert derived == pytest.approx(expected, rel=1E-7) else: - assert round(raw, 5) == expected + assert raw == pytest.approx(expected, rel=1E-7) @pytest.mark.parametrize( @@ -1588,7 +1651,6 @@ def test_enumerated_parameter_parsing(parameter_type, packet, expected): """"Test parsing enumerated parameters""" raw, derived = parameter_type.parse_value(packet) if derived: - # NOTE: These results are rounded due to the imprecise storage of floats assert derived == expected else: assert raw == expected @@ -1945,9 +2007,9 @@ def test_absolute_time_parameter_type(xml_string, expectation): ) def test_absolute_time_parameter_parsing(parameter_type, packet, expected_raw, expected_derived): raw, derived = parameter_type.parse_value(packet) - assert round(raw, 5) == round(expected_raw, 5) + assert raw == pytest.approx(expected_raw, rel=1E-6) # NOTE: derived values are rounded for comparison due to imprecise storage of floats - assert round(derived, 5) == round(expected_derived, 5) + assert derived == pytest.approx(expected_derived, rel=1E-6) # ---------------