From 01b9a9b57143bc916bfa2fa26102a787b4e558e9 Mon Sep 17 00:00:00 2001 From: Pavel Kirienko Date: Sat, 3 Jun 2023 16:56:06 +0300 Subject: [PATCH] Update the docs for Cyphal/UDP and Cyphal/serial (#301) This is a trivial update to the docs to remove duplication with the official Specification. --- pycyphal/_version.py | 2 +- pycyphal/application/_transport_factory.py | 4 +- pycyphal/transport/_transport.py | 2 +- pycyphal/transport/redundant/_tracer.py | 2 +- pycyphal/transport/serial/__init__.py | 110 +--------- pycyphal/transport/serial/_serial.py | 2 +- pycyphal/transport/udp/__init__.py | 237 +-------------------- pycyphal/transport/udp/_frame.py | 23 +- pycyphal/transport/udp/_udp.py | 3 +- 9 files changed, 27 insertions(+), 358 deletions(-) diff --git a/pycyphal/_version.py b/pycyphal/_version.py index 8b0c91569..ed6661eef 100644 --- a/pycyphal/_version.py +++ b/pycyphal/_version.py @@ -1 +1 @@ -__version__ = "1.15.1" +__version__ = "1.15.2" diff --git a/pycyphal/application/_transport_factory.py b/pycyphal/application/_transport_factory.py index e65a58699..b55f833fa 100644 --- a/pycyphal/application/_transport_factory.py +++ b/pycyphal/application/_transport_factory.py @@ -52,7 +52,7 @@ def make_transport( * - ``uavcan.udp.duplicate_service_transfers`` - ``bit[1]`` - - Apply deterministic data loss mitigation to RPC-service transfers by setting multiplication factor = 2. + - Apply forward error correction to RPC-service transfers by setting multiplication factor = 2. * - ``uavcan.udp.mtu`` - ``natural16[1]`` @@ -73,7 +73,7 @@ def make_transport( * - ``uavcan.serial.duplicate_service_transfers`` - ``bit[1]`` - - Apply deterministic data loss mitigation to RPC-service transfers by setting multiplication factor = 2. + - Apply forward error correction to RPC-service transfers by setting multiplication factor = 2. * - ``uavcan.serial.baudrate`` - ``natural32[1]`` diff --git a/pycyphal/transport/_transport.py b/pycyphal/transport/_transport.py index 09161d634..045e27d8a 100644 --- a/pycyphal/transport/_transport.py +++ b/pycyphal/transport/_transport.py @@ -207,7 +207,7 @@ def begin_capture(self, handler: CaptureCallback) -> None: Technically, the capture protocol, as you can see, does not present any requirements to the emitted events, so an implementation that pretends to enter the capture mode while not actually doing anything is compliant. - Since capture reflects actual network events, deterministic data loss mitigation will make the instance emit + Since capture reflects actual network events, FEC will make the instance emit duplicate frames for affected transfers (although this is probably obvious enough without this elaboration). It is not possible to disable capture. Once enabled, it will go on until the transport instance is destroyed. diff --git a/pycyphal/transport/redundant/_tracer.py b/pycyphal/transport/redundant/_tracer.py index 6a332c35b..184c53f61 100644 --- a/pycyphal/transport/redundant/_tracer.py +++ b/pycyphal/transport/redundant/_tracer.py @@ -47,7 +47,7 @@ def get_transport_type() -> typing.Type[pycyphal.transport.redundant.RedundantTr class RedundantDuplicateTransferTrace(pycyphal.transport.Trace): """ Indicates that the last capture object completed a valid transfer that was discarded as a duplicate - (either received from another redundant interface or deterministic data loss mitigation (DDLM) is employed). + (either received from another redundant interface or forward error correction is employed). Observe that it is NOT a subclass of :class:`pycyphal.transport.TransferTrace`! It shall not be one because duplicates should not be processed normally. diff --git a/pycyphal/transport/serial/__init__.py b/pycyphal/transport/serial/__init__.py index 7e1c3b51e..b0e01f344 100644 --- a/pycyphal/transport/serial/__init__.py +++ b/pycyphal/transport/serial/__init__.py @@ -3,118 +3,28 @@ # Author: Pavel Kirienko """ -Cyphal/Serial transport overview +Cyphal/serial transport overview ++++++++++++++++++++++++++++++++ -The Cyphal/Serial transport is designed for OSI L1 byte-level duplex serial links and tunnels: +The Cyphal/serial transport is designed for byte-level communication channels, such as: -- UART, RS-422/485/232 (duplex); the recommended rates are: 115200 bps, 921600 bps, 3 Mbps, 10 Mbps, 100 Mbps. -- USB CDC ACM. -- TCP/IP encapsulation. +- TCP/IP +- UART, RS-422/232 +- USB CDC ACM -It may also be suited for raw transport log storage, because one-dimensional flat binary files are structurally -similar to serial byte-level links. +It may also be suited for raw transport log storage. This transport module contains no media sublayers because the media abstraction is handled directly by the `PySerial `_ library and the underlying operating system. -The serial transport supports all transfer categories: +For the full protocol definition, please refer to the `Cyphal Specification `_. -+--------------------+--------------------------+---------------------------+ -| Supported transfers| Unicast | Broadcast | -+====================+==========================+===========================+ -|**Message** | Yes, non-spec extension | Yes | -+--------------------+--------------------------+---------------------------+ -|**Service** | Yes | Banned by Specification | -+--------------------+--------------------------+---------------------------+ +Forward error correction (FEC) +++++++++++++++++++++++++++++++ -Protocol definition -+++++++++++++++++++ - -The packet header is defined as follows (byte/bit ordering in this definition follow the DSDL specification: -least significant first):: - - uint4 version # = 1, Discard the frame if not. - void4 - - uint3 priority # 0 = highest, 7 = lowest; the rest are unused. - void5 - - uint16 source_node_id # 0xFFFF = anonymous. - uint16 destination_node_id # 0xFFFF = broadcast. - uint16 data_specifier # subject-ID | (service-ID + RNR (Request, Not Response)) - - uint64 transfer_id - - uint31 frame_index - bool end_of_transfer # Set if last frame of the transfer - - uint16 user_data # Opaque application-specific data with user-defined semantics. - # Generic implementations should ignore - - uint8[2] header_crc_be # CRC-16-CCITT of the header (all fields above). - # Most significant byte first. - -For message frames, the data specifier field contains the subject-ID value, -so that the most significant bit is always cleared. -For service frames, the most significant bit (15th) is always set, -and the second-to-most-significant bit (14th) is set for request transfers only; -the remaining 14 least significant bits contain the service-ID value. - -Total header size: 24 bytes (192 bits). - -The header is prepended before the frame payload; the resulting structure is -encoded into its serialized form using the following packet format: - -+-------------------------+--------------+---------------+--------------------------------+-------------------------+ -| Frame delimiter **0x00**|Escaped header|Escaped payload| Escaped CRC-32C of the payload | Frame delimiter **0x00**| -+=========================+==============+===============+================================+=========================+ -| 1 byte | 24 bytes | >=0 bytes | 4 bytes | 1 byte | -+-------------------------+--------------+---------------+--------------------------------+-------------------------+ -| Single-byte frame | | Four bytes long, little-endian | Same frame delimiter as | -| delimiter **0x00**. | | byte order; The CRC is | at the start. | -| Begins a new frame and | | computed over the unescaped | Terminates the current | -| possibly terminates the | | (i.e., original form) payload, | frame and possibly | -| previous frame. | | not including the header | begins the next frame. | -| | | (because the header has a | | -| | | dedicated CRC). | | -| +------------------------------+--------------------------------+ | -| | This part is escaped using COBS alorithm by Chesire and Baker | | -| | http://www.stuartcheshire.org/papers/COBSforToN.pdf. | | -| | A frame delimiter (0) is guaranteed to never occur here. | | -+-------------------------+------------------------------+--------------------------------+-------------------------+ - -The frame encoding overhead is 1 byte in every 254 bytes of the header+payload+CRC, which is about ~0.4%. -There is a somewhat relevant discussion at -https://forum.opencyphal.org/t/uavcan-serial-issues-with-dma-friendliness-and-bandwidth-overhead/846. - -The last four bytes of a multi-frame transfer payload contain the CRC32C (Castagnoli) hash of the transfer -payload in little-endian byte order. -The multi-frame transfer logic (decomposition and reassembly) is implemented in a separate -transport-agnostic module :mod:`pycyphal.transport.commons.high_overhead_transport`. -**Despite the fact that the support for multi-frame transfers is built into this transport, -it should not be relied on and it may be removed later.** -The reason is that serial links do not have native support for framing, and as such, -it is possible to configure the MTU to be arbitrarily high to avoid multi-frame transfers completely. -The lack of multi-frame transfers simplifies implementations drastically, which is important for -deeply-embedded systems. As such, all serial transfers should be single-frame transfers. - -Note that we use CRC-32C (Castagnoli) as the header/frame CRC instead of CRC-32K2 (Koopman-2) -which is superior at short data blocks offering the Hamming distance of 6 as opposed to 4. -This is because Castagnoli is superior for transfer CRC which is often sufficiently long -to flip the balance in favor of Castagnoli rather than Koopman. -We could use Koopman for the header/frame CRC and keep Castagnoli for the transfer CRC, -but such diversity is harmful because it would require implementers to keep two separate CRC tables -which may be costly in embedded applications and may deteriorate the performance of CPU caches. - - -Unreliable links and temporal redundancy -++++++++++++++++++++++++++++++++++++++++ - -The serial transport supports the deterministic data loss mitigation option, -where a transfer can be repeated several times to reduce the probability of its loss. +This transport supports optional FEC through full duplication of transfers. This feature is discussed in detail in the documentation for the UDP transport :mod:`pycyphal.transport.udp`. diff --git a/pycyphal/transport/serial/_serial.py b/pycyphal/transport/serial/_serial.py index e0565f820..8381ebefb 100644 --- a/pycyphal/transport/serial/_serial.py +++ b/pycyphal/transport/serial/_serial.py @@ -99,7 +99,7 @@ def __init__( This setting does not affect transfer reception -- the RX MTU is always set to the maximum valid MTU (i.e., practically unlimited). - :param service_transfer_multiplier: Deterministic data loss mitigation for service transfers. + :param service_transfer_multiplier: Forward error correction for service transfers. This parameter specifies the number of times each outgoing service transfer will be repeated. This setting does not affect message transfers. diff --git a/pycyphal/transport/udp/__init__.py b/pycyphal/transport/udp/__init__.py index 6d13b560b..90c3e991e 100644 --- a/pycyphal/transport/udp/__init__.py +++ b/pycyphal/transport/udp/__init__.py @@ -6,240 +6,26 @@ Cyphal/UDP transport overview +++++++++++++++++++++++++++++ -The Cyphal/UDP transport is essentially a trivial stateless UDP blaster based on IP multicasting. -This transport is intended for low-latency, high-throughput switched Ethernet networks with complex topologies. -In the spirit of Cyphal, it is designed to be simple and robust; -much of the data handling work is offloaded to the standard underlying UDP/IP stack. -Both IPv4 and IPv6 are supported by this design, -although it is expected that the advantages of IPv6 over IPv4 are less relevant in an intravehicular setting. - -Cyphal/UDP supports anonymous transfers (i.e., transfers without a source node-ID) with one limitation: -an anonymous node is only able to send Message transfers (but not Service transfers). +Please refer to the appropriate section of the `Cyphal Specification `_ +for the definition of the Cyphal/UDP transport. This transport module contains no media sublayers because the media abstraction is handled directly by the standard UDP/IP stack of the underlying operating system. -Per the Cyphal transport model provided in the Cyphal specification, the following transfer categories are supported: - -+--------------------+--------------------------+---------------------------+ -| Supported transfers| Point-to-point | Point-to-many | -+====================+==========================+===========================+ -|**Message** | No | Yes | -+--------------------+--------------------------+---------------------------+ -|**Service** | Yes | Banned by Specification | -+--------------------+--------------------------+---------------------------+ - - -Protocol definition -+++++++++++++++++++ - -The entirety of the session specifier (:class:`pycyphal.transport.SessionSpecifier`) -is reified through the standard UDP/IP stack without any special extensions. -The transfer-ID, transfer priority, and the multi-frame transfer reassembly metadata are allocated in the -Cyphal-specific UDP datagram header. - -There are two data types that model Cyphal/UDP protocol data: :class:`UDPFrame` and :class:`RawPacket`. -The latter is never used during normal operation but only during on-line capture sessions -for reporting captured packets (see :class:`UDPCaptured`). - -Cyphal uses a single UDP port for all transfers (9382). - -For more background information on how Cyphal/UDP came to be, please see the following thread in the OpenCyphal forum: -https://forum.opencyphal.org/t/1765 - - -IP address mapping -++++++++++++++++++ - -Message transfers -~~~~~~~~~~~~~~~~~ - -Message transfers are executed as IP multicast transfers. -The IPv4 multicast group address is computed statically as follows:: - - fixed subject-ID (Message) - (15 bits) res. (15 bits) - ______________ | ______________ - / \ v/ \ - 11101111.00000000.0sssssss.ssssssss - \__/ ^ ^ - (4 bits) Cyphal SNM - IPv4 UDP - multicast address - prefix version - \_______________________/ - (23 bits) - collision-free multicast - addressing limit of - Ethernet MAC for IPv4 - -SNM: Service, not Message - -From the most significant bit to the least significant bit, the IPv4 multicast group address components are as follows: - -- IPv4 multicast prefix is defined by RFC 1112. - -- The following 5 bits are set to 0b11110 by this Specification. The motivation is as follows: - - - Setting the four least significant bits of the most significant byte to 0b1111 moves the address range - into the administratively-scoped range (239.0.0.0/8, RFC 2365), - which ensures that there may be no conflicts with well-known multicast groups. - - - Setting the most significant bit of the second octet to zero ensures that there may be no conflict - with reserved sub-ranges within the administratively-scoped range. - The resulting range 239.0.0.0/9 is entirely ad-hoc defined. - - - Fixing the 5+4=9 most significant bits of the multicast group address ensures that the variability - is confined to the 23 least significant bits of the address only, - which is desirable because the IPv4 Ethernet MAC layer does not differentiate beyond the - 23 least significant bits of the multicast group address - (i.e., addresses that differ only in the 9 MSb collide at the MAC layer, - which is unacceptable in a real-time system; see RFC 1112 section 6.4). - Without this limitation, an engineer deploying a network might inadvertently create a configuration that - causes MAC-layer collisions which may be difficult to detect. - -- The next 6 bits complete the fixed part of the multicast group address, with the most significant bit - defining the Cyphal UDP address version (this can be used in case we want to make changes to the endpoint - mapping). - -- Last but not least, the remaining 17 bits are used to encode: - - - SNM: Service, not Message (1 bit), which is used to differentiate between a Message and Service address. - Set to zero in case of Message. - - - 1 reserved bit for future use. - - - The 15-bit subject-ID of the Message. - -Per RFC 1112, the default TTL is 1, which is unacceptable. -Therefore, publishers should use the TTL value of 16 by default, -which is chosen as a sensible default suitable for any intravehicular network. - -Per RFC 1112, in order to emit a multicast packet, a limited level-1 implementation without the full support of -IGMP and multicast-specific packet handling policies is sufficient. - -Due to the dependency on the dynamic IGMP configuration, -a newly configured subscriber may not immediately receive data from the subject -- -a brief *subscription initialization latency* may occur (typically it is well under one second). -This is because the underlying IP stack needs to inform the network switch/router about its interest in a particular -multicast group by sending an IGMP membership report first. -A high-integrity application may choose to rely on a static switch configuration, -in which case no initialization delay will take place. -Example:: +Forward error correction (FEC) +++++++++++++++++++++++++++++++ - Fixed prefix: 11101111 0000000x xxxxxxxx xxxxxxxx - - Service, : xxxxxxxx xxxxxxx0 xxxxxxxx xxxxxxxx - not Message - - Reserved: xxxxxxxx xxxxxxxx 0xxxxxxx xxxxxxxx - - Subject-ID (=42): xxxxxxxx xxxxxxxx x0000000 00101010 - - Multicast group: 11101111 00000000 00000000 00101010 - 239 0 0 42 - - -Service transfers -~~~~~~~~~~~~~~~~~ - -Service transfers are also executed as IP multicast transfers. -The IPv4 multicast group address is computed statically as follows:: - - fixed - (15 bits) - ______________ - / \ - 11101111.00000001.ssssssss.ssssssss - \__/ ^ ^ \_______________/ - (4 bits) Cyphal SNM (16 bits) - IPv4 UDP destination node-ID - multicast address - prefix version - \_______________________/ - (23 bits) - collision-free multicast - addressing limit of - Ethernet MAC for IPv4 - -Service transfers are distinguished from message transfers by the least significant bit of the second octet. -The 2 last octets define the destination node-ID of the service transfer. - -Example:: - - Fixed prefix: 11101111 0000000x xxxxxxxx xxxxxxxx - - Service, : xxxxxxxx xxxxxxx1 xxxxxxxx xxxxxxxx - not Message - - Reserved: xxxxxxxx xxxxxxxx 0xxxxxxx xxxxxxxx - - Subject-ID (=42): xxxxxxxx xxxxxxxx x0000000 00101010 - - Multicast group: 11101111 00000000 00000000 00101010 - 239 1 0 42 - -Datagram header format -~~~~~~~~~~~~~~~~~~~~~~ - -Every Cyphal/UDP frame contains the following header before the payload, -encoded in the little-endian byte order, expressed here in the DSDL notation:: - - uint8 version # =1 in this revision; ignore frame otherwise. - uint8 priority # Like in CAN: 0 -- highest priority, 7 -- lowest priority. - uint16 source_node_id # Cyphal node-ID of the origin. - uint32 frame_index_eot # MSB is set if the current frame is the last frame of the transfer. - uint64 transfer_id # The transfer-ID never overflows. - void64 # This space may be used later for runtime type identification. - - uint4 version # <- 1 - void4 - uint3 priority # Duplicates QoS for ease of access; 0 -- highest, 7 -- lowest. - void5 - uint16 source_node_id - uint16 destination_node_id - uint16 data_specifier # Like in Cyphal/serial: subject-ID | (service-ID + RNR (Request, Not Response)) - uint64 transfer_id - uint31 frame_index # Index of the current frame within the current transfer. - bool end_of_transfer - uint16 user_data - # Opaque application-specific data with user-defined semantics. Generic implementations should ignore - uint16 header_crc - @assert _offset_ / 8 == {24} # Fixed-size 24-byte header with natural alignment for each field ensured. - @sealed - -In the case of a Message frame, the ``data_specifier`` field contains the subject-ID of the message -(15 least significant bits) and the remaining most significant bit represents SNM. - -In the case of a Service frame, the ``data_specifier`` field contains the service-ID of the service -(14 least significant bits) and the remaining two most significant bits represent RNR and SNM -(second and most significant bits respectively). - -Also see the documentation for :class:`UDPFrame`. - -Please note: in addition to ``header_crc``, multi-frame transfers contain four bytes of CRC32-C (Castagnoli) -at the end of the payload computed over the entire transfer payload (payload_crc). -For more info on multi-frame transfers, please see -:class:`pycyphal.transport.commons.high_overhead_transport.TransferReassembler`. - -Unreliable networks and temporal redundancy -+++++++++++++++++++++++++++++++++++++++++++ - -For unreliable networks, deterministic data loss mitigation is supported. +For unreliable networks, optional forward error correction (FEC) is supported by this implementation. This measure is only available for service transfers, not for message transfers due to their different semantics. If the probability of a frame loss exceeds the desired reliability threshold, the transport can be configured to repeat every outgoing service transfer a specified number of times, on the assumption that the probability of losing any given frame is uncorrelated (or weakly correlated) with that of its neighbors. - Assuming that the probability of transfer loss ``P`` is time-invariant, -the influence of the multiplier ``M`` can be approximated as ``P' = P^M``. -For example, given a network that successfully delivers 99% of transfers, -and the probabilities of adjacent transfer loss are uncorrelated, -the multiplication factor of 2 can increase the link reliability up to ``100% - (100% - 99%)^2 = 99.99%``. +the influence of the FEC multiplier ``M`` can be approximated as ``P' = P^M``. -The duplicates are emitted immediately following the original transfer. +Duplicates are emitted immediately following the original transfer. For example, suppose that a service transfer contains three frames, F0 to F2, and the service transfer multiplication factor is two, then the resulting frame sequence would be as follows:: @@ -272,7 +58,7 @@ transfer Removal of duplicate transfers at the opposite end of the link is natively guaranteed by the Cyphal protocol; -no special activities are needed there (read the Cyphal Specification for background). +no special activities are needed there (refer to the Cyphal Specification for background). For time-deterministic (real-time) networks this strategy is preferred over the conventional confirmation-retry approach (e.g., the TCP model) because it results in more predictable @@ -280,9 +66,6 @@ about the state of other agents involved in data exchange). -Implementation-specific details -+++++++++++++++++++++++++++++++ - Usage +++++ @@ -330,13 +113,11 @@ >>> tr_0.close() >>> tr_1.close() -TODO Add Service example - Tooling +++++++ -Run Cyphal networks on the local loopback interface (``127.x.y.z/8``) or create virtual interfaces for testing. +Run Cyphal networks on the local loopback interface (``127.0.0.1``) or create virtual interfaces for testing. Use Wireshark for monitoring and inspection. diff --git a/pycyphal/transport/udp/_frame.py b/pycyphal/transport/udp/_frame.py index c330c9cf0..e2314b519 100644 --- a/pycyphal/transport/udp/_frame.py +++ b/pycyphal/transport/udp/_frame.py @@ -13,32 +13,11 @@ @dataclasses.dataclass(frozen=True, repr=False) class UDPFrame(pycyphal.transport.commons.high_overhead_transport.Frame): """ - The header format is up to debate until it's frozen in Specification. - An important thing to keep in mind is that the minimum size of an UDP/IPv4 payload when transferred over 100M Ethernet is 18 bytes, due to the minimum Ethernet frame size limit. That is, if the application payload requires less space, the missing bytes will be padded out to the minimum size. - The current header format enables encoding by trivial memory aliasing on any conventional little-endian platform:: - - struct Header { - uint4_t version; # <- 1 - uint4_t _reserved_a; - uint3_t priority; # Duplicates QoS for ease of access; 0 -- highest, 7 -- lowest. - uint5_t _reserved_b; - uint16_t source_node_id; # 0xFFFF == anonymous transfer - uint16_t destination_node_id; # 0xFFFF == broadcast - uint15_t data_specifier; # subject-ID | (service-ID + RNR (Request, Not Response)) - bool snm; # SNM (Service, Not Message) - uint64_t transfer_id; - uint31_t frame_index; - bool frame_index_eot; # End of transfer - uint16_t user_data; # Opaque application-specific data with user-defined semantics. - # Generic implementations should ignore - uint16_t header_crc; # Checksum of the header, excluding the CRC field itself - }; - static_assert(sizeof(struct Header) == 24, "Invalid layout"); # Fixed-size 24-byte header with - # natural alignment for each field ensured. + The current header format enables encoding by trivial memory aliasing on any conventional little-endian platform. +---------------+---------------+---------------+-----------------+------------------+ |**MAC header** | **IP header** |**UDP header** |**Cyphal header**|**Cyphal payload**| diff --git a/pycyphal/transport/udp/_udp.py b/pycyphal/transport/udp/_udp.py index c8afda885..1b1167585 100644 --- a/pycyphal/transport/udp/_udp.py +++ b/pycyphal/transport/udp/_udp.py @@ -93,7 +93,7 @@ def __init__( The default value is the smallest valid value for reasons of compatibility. - :param service_transfer_multiplier: Deterministic data loss mitigation is disabled by default. + :param service_transfer_multiplier: Forward error correction is disabled by default. This parameter specifies the number of times each outgoing service transfer will be repeated. This setting does not affect message transfers. @@ -207,7 +207,6 @@ def get_output_session( ) -> UDPOutputSession: self._ensure_not_closed() if specifier not in self._output_registry: - # check if anonymous, in that case no service transfers are allowed if self._anonymous and isinstance(specifier.data_specifier, pycyphal.transport.ServiceDataSpecifier): raise OperationNotDefinedForAnonymousNodeError(