add support for multipath (draft-05)

examples/echo.c

@@ -215,7 +215,7 @@ static int run_loop(int fd, quicly_conn_t *client)
                 tv.tv_sec = delta / 1000;
                 tv.tv_usec = (delta % 1000) * 1000;
             } else {
-                tv.tv_sec = 1000;
+                tv.tv_sec = 0;
                 tv.tv_usec = 0;
             }
             FD_ZERO(&readfds);

include/quicly.h

@@ -192,10 +192,11 @@ typedef struct st_quicly_crypto_engine_t {
      * header protection using `header_protect_ctx`. Quicly does not read or write the content of the UDP datagram payload after
      * this function is called. Therefore, an engine might retain the information provided by this function, and protect the packet
      * and the header at a later moment (e.g., hardware crypto offload).
+     * @param dcid specifies the CID sequence number for encrypting Multipath QUIC packets; will always be zero in QUIC v1
      */
     void (*encrypt_packet)(struct st_quicly_crypto_engine_t *engine, quicly_conn_t *conn, ptls_cipher_context_t *header_protect_ctx,
                            ptls_aead_context_t *packet_protect_ctx, ptls_iovec_t datagram, size_t first_byte_at,
-                           size_t payload_from, uint64_t packet_number, int coalesced);
+                           size_t payload_from, uint64_t dcid, uint64_t packet_number, int coalesced);
 } quicly_crypto_engine_t;
 
 /**
@@ -261,6 +262,10 @@ typedef struct st_quicly_transport_parameters_t {
      *
      */
     uint8_t disable_active_migration : 1;
+    /**
+     *
+     */
+    uint8_t enable_multipath : 1;
     /**
      *
      */
@@ -338,6 +343,10 @@ struct st_quicly_context_t {
      * expand client hello so that it does not fit into one datagram
      */
     unsigned expand_client_hello : 1;
+    /**
+     * whether to use ECN on the send side; ECN is always on on the receive side
+     */
+    unsigned enable_ecn : 1;
     /**
      *
      */
@@ -462,6 +471,14 @@ struct st_quicly_conn_streamgroup_state_t {
          * Total number of packets received out of order.                                                                          \
          */                                                                                                                        \
         uint64_t received_out_of_order;                                                                                            \
+        /**                                                                                                                        \
+         * connection-wide counters for ECT(0), ECT(1), CE                                                                         \
+         */                                                                                                                        \
+        uint64_t received_ecn_counts[3];                                                                                           \
+        /**                                                                                                                        \
+         * connection-wide ack-received counters for ECT(0), ECT(1), CE                                                            \
+         */                                                                                                                        \
+        uint64_t acked_ecn_counts[3];                                                                                              \
         /**                                                                                                                        \
          * Total number of packets sent on promoted paths.                                                                         \
          */                                                                                                                        \
@@ -522,6 +539,14 @@ struct st_quicly_conn_streamgroup_state_t {
          * number of alternate paths that were closed due to Connection ID being unavailable                                       \
          */                                                                                                                        \
         uint64_t closed_no_dcid;                                                                                                   \
+        /**                                                                                                                        \
+         * number of paths that were ECN-capable                                                                                   \
+         */                                                                                                                        \
+        uint64_t ecn_validated;                                                                                                    \
+        /**                                                                                                                        \
+         * number of paths that were deemed as ECN black holes                                                                     \
+         */                                                                                                                        \
+        uint64_t ecn_failed;                                                                                                       \
     } num_paths;                                                                                                                   \
     /**                                                                                                                            \
      * Total number of each frame being sent / received.                                                                           \
@@ -530,7 +555,7 @@ struct st_quicly_conn_streamgroup_state_t {
         uint64_t padding, ping, ack, reset_stream, stop_sending, crypto, new_token, stream, max_data, max_stream_data,             \
             max_streams_bidi, max_streams_uni, data_blocked, stream_data_blocked, streams_blocked, new_connection_id,              \
             retire_connection_id, path_challenge, path_response, transport_close, application_close, handshake_done, datagram,     \
-            ack_frequency;                                                                                                         \
+            ack_frequency, ack_mp, path_abandon, path_status;                                                                      \
     } num_frames_sent, num_frames_received;                                                                                        \
     /**                                                                                                                            \
      * Total number of PTOs observed during the connection.                                                                        \
@@ -848,6 +873,10 @@ typedef struct st_quicly_decoded_packet_t {
         uint64_t pn;
         uint64_t key_phase;
     } decrypted;
+    /**
+     * ECN bits
+     */
+    uint8_t ecn : 2;
     /**
      *
      */
@@ -1065,6 +1094,10 @@ size_t quicly_send_retry(quicly_context_t *ctx, ptls_aead_context_t *token_encry
  */
 int quicly_send(quicly_conn_t *conn, quicly_address_t *dest, quicly_address_t *src, struct iovec *datagrams, size_t *num_datagrams,
                 void *buf, size_t bufsize);
+/**
+ * returns ECN bits to be set for the packets built by the last invocation of `quicly_send`
+ */
+uint8_t quicly_send_get_ecn_bits(quicly_conn_t *conn);
 /**
  *
  */
@@ -1128,10 +1161,20 @@ int quicly_connect(quicly_conn_t **conn, quicly_context_t *ctx, const char *serv
 int quicly_accept(quicly_conn_t **conn, quicly_context_t *ctx, struct sockaddr *dest_addr, struct sockaddr *src_addr,
                   quicly_decoded_packet_t *packet, quicly_address_token_plaintext_t *address_token,
                   const quicly_cid_plaintext_t *new_cid, ptls_handshake_properties_t *handshake_properties, void *appdata);
+/**
+ * Adds a new path. Only usable when running as a client. Local must contain sufficient information to distinguish between the paths
+ * being establisished; i.e, either the port number should be different or if one port is shared then the IP addresses of each
+ * local address must be different.
+ */
+int quicly_add_path(quicly_conn_t *conn, struct sockaddr *local);
 /**
  *
  */
 ptls_t *quicly_get_tls(quicly_conn_t *conn);
+/**
+ *
+ */
+int quicly_is_multipath(quicly_conn_t *conn);
 /**
  * Resumes an async TLS handshake, and returns a pointer to the QUIC connection or NULL if the corresponding QUIC connection has
  * been discarded. See `quicly_async_handshake_t`.
@@ -1226,6 +1269,12 @@ int quicly_set_cc(quicly_conn_t *conn, quicly_cc_type_t *cc);
  *
  */
 void quicly_amend_ptls_context(ptls_context_t *ptls);
+/**
+ * Builds the IV prefix of used to encrypt / decrypt Multipath QUIC packets. Size of the supplied buffer (`iv`) must be no less than
+ * `PTLS_MAX_IV_SIZE`. Once the IV is built, that should be applied to AEAD using `ptls_aead_xor_iv` prior to calling the encryption
+ * function. After that, `ptls_aead_xor_iv` should be called again with the same arguments to nagate the changes to IV.
+ */
+static size_t quicly_build_multipath_iv(ptls_aead_algorithm_t *algo, uint64_t sequence, void *iv);
 /**
  * Encrypts an address token by serializing the plaintext structure and appending an authentication tag.
  *
@@ -1417,6 +1466,21 @@ inline void quicly_stream_set_receive_window(quicly_stream_t *stream, uint32_t w
     stream->_recv_aux.window = window;
 }
 
+inline size_t quicly_build_multipath_iv(ptls_aead_algorithm_t *algo, uint64_t sequence, void *_iv)
+{
+    size_t len = algo->iv_size - 8;
+    uint8_t *iv = (uint8_t *)_iv;
+
+    for (size_t i = 0; i + 4 < len; ++i)
+        *iv++ = 0;
+    *iv++ = (uint8_t)(sequence >> 24);
+    *iv++ = (uint8_t)(sequence >> 16);
+    *iv++ = (uint8_t)(sequence >> 8);
+    *iv++ = (uint8_t)sequence;
+
+    return len;
+}
+
 inline int quicly_stream_is_client_initiated(quicly_stream_id_t stream_id)
 {
     if (stream_id < 0)

include/quicly/cc.h

@@ -61,6 +61,10 @@ typedef struct st_quicly_cc_t {
      * Packet number indicating end of recovery period, if in recovery.
      */
     uint64_t recovery_end;
+    /**
+     * If the most recent loss episode was signalled by ECN only (i.e., no packet loss).
+     */
+    unsigned episode_by_ecn : 1;
     /**
      * State information specific to the congestion controller implementation.
      */
@@ -130,9 +134,13 @@ typedef struct st_quicly_cc_t {
      */
     uint32_t cwnd_maximum;
     /**
-     * Total number of number of loss episodes (congestion window reductions).
+     * Total number of loss episodes (congestion window reductions).
      */
     uint32_t num_loss_episodes;
+    /**
+     * Total number of loss episodes that was reported only by ECN (hence no packet loss).
+     */
+    uint32_t num_ecn_loss_episodes;
 } quicly_cc_t;
 
 struct st_quicly_cc_type_t {
@@ -150,8 +158,9 @@ struct st_quicly_cc_type_t {
     void (*cc_on_acked)(quicly_cc_t *cc, const quicly_loss_t *loss, uint32_t bytes, uint64_t largest_acked, uint32_t inflight,
                         uint64_t next_pn, int64_t now, uint32_t max_udp_payload_size);
     /**
-     * Called when a packet is detected as lost. |next_pn| is the next unsent packet number,
-     * used for setting the recovery window.
+     * Called when a packet is detected as lost.
+     * @param bytes    bytes declared lost, or zero iff ECN_CE is observed
+     * @param next_pn  the next unsent packet number, used for setting the recovery window
      */
     void (*cc_on_lost)(quicly_cc_t *cc, const quicly_loss_t *loss, uint32_t bytes, uint64_t lost_pn, uint64_t next_pn, int64_t now,
                        uint32_t max_udp_payload_size);
@@ -192,6 +201,27 @@ void quicly_cc_reno_on_lost(quicly_cc_t *cc, const quicly_loss_t *loss, uint32_t
                             int64_t now, uint32_t max_udp_payload_size);
 void quicly_cc_reno_on_persistent_congestion(quicly_cc_t *cc, const quicly_loss_t *loss, int64_t now);
 void quicly_cc_reno_on_sent(quicly_cc_t *cc, const quicly_loss_t *loss, uint32_t bytes, int64_t now);
+/**
+ * Updates ECN counter when loss is observed.
+ */
+static void quicly_cc__update_ecn_episodes(quicly_cc_t *cc, uint32_t lost_bytes, uint64_t lost_pn);
+
+/* inline definitions */
+
+inline void quicly_cc__update_ecn_episodes(quicly_cc_t *cc, uint32_t lost_bytes, uint64_t lost_pn)
+{
+    /* when it is a new loss episode, initially assume that all losses are due to ECN signalling ... */
+    if (lost_pn >= cc->recovery_end) {
+        ++cc->num_ecn_loss_episodes;
+        cc->episode_by_ecn = 1;
+    }
+
+    /* ... but if a loss is observed, decrement the ECN loss episode counter */
+    if (lost_bytes != 0 && cc->episode_by_ecn) {
+        --cc->num_ecn_loss_episodes;
+        cc->episode_by_ecn = 0;
+    }
+}
 
 #ifdef __cplusplus
 }

include/quicly/constants.h

@@ -97,6 +97,7 @@ extern "C" {
 #define QUICLY_TRANSPORT_ERROR_KEY_UPDATE QUICLY_ERROR_FROM_TRANSPORT_ERROR_CODE(0xe)
 #define QUICLY_TRANSPORT_ERROR_AEAD_LIMIT_REACHED QUICLY_ERROR_FROM_TRANSPORT_ERROR_CODE(0xf)
 #define QUICLY_TRANSPORT_ERROR_TLS_ALERT_BASE QUICLY_ERROR_FROM_TRANSPORT_ERROR_CODE(0x100)
+#define QUICLY_TRANSPORT_ERROR_MP_PROTOCOL_VIOLATION QUICLY_ERROR_FROM_TRANSPORT_ERROR_CODE(0xba01)
 
 /* internal error codes, used purely for signaling status to the application */
 #define QUICLY_ERROR_PACKET_IGNORED 0xff01
@@ -108,6 +109,7 @@ extern "C" {
 #define QUICLY_ERROR_STATE_EXHAUSTION 0xff07
 #define QUICLY_ERROR_INVALID_INITIAL_VERSION 0xff08
 #define QUICLY_ERROR_DECRYPTION_FAILED 0xff09
+#define QUICLY_ERROR_INVALID_PARAMETERS 0xff0a
 
 typedef int64_t quicly_stream_id_t;