Skip to content

Commit

Permalink
fix: hanging registration close due to a race in listener stop/close
Browse files Browse the repository at this point in the history
  • Loading branch information
@qzhuyan
qzhuyan committed May 3, 2024
1 parent 574417a commit 2276316
Show file tree
Hide file tree
Showing 3 changed files with 130 additions and 96 deletions.
26 changes: 19 additions & 7 deletions c_src/quicer_listener.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,17 @@ ServerListenerCallback(__unused_parm__ HQUIC Listener,
QuicerConnCTX *c_ctx = NULL;
BOOLEAN is_destroy = FALSE;

BOOLEAN is_worker = (enif_thread_type() == ERL_NIF_THR_UNDEFINED);

if (is_worker)
{
enif_mutex_lock(l_ctx->lock);
}

switch (Event->Type)
{
case QUIC_LISTENER_EVENT_NEW_CONNECTION:
enif_mutex_lock(l_ctx->lock);

//
// Note, c_ctx is newly init here, don't grab lock.
//
Expand Down Expand Up @@ -219,9 +226,8 @@ ServerListenerCallback(__unused_parm__ HQUIC Listener,
break;

case QUIC_LISTENER_EVENT_STOP_COMPLETE:
// **Note**, the callback in msquic for this event is called in the
// MsQuicListenerClose or MsQuicListenerStop. we assume caller should
// ensure the thread safty thus we don't hold lock
// **Note**, this callback event in msquic can be triggered by either
// MsQuicListenerClose or MsQuicListenerStop.
env = l_ctx->env;
enif_send(NULL,
&(l_ctx->listenerPid),
Expand All @@ -238,15 +244,21 @@ ServerListenerCallback(__unused_parm__ HQUIC Listener,
assert(!l_ctx->is_stopped);
is_destroy = TRUE;
}
else
{
l_ctx->is_stopped = TRUE;
}
enif_clear_env(env);
goto Exit2;
break;
default:
break;
}

Error:
enif_mutex_unlock(l_ctx->lock);
Exit2:
if (is_worker)
{
enif_mutex_unlock(l_ctx->lock);
}
if (is_destroy)
{
destroy_l_ctx(l_ctx);
Expand Down
2 changes: 1 addition & 1 deletion c_src/quicer_nif.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -804,7 +804,7 @@ resource_listener_down_callback(__unused_parm__ ErlNifEnv *env,
// b. Some pid could still close the stopped listener with nif
// handle.
// c. We close it in resource_listener_dealloc_callback anyway when
// Listener term get GC.
// Listener term get GCed.
*/
MsQuic->ListenerStop(l_ctx->Listener);
put_listener_handle(l_ctx);
Expand Down
198 changes: 110 additions & 88 deletions include/quicer.hrl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%--------------------------------------------------------------------
%%% % @noformat
%%% % noformat
-ifndef(QUICER_HRL).
-define(QUICER_HRL, true).

Expand All @@ -22,115 +22,136 @@
%%% ========================================

%% QUIC_STREAM_EVENT_TYPE
-define(QUIC_STREAM_EVENT_START_COMPLETE , 0).
-define(QUIC_STREAM_EVENT_RECEIVE , 1).
-define(QUIC_STREAM_EVENT_SEND_COMPLETE , 2).
-define(QUIC_STREAM_EVENT_PEER_SEND_SHUTDOWN , 3).
-define(QUIC_STREAM_EVENT_PEER_SEND_ABORTED , 4).
-define(QUIC_STREAM_EVENT_PEER_RECEIVE_ABORTED , 5).
-define(QUIC_STREAM_EVENT_SEND_SHUTDOWN_COMPLETE , 6).
-define(QUIC_STREAM_EVENT_SHUTDOWN_COMPLETE , 7).
-define(QUIC_STREAM_EVENT_IDEAL_SEND_BUFFER_SIZE , 8).


-define(QUIC_SEND_COMPLETE_SUCCESS , 0).
-define(QUIC_SEND_COMPLETE_CANCELLED , 1).
-define(QUIC_STREAM_EVENT_START_COMPLETE, 0).
-define(QUIC_STREAM_EVENT_RECEIVE, 1).
-define(QUIC_STREAM_EVENT_SEND_COMPLETE, 2).
-define(QUIC_STREAM_EVENT_PEER_SEND_SHUTDOWN, 3).
-define(QUIC_STREAM_EVENT_PEER_SEND_ABORTED, 4).
-define(QUIC_STREAM_EVENT_PEER_RECEIVE_ABORTED, 5).
-define(QUIC_STREAM_EVENT_SEND_SHUTDOWN_COMPLETE, 6).
-define(QUIC_STREAM_EVENT_SHUTDOWN_COMPLETE, 7).
-define(QUIC_STREAM_EVENT_IDEAL_SEND_BUFFER_SIZE, 8).

-define(QUIC_SEND_COMPLETE_SUCCESS, 0).
-define(QUIC_SEND_COMPLETE_CANCELLED, 1).
%% QUIC_LISTENER_EVENT_TYPE
-define(QUIC_LISTENER_EVENT_NEW_CONNECTION , 0).
-define(QUIC_LISTENER_EVENT_STOP_COMPLETE , 1).
-define(QUIC_LISTENER_EVENT_NEW_CONNECTION, 0).
-define(QUIC_LISTENER_EVENT_STOP_COMPLETE, 1).

%% QUIC_CONNECTION_EVENT_TYPE
-define(QUIC_CONNECTION_EVENT_CONNECTED , 0).
-define(QUIC_CONNECTION_EVENT_SHUTDOWN_INITIATED_BY_TRANSPORT , 1).
-define(QUIC_CONNECTION_EVENT_SHUTDOWN_INITIATED_BY_PEER , 2).
-define(QUIC_CONNECTION_EVENT_SHUTDOWN_COMPLETE , 3).
-define(QUIC_CONNECTION_EVENT_LOCAL_ADDRESS_CHANGED , 4).
-define(QUIC_CONNECTION_EVENT_PEER_ADDRESS_CHANGED , 5).
-define(QUIC_CONNECTION_EVENT_PEER_STREAM_STARTED , 6).
-define(QUIC_CONNECTION_EVENT_STREAMS_AVAILABLE , 7).
-define(QUIC_CONNECTION_EVENT_PEER_NEEDS_STREAMS , 8).
-define(QUIC_CONNECTION_EVENT_IDEAL_PROCESSOR_CHANGED , 9).
-define(QUIC_CONNECTION_EVENT_DATAGRAM_STATE_CHANGED , 10).
-define(QUIC_CONNECTION_EVENT_DATAGRAM_RECEIVED , 11).
-define(QUIC_CONNECTION_EVENT_DATAGRAM_SEND_STATE_CHANGED , 12).
-define(QUIC_CONNECTION_EVENT_RESUMED , 13).
-define(QUIC_CONNECTION_EVENT_RESUMPTION_TICKET_RECEIVED , 14).
-define(QUIC_CONNECTION_EVENT_PEER_CERTIFICATE_RECEIVED , 15).

-define(QUIC_CONNECTION_EVENT_CONNECTED, 0).
-define(QUIC_CONNECTION_EVENT_SHUTDOWN_INITIATED_BY_TRANSPORT, 1).
-define(QUIC_CONNECTION_EVENT_SHUTDOWN_INITIATED_BY_PEER, 2).
-define(QUIC_CONNECTION_EVENT_SHUTDOWN_COMPLETE, 3).
-define(QUIC_CONNECTION_EVENT_LOCAL_ADDRESS_CHANGED, 4).
-define(QUIC_CONNECTION_EVENT_PEER_ADDRESS_CHANGED, 5).
-define(QUIC_CONNECTION_EVENT_PEER_STREAM_STARTED, 6).
-define(QUIC_CONNECTION_EVENT_STREAMS_AVAILABLE, 7).
-define(QUIC_CONNECTION_EVENT_PEER_NEEDS_STREAMS, 8).
-define(QUIC_CONNECTION_EVENT_IDEAL_PROCESSOR_CHANGED, 9).
-define(QUIC_CONNECTION_EVENT_DATAGRAM_STATE_CHANGED, 10).
-define(QUIC_CONNECTION_EVENT_DATAGRAM_RECEIVED, 11).
-define(QUIC_CONNECTION_EVENT_DATAGRAM_SEND_STATE_CHANGED, 12).
-define(QUIC_CONNECTION_EVENT_RESUMED, 13).
-define(QUIC_CONNECTION_EVENT_RESUMPTION_TICKET_RECEIVED, 14).
-define(QUIC_CONNECTION_EVENT_PEER_CERTIFICATE_RECEIVED, 15).

%% STREAM OPEN FLAGS
-define(QUIC_STREAM_OPEN_FLAG_NONE , 0).
-define(QUIC_STREAM_OPEN_FLAG_UNIDIRECTIONAL , 1).
-define(QUIC_STREAM_OPEN_FLAG_0_RTT , 2).
-define(QUIC_STREAM_OPEN_FLAG_NONE, 0).
-define(QUIC_STREAM_OPEN_FLAG_UNIDIRECTIONAL, 1).
-define(QUIC_STREAM_OPEN_FLAG_0_RTT, 2).

%% STREAM START FLAGS
-define(QUIC_STREAM_START_FLAG_NONE , 0).
-define(QUIC_STREAM_START_FLAG_IMMEDIATE , 1).
-define(QUIC_STREAM_START_FLAG_FAIL_BLOCKED , 2).
-define(QUIC_STREAM_START_FLAG_SHUTDOWN_ON_FAIL , 4).
-define(QUIC_STREAM_START_FLAG_INDICATE_PEER_ACCEPT , 8).
-define(QUIC_STREAM_START_FLAG_NONE, 0).
-define(QUIC_STREAM_START_FLAG_IMMEDIATE, 1).
-define(QUIC_STREAM_START_FLAG_FAIL_BLOCKED, 2).
-define(QUIC_STREAM_START_FLAG_SHUTDOWN_ON_FAIL, 4).
-define(QUIC_STREAM_START_FLAG_INDICATE_PEER_ACCEPT, 8).
%% STREAM SHUTDOWN FLAGS
-define(QUIC_STREAM_SHUTDOWN_FLAG_NONE , 0).
-define(QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL , 1). % Cleanly closes the send path.
-define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND , 2). % Abruptly closes the send path.
-define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE , 4). % Abruptly closes the receive path.
-define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT , 6). % Abruptly closes both send and receive paths.
-define(QUIC_STREAM_SHUTDOWN_FLAG_IMMEDIATE , 8).

-define(QUIC_STREAM_SHUTDOWN_FLAG_NONE, 0).
% Cleanly closes the send path.
-define(QUIC_STREAM_SHUTDOWN_FLAG_GRACEFUL, 1).
% Abruptly closes the send path.
-define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT_SEND, 2).
% Abruptly closes the receive path.
-define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT_RECEIVE, 4).
% Abruptly closes both send and receive paths.
-define(QUIC_STREAM_SHUTDOWN_FLAG_ABORT, 6).
-define(QUIC_STREAM_SHUTDOWN_FLAG_IMMEDIATE, 8).

%% CONNECTION SHUTDOWN FLAGS
-define(QUIC_CONNECTION_SHUTDOWN_FLAG_NONE , 0).
-define(QUIC_CONNECTION_SHUTDOWN_FLAG_SILENT , 1).


-define(QUIC_CONNECTION_SHUTDOWN_FLAG_NONE, 0).
-define(QUIC_CONNECTION_SHUTDOWN_FLAG_SILENT, 1).

%% QUIC_CREDENTIAL_FLAGS
-define(QUIC_CREDENTIAL_FLAG_NONE , 16#00000000).
-define(QUIC_CREDENTIAL_FLAG_CLIENT , 16#00000001). %% Lack of client flag indicates server.
-define(QUIC_CREDENTIAL_FLAG_LOAD_ASYNCHRONOUS , 16#00000002).
-define(QUIC_CREDENTIAL_FLAG_NO_CERTIFICATE_VALIDATION , 16#00000004).
-define(QUIC_CREDENTIAL_FLAG_ENABLE_OCSP , 16#00000008). %% Schannel only currently
-define(QUIC_CREDENTIAL_FLAG_INDICATE_CERTIFICATE_RECEIVED , 16#00000010).
-define(QUIC_CREDENTIAL_FLAG_DEFER_CERTIFICATE_VALIDATION , 16#00000020). %% Schannel only currently
-define(QUIC_CREDENTIAL_FLAG_REQUIRE_CLIENT_AUTHENTICATION , 16#00000040). %% Schannel only currently
-define(QUIC_CREDENTIAL_FLAG_USE_TLS_BUILTIN_CERTIFICATE_VALIDATION , 16#00000080). %% OpenSSL only currently
-define(QUIC_CREDENTIAL_FLAG_REVOCATION_CHECK_END_CERT , 16#00000100). %% Schannel only currently
-define(QUIC_CREDENTIAL_FLAG_REVOCATION_CHECK_CHAIN , 16#00000200). %% Schannel only currently
-define(QUIC_CREDENTIAL_FLAG_REVOCATION_CHECK_CHAIN_EXCLUDE_ROOT , 16#00000400). %% Schannel only currently
-define(QUIC_CREDENTIAL_FLAG_IGNORE_NO_REVOCATION_CHECK , 16#00000800). %% Schannel only currently
-define(QUIC_CREDENTIAL_FLAG_IGNORE_REVOCATION_OFFLINE , 16#00001000). %% Schannel only currently
-define(QUIC_CREDENTIAL_FLAG_SET_ALLOWED_CIPHER_SUITES , 16#00002000).
-define(QUIC_CREDENTIAL_FLAG_USE_PORTABLE_CERTIFICATES , 16#00004000). %% OpenSSL only currently
-define(QUIC_CREDENTIAL_FLAG_NONE, 16#00000000).
%% Lack of client flag indicates server.
-define(QUIC_CREDENTIAL_FLAG_CLIENT, 16#00000001).
-define(QUIC_CREDENTIAL_FLAG_LOAD_ASYNCHRONOUS, 16#00000002).
-define(QUIC_CREDENTIAL_FLAG_NO_CERTIFICATE_VALIDATION, 16#00000004).
%% Schannel only currently
-define(QUIC_CREDENTIAL_FLAG_ENABLE_OCSP, 16#00000008).
-define(QUIC_CREDENTIAL_FLAG_INDICATE_CERTIFICATE_RECEIVED, 16#00000010).
%% Schannel only currently
-define(QUIC_CREDENTIAL_FLAG_DEFER_CERTIFICATE_VALIDATION, 16#00000020).
%% Schannel only currently
-define(QUIC_CREDENTIAL_FLAG_REQUIRE_CLIENT_AUTHENTICATION, 16#00000040).
%% OpenSSL only currently
-define(QUIC_CREDENTIAL_FLAG_USE_TLS_BUILTIN_CERTIFICATE_VALIDATION, 16#00000080).
%% Schannel only currently
-define(QUIC_CREDENTIAL_FLAG_REVOCATION_CHECK_END_CERT, 16#00000100).
%% Schannel only currently
-define(QUIC_CREDENTIAL_FLAG_REVOCATION_CHECK_CHAIN, 16#00000200).
%% Schannel only currently
-define(QUIC_CREDENTIAL_FLAG_REVOCATION_CHECK_CHAIN_EXCLUDE_ROOT, 16#00000400).
%% Schannel only currently
-define(QUIC_CREDENTIAL_FLAG_IGNORE_NO_REVOCATION_CHECK, 16#00000800).
%% Schannel only currently
-define(QUIC_CREDENTIAL_FLAG_IGNORE_REVOCATION_OFFLINE, 16#00001000).
-define(QUIC_CREDENTIAL_FLAG_SET_ALLOWED_CIPHER_SUITES, 16#00002000).
%% OpenSSL only currently
-define(QUIC_CREDENTIAL_FLAG_USE_PORTABLE_CERTIFICATES, 16#00004000).

%% QUICER_CONNECTION_EVENT_MASKS
-define(QUICER_CONNECTION_EVENT_MASK_NST , 16#00000001).
-define(QUICER_CONNECTION_EVENT_MASK_NO_STREAMS_AVAILABLE , 16#00000002).
-define(QUICER_CONNECTION_EVENT_MASK_NST, 16#00000001).
-define(QUICER_CONNECTION_EVENT_MASK_NO_STREAMS_AVAILABLE, 16#00000002).

%% QUICER_STREAM_EVENT_MASKS
-define(QUICER_STREAM_EVENT_MASK_START_COMPLETE , 16#00000001).
-define(QUICER_STREAM_EVENT_MASK_START_COMPLETE, 16#00000001).

%% QUIC SEND FLAGS
-define(QUIC_SEND_FLAG_NONE , 16#0000).
-define(QUIC_SEND_FLAG_ALLOW_0_RTT , 16#0001). % Allows the use of encrypting with 0-RTT key.
-define(QUIC_SEND_FLAG_START , 16#0002). % Asynchronously starts the stream with the sent data.
-define(QUIC_SEND_FLAG_FIN , 16#0004). % Indicates the request is the one last sent on the stream.
-define(QUIC_SEND_FLAG_DGRAM_PRIORITY , 16#0008). % Indicates the datagram is higher priority than others.
-define(QUIC_SEND_FLAG_DELAY_SEND , 16#0010).
-define(QUIC_SEND_FLAG_NONE, 16#0000).
% Allows the use of encrypting with 0-RTT key.
-define(QUIC_SEND_FLAG_ALLOW_0_RTT, 16#0001).
% Asynchronously starts the stream with the sent data.
-define(QUIC_SEND_FLAG_START, 16#0002).
% Indicates the request is the one last sent on the stream.
-define(QUIC_SEND_FLAG_FIN, 16#0004).
% Indicates the datagram is higher priority than others.
-define(QUIC_SEND_FLAG_DGRAM_PRIORITY, 16#0008).
-define(QUIC_SEND_FLAG_DELAY_SEND, 16#0010).
%% QUICER SEND FLAG
-define(QUICER_SEND_FLAG_SYNC , 16#1000).
-define(QUICER_SEND_FLAG_SYNC, 16#1000).

%% QUIC RECV FLAGS
-define(QUIC_RECEIVE_FLAG_NONE , 16#0000).
-define(QUIC_RECEIVE_FLAG_0_RTT , 16#0001).
-define(QUIC_RECEIVE_FLAG_FIN , 16#0002).
-define(QUIC_RECEIVE_FLAG_NONE, 16#0000).
-define(QUIC_RECEIVE_FLAG_0_RTT, 16#0001).
-define(QUIC_RECEIVE_FLAG_FIN, 16#0002).

%% QUIC DATAGRAM_SEND_STATE
-define(QUIC_DATAGRAM_SEND_UNKNOWN, dgram_send_unknown). %% Not yet sent.
-define(QUIC_DATAGRAM_SEND_SENT, dgram_send_sent). %% Sent and awaiting acknowledegment
-define(QUIC_DATAGRAM_SEND_LOST_SUSPECT, dgram_send_lost_suspect). %% Suspected as lost, but still tracked
-define(QUIC_DATAGRAM_SEND_LOST_DISCARDED, dgram_send_lost_discarded). %% Lost and not longer being tracked
-define(QUIC_DATAGRAM_SEND_ACKNOWLEDGED, dgram_send_acknowledged). %% Acknowledged
-define(QUIC_DATAGRAM_SEND_ACKNOWLEDGED_SPURIOUS, dgram_send_acknowledged_spurious). %% Acknowledged after being suspected lost

%% Not yet sent.
-define(QUIC_DATAGRAM_SEND_UNKNOWN, dgram_send_unknown).
%% Sent and awaiting acknowledegment
-define(QUIC_DATAGRAM_SEND_SENT, dgram_send_sent).
%% Suspected as lost, but still tracked
-define(QUIC_DATAGRAM_SEND_LOST_SUSPECT, dgram_send_lost_suspect).
%% Lost and not longer being tracked
-define(QUIC_DATAGRAM_SEND_LOST_DISCARDED, dgram_send_lost_discarded).
%% Acknowledged
-define(QUIC_DATAGRAM_SEND_ACKNOWLEDGED, dgram_send_acknowledged).
%% Acknowledged after being suspected lost
-define(QUIC_DATAGRAM_SEND_ACKNOWLEDGED_SPURIOUS, dgram_send_acknowledged_spurious).
-define(QUIC_DATAGRAM_SEND_CANCELED, dgram_send_canceled).

-record(quic_data, {
Expand All @@ -140,4 +161,5 @@
bin :: binary()
}).

-endif. %% QUICER_HRL
%% QUICER_HRL
-endif.

0 comments on commit 2276316

Please sign in to comment.