support nif module reload

.github/workflows/main.yml

@@ -3,6 +3,7 @@ on: [push, pull_request]
 
 jobs:
   mac:
+    timeout-minutes: 45
     strategy:
       fail-fast: false
       matrix:
@@ -62,6 +63,7 @@ jobs:
 
   linux:
     runs-on: ubuntu-22.04
+    timeout-minutes: 25
     strategy:
       fail-fast: false
       matrix:

CMakeLists.txt

@@ -2,9 +2,17 @@ cmake_minimum_required(VERSION 3.16)
 
 project(quicer)
 
+## NIF library ABI version
+## Bump manually when ABI changes
+set(QUICER_ABI_VERSION 1)
+
 SET(Erlang_EI_INCLUDE_DIRS   ${Erlang_OTP_LIB_DIR}/${Erlang_EI_DIR}/include)
 SET(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/priv/)
 
+if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+  set(CMAKE_INSTALL_PREFIX ${PROJECT_SOURCE_DIR}/priv/ CACHE PATH "..." FORCE)
+endif()
+
 # For cerl picking up the OTP_ROOT
 if (DEFINED ENV{Erlang_OTP_ROOT_DIR})
   SET(Erlang_OTP_ROOT_DIR $ENV{Erlang_OTP_ROOT_DIR})
@@ -15,6 +23,12 @@ EXECUTE_PROCESS(
         )
 endif()
 
+if (DEFINED ENV{QUICER_VERSION})
+  set(QUICER_VERSION $ENV{QUICER_VERSION})
+else()
+  set(QUICER_VERSION "0")
+endif()
+
 if (DEFINED ENV{CMAKE_BUILD_TYPE})
   set(CMAKE_BUILD_TYPE $ENV{CMAKE_BUILD_TYPE})
 else()
@@ -64,6 +78,16 @@ set(QUIC_BUILD_PERF "OFF")
 
 set(QUIC_TLS_SECRETS_SUPPORT "ON")
 
+configure_file(
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/quicer_vsn.hrl.in
+    ${CMAKE_CURRENT_SOURCE_DIR}/include/quicer_vsn.hrl
+)
+
+configure_file(
+    ${CMAKE_CURRENT_SOURCE_DIR}/c_src/quicer_vsn.h.in
+    ${CMAKE_CURRENT_BINARY_DIR}/c_src/quicer_vsn.h
+)
+
 # src files
 set(SOURCES
   c_src/quicer_nif.c
@@ -96,7 +120,8 @@ add_compile_options(-DSO_ATTACH_REUSEPORT_CBPF=51)
 
 # for lttng, quicer_tp.h
 include_directories(c_src)
-
+# for templ files
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/c_src/)
 add_subdirectory(msquic)
 
 add_library(quicer_static STATIC ${SOURCES})
@@ -138,10 +163,9 @@ add_dependencies(quicer_static msquic)
 
 set_target_properties(quicer_nif
     PROPERTIES
-        LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/priv/
+        LIBRARY_OUTPUT_NAME quicer_nif
+        VERSION ${QUICER_ABI_VERSION}-${QUICER_VERSION}
 )
+include(GNUInstallDirs)
+install(TARGETS quicer_nif LIBRARY DESTINATION ${PROJECT_SOURCE_DIR}/priv/)
 
-if (QUIC_ENABLE_LOGGING STREQUAL "ON" AND QUIC_LOGGING_TYPE STREQUAL "lttng")
-  set_target_properties(msquic.lttng PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/priv)
-  set_target_properties(msquic.lttng PROPERTIES LIBRARY_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_SOURCE_DIR}/priv)
-endif()

Makefile

@@ -1,5 +1,8 @@
 REBAR := rebar3
 
+QUICER_VERSION ?= $(shell git describe --tags --always)
+export QUICER_VERSION
+
 .PHONY: all
 all: compile
 

build.sh

@@ -17,6 +17,7 @@ build() {
     ./get-msquic.sh "$MSQUIC_VERSION"
     cmake -B c_build
     make -j "$JOBS" -C c_build
+    make install -C c_build
     ## MacOS
     if [ -f priv/libquicer_nif.dylib ]; then
         # https://developer.apple.com/forums/thread/696460

c_src/quicer_config.c

@@ -21,6 +21,7 @@ limitations under the License.
 #include <msquichelper.h>
 
 extern QuicerRegistrationCTX *G_r_ctx;
+extern pthread_mutex_t MsQuicLock;
 
 static ERL_NIF_TERM get_stream_opt(ErlNifEnv *env,
                                    QuicerStreamCTX *s_ctx,
@@ -768,7 +769,15 @@ getopt3(ErlNifEnv *env,
 
   if (IS_SAME_TERM(ATOM_QUIC_GLOBAL, ctx))
     {
-      res = get_global_opt(env, NULL, eopt);
+      pthread_mutex_lock(&MsQuicLock);
+      // In a env that while there is no allocated NIF resources (reg, conf,
+      // listener, conn, stream), VM may unload the module causes unloading DSO
+      // in parallel.
+      if (MsQuic)
+        {
+          res = get_global_opt(env, NULL, eopt);
+        }
+      pthread_mutex_unlock(&MsQuicLock);
     }
   else if (enif_get_resource(env, ctx, ctx_stream_t, &q_ctx))
     {

c_src/quicer_connection.c

@@ -327,7 +327,6 @@ _IRQL_requires_max_(DISPATCH_LEVEL)
       // @see async_connect3
       status = handle_connection_event_shutdown_complete(c_ctx, Event);
       is_destroy = TRUE;
-      c_ctx->is_closed = TRUE; // client shutdown completed
       break;
 
     case QUIC_CONNECTION_EVENT_PEER_ADDRESS_CHANGED:
@@ -372,10 +371,23 @@ _IRQL_requires_max_(DISPATCH_LEVEL)
       break;
     }
   enif_clear_env(env);
+
+  QuicerConfigCTX *conf_ctx = c_ctx->config_resource;
+  if (is_destroy)
+    {
+      c_ctx->is_closed = TRUE; // client shutdown completed
+      c_ctx->Connection = NULL;
+      c_ctx->config_resource = NULL;
+    }
   enif_mutex_unlock(c_ctx->lock);
 
   if (is_destroy)
     {
+      MsQuic->ConnectionClose(Connection);
+      if (conf_ctx)
+        {
+          enif_release_resource(conf_ctx);
+        }
       destroy_c_ctx(c_ctx);
     }
   return status;
@@ -434,7 +446,6 @@ ServerConnectionCallback(HQUIC Connection,
       // safely cleaned up.
       //
       status = handle_connection_event_shutdown_complete(c_ctx, Event);
-      c_ctx->is_closed = TRUE; // server shutdown_complete
       is_destroy = TRUE;
       break;
     case QUIC_CONNECTION_EVENT_PEER_STREAM_STARTED:
@@ -481,10 +492,23 @@ ServerConnectionCallback(HQUIC Connection,
       break;
     }
   enif_clear_env(env);
+
+  QuicerConfigCTX *conf_ctx = c_ctx->config_resource;
+  if (is_destroy)
+    {
+      c_ctx->Connection = NULL;
+      c_ctx->is_closed = TRUE; // server shutdown_complete
+      c_ctx->config_resource = NULL;
+    }
   enif_mutex_unlock(c_ctx->lock);
 
   if (is_destroy)
     {
+      MsQuic->ConnectionClose(Connection);
+      if (conf_ctx)
+        {
+          enif_release_resource(conf_ctx);
+        }
       destroy_c_ctx(c_ctx);
     }
 
@@ -503,7 +527,7 @@ open_connectionX(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
   ERL_NIF_TERM res = ERROR_TUPLE_2(ATOM_ERROR_INTERNAL_ERROR);
   QuicerRegistrationCTX *r_ctx = NULL;
   HQUIC registration = NULL;
-  ERL_NIF_TERM options = argv[1];
+  ERL_NIF_TERM options = argv[0];
 
   if (argc == 0)
     {
@@ -799,14 +823,15 @@ async_connect3(ErlNifEnv *env,
       AcceptorDestroy(c_ctx->owner);
       c_ctx->owner = NULL;
 
-      /* Although MsQuic internally close the connection after failed to start,
-         we still do not need to set is_closed here, we expect callback to set
-         it while handling the shutdown complete event otherwise could cause
-         race cond.
-      */
-      // c_ctx->is_closed = TRUE;
+      if (Status != QUIC_STATUS_INVALID_PARAMETER)
+        {
+          c_ctx->Connection = NULL;
+        }
 
-      c_ctx->Connection = NULL;
+      if (c_ctx->config_resource)
+        {
+          destroy_config_ctx(c_ctx->config_resource);
+        }
 
       res = ERROR_TUPLE_2(ATOM_CONN_START_ERROR);
       TP_NIF_3(start_fail, (uintptr_t)(c_ctx->Connection), Status);
@@ -1056,6 +1081,11 @@ continue_connection_handshake(QuicerConnCTX *c_ctx)
       return QUIC_STATUS_INTERNAL_ERROR;
     }
 
+  if (!c_ctx->Connection)
+    {
+      return QUIC_STATUS_INVALID_STATE;
+    }
+
   if (QUIC_FAILED(
           Status = MsQuic->ConnectionSetConfiguration(
               c_ctx->Connection, c_ctx->config_resource->Configuration)))

c_src/quicer_ctx.c

@@ -215,7 +215,10 @@ deinit_config_ctx(QuicerConfigCTX *config_ctx)
 void
 destroy_config_ctx(QuicerConfigCTX *config_ctx)
 {
-  enif_release_resource(config_ctx);
+  if (config_ctx)
+    {
+      enif_release_resource(config_ctx);
+    }
 }
 
 QuicerStreamCTX *
@@ -254,10 +257,8 @@ deinit_s_ctx(QuicerStreamCTX *s_ctx)
 void
 destroy_s_ctx(QuicerStreamCTX *s_ctx)
 {
+  assert(!s_ctx->Stream);
   enif_free_env(s_ctx->imm_env);
-  // Since enif_release_resource is async call,
-  // we should demon the owner now!
-  enif_demonitor_process(s_ctx->env, s_ctx, &s_ctx->owner_mon);
   enif_release_resource(s_ctx);
 }
 

c_src/quicer_listener.c

@@ -23,6 +23,7 @@ limitations under the License.
 #include <openssl/x509.h>
 
 extern QuicerRegistrationCTX *G_r_ctx;
+extern pthread_mutex_t GRegLock;
 
 BOOLEAN parse_registration(ErlNifEnv *env,
                            ERL_NIF_TERM options,
@@ -279,6 +280,7 @@ listen2(ErlNifEnv *env, __unused_parm__ int argc, const ERL_NIF_TERM argv[])
     {
       // TLS opt error not file content error
       free(cacertfile);
+      free_certificate(&CredConfig);
       return ERROR_TUPLE_2(ATOM_CACERTFILE);
     }
 
@@ -288,6 +290,7 @@ listen2(ErlNifEnv *env, __unused_parm__ int argc, const ERL_NIF_TERM argv[])
   if (!l_ctx)
     {
       free(cacertfile);
+      free_certificate(&CredConfig);
       return ERROR_TUPLE_2(ATOM_ERROR_NOT_ENOUGH_MEMORY);
     }
 
@@ -306,6 +309,10 @@ listen2(ErlNifEnv *env, __unused_parm__ int argc, const ERL_NIF_TERM argv[])
           goto exit;
         }
     }
+  else
+    { // since we don't use cacertfile, free it
+      free(cacertfile);
+    }
 
   // Set owner for l_ctx
   if (!enif_self(env, &(l_ctx->listenerPid)))
@@ -576,8 +583,15 @@ ERL_NIF_TERM
 get_listenersX(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
 {
   QuicerRegistrationCTX *r_ctx = NULL;
-  if (argc == 0)
+  ERL_NIF_TERM res = enif_make_list(env, 0);
+  if (argc == 0) // use global registration
     {
+      pthread_mutex_lock(&GRegLock);
+      if (!G_r_ctx)
+        {
+          pthread_mutex_unlock(&GRegLock);
+          return res;
+        }
       r_ctx = G_r_ctx;
     }
   else
@@ -587,8 +601,6 @@ get_listenersX(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
           return ERROR_TUPLE_2(ATOM_BADARG);
         }
     }
-  ERL_NIF_TERM res = enif_make_list(env, 0);
-
   enif_mutex_lock(r_ctx->lock);
   CXPLAT_LIST_ENTRY *Entry = r_ctx->Listeners.Flink;
   while (Entry != &r_ctx->Listeners)
@@ -600,5 +612,9 @@ get_listenersX(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
     }
   enif_mutex_unlock(r_ctx->lock);
 
+  if (argc == 0) // use global registration
+    {
+      pthread_mutex_unlock(&GRegLock);
+    }
   return res;
 }