diff --git a/c_src/quicer_listener.c b/c_src/quicer_listener.c
index 1cbd9a2f..8da6c0e9 100644
--- a/c_src/quicer_listener.c
+++ b/c_src/quicer_listener.c
@@ -39,10 +39,17 @@ ServerListenerCallback(__unused_parm__ HQUIC Listener,
   QuicerConnCTX *c_ctx = NULL;
   BOOLEAN is_destroy = FALSE;
 
+  BOOLEAN is_worker = (enif_thread_type() == ERL_NIF_THR_UNDEFINED);
+
+  if (is_worker)
+    {
+      enif_mutex_lock(l_ctx->lock);
+    }
+
   switch (Event->Type)
     {
     case QUIC_LISTENER_EVENT_NEW_CONNECTION:
-      enif_mutex_lock(l_ctx->lock);
+
       //
       // Note, c_ctx is newly init here, don't grab lock.
       //
@@ -219,9 +226,8 @@ ServerListenerCallback(__unused_parm__ HQUIC Listener,
       break;
 
     case QUIC_LISTENER_EVENT_STOP_COMPLETE:
-      // **Note**, the callback in msquic for this event is called in the
-      // MsQuicListenerClose or MsQuicListenerStop. we assume caller should
-      // ensure the thread safty thus we don't hold lock
+      // **Note**, this callback event in msquic can be triggered by either
+      // MsQuicListenerClose or MsQuicListenerStop.
       env = l_ctx->env;
       enif_send(NULL,
                 &(l_ctx->listenerPid),
@@ -238,15 +244,21 @@ ServerListenerCallback(__unused_parm__ HQUIC Listener,
           assert(!l_ctx->is_stopped);
           is_destroy = TRUE;
         }
+      else
+        {
+          l_ctx->is_stopped = TRUE;
+        }
       enif_clear_env(env);
-      goto Exit2;
+      break;
     default:
       break;
     }
 
 Error:
-  enif_mutex_unlock(l_ctx->lock);
-Exit2:
+  if (is_worker)
+    {
+      enif_mutex_unlock(l_ctx->lock);
+    }
   if (is_destroy)
     {
       destroy_l_ctx(l_ctx);
diff --git a/c_src/quicer_nif.c b/c_src/quicer_nif.c
index 8b10a472..1a9a1239 100644
--- a/c_src/quicer_nif.c
+++ b/c_src/quicer_nif.c
@@ -804,7 +804,7 @@ resource_listener_down_callback(__unused_parm__ ErlNifEnv *env,
       // b. Some pid could still close the stopped listener with nif
       // handle.
       // c. We close it in resource_listener_dealloc_callback anyway when
-      // Listener term get GC.
+      // Listener term get GCed.
       */
       MsQuic->ListenerStop(l_ctx->Listener);
       put_listener_handle(l_ctx);