External Execution Interface

src/cs/lib/msquic_generated.cs

@@ -234,6 +234,22 @@ internal unsafe partial struct QUIC_EXECUTION_CONFIG
         internal fixed ushort ProcessorList[1];
     }
 
+    internal unsafe partial struct QUIC_EXECUTION_CONTEXT_CONFIG
+    {
+        [NativeTypeName("uint32_t")]
+        internal uint IdealProcessor;
+
+        [NativeTypeName("uint32_t")]
+        internal uint PollingIdleTimeoutUs;
+
+        [NativeTypeName("QUIC_EVENTQ *")]
+        internal void** EventQ;
+    }
+
+    internal partial struct QUIC_EXECUTION_CONTEXT
+    {
+    }
+
     internal unsafe partial struct QUIC_REGISTRATION_CONFIG
     {
         [NativeTypeName("const char *")]
@@ -3250,6 +3266,15 @@ internal unsafe partial struct QUIC_API_TABLE
 
         [NativeTypeName("QUIC_CONNECTION_COMP_CERT_FN")]
         internal delegate* unmanaged[Cdecl]<QUIC_HANDLE*, byte, QUIC_TLS_ALERT_CODES, int> ConnectionCertificateValidationComplete;
+
+        [NativeTypeName("QUIC_EXECUTION_CREATE_FN")]
+        internal delegate* unmanaged[Cdecl]<QUIC_EXECUTION_CONFIG_FLAGS, uint, QUIC_EXECUTION_CONTEXT_CONFIG*, QUIC_EXECUTION_CONTEXT**, int> ExecutionCreate;
+
+        [NativeTypeName("QUIC_EXECUTION_DELETE_FN")]
+        internal delegate* unmanaged[Cdecl]<uint, QUIC_EXECUTION_CONTEXT**, void> ExecutionDelete;
+
+        [NativeTypeName("QUIC_EXECUTION_POLL_FN")]
+        internal delegate* unmanaged[Cdecl]<QUIC_EXECUTION_CONTEXT*, uint> ExecutionPoll;
     }
 
     internal static unsafe partial class MsQuic

src/inc/msquic.h

@@ -264,15 +264,15 @@ typedef enum QUIC_DATAGRAM_SEND_STATE {
 #define QUIC_DATAGRAM_SEND_STATE_IS_FINAL(State) \
     ((State) >= QUIC_DATAGRAM_SEND_LOST_DISCARDED)
 
+#ifdef QUIC_API_ENABLE_PREVIEW_FEATURES
+
 typedef enum QUIC_EXECUTION_CONFIG_FLAGS {
     QUIC_EXECUTION_CONFIG_FLAG_NONE             = 0x0000,
-#ifdef QUIC_API_ENABLE_PREVIEW_FEATURES
     QUIC_EXECUTION_CONFIG_FLAG_QTIP             = 0x0001,
     QUIC_EXECUTION_CONFIG_FLAG_RIO              = 0x0002,
     QUIC_EXECUTION_CONFIG_FLAG_XDP              = 0x0004,
     QUIC_EXECUTION_CONFIG_FLAG_NO_IDEAL_PROC    = 0x0008,
     QUIC_EXECUTION_CONFIG_FLAG_HIGH_PRIORITY    = 0x0010,
-#endif
 } QUIC_EXECUTION_CONFIG_FLAGS;
 
 DEFINE_ENUM_FLAG_OPERATORS(QUIC_EXECUTION_CONFIG_FLAGS)
@@ -293,6 +293,62 @@ typedef struct QUIC_EXECUTION_CONFIG {
 #define QUIC_EXECUTION_CONFIG_MIN_SIZE \
     (uint32_t)FIELD_OFFSET(QUIC_EXECUTION_CONFIG, ProcessorList)
 
+#ifndef _KERNEL_MODE
+
+//
+// Execution Context abstraction, which allows the application layer to
+// completely control execution of all MsQuic work.
+//
+
+typedef struct QUIC_EXECUTION_CONTEXT_CONFIG {
+    uint32_t IdealProcessor;
+    uint32_t PollingIdleTimeoutUs;
+    QUIC_EVENTQ* EventQ;
+} QUIC_EXECUTION_CONTEXT_CONFIG;
+
+typedef struct QUIC_EXECUTION_CONTEXT QUIC_EXECUTION_CONTEXT;
+
+//
+// This is called create the execution contexts.
+//
+typedef
+_IRQL_requires_max_(PASSIVE_LEVEL)
+QUIC_STATUS
+(QUIC_API * QUIC_EXECUTION_CREATE_FN)(
+    _In_ QUIC_EXECUTION_CONFIG_FLAGS Flags, // Used for datapath type
+    _In_ uint32_t Count,
+    _In_reads_(Count) QUIC_EXECUTION_CONTEXT_CONFIG* Configs,
+    _Out_writes_(Count) QUIC_EXECUTION_CONTEXT** ExecutionContexts
+    );
+
+//
+// This is called to delete the execution contexts.
+//
+typedef
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+(QUIC_API * QUIC_EXECUTION_DELETE_FN)(
+    _In_ uint32_t Count,
+    _In_reads_(Count) QUIC_EXECUTION_CONTEXT** ExecutionContexts
+    );
+
+//
+// This is called to allow MsQuic to process any polling work. It returns the
+// number of milliseconds until the next scheduled timer expiration.
+//
+// TODO: Should it return an indication for if we should yield?
+//
+typedef
+_IRQL_requires_max_(PASSIVE_LEVEL)
+uint32_t
+(QUIC_API * QUIC_EXECUTION_POLL_FN)(
+    _In_ QUIC_EXECUTION_CONTEXT* ExecutionContext
+    );
+
+#endif // _KERNEL_MODE
+
+#endif // QUIC_API_ENABLE_PREVIEW_FEATURES
+
 typedef struct QUIC_REGISTRATION_CONFIG { // All fields may be NULL/zero.
     const char* AppName;
     QUIC_EXECUTION_PROFILE ExecutionProfile;
@@ -857,6 +913,7 @@ void
 #endif
 #define QUIC_PARAM_GLOBAL_TLS_PROVIDER                  0x0100000A  // QUIC_TLS_PROVIDER
 #define QUIC_PARAM_GLOBAL_STATELESS_RESET_KEY           0x0100000B  // uint8_t[] - Array size is QUIC_STATELESS_RESET_KEY_LENGTH
+
 //
 // Parameters for Registration.
 //
@@ -1626,6 +1683,12 @@ typedef struct QUIC_API_TABLE {
     QUIC_CONNECTION_COMP_RESUMPTION_FN  ConnectionResumptionTicketValidationComplete; // Available from v2.2
     QUIC_CONNECTION_COMP_CERT_FN        ConnectionCertificateValidationComplete;      // Available from v2.2
 
+#ifndef _KERNEL_MODE
+    QUIC_EXECUTION_CREATE_FN            ExecutionCreate;    // Available from v2.5
+    QUIC_EXECUTION_DELETE_FN            ExecutionDelete;    // Available from v2.5
+    QUIC_EXECUTION_POLL_FN              ExecutionPoll;      // Available from v2.5
+#endif
+
 } QUIC_API_TABLE;
 
 #define QUIC_API_VERSION_1      1 // Not supported any more

src/inc/msquic_posix.h

@@ -515,6 +515,34 @@ QuicAddrToString(
     return TRUE;
 }
 
+//
+// Event Queue Abstraction
+//
+
+#if __linux__ // epoll
+
+typedef int QUIC_EVENTQ;
+
+typedef struct QUIC_CQE {
+    struct epoll_event Event;
+    void (*Completion)(struct QUIC_CQE *Cqe);
+} QUIC_CQE;
+
+#elif __APPLE__ || __FreeBSD__ // kqueue
+
+typedef int QUIC_EVENTQ;
+
+typedef struct QUIC_CQE {
+    struct kevent Event;
+    void (*Completion)(struct QUIC_CQE *Cqe);
+} QUIC_CQE;
+
+#else
+
+#error Unsupported Platform
+
+#endif
+
 #if defined(__cplusplus)
 }
 #endif

src/inc/msquic_winuser.h

@@ -373,4 +373,15 @@ QuicAddrToString(
 
 #endif // WINAPI_FAMILY != WINAPI_FAMILY_GAMES
 
+//
+// Event Queue Abstraction
+//
+
+typedef HANDLE QUIC_EVENTQ;
+
+typedef struct QUIC_CQE {
+    OVERLAPPED_ENTRY Overlapped;
+    void (*Completion)(struct QUIC_CQE *Cqe);
+} QUIC_CQE;
+
 #endif // _MSQUIC_WINUSER_

src/tools/execution/CMakeLists.txt

@@ -0,0 +1,5 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+add_quic_tool(quicexecution execution_windows.cpp)
+quic_tool_warnings(quicexecution)

src/tools/execution/execution_windows.cpp

@@ -0,0 +1,91 @@
+/*++
+
+    Copyright (c) Microsoft Corporation.
+    Licensed under the MIT License.
+
+Abstract:
+
+    Provides simple client MsQuic example that leverages custom execution.
+
+--*/
+
+#define QUIC_API_ENABLE_PREVIEW_FEATURES 1
+
+#include "msquic.hpp"
+#include <stdio.h>
+
+void PrintUsage()
+{
+    printf(
+        "\n"
+        "quicexec is a simple app that can connect to an HTTP/3 server.\n"
+        "\n"
+        "Usage:\n"
+        "\n"
+        "  quicexec <host or ip>\n"
+        );
+}
+
+int
+QUIC_MAIN_EXPORT
+main(
+    _In_ int argc,
+    _In_reads_(argc) _Null_terminated_ char* argv[]
+    )
+{
+    MsQuicApi _MsQuic;
+    if (!_MsQuic.IsValid()) { return 1; }
+    MsQuic = &_MsQuic;
+
+    QUIC_STATUS Status;
+    HANDLE IOCP = CreateIoCompletionPort(INVALID_HANDLE_VALUE, nullptr, 0, 1);
+    QUIC_EXECUTION_CONTEXT_CONFIG ExecConfig = { 0, 0, &IOCP };
+    QUIC_EXECUTION_CONTEXT* ExecContext = nullptr;
+    if (QUIC_FAILED(Status = MsQuic->ExecutionCreate(QUIC_EXECUTION_CONFIG_FLAG_NONE, 1, &ExecConfig, &ExecContext))) {
+        return 1;
+    }
+
+    do {
+        bool AllDone = false;
+        MsQuicRegistration Registration("quicexec");
+        MsQuicSettings Settings;
+        Settings.SetPeerUnidiStreamCount(3); // required for H3
+        MsQuicConfiguration Configuration(Registration, "h3", Settings, MsQuicCredentialConfig());
+        if (!Configuration.IsValid()) { break; }
+
+        struct ConnectionCallback {
+            static QUIC_STATUS MsQuicConnectionCallback(_In_ struct MsQuicConnection* Connection, _In_opt_ void* Context, _Inout_ QUIC_CONNECTION_EVENT* Event) {
+                if (Event->Type == QUIC_CONNECTION_EVENT_CONNECTED) {
+                    Connection->Shutdown(0);
+                } else if (Event->Type == QUIC_CONNECTION_EVENT_SHUTDOWN_COMPLETE) {
+                    *((bool*)Context) = true;
+                }
+            }
+        };
+
+        MsQuicConnection Connection(Registration, CleanUpManual, ConnectionCallback::MsQuicConnectionCallback, &AllDone);
+        if (QUIC_FAILED(
+            Status = Connection.Start(Configuration, argv[1], 443))) {
+            break;
+        }
+
+        while (!AllDone) {
+            uint32_t WaitTime = MsQuic->ExecutionPoll(ExecContext);
+
+            OVERLAPPED_ENTRY Overlapped[8];
+            ULONG OverlappedCount = 0;
+            if (GetQueuedCompletionStatusEx(IOCP, Overlapped, ARRAYSIZE(Overlapped), &OverlappedCount, WaitTime, FALSE)) {
+                for (ULONG i = 0; i < OverlappedCount; ++i) {
+                    QUIC_CQE* Cqe = CONTAINING_RECORD(Overlapped[i].lpOverlapped, QUIC_CQE, Overlapped);
+                    Cqe->Completion(Cqe);
+                }
+            }
+        }
+
+    } while (false);
+
+    MsQuic->ExecutionDelete(1, &ExecContext);
+    CloseHandle(IOCP);
+
+    return 0;
+}