From 0a7fddddea28338b69fb91d8183a6bc02eaf52ad Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Tue, 12 Nov 2024 19:22:35 -0800 Subject: [PATCH] Speedup LRU by using the faster timestamp API (#3990) * Speedup LRU by using the faster timestamp API Signed-off-by: Alan Jowett * PR feedback Signed-off-by: Alan Jowett * PR feedback Signed-off-by: Alan Jowett --------- Signed-off-by: Alan Jowett Co-authored-by: Alan Jowett --- libs/execution_context/ebpf_core.c | 8 ++++---- libs/execution_context/ebpf_maps.c | 8 +++++--- libs/execution_context/ebpf_program.c | 8 ++++---- libs/runtime/ebpf_platform.c | 13 ++++++++++++- libs/runtime/ebpf_platform.h | 15 ++++++++++++++- tests/performance/platform.cpp | 4 ++-- 6 files changed, 41 insertions(+), 15 deletions(-) diff --git a/libs/execution_context/ebpf_core.c b/libs/execution_context/ebpf_core.c index 11e87d6d21..8a1ee63219 100644 --- a/libs/execution_context/ebpf_core.c +++ b/libs/execution_context/ebpf_core.c @@ -2178,17 +2178,17 @@ _ebpf_core_random_uint32() static uint64_t _ebpf_core_get_time_since_boot_ns() { - // ebpf_query_time_since_boot returns time elapsed since + // ebpf_query_time_since_boot_precise returns time elapsed since // boot in units of 100 ns. - return ebpf_query_time_since_boot(true) * EBPF_NS_PER_FILETIME; + return ebpf_query_time_since_boot_precise(true) * EBPF_NS_PER_FILETIME; } static uint64_t _ebpf_core_get_time_ns() { - // ebpf_query_time_since_boot returns time elapsed since + // ebpf_query_time_since_boot_precise returns time elapsed since // boot in units of 100 ns. - return ebpf_query_time_since_boot(false) * EBPF_NS_PER_FILETIME; + return ebpf_query_time_since_boot_precise(false) * EBPF_NS_PER_FILETIME; } static uint64_t diff --git a/libs/execution_context/ebpf_maps.c b/libs/execution_context/ebpf_maps.c index cd2e0d4767..9882e84046 100644 --- a/libs/execution_context/ebpf_maps.c +++ b/libs/execution_context/ebpf_maps.c @@ -166,6 +166,8 @@ __declspec(align(EBPF_CACHE_LINE_SIZE)) typedef struct _ebpf_lru_partition size_t hot_list_limit; //< Maximum size of the hot list. } ebpf_lru_partition_t; +static_assert(sizeof(ebpf_lru_partition_t) % EBPF_CACHE_LINE_SIZE == 0, "ebpf_core_lru_map_t is not cache aligned."); + /** * @brief The map definition for an LRU map. */ @@ -1203,7 +1205,7 @@ _insert_into_hot_list(_Inout_ ebpf_core_lru_map_t* map, size_t partition, _Inout switch (key_state) { case EBPF_LRU_KEY_UNINITIALIZED: EBPF_LRU_ENTRY_GENERATION_PTR(map, entry)[partition] = map->partitions[partition].current_generation; - EBPF_LRU_ENTRY_LAST_USED_TIME_PTR(map, entry)[partition] = ebpf_query_time_since_boot(false); + EBPF_LRU_ENTRY_LAST_USED_TIME_PTR(map, entry)[partition] = ebpf_query_time_since_boot_approximate(false); ebpf_list_insert_tail( &map->partitions[partition].hot_list, &EBPF_LRU_ENTRY_LIST_ENTRY_PTR(map, entry)[partition]); map->partitions[partition].hot_list_size++; @@ -1211,7 +1213,7 @@ _insert_into_hot_list(_Inout_ ebpf_core_lru_map_t* map, size_t partition, _Inout case EBPF_LRU_KEY_COLD: // Remove from cold list. EBPF_LRU_ENTRY_GENERATION_PTR(map, entry)[partition] = map->partitions[partition].current_generation; - EBPF_LRU_ENTRY_LAST_USED_TIME_PTR(map, entry)[partition] = ebpf_query_time_since_boot(false); + EBPF_LRU_ENTRY_LAST_USED_TIME_PTR(map, entry)[partition] = ebpf_query_time_since_boot_approximate(false); ebpf_list_remove_entry(&EBPF_LRU_ENTRY_LIST_ENTRY_PTR(map, entry)[partition]); ebpf_list_insert_tail( &map->partitions[partition].hot_list, &EBPF_LRU_ENTRY_LIST_ENTRY_PTR(map, entry)[partition]); @@ -1255,7 +1257,7 @@ _initialize_lru_entry( // Only insert into the current partition's hot list. ebpf_lock_state_t state = ebpf_lock_lock(&map->partitions[partition].lock); EBPF_LRU_ENTRY_GENERATION_PTR(map, entry)[partition] = map->partitions[partition].current_generation; - EBPF_LRU_ENTRY_LAST_USED_TIME_PTR(map, entry)[partition] = ebpf_query_time_since_boot(false); + EBPF_LRU_ENTRY_LAST_USED_TIME_PTR(map, entry)[partition] = ebpf_query_time_since_boot_approximate(false); ebpf_list_insert_tail(&map->partitions[partition].hot_list, &EBPF_LRU_ENTRY_LIST_ENTRY_PTR(map, entry)[partition]); map->partitions[partition].hot_list_size++; diff --git a/libs/execution_context/ebpf_program.c b/libs/execution_context/ebpf_program.c index ed000b75b0..163c22d617 100644 --- a/libs/execution_context/ebpf_program.c +++ b/libs/execution_context/ebpf_program.c @@ -2435,7 +2435,7 @@ _ebpf_program_test_run_work_item(_In_ cxplat_preemptible_work_item_t* work_item, state_stored = true; } - uint64_t start_time = ebpf_query_time_since_boot(false); + uint64_t start_time = ebpf_query_time_since_boot_precise(false); // Use a counter instead of performing a modulus operation to determine when to start a new epoch. // This is because the modulus operation is expensive and we want to minimize the overhead of // the test run. @@ -2448,7 +2448,7 @@ _ebpf_program_test_run_work_item(_In_ cxplat_preemptible_work_item_t* work_item, ebpf_epoch_exit(&epoch_state); if (ebpf_should_yield_processor()) { // Compute the elapsed time since the last yield. - end_time = ebpf_query_time_since_boot(false); + end_time = ebpf_query_time_since_boot_precise(false); // Add the elapsed time to the cumulative time. cumulative_time += end_time - start_time; @@ -2460,7 +2460,7 @@ _ebpf_program_test_run_work_item(_In_ cxplat_preemptible_work_item_t* work_item, old_irql = ebpf_raise_irql(context->required_irql); // Reset the start time. - start_time = ebpf_query_time_since_boot(false); + start_time = ebpf_query_time_since_boot_precise(false); } ebpf_epoch_enter(&epoch_state); } @@ -2470,7 +2470,7 @@ _ebpf_program_test_run_work_item(_In_ cxplat_preemptible_work_item_t* work_item, break; } } - end_time = ebpf_query_time_since_boot(false); + end_time = ebpf_query_time_since_boot_precise(false); cumulative_time += end_time - start_time; diff --git a/libs/runtime/ebpf_platform.c b/libs/runtime/ebpf_platform.c index 2a358dc263..1760fe5dc8 100644 --- a/libs/runtime/ebpf_platform.c +++ b/libs/runtime/ebpf_platform.c @@ -221,7 +221,7 @@ ebpf_allocate_process_state() } uint64_t -ebpf_query_time_since_boot(bool include_suspended_time) +ebpf_query_time_since_boot_precise(bool include_suspended_time) { uint64_t qpc_time; if (include_suspended_time) { @@ -237,6 +237,17 @@ ebpf_query_time_since_boot(bool include_suspended_time) } } +uint64_t +ebpf_query_time_since_boot_approximate(bool include_suspend_time) +{ + if (include_suspend_time) { + ebpf_assert(!"Include suspend time not supported on this platform."); + return 0; + } else { + return KeQueryInterruptTime(); + } +} + MDL* ebpf_map_memory(size_t length) { diff --git a/libs/runtime/ebpf_platform.h b/libs/runtime/ebpf_platform.h index 2233846f73..74ab5471c1 100644 --- a/libs/runtime/ebpf_platform.h +++ b/libs/runtime/ebpf_platform.h @@ -654,7 +654,20 @@ extern "C" */ EBPF_INLINE_HINT uint64_t - ebpf_query_time_since_boot(bool include_suspended_time); + ebpf_query_time_since_boot_precise(bool include_suspended_time); + + /** + * @brief Return time elapsed since boot in units of 100 nanoseconds. + * This function is faster than ebpf_query_time_since_boot_precise() but may not + * be as accurate. + * + * @param[in] include_suspended_time Include time the system spent in a suspended state. + * + * @return Time elapsed since boot in 100 nanosecond units. + */ + EBPF_INLINE_HINT + uint64_t + ebpf_query_time_since_boot_approximate(bool include_suspended_time); /** * @brief Affinitize the current thread to a specific CPU by index and return the old affinity. diff --git a/tests/performance/platform.cpp b/tests/performance/platform.cpp index 61a352b6fb..9c78566e42 100644 --- a/tests/performance/platform.cpp +++ b/tests/performance/platform.cpp @@ -46,7 +46,7 @@ _perf_bpf_ktime_get_boot_ns() uint64_t time; ebpf_epoch_state_t epoch_state; ebpf_epoch_enter(&epoch_state); - time = ebpf_query_time_since_boot(true) * EBPF_NS_PER_FILETIME; + time = ebpf_query_time_since_boot_precise(true) * EBPF_NS_PER_FILETIME; ebpf_epoch_exit(&epoch_state); } @@ -56,7 +56,7 @@ _perf_bpf_ktime_get_ns() uint64_t time; ebpf_epoch_state_t epoch_state; ebpf_epoch_enter(&epoch_state); - time = ebpf_query_time_since_boot(false) * EBPF_NS_PER_FILETIME; + time = ebpf_query_time_since_boot_precise(false) * EBPF_NS_PER_FILETIME; ebpf_epoch_exit(&epoch_state); }