From bcffe095b67117385879dee7681d41484f8c0dff Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Fri, 8 Dec 2023 20:10:19 +0800 Subject: [PATCH 01/21] =?UTF-8?q?=E4=BF=AE=E6=94=B9action=EF=BC=8C?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=99=9A=E6=8B=9F=E6=9C=BA=E8=84=8F=E9=A1=B5?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E7=BB=9F=E8=AE=A1=EF=BC=8C=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/kvm_watcher.yml | 1 + eBPF_Supermarket/kvm_watcher/Makefile | 2 +- .../kvm_watcher/include/kvm_vcpu.h | 58 +++++-- .../kvm_watcher/include/kvm_watcher.h | 29 ++++ .../kvm_watcher/src/kvm_watcher.bpf.c | 25 ++- .../kvm_watcher/src/kvm_watcher.c | 147 +++++++++++++++--- 6 files changed, 209 insertions(+), 53 deletions(-) diff --git a/.github/workflows/kvm_watcher.yml b/.github/workflows/kvm_watcher.yml index 861db6f2b..5f7e70076 100644 --- a/.github/workflows/kvm_watcher.yml +++ b/.github/workflows/kvm_watcher.yml @@ -34,5 +34,6 @@ jobs: sudo ./kvm_watcher -w -t 5 sudo ./kvm_watcher -e -t 5 -s sudo ./kvm_watcher -n -t 5 + sudo ./kvm_watcher -d -t 5 make clean diff --git a/eBPF_Supermarket/kvm_watcher/Makefile b/eBPF_Supermarket/kvm_watcher/Makefile index 8a10f9b44..2878e2af7 100644 --- a/eBPF_Supermarket/kvm_watcher/Makefile +++ b/eBPF_Supermarket/kvm_watcher/Makefile @@ -21,4 +21,4 @@ endif clean: cd src && rm -f *.o *.skel.h - rm -f $(notdir $(APP)) include/vmlinux.h \ No newline at end of file + sudo rm -rf $(notdir $(APP)) include/vmlinux.h temp \ No newline at end of file diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h b/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h index 5f18cd7f4..8c207da84 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h @@ -14,7 +14,7 @@ // // author: nanshuaibo811@163.com // -// Kernel space BPF program used for monitoring data for vCPU HLT. +// Kernel space BPF program used for monitoring data for vCPU. #ifndef __KVM_VCPU_H #define __KVM_VCPU_H @@ -40,16 +40,19 @@ struct halt_poll_ns{ unsigned int old; }; +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 8192); + __type(key, u64); + __type(value, u32); +} count_dirty_map SEC(".maps"); + static int trace_kvm_vcpu_wakeup(struct vcpu_wakeup *ctx,void *rb,pid_t vm_pid) { - unsigned pid = bpf_get_current_pid_tgid() >> 32; - if (vm_pid < 0 || pid == vm_pid){ + CHECK_PID(vm_pid){ u32 tid = bpf_get_current_pid_tgid(); struct vcpu_wakeup_event *e; - e = bpf_ringbuf_reserve(rb, sizeof(*e), 0); - if (!e){ - return 0; - } + RESERVE_RINGBUF_ENTRY(rb, e); u64 hlt_time = bpf_ktime_get_ns(); e->waited = ctx->waited; e->process.pid = pid; @@ -64,13 +67,10 @@ static int trace_kvm_vcpu_wakeup(struct vcpu_wakeup *ctx,void *rb,pid_t vm_pid) static int trace_kvm_halt_poll_ns(struct halt_poll_ns *ctx,void *rb,pid_t vm_pid) { - unsigned pid = bpf_get_current_pid_tgid() >> 32; - if (vm_pid < 0 || pid == vm_pid){ + CHECK_PID(vm_pid){ u32 tid = bpf_get_current_pid_tgid(); struct halt_poll_ns_event *e; - e = bpf_ringbuf_reserve(rb, sizeof(*e), 0); - if (!e) - return 0; + RESERVE_RINGBUF_ENTRY(rb, e); u64 time = bpf_ktime_get_ns(); e->process.pid = pid; e->process.tid = tid; @@ -84,4 +84,38 @@ static int trace_kvm_halt_poll_ns(struct halt_poll_ns *ctx,void *rb,pid_t vm_pid return 0; } +static int trace_mark_page_dirty_in_slot(struct kvm *kvm,const struct kvm_memory_slot *memslot,gfn_t gfn,void *rb,pid_t vm_pid) +{ + CHECK_PID(vm_pid){ + u32 flags; + bpf_probe_read_kernel(&flags,sizeof(memslot->flags),&memslot->flags); + if(flags & KVM_MEM_LOG_DIRTY_PAGES){// 检查memslot是否启用了脏页追踪 + gfn_t gfnum=gfn; + u32 *count = bpf_map_lookup_elem(&count_dirty_map, &gfnum); + if (count){ + *count += 1; + }else{ + u32 init_count = 1; + bpf_map_update_elem(&count_dirty_map, &gfnum, &init_count, BPF_ANY); + } + u32 tid = bpf_get_current_pid_tgid(); + unsigned long base_gfn; + struct mark_page_dirty_in_slot_event *e; + RESERVE_RINGBUF_ENTRY(rb, e); + u64 time = bpf_ktime_get_ns(); + e->process.pid = pid; + e->process.tid = tid; + e->time = time; + e->gfn=gfn; + bpf_probe_read_kernel(&base_gfn,sizeof(memslot->base_gfn),&memslot->base_gfn); + e->rel_gfn = gfn - base_gfn; + bpf_probe_read_kernel(&e->npages,sizeof(memslot->npages),&memslot->npages); + bpf_probe_read_kernel(&e->userspace_addr,sizeof(memslot->userspace_addr),&memslot->userspace_addr); + bpf_probe_read_kernel(&e->slot_id,sizeof(memslot->id),&memslot->id); + bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm)); + bpf_ringbuf_submit(e, 0); + } + } + return 0; +} #endif /* __KVM_VCPU_H */ diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h index 135d43ae9..3ff630d68 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h @@ -20,6 +20,25 @@ #define __KVM_WATCHER_H #define TASK_COMM_LEN 16 +#define KVM_MEM_LOG_DIRTY_PAGES (1UL << 0) + +#define PRINT_USAGE_ERR() \ + do { \ + fprintf(stderr, "Use either the -w, -p, -d, or -e option.\n"); \ + argp_usage(state); \ + } while (0) + +#define RESERVE_RINGBUF_ENTRY(rb, e) \ + do { \ + typeof(e) _tmp = bpf_ringbuf_reserve(rb, sizeof(*e), 0); \ + if (!_tmp) \ + return 0; \ + e = _tmp; \ + } while (0) + +#define CHECK_PID(vm_pid) \ + unsigned pid = bpf_get_current_pid_tgid() >> 32; \ + if ((vm_pid) < 0 || pid == (vm_pid)) struct process{ unsigned pid; @@ -59,4 +78,14 @@ struct halt_poll_ns_event { unsigned int old; unsigned long long time; }; + +struct mark_page_dirty_in_slot_event{ + struct process process; + unsigned long long time; + unsigned long npages; + unsigned long userspace_addr; + unsigned long long rel_gfn; + unsigned long long gfn; + short slot_id; +}; #endif /* __KVM_WATCHER_H */ \ No newline at end of file diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c index 37cf689af..caf3ad9b9 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c @@ -27,9 +27,6 @@ char LICENSE[] SEC("license") = "Dual BSD/GPL"; const volatile pid_t vm_pid = -1; -const volatile bool execute_vcpu_wakeup=false; -const volatile bool execute_exit=false; -const volatile bool execute_halt_poll_ns=false; struct { __uint(type, BPF_MAP_TYPE_RINGBUF); @@ -39,35 +36,33 @@ struct { SEC("tp/kvm/kvm_vcpu_wakeup") int tp_vcpu_wakeup(struct vcpu_wakeup *ctx) { - if(execute_vcpu_wakeup){ - trace_kvm_vcpu_wakeup(ctx,&rb,vm_pid); - } + trace_kvm_vcpu_wakeup(ctx,&rb,vm_pid); return 0; } SEC("tp/kvm/kvm_halt_poll_ns") int tp_kvm_halt_poll_ns(struct halt_poll_ns *ctx) { - if(execute_halt_poll_ns){ - trace_kvm_halt_poll_ns(ctx,&rb,vm_pid); - } + trace_kvm_halt_poll_ns(ctx,&rb,vm_pid); return 0; } SEC("tp/kvm/kvm_exit") int tp_exit(struct exit *ctx) { - if(execute_exit){ - trace_kvm_exit(ctx,vm_pid); - } + trace_kvm_exit(ctx,vm_pid); return 0; } SEC("tp/kvm/kvm_entry") int tp_entry(struct exit *ctx) { - if(execute_exit){ - trace_kvm_entry(&rb); - } + trace_kvm_entry(&rb); return 0; } +SEC("kprobe/mark_page_dirty_in_slot") +int BPF_KPROBE(kp_mark_page_dirty_in_slot,struct kvm *kvm,const struct kvm_memory_slot *memslot,gfn_t gfn) +{ + trace_mark_page_dirty_in_slot(kvm,memslot,gfn,&rb,vm_pid); + return 0; +} \ No newline at end of file diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index f2e371843..41cae9a56 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -24,12 +24,13 @@ #include #include #include +#include #include #include +#include #include "kvm_watcher.skel.h" #include "../include/kvm_watcher.h" - struct ExitReason exitReasons[] = { {0, "EXCEPTION_NMI"}, {1, "EXTERNAL_INTERRUPT"}, @@ -134,8 +135,8 @@ void addExitInfo(Node** head, int exit_reason, const char* info,unsigned long lo if (current->data.exit_reason == exit_reason) { // 更新已存在的信息 strncpy(current->data.info, info, sizeof(current->data.info)); - current->data.total_dur=dur+current->data.total_dur; - current->data.avg_dur=current->data.total_dur/count; + current->data.total_dur=dur+current->data.total_dur; + current->data.avg_dur=current->data.total_dur/count; free(newNode); // 释放新节点，因为信息已经更新 return; } @@ -173,6 +174,7 @@ void freeExitInfoList(Node* head) { void printExitInfo(Node* head) { Node* current = head; + printf("\n---------------------------------------------------------------------------\n"); printf("%-23s %-10s %-15s %-8s %-13s \n", "EXIT_REASON", "COMM","PID/TID","COUNT","AVG_DURATION(ns)"); while (current != NULL) { printf("%-2d/%-20s %-33s %-13llu \n", current->data.exit_reason,getExitReasonName(current->data.exit_reason), current->data.info,current->data.avg_dur); @@ -206,11 +208,89 @@ int doesVmProcessExist(pid_t pid) { return 0; // VmProcess with the given PID not found } +// 结构用于保存键值对 +struct KeyValPair { + unsigned long long key; + unsigned int value; +}; + +// 比较函数，用于 qsort +int compare(const void *a, const void *b) { + return ((struct KeyValPair *)b)->value - ((struct KeyValPair *)a)->value; +} + +int save_count_dirtypagemap_to_file(struct bpf_map *map) { + const char *directory = "./temp"; + const char *filename = "./temp/dirty_temp"; + + // 创建目录，如果不存在 + if (mkdir(directory, 0777) == -1) { + // 如果目录已经存在，这里的错误是预期的，可以忽略 + // 否则，打印错误信息并返回 + if (errno != EEXIST) { + perror("Failed to create directory"); + return -1; + } + } + + FILE *output = fopen(filename, "w"); + if (!output) { + perror("Failed to open output file"); + return -1; + } + + int count_dirty_fd = bpf_map__fd(map); + unsigned long long lookup_key = -1, next_key; + unsigned int dirty_counts; + + // 保存键值对到数组 + struct KeyValPair *pairs = NULL; + size_t size = 0; + + while (!bpf_map_get_next_key(count_dirty_fd, &lookup_key, &next_key)) { + int err = bpf_map_lookup_elem(count_dirty_fd, &next_key, &dirty_counts); + if (err < 0) { + fprintf(stderr, "failed to lookup dirty page: %d\n", err); + fclose(output); + free(pairs); + return -1; + } + + // 保存到数组 + pairs = realloc(pairs, (size + 1) * sizeof(struct KeyValPair)); + pairs[size].key = next_key; + pairs[size].value = dirty_counts; + size++; + + // 删除元素 + err = bpf_map_delete_elem(count_dirty_fd, &next_key); + if (err < 0) { + fprintf(stderr, "failed to cleanup dirty page: %d\n", err); + fclose(output); + free(pairs); + return -1; + } + } + + // 对数组进行排序 + qsort(pairs, size, sizeof(struct KeyValPair), compare); + + // 输出到文件 + for (size_t i = 0; i < size; i++) { + fprintf(output, "%llx %d\n", pairs[i].key, pairs[i].value); + } + + fclose(output); + free(pairs); + return 0; +} + static struct env { bool execute_vcpu_wakeup; bool execute_exit; bool ShowStats; bool execute_halt_poll_ns; + bool execute_mark_page_dirty; int monitoring_time; pid_t vm_pid; } env={ @@ -218,6 +298,7 @@ static struct env { .execute_exit=false, .ShowStats=false, .execute_halt_poll_ns=false, + .execute_mark_page_dirty=false, .monitoring_time=0, .vm_pid=-1, }; @@ -229,8 +310,9 @@ int option_selected = 0; // 功能标志变量,确保激活子功能 static const struct argp_option opts[] = { { "vcpu_wakeup", 'w', NULL, 0, "Monitoring the wakeup of vcpu." }, - { "vm_exit)", 'e', NULL, 0, "Monitoring the event of vm exit." }, - { "vcpu_halt_poll_ns)", 'n', NULL, 0, "Monitoring the variation in vCPU polling time." }, + { "vm_exit", 'e', NULL, 0, "Monitoring the event of vm exit." }, + { "vcpu_halt_poll_ns", 'n', NULL, 0, "Monitoring the variation in vCPU polling time." }, + { "mark_page_dirty", 'd', NULL, 0, "Monitor virtual machine dirty page information." }, { "stat",'s',NULL,0,"Display statistical data.(The -e option must be specified.)" }, { "vm_pid", 'p', "PID", 0, "Specify the virtual machine pid to monitor." }, { "monitoring_time", 't', "SEC", 0, "Time for monitoring event." }, @@ -245,8 +327,7 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) env.execute_vcpu_wakeup = true; option_selected = 1; } else { - fprintf(stderr, "Use either the -w ,-p or -e option.\n"); - argp_usage(state); + PRINT_USAGE_ERR(); } break; case 'e': @@ -254,8 +335,7 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) env.execute_exit = true; option_selected = 1; } else { - fprintf(stderr, "Use either the -w ,p or -e option.\n"); - argp_usage(state); + PRINT_USAGE_ERR(); } break; case 'n': @@ -263,10 +343,17 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) env.execute_halt_poll_ns = true; option_selected = 1; } else { - fprintf(stderr, "Use either the -w ,-n or -e option.\n"); - argp_usage(state); + PRINT_USAGE_ERR(); } break; + case 'd': + if (option_selected == 0) { + env.execute_mark_page_dirty = true; + option_selected = 1; + } else { + PRINT_USAGE_ERR(); + } + break; case 's': if(env.execute_exit){ env.ShowStats=true; @@ -327,8 +414,7 @@ static int handle_event(void *ctx, void *data, size_t data_sz) if(env.execute_vcpu_wakeup){ const struct vcpu_wakeup_event *e = data; printf("%-18llu %-20llu %-15s %-6d/%-8d %-10s\n", e->hlt_time,e->dur_hlt_ns, e->process.comm, e->process.pid,e->process.tid,e->waited ? "wait" : "poll"); - } - if(env.execute_exit){ + }else if(env.execute_exit){ char info_buffer[256]; const struct exit_event *e = data; printf("%-2d/%-20s %-10s %-6u/%-8u %-8d %-13llu \n", e->reason_number,getExitReasonName(e->reason_number), e->process.comm, e->process.pid,e->process.tid,e->count,e->duration_ns); @@ -336,10 +422,12 @@ static int handle_event(void *ctx, void *data, size_t data_sz) snprintf(info_buffer, sizeof(info_buffer), "%-10s %-6u/%-8u %-8d", e->process.comm, e->process.pid,e->process.tid,e->count); addExitInfo(&exitInfoBuffer,e->reason_number,info_buffer,e->duration_ns,e->count); } - } - if(env.execute_halt_poll_ns){ + }else if(env.execute_halt_poll_ns){ const struct halt_poll_ns_event *e = data; printf("%-18llu %-15s %-6d/%-8d %-10s %-7d --> %d \n", e->time, e->process.comm, e->process.pid,e->process.tid,e->grow ? "grow" : "shrink",e->old,e->new); + }else if(env.execute_mark_page_dirty){ + const struct mark_page_dirty_in_slot_event *e = data; + printf("%-18llu %-15s %-6d/%-8d %-10llx %-10llx %-10lu %-15lx %d \n", e->time, e->process.comm, e->process.pid,e->process.tid,e->gfn,e->rel_gfn,e->npages,e->userspace_addr,e->slot_id); } return 0; } @@ -371,10 +459,14 @@ int main(int argc, char **argv) /* Parameterize BPF code with parameter */ skel->rodata->vm_pid= env.vm_pid; - skel->rodata->execute_vcpu_wakeup = env.execute_vcpu_wakeup; - skel->rodata->execute_exit = env.execute_exit; - skel->rodata->execute_halt_poll_ns=env.execute_halt_poll_ns; + /* Disable or load kernel hook functions */ + bpf_program__set_autoload(skel->progs.tp_vcpu_wakeup, env.execute_vcpu_wakeup ? true : false); + bpf_program__set_autoload(skel->progs.tp_exit, env.execute_exit ? true : false); + bpf_program__set_autoload(skel->progs.tp_entry, env.execute_exit ? true : false); + bpf_program__set_autoload(skel->progs.tp_kvm_halt_poll_ns, env.execute_halt_poll_ns ? true : false); + bpf_program__set_autoload(skel->progs.kp_mark_page_dirty_in_slot, env.execute_mark_page_dirty ? true : false); + /* Load & verify BPF programs */ err = kvm_watcher_bpf__load(skel); if (err) { @@ -389,22 +481,22 @@ int main(int argc, char **argv) goto cleanup; } - /* Set up ring buffer polling */ + /* Set up ring buffer polling */ rb = ring_buffer__new(bpf_map__fd(skel->maps.rb), handle_event, NULL, NULL); if (!rb) { err = -1; fprintf(stderr, "Failed to create ring buffer\n"); goto cleanup; } - /* Process events */ + /* Process events */ if(env.execute_vcpu_wakeup){ printf("%-18s %-20s %-15s %-15s %-10s\n", "HLT_TIME(ns)", "DURATIONS_TIME(ns)","VCPUID/COMM","PID/TID","WAIT/POLL"); - } - if(env.execute_exit){ + }else if(env.execute_exit){ printf("%-23s %-10s %-15s %-8s %-13s \n", "EXIT_REASON", "COMM","PID/TID","COUNT","DURATION(ns)"); - } - if(env.execute_halt_poll_ns){ + }else if(env.execute_halt_poll_ns){ printf("%-18s %-15s %-15s %-10s %-11s %-10s\n", "TIME(ns)", "VCPUID/COMM","PID/TID","TYPE","OLD(ns)","NEW(ns)"); + }else if(env.execute_mark_page_dirty){ + printf("%-18s %-15s %-15s %-10s %-11s %-10s %-10s %-10s\n", "TIME(ns)", "VCPUID/COMM","PID/TID","GFN","REL_GFN","NPAGES","USERSPACE_ADDR","SLOT_ID"); } while (!exiting) { err = ring_buffer__poll(rb, 10 /* timeout, ms */); @@ -419,9 +511,14 @@ int main(int argc, char **argv) } } if(env.ShowStats){ - printf("\n---------------------------------------------------------------------------\n"); printExitInfo(exitInfoBuffer); freeExitInfoList(exitInfoBuffer); + }else if(env.execute_mark_page_dirty){ + err = save_count_dirtypagemap_to_file(skel->maps.count_dirty_map); + if (err < 0) { + printf("Save count dirty page map to file fail: %d\n", err); + goto cleanup; + } } cleanup: ring_buffer__free(rb); From 72aa75ecdc3b01cc96bbd68b83d18ce1ce3dd344 Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Sat, 9 Dec 2023 16:18:09 +0800 Subject: [PATCH 02/21] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E5=8C=96=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eBPF_Supermarket/kvm_watcher/.clang-format | 50 ++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 eBPF_Supermarket/kvm_watcher/.clang-format diff --git a/eBPF_Supermarket/kvm_watcher/.clang-format b/eBPF_Supermarket/kvm_watcher/.clang-format new file mode 100644 index 000000000..1f94a30cd --- /dev/null +++ b/eBPF_Supermarket/kvm_watcher/.clang-format @@ -0,0 +1,50 @@ +# .clang-format for kvm_watcher project +# +# This configuration file defines the coding style for the kvm_watcher project. +# It is based on the Google style with some specific modifications. +# +# Author: nanshuaibo811@163.com +# Project: kvm_watcher +# Description: This file defines the coding style for the kvm_watcher project. + + +# 使用Google风格作为基础 +BasedOnStyle: Google + +# 设置缩进为4个空格 +IndentWidth: 4 + +# 禁用使用制表符 +UseTab: Never + +# 左大括号放在同一行 +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterStruct: false + AfterUnion: false + BeforeCatch: true + BeforeElse: true + IndentBraces: false + +# switch下的case语句缩进 +IndentCaseLabels: true + +# 不允许将简短的循环写在同一行 +AllowShortLoopsOnASingleLine: false + +# 不允许将简短的函数写在同一行 +AllowShortFunctionsOnASingleLine: false + +# if 语句和 return 语句不能在同一行 +AllowShortIfStatementsOnASingleLine: false + +# 设置一行的字符数限制 +ColumnLimit: 80 + +# 保留原始的 #include 排序 +SortIncludes: false + From fe48e5e58562ea27f5c7fe6a05d47aee7b4375b7 Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Sat, 9 Dec 2023 16:18:40 +0800 Subject: [PATCH 03/21] =?UTF-8?q?=E4=BD=BF=E7=94=A8clang-format=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kvm_watcher/include/kvm_exits.h | 61 ++- .../kvm_watcher/include/kvm_vcpu.h | 65 +-- .../kvm_watcher/include/kvm_watcher.h | 32 +- .../kvm_watcher/src/kvm_watcher.bpf.c | 30 +- .../kvm_watcher/src/kvm_watcher.c | 456 +++++++++--------- 5 files changed, 332 insertions(+), 312 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h b/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h index a676c7386..b19c0668a 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h @@ -39,7 +39,7 @@ struct { __type(value, u32); } counts SEC(".maps"); -struct exit{ +struct exit { u64 pad; unsigned int exit_reason; unsigned long guest_rip; @@ -51,29 +51,28 @@ struct exit{ unsigned int vcpu_id; }; -int total=0; +int total = 0; -static int trace_kvm_exit(struct exit *ctx, pid_t vm_pid) -{ - u64 id,ts; +static int trace_kvm_exit(struct exit *ctx, pid_t vm_pid) { + u64 id, ts; id = bpf_get_current_pid_tgid(); pid_t tid = (u32)id; pid_t pid = id >> 32; - if (vm_pid < 0 || pid == vm_pid){ + if (vm_pid < 0 || pid == vm_pid) { ts = bpf_ktime_get_ns(); u32 reason; - reason=(u32)ctx->exit_reason; - struct reason_info reas={}; - reas.reason=reason; - reas.time=ts; + reason = (u32)ctx->exit_reason; + struct reason_info reas = {}; + reas.reason = reason; + reas.time = ts; u32 *count; - count=bpf_map_lookup_elem(&counts, &reason); - if(count){ + count = bpf_map_lookup_elem(&counts, &reason); + if (count) { (*count)++; - reas.count=*count; - }else{ + reas.count = *count; + } else { u32 new_count = 1; - reas.count=new_count; + reas.count = new_count; bpf_map_update_elem(&counts, &reason, &new_count, BPF_ANY); } bpf_map_update_elem(×, &tid, &reas, BPF_ANY); @@ -81,33 +80,32 @@ static int trace_kvm_exit(struct exit *ctx, pid_t vm_pid) return 0; } -static int trace_kvm_entry(void *rb) -{ +static int trace_kvm_entry(void *rb) { struct reason_info *reas; pid_t pid, tid; - u64 id, ts, *start_ts, duration_ns=0; + u64 id, ts, *start_ts, duration_ns = 0; id = bpf_get_current_pid_tgid(); pid = id >> 32; tid = (u32)id; reas = bpf_map_lookup_elem(×, &tid); - if(reas){ + if (reas) { u32 reason; struct exit_event *e; - int count=0; - duration_ns=bpf_ktime_get_ns() - reas->time; + int count = 0; + duration_ns = bpf_ktime_get_ns() - reas->time; bpf_map_delete_elem(×, &tid); - reason=reas->reason; - count=reas->count; + reason = reas->reason; + count = reas->count; e = bpf_ringbuf_reserve(rb, sizeof(*e), 0); - if (!e){ + if (!e) { return 0; } - e->reason_number=reason; - e->process.pid=pid; + e->reason_number = reason; + e->process.pid = pid; e->duration_ns = duration_ns; bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm)); - e->process.tid=tid; - e->total=++total; + e->process.tid = tid; + e->total = ++total; if (count) { e->count = count; } else { @@ -115,11 +113,8 @@ static int trace_kvm_entry(void *rb) } bpf_ringbuf_submit(e, 0); return 0; - } - else{ + } else { return 0; - } + } } #endif /* __KVM_EXITS_H */ - - diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h b/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h index 8c207da84..f5246e08a 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h @@ -25,14 +25,14 @@ #include #include -struct vcpu_wakeup{ +struct vcpu_wakeup { u64 pad; __u64 ns; bool waited; bool vaild; }; -struct halt_poll_ns{ +struct halt_poll_ns { u64 pad; bool grow; unsigned int vcpu_id; @@ -41,15 +41,15 @@ struct halt_poll_ns{ }; struct { - __uint(type, BPF_MAP_TYPE_HASH); - __uint(max_entries, 8192); - __type(key, u64); - __type(value, u32); + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 8192); + __type(key, u64); + __type(value, u32); } count_dirty_map SEC(".maps"); -static int trace_kvm_vcpu_wakeup(struct vcpu_wakeup *ctx,void *rb,pid_t vm_pid) -{ - CHECK_PID(vm_pid){ +static int trace_kvm_vcpu_wakeup(struct vcpu_wakeup *ctx, void *rb, + pid_t vm_pid) { + CHECK_PID(vm_pid) { u32 tid = bpf_get_current_pid_tgid(); struct vcpu_wakeup_event *e; RESERVE_RINGBUF_ENTRY(rb, e); @@ -65,9 +65,9 @@ static int trace_kvm_vcpu_wakeup(struct vcpu_wakeup *ctx,void *rb,pid_t vm_pid) return 0; } -static int trace_kvm_halt_poll_ns(struct halt_poll_ns *ctx,void *rb,pid_t vm_pid) -{ - CHECK_PID(vm_pid){ +static int trace_kvm_halt_poll_ns(struct halt_poll_ns *ctx, void *rb, + pid_t vm_pid) { + CHECK_PID(vm_pid) { u32 tid = bpf_get_current_pid_tgid(); struct halt_poll_ns_event *e; RESERVE_RINGBUF_ENTRY(rb, e); @@ -75,28 +75,30 @@ static int trace_kvm_halt_poll_ns(struct halt_poll_ns *ctx,void *rb,pid_t vm_pid e->process.pid = pid; e->process.tid = tid; e->time = time; - e->grow=ctx->grow; - e->old=ctx->old; - e->new=ctx->new; + e->grow = ctx->grow; + e->old = ctx->old; + e->new = ctx->new; bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm)); bpf_ringbuf_submit(e, 0); } return 0; } -static int trace_mark_page_dirty_in_slot(struct kvm *kvm,const struct kvm_memory_slot *memslot,gfn_t gfn,void *rb,pid_t vm_pid) -{ - CHECK_PID(vm_pid){ +static int trace_mark_page_dirty_in_slot(struct kvm *kvm, + const struct kvm_memory_slot *memslot, + gfn_t gfn, void *rb, pid_t vm_pid) { + CHECK_PID(vm_pid) { u32 flags; - bpf_probe_read_kernel(&flags,sizeof(memslot->flags),&memslot->flags); - if(flags & KVM_MEM_LOG_DIRTY_PAGES){// 检查memslot是否启用了脏页追踪 - gfn_t gfnum=gfn; + bpf_probe_read_kernel(&flags, sizeof(memslot->flags), &memslot->flags); + if (flags & KVM_MEM_LOG_DIRTY_PAGES) { // 检查memslot是否启用了脏页追踪 + gfn_t gfnum = gfn; u32 *count = bpf_map_lookup_elem(&count_dirty_map, &gfnum); - if (count){ + if (count) { *count += 1; - }else{ + } else { u32 init_count = 1; - bpf_map_update_elem(&count_dirty_map, &gfnum, &init_count, BPF_ANY); + bpf_map_update_elem(&count_dirty_map, &gfnum, &init_count, + BPF_ANY); } u32 tid = bpf_get_current_pid_tgid(); unsigned long base_gfn; @@ -106,12 +108,17 @@ static int trace_mark_page_dirty_in_slot(struct kvm *kvm,const struct kvm_memory e->process.pid = pid; e->process.tid = tid; e->time = time; - e->gfn=gfn; - bpf_probe_read_kernel(&base_gfn,sizeof(memslot->base_gfn),&memslot->base_gfn); + e->gfn = gfn; + bpf_probe_read_kernel(&base_gfn, sizeof(memslot->base_gfn), + &memslot->base_gfn); e->rel_gfn = gfn - base_gfn; - bpf_probe_read_kernel(&e->npages,sizeof(memslot->npages),&memslot->npages); - bpf_probe_read_kernel(&e->userspace_addr,sizeof(memslot->userspace_addr),&memslot->userspace_addr); - bpf_probe_read_kernel(&e->slot_id,sizeof(memslot->id),&memslot->id); + bpf_probe_read_kernel(&e->npages, sizeof(memslot->npages), + &memslot->npages); + bpf_probe_read_kernel(&e->userspace_addr, + sizeof(memslot->userspace_addr), + &memslot->userspace_addr); + bpf_probe_read_kernel(&e->slot_id, sizeof(memslot->id), + &memslot->id); bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm)); bpf_ringbuf_submit(e, 0); } diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h index 3ff630d68..0e08dd3cb 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h @@ -19,28 +19,28 @@ #ifndef __KVM_WATCHER_H #define __KVM_WATCHER_H -#define TASK_COMM_LEN 16 -#define KVM_MEM_LOG_DIRTY_PAGES (1UL << 0) +#define TASK_COMM_LEN 16 +#define KVM_MEM_LOG_DIRTY_PAGES (1UL << 0) -#define PRINT_USAGE_ERR() \ - do { \ +#define PRINT_USAGE_ERR() \ + do { \ fprintf(stderr, "Use either the -w, -p, -d, or -e option.\n"); \ - argp_usage(state); \ + argp_usage(state); \ } while (0) -#define RESERVE_RINGBUF_ENTRY(rb, e) \ - do { \ +#define RESERVE_RINGBUF_ENTRY(rb, e) \ + do { \ typeof(e) _tmp = bpf_ringbuf_reserve(rb, sizeof(*e), 0); \ - if (!_tmp) \ - return 0; \ - e = _tmp; \ + if (!_tmp) \ + return 0; \ + e = _tmp; \ } while (0) -#define CHECK_PID(vm_pid) \ +#define CHECK_PID(vm_pid) \ unsigned pid = bpf_get_current_pid_tgid() >> 32; \ if ((vm_pid) < 0 || pid == (vm_pid)) -struct process{ +struct process { unsigned pid; unsigned tid; char comm[TASK_COMM_LEN]; @@ -62,12 +62,12 @@ struct exit_event { struct ExitReason { int number; - const char* name; + const char *name; }; struct reason_info { unsigned long long time; - unsigned long reason; + unsigned long reason; int count; }; @@ -75,11 +75,11 @@ struct halt_poll_ns_event { struct process process; bool grow; unsigned int new; - unsigned int old; + unsigned int old; unsigned long long time; }; -struct mark_page_dirty_in_slot_event{ +struct mark_page_dirty_in_slot_event { struct process process; unsigned long long time; unsigned long npages; diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c index caf3ad9b9..7601882c4 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c @@ -17,12 +17,12 @@ // Kernel space BPF program used for monitoring data for KVM event. #include "../include/vmlinux.h" -#include #include +#include #include -#include "../include/kvm_watcher.h" -#include "../include/kvm_vcpu.h" #include "../include/kvm_exits.h" +#include "../include/kvm_vcpu.h" +#include "../include/kvm_watcher.h" char LICENSE[] SEC("license") = "Dual BSD/GPL"; @@ -34,35 +34,31 @@ struct { } rb SEC(".maps"); SEC("tp/kvm/kvm_vcpu_wakeup") -int tp_vcpu_wakeup(struct vcpu_wakeup *ctx) -{ - trace_kvm_vcpu_wakeup(ctx,&rb,vm_pid); +int tp_vcpu_wakeup(struct vcpu_wakeup *ctx) { + trace_kvm_vcpu_wakeup(ctx, &rb, vm_pid); return 0; } SEC("tp/kvm/kvm_halt_poll_ns") -int tp_kvm_halt_poll_ns(struct halt_poll_ns *ctx) -{ - trace_kvm_halt_poll_ns(ctx,&rb,vm_pid); +int tp_kvm_halt_poll_ns(struct halt_poll_ns *ctx) { + trace_kvm_halt_poll_ns(ctx, &rb, vm_pid); return 0; } SEC("tp/kvm/kvm_exit") -int tp_exit(struct exit *ctx) -{ - trace_kvm_exit(ctx,vm_pid); +int tp_exit(struct exit *ctx) { + trace_kvm_exit(ctx, vm_pid); return 0; } SEC("tp/kvm/kvm_entry") -int tp_entry(struct exit *ctx) -{ +int tp_entry(struct exit *ctx) { trace_kvm_entry(&rb); return 0; } SEC("kprobe/mark_page_dirty_in_slot") -int BPF_KPROBE(kp_mark_page_dirty_in_slot,struct kvm *kvm,const struct kvm_memory_slot *memslot,gfn_t gfn) -{ - trace_mark_page_dirty_in_slot(kvm,memslot,gfn,&rb,vm_pid); +int BPF_KPROBE(kp_mark_page_dirty_in_slot, struct kvm *kvm, + const struct kvm_memory_slot *memslot, gfn_t gfn) { + trace_mark_page_dirty_in_slot(kvm, memslot, gfn, &rb, vm_pid); return 0; } \ No newline at end of file diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 41cae9a56..46901d90e 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -16,98 +16,95 @@ // // User space BPF program used for monitoring KVM event. - +#include +#include +#include +#include #include -#include #include #include -#include #include -#include #include -#include -#include -#include -#include "kvm_watcher.skel.h" +#include +#include #include "../include/kvm_watcher.h" +#include "kvm_watcher.skel.h" -struct ExitReason exitReasons[] = { - {0, "EXCEPTION_NMI"}, - {1, "EXTERNAL_INTERRUPT"}, - {2, "TRIPLE_FAULT"}, - {3, "INIT_SIGNAL"}, - {4, "SIPI_SIGNAL"}, - {7, "INTERRUPT_WINDOW"}, - {8, "NMI_WINDOW"}, - {9, "TASK_SWITCH"}, - {10, "CPUID"}, - {12, "HLT"}, - {13, "INVD"}, - {14, "INVLPG"}, - {15, "RDPMC"}, - {16, "RDTSC"}, - {18, "VMCALL"}, - {19, "VMCLEAR"}, - {20, "VMLAUNCH"}, - {21, "VMPTRLD"}, - {22, "VMPTRST"}, - {23, "VMREAD"}, - {24, "VMRESUME"}, - {25, "VMWRITE"}, - {26, "VMOFF"}, - {27, "VMON"}, - {28, "CR_ACCESS"}, - {29, "DR_ACCESS"}, - {30, "IO_INSTRUCTION"}, - {31, "MSR_READ"}, - {32, "MSR_WRITE"}, - {33, "INVALID_STATE"}, - {34, "MSR_LOAD_FAIL"}, - {36, "MWAIT_INSTRUCTION"}, - {37, "MONITOR_TRAP_FLAG"}, - {39, "MONITOR_INSTRUCTION"}, - {40, "PAUSE_INSTRUCTION"}, - {41, "MCE_DURING_VMENTRY"}, - {43, "TPR_BELOW_THRESHOLD"}, - {44, "APIC_ACCESS"}, - {45, "EOI_INDUCED"}, - {46, "GDTR_IDTR"}, - {47, "LDTR_TR"}, - {48, "EPT_VIOLATION"}, - {49, "EPT_MISCONFIG"}, - {50, "INVEPT"}, - {51, "RDTSCP"}, - {52, "PREEMPTION_TIMER"}, - {53, "INVVPID"}, - {54, "WBINVD"}, - {55, "XSETBV"}, - {56, "APIC_WRITE"}, - {57, "RDRAND"}, - {58, "INVPCID"}, - {59, "VMFUNC"}, - {60, "ENCLS"}, - {61, "RDSEED"}, - {62, "PML_FULL"}, - {63, "XSAVES"}, - {64, "XRSTORS"}, - {67, "UMWAIT"}, - {68, "TPAUSE"}, - {74, "BUS_LOCK"}, - {75, "NOTIFY"} -}; - -const char* getExitReasonName(int number) { +struct ExitReason exitReasons[] = {{0, "EXCEPTION_NMI"}, + {1, "EXTERNAL_INTERRUPT"}, + {2, "TRIPLE_FAULT"}, + {3, "INIT_SIGNAL"}, + {4, "SIPI_SIGNAL"}, + {7, "INTERRUPT_WINDOW"}, + {8, "NMI_WINDOW"}, + {9, "TASK_SWITCH"}, + {10, "CPUID"}, + {12, "HLT"}, + {13, "INVD"}, + {14, "INVLPG"}, + {15, "RDPMC"}, + {16, "RDTSC"}, + {18, "VMCALL"}, + {19, "VMCLEAR"}, + {20, "VMLAUNCH"}, + {21, "VMPTRLD"}, + {22, "VMPTRST"}, + {23, "VMREAD"}, + {24, "VMRESUME"}, + {25, "VMWRITE"}, + {26, "VMOFF"}, + {27, "VMON"}, + {28, "CR_ACCESS"}, + {29, "DR_ACCESS"}, + {30, "IO_INSTRUCTION"}, + {31, "MSR_READ"}, + {32, "MSR_WRITE"}, + {33, "INVALID_STATE"}, + {34, "MSR_LOAD_FAIL"}, + {36, "MWAIT_INSTRUCTION"}, + {37, "MONITOR_TRAP_FLAG"}, + {39, "MONITOR_INSTRUCTION"}, + {40, "PAUSE_INSTRUCTION"}, + {41, "MCE_DURING_VMENTRY"}, + {43, "TPR_BELOW_THRESHOLD"}, + {44, "APIC_ACCESS"}, + {45, "EOI_INDUCED"}, + {46, "GDTR_IDTR"}, + {47, "LDTR_TR"}, + {48, "EPT_VIOLATION"}, + {49, "EPT_MISCONFIG"}, + {50, "INVEPT"}, + {51, "RDTSCP"}, + {52, "PREEMPTION_TIMER"}, + {53, "INVVPID"}, + {54, "WBINVD"}, + {55, "XSETBV"}, + {56, "APIC_WRITE"}, + {57, "RDRAND"}, + {58, "INVPCID"}, + {59, "VMFUNC"}, + {60, "ENCLS"}, + {61, "RDSEED"}, + {62, "PML_FULL"}, + {63, "XSAVES"}, + {64, "XRSTORS"}, + {67, "UMWAIT"}, + {68, "TPAUSE"}, + {74, "BUS_LOCK"}, + {75, "NOTIFY"}}; + +const char *getExitReasonName(int number) { for (int i = 0; i < sizeof(exitReasons) / sizeof(exitReasons[0]); i++) { if (exitReasons[i].number == number) { return exitReasons[i].name; } } - return "Unknown"; // 如果找不到对应的退出原因，返回一个默认值 + return "Unknown"; // 如果找不到对应的退出原因，返回一个默认值 } typedef struct { int exit_reason; - char info[256]; // 替换成适当的大小 + char info[256]; // 替换成适当的大小 unsigned long long total_dur; unsigned long long avg_dur; } ExitInfo; @@ -115,13 +112,14 @@ typedef struct { // 链表节点 typedef struct Node { ExitInfo data; - struct Node* next; + struct Node *next; } Node; -Node* exitInfoBuffer = NULL; +Node *exitInfoBuffer = NULL; -void addExitInfo(Node** head, int exit_reason, const char* info,unsigned long long dur,int count) { - Node* newNode = (Node*)malloc(sizeof(Node)); +void addExitInfo(Node **head, int exit_reason, const char *info, + unsigned long long dur, int count) { + Node *newNode = (Node *)malloc(sizeof(Node)); newNode->data.exit_reason = exit_reason; strncpy(newNode->data.info, info, sizeof(newNode->data.info)); newNode->next = NULL; @@ -129,15 +127,15 @@ void addExitInfo(Node** head, int exit_reason, const char* info,unsigned long lo newNode->data.avg_dur = dur / count; // 检查是否已经存在相同 exit reason 的信息 - Node* current = *head; - Node* previous = NULL; + Node *current = *head; + Node *previous = NULL; while (current != NULL) { if (current->data.exit_reason == exit_reason) { // 更新已存在的信息 strncpy(current->data.info, info, sizeof(current->data.info)); - current->data.total_dur=dur+current->data.total_dur; - current->data.avg_dur=current->data.total_dur/count; - free(newNode); // 释放新节点，因为信息已经更新 + current->data.total_dur = dur + current->data.total_dur; + current->data.avg_dur = current->data.total_dur / count; + free(newNode); // 释放新节点，因为信息已经更新 return; } previous = current; @@ -152,8 +150,8 @@ void addExitInfo(Node** head, int exit_reason, const char* info,unsigned long lo } // 查找指定退出原因的信息 -const char* findExitInfo(Node* head, int exit_reason) { - Node* current = head; +const char *findExitInfo(Node *head, int exit_reason) { + Node *current = head; while (current != NULL) { if (current->data.exit_reason == exit_reason) { return current->data.info; @@ -164,21 +162,26 @@ const char* findExitInfo(Node* head, int exit_reason) { } // 释放链表 -void freeExitInfoList(Node* head) { +void freeExitInfoList(Node *head) { while (head != NULL) { - Node* temp = head; + Node *temp = head; head = head->next; free(temp); } } -void printExitInfo(Node* head) { - Node* current = head; - printf("\n---------------------------------------------------------------------------\n"); - printf("%-23s %-10s %-15s %-8s %-13s \n", "EXIT_REASON", "COMM","PID/TID","COUNT","AVG_DURATION(ns)"); +void printExitInfo(Node *head) { + Node *current = head; + printf( + "\n-----------------------------------------------------------------" + "----------\n"); + printf("%-23s %-10s %-15s %-8s %-13s \n", "EXIT_REASON", "COMM", "PID/TID", + "COUNT", "AVG_DURATION(ns)"); while (current != NULL) { - printf("%-2d/%-20s %-33s %-13llu \n", current->data.exit_reason,getExitReasonName(current->data.exit_reason), current->data.info,current->data.avg_dur); - current = current->next; + printf("%-2d/%-20s %-33s %-13llu \n", current->data.exit_reason, + getExitReasonName(current->data.exit_reason), current->data.info, + current->data.avg_dur); + current = current->next; } } @@ -198,7 +201,8 @@ int doesVmProcessExist(pid_t pid) { return 1; // VmProcess name contains the target string } else { fclose(file); - fprintf(stderr, "Process exist!but is not vmprocess: %d\n", pid); + fprintf(stderr, "Process exist!but is not vmprocess: %d\n", + pid); return 0; // VmProcess name does not contain the target string } } @@ -242,7 +246,7 @@ int save_count_dirtypagemap_to_file(struct bpf_map *map) { int count_dirty_fd = bpf_map__fd(map); unsigned long long lookup_key = -1, next_key; unsigned int dirty_counts; - + // 保存键值对到数组 struct KeyValPair *pairs = NULL; size_t size = 0; @@ -293,99 +297,101 @@ static struct env { bool execute_mark_page_dirty; int monitoring_time; pid_t vm_pid; -} env={ - .execute_vcpu_wakeup=false, - .execute_exit=false, - .ShowStats=false, - .execute_halt_poll_ns=false, - .execute_mark_page_dirty=false, - .monitoring_time=0, - .vm_pid=-1, +} env = { + .execute_vcpu_wakeup = false, + .execute_exit = false, + .ShowStats = false, + .execute_halt_poll_ns = false, + .execute_mark_page_dirty = false, + .monitoring_time = 0, + .vm_pid = -1, }; const char *argp_program_version = "kvm_watcher 1.0"; const char *argp_program_bug_address = ""; const char argp_program_doc[] = "BPF program used for monitoring KVM event\n"; -int option_selected = 0; // 功能标志变量,确保激活子功能 +int option_selected = 0; // 功能标志变量,确保激活子功能 static const struct argp_option opts[] = { - { "vcpu_wakeup", 'w', NULL, 0, "Monitoring the wakeup of vcpu." }, - { "vm_exit", 'e', NULL, 0, "Monitoring the event of vm exit." }, - { "vcpu_halt_poll_ns", 'n', NULL, 0, "Monitoring the variation in vCPU polling time." }, - { "mark_page_dirty", 'd', NULL, 0, "Monitor virtual machine dirty page information." }, - { "stat",'s',NULL,0,"Display statistical data.(The -e option must be specified.)" }, - { "vm_pid", 'p', "PID", 0, "Specify the virtual machine pid to monitor." }, - { "monitoring_time", 't', "SEC", 0, "Time for monitoring event." }, + {"vcpu_wakeup", 'w', NULL, 0, "Monitoring the wakeup of vcpu."}, + {"vm_exit", 'e', NULL, 0, "Monitoring the event of vm exit."}, + {"vcpu_halt_poll_ns", 'n', NULL, 0, + "Monitoring the variation in vCPU polling time."}, + {"mark_page_dirty", 'd', NULL, 0, + "Monitor virtual machine dirty page information."}, + {"stat", 's', NULL, 0, + "Display statistical data.(The -e option must be specified.)"}, + {"vm_pid", 'p', "PID", 0, "Specify the virtual machine pid to monitor."}, + {"monitoring_time", 't', "SEC", 0, "Time for monitoring event."}, {}, }; -static error_t parse_arg(int key, char *arg, struct argp_state *state) -{ +static error_t parse_arg(int key, char *arg, struct argp_state *state) { switch (key) { - case 'w': - if (option_selected == 0) { - env.execute_vcpu_wakeup = true; - option_selected = 1; - } else { - PRINT_USAGE_ERR(); - } - break; - case 'e': - if (option_selected == 0) { - env.execute_exit = true; - option_selected = 1; - } else { - PRINT_USAGE_ERR(); - } - break; - case 'n': - if (option_selected == 0) { - env.execute_halt_poll_ns = true; - option_selected = 1; - } else { - PRINT_USAGE_ERR(); - } - break; - case 'd': - if (option_selected == 0) { - env.execute_mark_page_dirty = true; - option_selected = 1; - } else { - PRINT_USAGE_ERR(); - } - break; - case 's': - if(env.execute_exit){ - env.ShowStats=true; - }else{ - fprintf(stderr, "The -e option must be specified.\n"); - argp_usage(state); - } - break; - case 't': - env.monitoring_time = strtol(arg, NULL, 10); - if (env.monitoring_time <= 0) { - fprintf(stderr, "Invalid duration: %s\n", arg); - argp_usage(state); - } else if (!option_selected) { - fprintf(stderr, "No monitoring options activated!\n"); - argp_usage(state); - }else{ - alarm(env.monitoring_time); - } - break; - case 'p': - env.vm_pid=strtol(arg, NULL, 10); - if(env.vm_pid<=0 || doesVmProcessExist(env.vm_pid)==0 ){ - fprintf(stderr, "Invalid vm_pid: %s\n", arg); + case 'w': + if (option_selected == 0) { + env.execute_vcpu_wakeup = true; + option_selected = 1; + } else { + PRINT_USAGE_ERR(); + } + break; + case 'e': + if (option_selected == 0) { + env.execute_exit = true; + option_selected = 1; + } else { + PRINT_USAGE_ERR(); + } + break; + case 'n': + if (option_selected == 0) { + env.execute_halt_poll_ns = true; + option_selected = 1; + } else { + PRINT_USAGE_ERR(); + } + break; + case 'd': + if (option_selected == 0) { + env.execute_mark_page_dirty = true; + option_selected = 1; + } else { + PRINT_USAGE_ERR(); + } + break; + case 's': + if (env.execute_exit) { + env.ShowStats = true; + } else { + fprintf(stderr, "The -e option must be specified.\n"); + argp_usage(state); + } + break; + case 't': + env.monitoring_time = strtol(arg, NULL, 10); + if (env.monitoring_time <= 0) { + fprintf(stderr, "Invalid duration: %s\n", arg); + argp_usage(state); + } else if (!option_selected) { + fprintf(stderr, "No monitoring options activated!\n"); + argp_usage(state); + } else { + alarm(env.monitoring_time); + } + break; + case 'p': + env.vm_pid = strtol(arg, NULL, 10); + if (env.vm_pid <= 0 || doesVmProcessExist(env.vm_pid) == 0) { + fprintf(stderr, "Invalid vm_pid: %s\n", arg); + argp_usage(state); + } + break; + case ARGP_KEY_ARG: argp_usage(state); - } - break; - case ARGP_KEY_ARG: - argp_usage(state); - break; - default: - return ARGP_ERR_UNKNOWN; + break; + default: + return ARGP_ERR_UNKNOWN; } return 0; } @@ -396,44 +402,50 @@ static const struct argp argp = { .doc = argp_program_doc, }; - -static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args) -{ +static int libbpf_print_fn(enum libbpf_print_level level, const char *format, + va_list args) { return vfprintf(stderr, format, args); } static volatile bool exiting = false; -static void sig_handler(int sig) -{ +static void sig_handler(int sig) { exiting = true; } -static int handle_event(void *ctx, void *data, size_t data_sz) -{ - if(env.execute_vcpu_wakeup){ +static int handle_event(void *ctx, void *data, size_t data_sz) { + if (env.execute_vcpu_wakeup) { const struct vcpu_wakeup_event *e = data; - printf("%-18llu %-20llu %-15s %-6d/%-8d %-10s\n", e->hlt_time,e->dur_hlt_ns, e->process.comm, e->process.pid,e->process.tid,e->waited ? "wait" : "poll"); - }else if(env.execute_exit){ + printf("%-18llu %-20llu %-15s %-6d/%-8d %-10s\n", e->hlt_time, + e->dur_hlt_ns, e->process.comm, e->process.pid, e->process.tid, + e->waited ? "wait" : "poll"); + } else if (env.execute_exit) { char info_buffer[256]; const struct exit_event *e = data; - printf("%-2d/%-20s %-10s %-6u/%-8u %-8d %-13llu \n", e->reason_number,getExitReasonName(e->reason_number), e->process.comm, e->process.pid,e->process.tid,e->count,e->duration_ns); - if(env.ShowStats){ - snprintf(info_buffer, sizeof(info_buffer), "%-10s %-6u/%-8u %-8d", e->process.comm, e->process.pid,e->process.tid,e->count); - addExitInfo(&exitInfoBuffer,e->reason_number,info_buffer,e->duration_ns,e->count); + printf("%-2d/%-20s %-10s %-6u/%-8u %-8d %-13llu \n", e->reason_number, + getExitReasonName(e->reason_number), e->process.comm, + e->process.pid, e->process.tid, e->count, e->duration_ns); + if (env.ShowStats) { + snprintf(info_buffer, sizeof(info_buffer), "%-10s %-6u/%-8u %-8d", + e->process.comm, e->process.pid, e->process.tid, e->count); + addExitInfo(&exitInfoBuffer, e->reason_number, info_buffer, + e->duration_ns, e->count); } - }else if(env.execute_halt_poll_ns){ + } else if (env.execute_halt_poll_ns) { const struct halt_poll_ns_event *e = data; - printf("%-18llu %-15s %-6d/%-8d %-10s %-7d --> %d \n", e->time, e->process.comm, e->process.pid,e->process.tid,e->grow ? "grow" : "shrink",e->old,e->new); - }else if(env.execute_mark_page_dirty){ + printf("%-18llu %-15s %-6d/%-8d %-10s %-7d --> %d \n", e->time, + e->process.comm, e->process.pid, e->process.tid, + e->grow ? "grow" : "shrink", e->old, e->new); + } else if (env.execute_mark_page_dirty) { const struct mark_page_dirty_in_slot_event *e = data; - printf("%-18llu %-15s %-6d/%-8d %-10llx %-10llx %-10lu %-15lx %d \n", e->time, e->process.comm, e->process.pid,e->process.tid,e->gfn,e->rel_gfn,e->npages,e->userspace_addr,e->slot_id); + printf("%-18llu %-15s %-6d/%-8d %-10llx %-10llx %-10lu %-15lx %d \n", + e->time, e->process.comm, e->process.pid, e->process.tid, e->gfn, + e->rel_gfn, e->npages, e->userspace_addr, e->slot_id); } return 0; } -int main(int argc, char **argv) -{ +int main(int argc, char **argv) { struct ring_buffer *rb = NULL; struct kvm_watcher_bpf *skel; int err; @@ -458,14 +470,19 @@ int main(int argc, char **argv) } /* Parameterize BPF code with parameter */ - skel->rodata->vm_pid= env.vm_pid; - + skel->rodata->vm_pid = env.vm_pid; + /* Disable or load kernel hook functions */ - bpf_program__set_autoload(skel->progs.tp_vcpu_wakeup, env.execute_vcpu_wakeup ? true : false); - bpf_program__set_autoload(skel->progs.tp_exit, env.execute_exit ? true : false); - bpf_program__set_autoload(skel->progs.tp_entry, env.execute_exit ? true : false); - bpf_program__set_autoload(skel->progs.tp_kvm_halt_poll_ns, env.execute_halt_poll_ns ? true : false); - bpf_program__set_autoload(skel->progs.kp_mark_page_dirty_in_slot, env.execute_mark_page_dirty ? true : false); + bpf_program__set_autoload(skel->progs.tp_vcpu_wakeup, + env.execute_vcpu_wakeup ? true : false); + bpf_program__set_autoload(skel->progs.tp_exit, + env.execute_exit ? true : false); + bpf_program__set_autoload(skel->progs.tp_entry, + env.execute_exit ? true : false); + bpf_program__set_autoload(skel->progs.tp_kvm_halt_poll_ns, + env.execute_halt_poll_ns ? true : false); + bpf_program__set_autoload(skel->progs.kp_mark_page_dirty_in_slot, + env.execute_mark_page_dirty ? true : false); /* Load & verify BPF programs */ err = kvm_watcher_bpf__load(skel); @@ -480,7 +497,7 @@ int main(int argc, char **argv) fprintf(stderr, "Failed to attach BPF skeleton\n"); goto cleanup; } - + /* Set up ring buffer polling */ rb = ring_buffer__new(bpf_map__fd(skel->maps.rb), handle_event, NULL, NULL); if (!rb) { @@ -489,14 +506,19 @@ int main(int argc, char **argv) goto cleanup; } /* Process events */ - if(env.execute_vcpu_wakeup){ - printf("%-18s %-20s %-15s %-15s %-10s\n", "HLT_TIME(ns)", "DURATIONS_TIME(ns)","VCPUID/COMM","PID/TID","WAIT/POLL"); - }else if(env.execute_exit){ - printf("%-23s %-10s %-15s %-8s %-13s \n", "EXIT_REASON", "COMM","PID/TID","COUNT","DURATION(ns)"); - }else if(env.execute_halt_poll_ns){ - printf("%-18s %-15s %-15s %-10s %-11s %-10s\n", "TIME(ns)", "VCPUID/COMM","PID/TID","TYPE","OLD(ns)","NEW(ns)"); - }else if(env.execute_mark_page_dirty){ - printf("%-18s %-15s %-15s %-10s %-11s %-10s %-10s %-10s\n", "TIME(ns)", "VCPUID/COMM","PID/TID","GFN","REL_GFN","NPAGES","USERSPACE_ADDR","SLOT_ID"); + if (env.execute_vcpu_wakeup) { + printf("%-18s %-20s %-15s %-15s %-10s\n", "HLT_TIME(ns)", + "DURATIONS_TIME(ns)", "VCPUID/COMM", "PID/TID", "WAIT/POLL"); + } else if (env.execute_exit) { + printf("%-23s %-10s %-15s %-8s %-13s \n", "EXIT_REASON", "COMM", + "PID/TID", "COUNT", "DURATION(ns)"); + } else if (env.execute_halt_poll_ns) { + printf("%-18s %-15s %-15s %-10s %-11s %-10s\n", "TIME(ns)", + "VCPUID/COMM", "PID/TID", "TYPE", "OLD(ns)", "NEW(ns)"); + } else if (env.execute_mark_page_dirty) { + printf("%-18s %-15s %-15s %-10s %-11s %-10s %-10s %-10s\n", "TIME(ns)", + "VCPUID/COMM", "PID/TID", "GFN", "REL_GFN", "NPAGES", + "USERSPACE_ADDR", "SLOT_ID"); } while (!exiting) { err = ring_buffer__poll(rb, 10 /* timeout, ms */); @@ -510,15 +532,15 @@ int main(int argc, char **argv) break; } } - if(env.ShowStats){ + if (env.ShowStats) { printExitInfo(exitInfoBuffer); freeExitInfoList(exitInfoBuffer); - }else if(env.execute_mark_page_dirty){ + } else if (env.execute_mark_page_dirty) { err = save_count_dirtypagemap_to_file(skel->maps.count_dirty_map); if (err < 0) { printf("Save count dirty page map to file fail: %d\n", err); goto cleanup; - } + } } cleanup: ring_buffer__free(rb); From b62c10a208ddc7242babe99eb2b3cb2de6b2c9a8 Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Sun, 10 Dec 2023 15:35:00 +0800 Subject: [PATCH 04/21] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kvm_watcher/include/kvm_watcher.h | 10 ++++++ .../kvm_watcher/src/kvm_watcher.c | 31 +++++-------------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h index 0e08dd3cb..67c7c8eea 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h @@ -28,6 +28,16 @@ argp_usage(state); \ } while (0) +#define SET_OPTION_AND_CHECK_USAGE(option, value) \ + do { \ + if (option == 0) { \ + value = true; \ + option = 1; \ + } else { \ + PRINT_USAGE_ERR(); \ + } \ + } while (0) + #define RESERVE_RINGBUF_ENTRY(rb, e) \ do { \ typeof(e) _tmp = bpf_ringbuf_reserve(rb, sizeof(*e), 0); \ diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 46901d90e..849cc15e0 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -329,36 +329,19 @@ static const struct argp_option opts[] = { static error_t parse_arg(int key, char *arg, struct argp_state *state) { switch (key) { case 'w': - if (option_selected == 0) { - env.execute_vcpu_wakeup = true; - option_selected = 1; - } else { - PRINT_USAGE_ERR(); - } + SET_OPTION_AND_CHECK_USAGE(option_selected, + env.execute_vcpu_wakeup); break; case 'e': - if (option_selected == 0) { - env.execute_exit = true; - option_selected = 1; - } else { - PRINT_USAGE_ERR(); - } + SET_OPTION_AND_CHECK_USAGE(option_selected, env.execute_exit); break; case 'n': - if (option_selected == 0) { - env.execute_halt_poll_ns = true; - option_selected = 1; - } else { - PRINT_USAGE_ERR(); - } + SET_OPTION_AND_CHECK_USAGE(option_selected, + env.execute_halt_poll_ns); break; case 'd': - if (option_selected == 0) { - env.execute_mark_page_dirty = true; - option_selected = 1; - } else { - PRINT_USAGE_ERR(); - } + SET_OPTION_AND_CHECK_USAGE(option_selected, + env.execute_mark_page_dirty); break; case 's': if (env.execute_exit) { From 5da0c6bb049673cfb64ae2f6c66761f97723f1bb Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Fri, 15 Dec 2023 21:48:29 +0800 Subject: [PATCH 05/21] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kvmmmu=5Fpaga=5Ffault?= =?UTF-8?q?=E6=8C=87=E6=A0=87=E6=8F=90=E5=8F=96=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eBPF_Supermarket/kvm_watcher/Makefile | 2 +- .../kvm_watcher/include/kvm_exits.h | 12 +-- .../kvm_watcher/include/kvm_mmu.h | 93 +++++++++++++++++++ .../kvm_watcher/include/kvm_watcher.h | 30 +++++- .../kvm_watcher/src/kvm_watcher.bpf.c | 17 +++- .../kvm_watcher/src/kvm_watcher.c | 57 ++++++++++-- 6 files changed, 189 insertions(+), 22 deletions(-) create mode 100644 eBPF_Supermarket/kvm_watcher/include/kvm_mmu.h diff --git a/eBPF_Supermarket/kvm_watcher/Makefile b/eBPF_Supermarket/kvm_watcher/Makefile index 2878e2af7..086964505 100644 --- a/eBPF_Supermarket/kvm_watcher/Makefile +++ b/eBPF_Supermarket/kvm_watcher/Makefile @@ -12,7 +12,7 @@ all: $(APP) .PHONY: $(APP) $(APP): ifeq ($(wildcard ./include/vmlinux.h),) - bpftool btf dump file /sys/kernel/btf/vmlinux format c > ./include/vmlinux.h + bpftool btf dump file /sys/kernel/btf/kvm format c > ./include/vmlinux.h endif clang -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH) -I/usr/include/x86_64-linux-gnu -I. -c $@.bpf.c -o $@.bpf.o bpftool gen skeleton $@.bpf.o > $@.skel.h diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h b/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h index b19c0668a..8b98a4e0c 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_exits.h @@ -96,21 +96,15 @@ static int trace_kvm_entry(void *rb) { bpf_map_delete_elem(×, &tid); reason = reas->reason; count = reas->count; - e = bpf_ringbuf_reserve(rb, sizeof(*e), 0); - if (!e) { - return 0; - } + RESERVE_RINGBUF_ENTRY(rb, e); e->reason_number = reason; e->process.pid = pid; e->duration_ns = duration_ns; bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm)); e->process.tid = tid; e->total = ++total; - if (count) { - e->count = count; - } else { - e->count = 1; - } + e->count = count; + e->time = reas->time; bpf_ringbuf_submit(e, 0); return 0; } else { diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_mmu.h b/eBPF_Supermarket/kvm_watcher/include/kvm_mmu.h new file mode 100644 index 000000000..cc3292eb3 --- /dev/null +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_mmu.h @@ -0,0 +1,93 @@ +// Copyright 2023 The LMP Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://github.com/linuxkerneltravel/lmp/blob/develop/LICENSE +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// author: nanshuaibo811@163.com +// +// Kernel space BPF program used for monitoring data for KVM MMU. + +#ifndef __KVM_MMU_H +#define __KVM_MMU_H + +#include "kvm_watcher.h" +#include "vmlinux.h" +#include +#include +#include + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 8192); + __type(key, u64); + __type(value, u64); +} pf_delay SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 8192); + __type(key, u64); + __type(value, u32); +} pf_count SEC(".maps"); + +static int trace_page_fault(struct trace_event_raw_kvm_page_fault *ctx, + pid_t vm_pid) { + CHECK_PID(vm_pid) { + u64 ts = bpf_ktime_get_ns(); + u64 addr = ctx->fault_address; + bpf_map_update_elem(&pf_delay, &addr, &ts, BPF_ANY); + } + return 0; +} + +static int trace_direct_page_fault(struct kvm_vcpu *vcpu, + struct kvm_page_fault *fault, void *rb) { + u64 addr; + bpf_probe_read_kernel(&addr, sizeof(u64), &fault->addr); + u64 *ts; + ts = bpf_map_lookup_elem(&pf_delay, &addr); + if (ts) { + u32 *count; + u32 new_count = 1; + u32 error_code; + u64 hva, pfn; + bpf_probe_read_kernel(&error_code, sizeof(u32), &fault->error_code); + bpf_probe_read_kernel(&hva, sizeof(u64), &fault->hva); + bpf_probe_read_kernel(&pfn, sizeof(u64), &fault->pfn); + short memslot_id = BPF_CORE_READ(fault, slot, id); + u64 delay = bpf_ktime_get_ns() - *ts; + bpf_map_delete_elem(&pf_delay, &addr); + struct page_fault_event *e; + RESERVE_RINGBUF_ENTRY(rb, e); + count = bpf_map_lookup_elem(&pf_count, &addr); + if (count) { + (*count)++; + e->count = *count; + bpf_map_update_elem(&pf_count, &addr, count, BPF_ANY); + } else { + e->count = 1; + bpf_map_update_elem(&pf_count, &addr, &new_count, BPF_ANY); + } + e->delay = delay; + e->addr = addr; + e->error_code = error_code; + e->hva = hva; + e->pfn = pfn; + e->memslot_id = memslot_id; + e->process.pid = bpf_get_current_pid_tgid() >> 32; + bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm)); + e->time = *ts; + bpf_ringbuf_submit(e, 0); + } + return 0; +} +#endif /* __KVM_MMU_H */ \ No newline at end of file diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h index 67c7c8eea..0fdbee2f9 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h @@ -22,10 +22,18 @@ #define TASK_COMM_LEN 16 #define KVM_MEM_LOG_DIRTY_PAGES (1UL << 0) -#define PRINT_USAGE_ERR() \ - do { \ - fprintf(stderr, "Use either the -w, -p, -d, or -e option.\n"); \ - argp_usage(state); \ +#define PFERR_PRESENT_BIT 0 +#define PFERR_WRITE_BIT 1 +#define PFERR_USER_BIT 2 +#define PFERR_RSVD_BIT 3 +#define PFERR_FETCH_BIT 4 +#define PFERR_PK_BIT 5 +#define PFERR_SGX_BIT 15 + +#define PRINT_USAGE_ERR() \ + do { \ + fprintf(stderr, "Use either the -w, -p, -d,-f or -e option.\n"); \ + argp_usage(state); \ } while (0) #define SET_OPTION_AND_CHECK_USAGE(option, value) \ @@ -68,6 +76,7 @@ struct exit_event { unsigned long long duration_ns; int count; int total; + unsigned long long time; }; struct ExitReason { @@ -98,4 +107,17 @@ struct mark_page_dirty_in_slot_event { unsigned long long gfn; short slot_id; }; + +struct page_fault_event { + struct process process; + unsigned long long time; + unsigned long long delay; + unsigned long long error_code; + unsigned long long addr; + unsigned long long pfn; + unsigned long long hva; + unsigned count; + short memslot_id; +}; + #endif /* __KVM_WATCHER_H */ \ No newline at end of file diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c index 7601882c4..de5748498 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.bpf.c @@ -22,6 +22,7 @@ #include #include "../include/kvm_exits.h" #include "../include/kvm_vcpu.h" +#include "../include/kvm_mmu.h" #include "../include/kvm_watcher.h" char LICENSE[] SEC("license") = "Dual BSD/GPL"; @@ -56,9 +57,23 @@ int tp_entry(struct exit *ctx) { trace_kvm_entry(&rb); return 0; } + SEC("kprobe/mark_page_dirty_in_slot") int BPF_KPROBE(kp_mark_page_dirty_in_slot, struct kvm *kvm, const struct kvm_memory_slot *memslot, gfn_t gfn) { trace_mark_page_dirty_in_slot(kvm, memslot, gfn, &rb, vm_pid); return 0; -} \ No newline at end of file +} + +SEC("tp/kvm/kvm_page_fault") +int tp_page_fault(struct trace_event_raw_kvm_page_fault *ctx) { + trace_page_fault(ctx, vm_pid); + return 0; +} + +SEC("fexit/direct_page_fault") +int BPF_PROG(fexit_direct_page_fault, struct kvm_vcpu *vcpu, + struct kvm_page_fault *fault) { + trace_direct_page_fault(vcpu, fault, &rb); + return 0; +} diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 849cc15e0..6d46c26a1 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -104,7 +104,7 @@ const char *getExitReasonName(int number) { typedef struct { int exit_reason; - char info[256]; // 替换成适当的大小 + char info[256]; unsigned long long total_dur; unsigned long long avg_dur; } ExitInfo; @@ -295,6 +295,7 @@ static struct env { bool ShowStats; bool execute_halt_poll_ns; bool execute_mark_page_dirty; + bool execute_page_fault; int monitoring_time; pid_t vm_pid; } env = { @@ -303,6 +304,7 @@ static struct env { .ShowStats = false, .execute_halt_poll_ns = false, .execute_mark_page_dirty = false, + .execute_page_fault = false, .monitoring_time = 0, .vm_pid = -1, }; @@ -319,6 +321,8 @@ static const struct argp_option opts[] = { "Monitoring the variation in vCPU polling time."}, {"mark_page_dirty", 'd', NULL, 0, "Monitor virtual machine dirty page information."}, + {"kvmmmu_page_fault", 'f', NULL, 0, + "Monitoring the date of kvmmmu page fault."}, {"stat", 's', NULL, 0, "Display statistical data.(The -e option must be specified.)"}, {"vm_pid", 'p', "PID", 0, "Specify the virtual machine pid to monitor."}, @@ -343,6 +347,9 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) { SET_OPTION_AND_CHECK_USAGE(option_selected, env.execute_mark_page_dirty); break; + case 'f': + SET_OPTION_AND_CHECK_USAGE(option_selected, env.execute_page_fault); + break; case 's': if (env.execute_exit) { env.ShowStats = true; @@ -405,9 +412,10 @@ static int handle_event(void *ctx, void *data, size_t data_sz) { } else if (env.execute_exit) { char info_buffer[256]; const struct exit_event *e = data; - printf("%-2d/%-20s %-10s %-6u/%-8u %-8d %-13llu \n", e->reason_number, - getExitReasonName(e->reason_number), e->process.comm, - e->process.pid, e->process.tid, e->count, e->duration_ns); + printf("%-18llu %-2d/%-20s %-10s %-6u/%-8u %-8d %-13llu \n", e->time, + e->reason_number, getExitReasonName(e->reason_number), + e->process.comm, e->process.pid, e->process.tid, e->count, + e->duration_ns); if (env.ShowStats) { snprintf(info_buffer, sizeof(info_buffer), "%-10s %-6u/%-8u %-8d", e->process.comm, e->process.pid, e->process.tid, e->count); @@ -424,6 +432,34 @@ static int handle_event(void *ctx, void *data, size_t data_sz) { printf("%-18llu %-15s %-6d/%-8d %-10llx %-10llx %-10lu %-15lx %d \n", e->time, e->process.comm, e->process.pid, e->process.tid, e->gfn, e->rel_gfn, e->npages, e->userspace_addr, e->slot_id); + } else if (env.execute_page_fault) { + const struct page_fault_event *e = data; + printf( + "%-18llu %-10s %-10u %-12llx %-6u %-10llu %-20llx %-17llx %-10d ", + e->time, e->process.comm, e->process.pid, e->addr, e->count, + e->delay, e->hva, e->pfn, e->memslot_id); + if (e->error_code & (1ULL << PFERR_PRESENT_BIT)) { + printf("Present "); + } + if (e->error_code & (1ULL << PFERR_WRITE_BIT)) { + printf("Write "); + } + if (e->error_code & (1ULL << PFERR_USER_BIT)) { + printf("User "); + } + if (e->error_code & (1ULL << PFERR_RSVD_BIT)) { + printf("Reserved "); + } + if (e->error_code & (1ULL << PFERR_FETCH_BIT)) { + printf("Exec "); + } + if (e->error_code & (1ULL << PFERR_PK_BIT)) { + printf("Protection-Key "); + } + if (e->error_code & (1ULL << PFERR_SGX_BIT)) { + printf("SGX "); + } + printf("\n"); } return 0; } @@ -466,7 +502,10 @@ int main(int argc, char **argv) { env.execute_halt_poll_ns ? true : false); bpf_program__set_autoload(skel->progs.kp_mark_page_dirty_in_slot, env.execute_mark_page_dirty ? true : false); - + bpf_program__set_autoload(skel->progs.tp_page_fault, + env.execute_page_fault ? true : false); + bpf_program__set_autoload(skel->progs.fexit_direct_page_fault, + env.execute_page_fault ? true : false); /* Load & verify BPF programs */ err = kvm_watcher_bpf__load(skel); if (err) { @@ -493,8 +532,8 @@ int main(int argc, char **argv) { printf("%-18s %-20s %-15s %-15s %-10s\n", "HLT_TIME(ns)", "DURATIONS_TIME(ns)", "VCPUID/COMM", "PID/TID", "WAIT/POLL"); } else if (env.execute_exit) { - printf("%-23s %-10s %-15s %-8s %-13s \n", "EXIT_REASON", "COMM", - "PID/TID", "COUNT", "DURATION(ns)"); + printf("%-18s %-23s %-10s %-15s %-8s %-13s \n", "TIME", "EXIT_REASON", + "COMM", "PID/TID", "COUNT", "DURATION(ns)"); } else if (env.execute_halt_poll_ns) { printf("%-18s %-15s %-15s %-10s %-11s %-10s\n", "TIME(ns)", "VCPUID/COMM", "PID/TID", "TYPE", "OLD(ns)", "NEW(ns)"); @@ -502,6 +541,10 @@ int main(int argc, char **argv) { printf("%-18s %-15s %-15s %-10s %-11s %-10s %-10s %-10s\n", "TIME(ns)", "VCPUID/COMM", "PID/TID", "GFN", "REL_GFN", "NPAGES", "USERSPACE_ADDR", "SLOT_ID"); + } else if (env.execute_page_fault) { + printf("%-18s %-10s %-10s %-12s %-6s %-10s %-20s %-17s %-10s %-10s\n", + "TIMESTAMP", "COMM", "PID", "ADDRESS", "COUNT", "DELAY", "HVA", + "PFN", "MEM_SLOTID", "ERROR_TYPE"); } while (!exiting) { err = ring_buffer__poll(rb, 10 /* timeout, ms */); From 2b8b19b13646997d1cd2d4dadb3561b86c461bd6 Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Fri, 15 Dec 2023 21:52:22 +0800 Subject: [PATCH 06/21] =?UTF-8?q?=E4=BF=AE=E6=94=B9action?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/kvm_watcher.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/kvm_watcher.yml b/.github/workflows/kvm_watcher.yml index 5f7e70076..ecf7d4f93 100644 --- a/.github/workflows/kvm_watcher.yml +++ b/.github/workflows/kvm_watcher.yml @@ -26,6 +26,8 @@ jobs: sudo apt install libbpf-dev sudo apt install linux-tools-$(uname -r) sudo apt install linux-cloud-tools-$(uname -r) + - name: Load KVM mod + run: | sudo modprobe kvm - name: Run kvm_watcher run: | @@ -35,5 +37,6 @@ jobs: sudo ./kvm_watcher -e -t 5 -s sudo ./kvm_watcher -n -t 5 sudo ./kvm_watcher -d -t 5 + sudo ./kvm_watcher -f -t 5 make clean From 22b87ed939147bbf0cdbdf2211f9ac2ca8fd5fd6 Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Fri, 15 Dec 2023 22:03:04 +0800 Subject: [PATCH 07/21] =?UTF-8?q?=E4=BF=AE=E6=94=B9action?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/kvm_watcher.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kvm_watcher.yml b/.github/workflows/kvm_watcher.yml index ecf7d4f93..52aabd5b1 100644 --- a/.github/workflows/kvm_watcher.yml +++ b/.github/workflows/kvm_watcher.yml @@ -28,7 +28,7 @@ jobs: sudo apt install linux-cloud-tools-$(uname -r) - name: Load KVM mod run: | - sudo modprobe kvm + sudo modprobe kvm && sudo modprobe kvm-intel - name: Run kvm_watcher run: | cd eBPF_Supermarket/kvm_watcher/ From f31c02e977a66a88f10fdbec11e423f6dfd259d1 Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Fri, 15 Dec 2023 22:10:52 +0800 Subject: [PATCH 08/21] =?UTF-8?q?=E4=BF=AE=E6=94=B9action?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/kvm_watcher.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/kvm_watcher.yml b/.github/workflows/kvm_watcher.yml index 52aabd5b1..93d468ea9 100644 --- a/.github/workflows/kvm_watcher.yml +++ b/.github/workflows/kvm_watcher.yml @@ -26,17 +26,24 @@ jobs: sudo apt install libbpf-dev sudo apt install linux-tools-$(uname -r) sudo apt install linux-cloud-tools-$(uname -r) - - name: Load KVM mod + sudo apt-get update && sudo apt-get install -y qemu-kvm + - name: Download Cirros image + run: | + wget http://download.cirros-cloud.net/0.5.1/cirros-0.5.1-x86_64-disk.img + - name: Load KVM module run: | sudo modprobe kvm && sudo modprobe kvm-intel + - name: Run QEMU to start VM + run: | + qemu-system-x86_64 -enable-kvm -cpu host -m 2048 -drive file=cirros-0.5.1-x86_64-disk.img,format=qcow2 -boot c - name: Run kvm_watcher run: | cd eBPF_Supermarket/kvm_watcher/ make - sudo ./kvm_watcher -w -t 5 - sudo ./kvm_watcher -e -t 5 -s - sudo ./kvm_watcher -n -t 5 - sudo ./kvm_watcher -d -t 5 - sudo ./kvm_watcher -f -t 5 + sudo ./kvm_watcher -w -t 10 + sudo ./kvm_watcher -e -t 10 -s + sudo ./kvm_watcher -n -t 10 + sudo ./kvm_watcher -d -t 10 + sudo ./kvm_watcher -f -t 10 make clean From a762bb4dc6d641d7534313d03f3dd9bfb4dd81a5 Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Fri, 15 Dec 2023 22:14:13 +0800 Subject: [PATCH 09/21] update action --- .github/workflows/kvm_watcher.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kvm_watcher.yml b/.github/workflows/kvm_watcher.yml index 93d468ea9..6bac13f70 100644 --- a/.github/workflows/kvm_watcher.yml +++ b/.github/workflows/kvm_watcher.yml @@ -35,7 +35,7 @@ jobs: sudo modprobe kvm && sudo modprobe kvm-intel - name: Run QEMU to start VM run: | - qemu-system-x86_64 -enable-kvm -cpu host -m 2048 -drive file=cirros-0.5.1-x86_64-disk.img,format=qcow2 -boot c + sudo qemu-system-x86_64 -enable-kvm -cpu host -m 2048 -drive file=cirros-0.5.1-x86_64-disk.img,format=qcow2 -boot c - name: Run kvm_watcher run: | cd eBPF_Supermarket/kvm_watcher/ From 5ca4539bb374795e735f03e32c059f2bca2c8e7b Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Fri, 15 Dec 2023 22:16:06 +0800 Subject: [PATCH 10/21] update action --- .github/workflows/kvm_watcher.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kvm_watcher.yml b/.github/workflows/kvm_watcher.yml index 6bac13f70..a656c1f8c 100644 --- a/.github/workflows/kvm_watcher.yml +++ b/.github/workflows/kvm_watcher.yml @@ -35,7 +35,7 @@ jobs: sudo modprobe kvm && sudo modprobe kvm-intel - name: Run QEMU to start VM run: | - sudo qemu-system-x86_64 -enable-kvm -cpu host -m 2048 -drive file=cirros-0.5.1-x86_64-disk.img,format=qcow2 -boot c + sudo qemu-system-x86_64 -enable-kvm -cpu host -m 2048 -drive file=cirros-0.5.1-x86_64-disk.img,format=qcow2 -boot c -nographic - name: Run kvm_watcher run: | cd eBPF_Supermarket/kvm_watcher/ From 1ebbb7e68669528747613521ef36640fce290f83 Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Fri, 15 Dec 2023 22:21:32 +0800 Subject: [PATCH 11/21] update action --- .github/workflows/kvm_watcher.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kvm_watcher.yml b/.github/workflows/kvm_watcher.yml index a656c1f8c..e2e4b84fc 100644 --- a/.github/workflows/kvm_watcher.yml +++ b/.github/workflows/kvm_watcher.yml @@ -35,7 +35,7 @@ jobs: sudo modprobe kvm && sudo modprobe kvm-intel - name: Run QEMU to start VM run: | - sudo qemu-system-x86_64 -enable-kvm -cpu host -m 2048 -drive file=cirros-0.5.1-x86_64-disk.img,format=qcow2 -boot c -nographic + sudo qemu-system-x86_64 -enable-kvm -cpu host -m 2048 -drive file=cirros-0.5.1-x86_64-disk.img,format=qcow2 -boot c -nographic & - name: Run kvm_watcher run: | cd eBPF_Supermarket/kvm_watcher/ From b70a27bf5bd1b19230c81bafa4275748a0b67043 Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Fri, 15 Dec 2023 22:38:19 +0800 Subject: [PATCH 12/21] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A0=BC=E5=BC=8F?= =?UTF-8?q?=E5=8C=96=E8=BE=93=E5=87=BA=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kvm_watcher/src/kvm_watcher.c | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 6d46c26a1..eecddf612 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -175,7 +175,7 @@ void printExitInfo(Node *head) { printf( "\n-----------------------------------------------------------------" "----------\n"); - printf("%-23s %-10s %-15s %-8s %-13s \n", "EXIT_REASON", "COMM", "PID/TID", + printf("%-23s %-18s %-15s %-8s %-13s \n", "EXIT_REASON", "COMM", "PID/TID", "COUNT", "AVG_DURATION(ns)"); while (current != NULL) { printf("%-2d/%-20s %-33s %-13llu \n", current->data.exit_reason, @@ -406,36 +406,36 @@ static void sig_handler(int sig) { static int handle_event(void *ctx, void *data, size_t data_sz) { if (env.execute_vcpu_wakeup) { const struct vcpu_wakeup_event *e = data; - printf("%-18llu %-20llu %-15s %-6d/%-8d %-10s\n", e->hlt_time, + printf("%-18llu %-20llu %-18s %-6d/%-8d %-10s\n", e->hlt_time, e->dur_hlt_ns, e->process.comm, e->process.pid, e->process.tid, e->waited ? "wait" : "poll"); } else if (env.execute_exit) { char info_buffer[256]; const struct exit_event *e = data; - printf("%-18llu %-2d/%-20s %-10s %-6u/%-8u %-8d %-13llu \n", e->time, + printf("%-18llu %-2d/%-20s %-18s %-6u/%-8u %-8d %-13llu \n", e->time, e->reason_number, getExitReasonName(e->reason_number), e->process.comm, e->process.pid, e->process.tid, e->count, e->duration_ns); if (env.ShowStats) { - snprintf(info_buffer, sizeof(info_buffer), "%-10s %-6u/%-8u %-8d", + snprintf(info_buffer, sizeof(info_buffer), "%-18s %-6u/%-8u %-8d", e->process.comm, e->process.pid, e->process.tid, e->count); addExitInfo(&exitInfoBuffer, e->reason_number, info_buffer, e->duration_ns, e->count); } } else if (env.execute_halt_poll_ns) { const struct halt_poll_ns_event *e = data; - printf("%-18llu %-15s %-6d/%-8d %-10s %-7d --> %d \n", e->time, + printf("%-18llu %-18s %-6d/%-8d %-10s %-7d --> %d \n", e->time, e->process.comm, e->process.pid, e->process.tid, e->grow ? "grow" : "shrink", e->old, e->new); } else if (env.execute_mark_page_dirty) { const struct mark_page_dirty_in_slot_event *e = data; - printf("%-18llu %-15s %-6d/%-8d %-10llx %-10llx %-10lu %-15lx %d \n", + printf("%-18llu %-18s %-6d/%-8d %-10llx %-10llx %-10lu %-15lx %d \n", e->time, e->process.comm, e->process.pid, e->process.tid, e->gfn, e->rel_gfn, e->npages, e->userspace_addr, e->slot_id); } else if (env.execute_page_fault) { const struct page_fault_event *e = data; printf( - "%-18llu %-10s %-10u %-12llx %-6u %-10llu %-20llx %-17llx %-10d ", + "%-18llu %-18s %-10u %-12llx %-6u %-10llu %-20llx %-17llx %-10d ", e->time, e->process.comm, e->process.pid, e->addr, e->count, e->delay, e->hva, e->pfn, e->memslot_id); if (e->error_code & (1ULL << PFERR_PRESENT_BIT)) { @@ -529,20 +529,20 @@ int main(int argc, char **argv) { } /* Process events */ if (env.execute_vcpu_wakeup) { - printf("%-18s %-20s %-15s %-15s %-10s\n", "HLT_TIME(ns)", - "DURATIONS_TIME(ns)", "VCPUID/COMM", "PID/TID", "WAIT/POLL"); + printf("%-18s %-20s %-18s %-15s %-10s\n", "HLT_TIME(ns)", + "DURATIONS_TIME(ns)", "COMM", "PID/TID", "WAIT/POLL"); } else if (env.execute_exit) { - printf("%-18s %-23s %-10s %-15s %-8s %-13s \n", "TIME", "EXIT_REASON", + printf("%-18s %-23s %-18s %-15s %-8s %-13s \n", "TIME", "EXIT_REASON", "COMM", "PID/TID", "COUNT", "DURATION(ns)"); } else if (env.execute_halt_poll_ns) { - printf("%-18s %-15s %-15s %-10s %-11s %-10s\n", "TIME(ns)", - "VCPUID/COMM", "PID/TID", "TYPE", "OLD(ns)", "NEW(ns)"); + printf("%-18s %-18s %-15s %-10s %-11s %-10s\n", "TIME(ns)", + "COMM", "PID/TID", "TYPE", "OLD(ns)", "NEW(ns)"); } else if (env.execute_mark_page_dirty) { - printf("%-18s %-15s %-15s %-10s %-11s %-10s %-10s %-10s\n", "TIME(ns)", - "VCPUID/COMM", "PID/TID", "GFN", "REL_GFN", "NPAGES", + printf("%-18s %-18s %-15s %-10s %-11s %-10s %-10s %-10s\n", "TIME(ns)", + "COMM", "PID/TID", "GFN", "REL_GFN", "NPAGES", "USERSPACE_ADDR", "SLOT_ID"); } else if (env.execute_page_fault) { - printf("%-18s %-10s %-10s %-12s %-6s %-10s %-20s %-17s %-10s %-10s\n", + printf("%-18s %-18s %-10s %-12s %-6s %-10s %-20s %-17s %-10s %-10s\n", "TIMESTAMP", "COMM", "PID", "ADDRESS", "COUNT", "DELAY", "HVA", "PFN", "MEM_SLOTID", "ERROR_TYPE"); } From ebee0700fa779f6b284ad79d37cd958dd845ae18 Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Fri, 15 Dec 2023 22:43:07 +0800 Subject: [PATCH 13/21] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A0=BC=E5=BC=8F?= =?UTF-8?q?=E5=8C=96=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index eecddf612..790a39669 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -175,10 +175,10 @@ void printExitInfo(Node *head) { printf( "\n-----------------------------------------------------------------" "----------\n"); - printf("%-23s %-18s %-15s %-8s %-13s \n", "EXIT_REASON", "COMM", "PID/TID", + printf("%-25s %-18s %-15s %-8s %-13s \n", "EXIT_REASON", "COMM", "PID/TID", "COUNT", "AVG_DURATION(ns)"); while (current != NULL) { - printf("%-2d/%-20s %-33s %-13llu \n", current->data.exit_reason, + printf("%-2d/%-22s %-33s %-13llu \n", current->data.exit_reason, getExitReasonName(current->data.exit_reason), current->data.info, current->data.avg_dur); current = current->next; @@ -412,7 +412,7 @@ static int handle_event(void *ctx, void *data, size_t data_sz) { } else if (env.execute_exit) { char info_buffer[256]; const struct exit_event *e = data; - printf("%-18llu %-2d/%-20s %-18s %-6u/%-8u %-8d %-13llu \n", e->time, + printf("%-18llu %-2d/%-22s %-18s %-6u/%-8u %-8d %-13llu \n", e->time, e->reason_number, getExitReasonName(e->reason_number), e->process.comm, e->process.pid, e->process.tid, e->count, e->duration_ns); @@ -532,7 +532,7 @@ int main(int argc, char **argv) { printf("%-18s %-20s %-18s %-15s %-10s\n", "HLT_TIME(ns)", "DURATIONS_TIME(ns)", "COMM", "PID/TID", "WAIT/POLL"); } else if (env.execute_exit) { - printf("%-18s %-23s %-18s %-15s %-8s %-13s \n", "TIME", "EXIT_REASON", + printf("%-18s %-25s %-18s %-15s %-8s %-13s \n", "TIME", "EXIT_REASON", "COMM", "PID/TID", "COUNT", "DURATION(ns)"); } else if (env.execute_halt_poll_ns) { printf("%-18s %-18s %-15s %-10s %-11s %-10s\n", "TIME(ns)", From 80d1941461e9308735b656d72919273dea79d952 Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Sat, 16 Dec 2023 12:51:45 +0800 Subject: [PATCH 14/21] =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E5=8C=96=E8=BE=93?= =?UTF-8?q?=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 790a39669..9b9db7181 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -175,7 +175,7 @@ void printExitInfo(Node *head) { printf( "\n-----------------------------------------------------------------" "----------\n"); - printf("%-25s %-18s %-15s %-8s %-13s \n", "EXIT_REASON", "COMM", "PID/TID", + printf("%-20s %-18s %-15s %-8s %-13s \n", "EXIT_REASON", "COMM", "PID/TID", "COUNT", "AVG_DURATION(ns)"); while (current != NULL) { printf("%-2d/%-22s %-33s %-13llu \n", current->data.exit_reason, @@ -412,7 +412,7 @@ static int handle_event(void *ctx, void *data, size_t data_sz) { } else if (env.execute_exit) { char info_buffer[256]; const struct exit_event *e = data; - printf("%-18llu %-2d/%-22s %-18s %-6u/%-8u %-8d %-13llu \n", e->time, + printf("%-18llu %-4d/%-15s %-18s %-6u/%-8u %-8d %-13llu \n", e->time, e->reason_number, getExitReasonName(e->reason_number), e->process.comm, e->process.pid, e->process.tid, e->count, e->duration_ns); @@ -532,7 +532,7 @@ int main(int argc, char **argv) { printf("%-18s %-20s %-18s %-15s %-10s\n", "HLT_TIME(ns)", "DURATIONS_TIME(ns)", "COMM", "PID/TID", "WAIT/POLL"); } else if (env.execute_exit) { - printf("%-18s %-25s %-18s %-15s %-8s %-13s \n", "TIME", "EXIT_REASON", + printf("%-18s %-20s %-18s %-15s %-8s %-13s \n", "TIME", "EXIT_REASON", "COMM", "PID/TID", "COUNT", "DURATION(ns)"); } else if (env.execute_halt_poll_ns) { printf("%-18s %-18s %-15s %-10s %-11s %-10s\n", "TIME(ns)", From a76e3cfb621500c5c25b55cc6f0f9f3c85184313 Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Sat, 16 Dec 2023 12:56:26 +0800 Subject: [PATCH 15/21] =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E5=8C=96=E8=BE=93?= =?UTF-8?q?=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 9b9db7181..bff9a5ef3 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -178,7 +178,7 @@ void printExitInfo(Node *head) { printf("%-20s %-18s %-15s %-8s %-13s \n", "EXIT_REASON", "COMM", "PID/TID", "COUNT", "AVG_DURATION(ns)"); while (current != NULL) { - printf("%-2d/%-22s %-33s %-13llu \n", current->data.exit_reason, + printf("%-4d/%-15s %-33s %-13llu \n", current->data.exit_reason, getExitReasonName(current->data.exit_reason), current->data.info, current->data.avg_dur); current = current->next; From 990732c7f66d1035f9b7c7931a5e083bd42c9ff3 Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Sat, 16 Dec 2023 13:09:02 +0800 Subject: [PATCH 16/21] revert --- .../kvm_watcher/src/kvm_watcher.c | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index bff9a5ef3..40fd1c814 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -406,7 +406,7 @@ static void sig_handler(int sig) { static int handle_event(void *ctx, void *data, size_t data_sz) { if (env.execute_vcpu_wakeup) { const struct vcpu_wakeup_event *e = data; - printf("%-18llu %-20llu %-18s %-6d/%-8d %-10s\n", e->hlt_time, + printf("%-18llu %-20llu %-15s %-6d/%-8d %-10s\n", e->hlt_time, e->dur_hlt_ns, e->process.comm, e->process.pid, e->process.tid, e->waited ? "wait" : "poll"); } else if (env.execute_exit) { @@ -417,25 +417,25 @@ static int handle_event(void *ctx, void *data, size_t data_sz) { e->process.comm, e->process.pid, e->process.tid, e->count, e->duration_ns); if (env.ShowStats) { - snprintf(info_buffer, sizeof(info_buffer), "%-18s %-6u/%-8u %-8d", + snprintf(info_buffer, sizeof(info_buffer), "%-10s %-6u/%-8u %-8d", e->process.comm, e->process.pid, e->process.tid, e->count); addExitInfo(&exitInfoBuffer, e->reason_number, info_buffer, e->duration_ns, e->count); } } else if (env.execute_halt_poll_ns) { const struct halt_poll_ns_event *e = data; - printf("%-18llu %-18s %-6d/%-8d %-10s %-7d --> %d \n", e->time, + printf("%-18llu %-15s %-6d/%-8d %-10s %-7d --> %d \n", e->time, e->process.comm, e->process.pid, e->process.tid, e->grow ? "grow" : "shrink", e->old, e->new); } else if (env.execute_mark_page_dirty) { const struct mark_page_dirty_in_slot_event *e = data; - printf("%-18llu %-18s %-6d/%-8d %-10llx %-10llx %-10lu %-15lx %d \n", + printf("%-18llu %-15s %-6d/%-8d %-10llx %-10llx %-10lu %-15lx %d \n", e->time, e->process.comm, e->process.pid, e->process.tid, e->gfn, e->rel_gfn, e->npages, e->userspace_addr, e->slot_id); } else if (env.execute_page_fault) { const struct page_fault_event *e = data; printf( - "%-18llu %-18s %-10u %-12llx %-6u %-10llu %-20llx %-17llx %-10d ", + "%-18llu %-10s %-10u %-12llx %-6u %-10llu %-20llx %-17llx %-10d ", e->time, e->process.comm, e->process.pid, e->addr, e->count, e->delay, e->hva, e->pfn, e->memslot_id); if (e->error_code & (1ULL << PFERR_PRESENT_BIT)) { @@ -529,20 +529,20 @@ int main(int argc, char **argv) { } /* Process events */ if (env.execute_vcpu_wakeup) { - printf("%-18s %-20s %-18s %-15s %-10s\n", "HLT_TIME(ns)", - "DURATIONS_TIME(ns)", "COMM", "PID/TID", "WAIT/POLL"); + printf("%-18s %-20s %-15s %-15s %-10s\n", "HLT_TIME(ns)", + "DURATIONS_TIME(ns)", "VCPUID/COMM", "PID/TID", "WAIT/POLL"); } else if (env.execute_exit) { printf("%-18s %-20s %-18s %-15s %-8s %-13s \n", "TIME", "EXIT_REASON", "COMM", "PID/TID", "COUNT", "DURATION(ns)"); } else if (env.execute_halt_poll_ns) { - printf("%-18s %-18s %-15s %-10s %-11s %-10s\n", "TIME(ns)", - "COMM", "PID/TID", "TYPE", "OLD(ns)", "NEW(ns)"); + printf("%-18s %-15s %-15s %-10s %-11s %-10s\n", "TIME(ns)", + "VCPUID/COMM", "PID/TID", "TYPE", "OLD(ns)", "NEW(ns)"); } else if (env.execute_mark_page_dirty) { - printf("%-18s %-18s %-15s %-10s %-11s %-10s %-10s %-10s\n", "TIME(ns)", - "COMM", "PID/TID", "GFN", "REL_GFN", "NPAGES", + printf("%-18s %-15s %-15s %-10s %-11s %-10s %-10s %-10s\n", "TIME(ns)", + "VCPUID/COMM", "PID/TID", "GFN", "REL_GFN", "NPAGES", "USERSPACE_ADDR", "SLOT_ID"); } else if (env.execute_page_fault) { - printf("%-18s %-18s %-10s %-12s %-6s %-10s %-20s %-17s %-10s %-10s\n", + printf("%-18s %-10s %-10s %-12s %-6s %-10s %-20s %-17s %-10s %-10s\n", "TIMESTAMP", "COMM", "PID", "ADDRESS", "COUNT", "DELAY", "HVA", "PFN", "MEM_SLOTID", "ERROR_TYPE"); } From c682e64f415cb83af9878cd066a025fffee7692e Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Sat, 16 Dec 2023 13:19:21 +0800 Subject: [PATCH 17/21] update kvm_watcher.c --- eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 40fd1c814..a71daafb0 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -175,10 +175,10 @@ void printExitInfo(Node *head) { printf( "\n-----------------------------------------------------------------" "----------\n"); - printf("%-20s %-18s %-15s %-8s %-13s \n", "EXIT_REASON", "COMM", "PID/TID", + printf("%-21s %-18s %-8s %-8s %-13s \n", "EXIT_REASON", "COMM", "PID", "COUNT", "AVG_DURATION(ns)"); while (current != NULL) { - printf("%-4d/%-15s %-33s %-13llu \n", current->data.exit_reason, + printf("%-2d/%-18s %-33s %-13llu \n", current->data.exit_reason, getExitReasonName(current->data.exit_reason), current->data.info, current->data.avg_dur); current = current->next; @@ -412,13 +412,13 @@ static int handle_event(void *ctx, void *data, size_t data_sz) { } else if (env.execute_exit) { char info_buffer[256]; const struct exit_event *e = data; - printf("%-18llu %-4d/%-15s %-18s %-6u/%-8u %-8d %-13llu \n", e->time, + printf("%-18llu %-2d/%-18s %-18s %-6u/%-8u %-8d %-13llu \n", e->time, e->reason_number, getExitReasonName(e->reason_number), e->process.comm, e->process.pid, e->process.tid, e->count, e->duration_ns); if (env.ShowStats) { - snprintf(info_buffer, sizeof(info_buffer), "%-10s %-6u/%-8u %-8d", - e->process.comm, e->process.pid, e->process.tid, e->count); + snprintf(info_buffer, sizeof(info_buffer), "%-18s %-8u %-8d", + e->process.comm, e->process.pid, e->count); addExitInfo(&exitInfoBuffer, e->reason_number, info_buffer, e->duration_ns, e->count); } @@ -532,7 +532,7 @@ int main(int argc, char **argv) { printf("%-18s %-20s %-15s %-15s %-10s\n", "HLT_TIME(ns)", "DURATIONS_TIME(ns)", "VCPUID/COMM", "PID/TID", "WAIT/POLL"); } else if (env.execute_exit) { - printf("%-18s %-20s %-18s %-15s %-8s %-13s \n", "TIME", "EXIT_REASON", + printf("%-18s %-21s %-18s %-15s %-8s %-13s \n", "TIME", "EXIT_REASON", "COMM", "PID/TID", "COUNT", "DURATION(ns)"); } else if (env.execute_halt_poll_ns) { printf("%-18s %-15s %-15s %-10s %-11s %-10s\n", "TIME(ns)", From d9ea4ec1090b2daa85822dde84175a267e21f4e9 Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Sat, 16 Dec 2023 16:06:55 +0800 Subject: [PATCH 18/21] update kvm_watcher.c --- eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index a71daafb0..490a223bb 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -530,16 +530,16 @@ int main(int argc, char **argv) { /* Process events */ if (env.execute_vcpu_wakeup) { printf("%-18s %-20s %-15s %-15s %-10s\n", "HLT_TIME(ns)", - "DURATIONS_TIME(ns)", "VCPUID/COMM", "PID/TID", "WAIT/POLL"); + "DURATIONS_TIME(ns)", "COMM", "PID/TID", "WAIT/POLL"); } else if (env.execute_exit) { printf("%-18s %-21s %-18s %-15s %-8s %-13s \n", "TIME", "EXIT_REASON", "COMM", "PID/TID", "COUNT", "DURATION(ns)"); } else if (env.execute_halt_poll_ns) { printf("%-18s %-15s %-15s %-10s %-11s %-10s\n", "TIME(ns)", - "VCPUID/COMM", "PID/TID", "TYPE", "OLD(ns)", "NEW(ns)"); + "COMM", "PID/TID", "TYPE", "OLD(ns)", "NEW(ns)"); } else if (env.execute_mark_page_dirty) { printf("%-18s %-15s %-15s %-10s %-11s %-10s %-10s %-10s\n", "TIME(ns)", - "VCPUID/COMM", "PID/TID", "GFN", "REL_GFN", "NPAGES", + "COMM", "PID/TID", "GFN", "REL_GFN", "NPAGES", "USERSPACE_ADDR", "SLOT_ID"); } else if (env.execute_page_fault) { printf("%-18s %-10s %-10s %-12s %-6s %-10s %-20s %-17s %-10s %-10s\n", From 013caa8d8680d7db557f5cd2560cb43f6b0c1184 Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Sat, 16 Dec 2023 16:11:17 +0800 Subject: [PATCH 19/21] update kvm_watcher.c --- eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 490a223bb..2b6b975f1 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -435,7 +435,7 @@ static int handle_event(void *ctx, void *data, size_t data_sz) { } else if (env.execute_page_fault) { const struct page_fault_event *e = data; printf( - "%-18llu %-10s %-10u %-12llx %-6u %-10llu %-20llx %-17llx %-10d ", + "%-18llu %-15s %-10u %-12llx %-6u %-10llu %-20llx %-17llx %-10d ", e->time, e->process.comm, e->process.pid, e->addr, e->count, e->delay, e->hva, e->pfn, e->memslot_id); if (e->error_code & (1ULL << PFERR_PRESENT_BIT)) { @@ -542,7 +542,7 @@ int main(int argc, char **argv) { "COMM", "PID/TID", "GFN", "REL_GFN", "NPAGES", "USERSPACE_ADDR", "SLOT_ID"); } else if (env.execute_page_fault) { - printf("%-18s %-10s %-10s %-12s %-6s %-10s %-20s %-17s %-10s %-10s\n", + printf("%-18s %-15s %-10s %-12s %-6s %-10s %-20s %-17s %-10s %-10s\n", "TIMESTAMP", "COMM", "PID", "ADDRESS", "COUNT", "DELAY", "HVA", "PFN", "MEM_SLOTID", "ERROR_TYPE"); } From 5bcbab655f60ca507d8d09617b09e0d9bcf9fadf Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Mon, 18 Dec 2023 17:11:03 +0800 Subject: [PATCH 20/21] =?UTF-8?q?=E5=AE=8C=E5=96=84=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h b/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h index f5246e08a..2a161238b 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h @@ -89,8 +89,10 @@ static int trace_mark_page_dirty_in_slot(struct kvm *kvm, gfn_t gfn, void *rb, pid_t vm_pid) { CHECK_PID(vm_pid) { u32 flags; + struct kvm_memory_slot *slot; + bpf_probe_read_kernel(&slot, sizeof(memslot), &memslot); bpf_probe_read_kernel(&flags, sizeof(memslot->flags), &memslot->flags); - if (flags & KVM_MEM_LOG_DIRTY_PAGES) { // 检查memslot是否启用了脏页追踪 + if (slot && (flags & KVM_MEM_LOG_DIRTY_PAGES)) { // 检查memslot是否启用了脏页追踪 gfn_t gfnum = gfn; u32 *count = bpf_map_lookup_elem(&count_dirty_map, &gfnum); if (count) { From 53ff52ec40088e314deb57fba92c0cb9f78f21ed Mon Sep 17 00:00:00 2001 From: nanshuaibo Date: Thu, 21 Dec 2023 21:41:00 +0800 Subject: [PATCH 21/21] =?UTF-8?q?=E5=AE=8C=E5=96=84=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h | 1 + eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h | 1 + eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c | 8 ++++---- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h b/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h index 2a161238b..8a7167f7c 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_vcpu.h @@ -78,6 +78,7 @@ static int trace_kvm_halt_poll_ns(struct halt_poll_ns *ctx, void *rb, e->grow = ctx->grow; e->old = ctx->old; e->new = ctx->new; + e->vcpu_id = ctx->vcpu_id; bpf_get_current_comm(&e->process.comm, sizeof(e->process.comm)); bpf_ringbuf_submit(e, 0); } diff --git a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h index 0fdbee2f9..2fc23538b 100644 --- a/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h +++ b/eBPF_Supermarket/kvm_watcher/include/kvm_watcher.h @@ -96,6 +96,7 @@ struct halt_poll_ns_event { unsigned int new; unsigned int old; unsigned long long time; + unsigned vcpu_id; }; struct mark_page_dirty_in_slot_event { diff --git a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c index 2b6b975f1..c96a51c53 100644 --- a/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c +++ b/eBPF_Supermarket/kvm_watcher/src/kvm_watcher.c @@ -424,9 +424,9 @@ static int handle_event(void *ctx, void *data, size_t data_sz) { } } else if (env.execute_halt_poll_ns) { const struct halt_poll_ns_event *e = data; - printf("%-18llu %-15s %-6d/%-8d %-10s %-7d --> %d \n", e->time, + printf("%-18llu %-15s %-6d/%-8d %-10s %-7d %-7d --> %d \n", e->time, e->process.comm, e->process.pid, e->process.tid, - e->grow ? "grow" : "shrink", e->old, e->new); + e->grow ? "grow" : "shrink", e->vcpu_id, e->old, e->new); } else if (env.execute_mark_page_dirty) { const struct mark_page_dirty_in_slot_event *e = data; printf("%-18llu %-15s %-6d/%-8d %-10llx %-10llx %-10lu %-15lx %d \n", @@ -535,8 +535,8 @@ int main(int argc, char **argv) { printf("%-18s %-21s %-18s %-15s %-8s %-13s \n", "TIME", "EXIT_REASON", "COMM", "PID/TID", "COUNT", "DURATION(ns)"); } else if (env.execute_halt_poll_ns) { - printf("%-18s %-15s %-15s %-10s %-11s %-10s\n", "TIME(ns)", - "COMM", "PID/TID", "TYPE", "OLD(ns)", "NEW(ns)"); + printf("%-18s %-15s %-15s %-10s %-7s %-11s %-10s\n", "TIME(ns)", + "COMM", "PID/TID", "TYPE", "VCPU_ID", "OLD(ns)", "NEW(ns)"); } else if (env.execute_mark_page_dirty) { printf("%-18s %-15s %-15s %-10s %-11s %-10s %-10s %-10s\n", "TIME(ns)", "COMM", "PID/TID", "GFN", "REL_GFN", "NPAGES",