From cc2b8ed1369592fb84609e920f99a5659a6445f7 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 23 Nov 2018 18:29:02 +0100 Subject: [PATCH 01/51] arm64/bpf: use movn/movk/movk sequence to generate kernel addresses On arm64, all executable code is guaranteed to reside in the vmalloc space (or the module space), and so jump targets will only use 48 bits at most, and the remaining bits are guaranteed to be 0x1. This means we can generate an immediate jump address using a sequence of one MOVN (move wide negated) and two MOVK instructions, where the first one sets the lower 16 bits but also sets all top bits to 0x1. Signed-off-by: Ard Biesheuvel Acked-by: Will Deacon Acked-by: Daniel Borkmann Signed-off-by: Daniel Borkmann --- arch/arm64/net/bpf_jit_comp.c | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 89198017e8e68..b872859240234 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -134,10 +134,9 @@ static inline void emit_a64_mov_i64(const int reg, const u64 val, } /* - * This is an unoptimized 64 immediate emission used for BPF to BPF call - * addresses. It will always do a full 64 bit decomposition as otherwise - * more complexity in the last extra pass is required since we previously - * reserved 4 instructions for the address. + * Kernel addresses in the vmalloc space use at most 48 bits, and the + * remaining bits are guaranteed to be 0x1. So we can compose the address + * with a fixed length movn/movk/movk sequence. */ static inline void emit_addr_mov_i64(const int reg, const u64 val, struct jit_ctx *ctx) @@ -145,8 +144,8 @@ static inline void emit_addr_mov_i64(const int reg, const u64 val, u64 tmp = val; int shift = 0; - emit(A64_MOVZ(1, reg, tmp & 0xffff, shift), ctx); - for (;shift < 48;) { + emit(A64_MOVN(1, reg, ~tmp & 0xffff, shift), ctx); + while (shift < 32) { tmp >>= 16; shift += 16; emit(A64_MOVK(1, reg, tmp & 0xffff, shift), ctx); @@ -634,11 +633,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, &func_addr, &func_addr_fixed); if (ret < 0) return ret; - if (func_addr_fixed) - /* We can use optimized emission here. */ - emit_a64_mov_i64(tmp, func_addr, ctx); - else - emit_addr_mov_i64(tmp, func_addr, ctx); + emit_addr_mov_i64(tmp, func_addr, ctx); emit(A64_BLR(tmp), ctx); emit(A64_MOV(1, r0, A64_R(0)), ctx); break; From 327e5dab6dd8559ba28739ec8873350d7b5bb080 Mon Sep 17 00:00:00 2001 From: Quentin Monnet Date: Fri, 30 Nov 2018 16:25:44 +0000 Subject: [PATCH 02/51] tools: bpftool: use "/proc/self/" i.o. crafting links with getpid() The getpid() function is called in a couple of places in bpftool to craft links of the shape "/proc//...". Instead, it is possible to use the "/proc/self/" shortcut, which makes things a bit easier, in particular in jit_disasm.c. Do the replacement, and remove the includes of from the relevant files, now we do not use getpid() anymore. Signed-off-by: Quentin Monnet Reviewed-by: Jakub Kicinski Signed-off-by: Alexei Starovoitov --- tools/bpf/bpftool/common.c | 5 ++--- tools/bpf/bpftool/jit_disasm.c | 11 +---------- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c index 4e217d57118e3..4349b6683ca85 100644 --- a/tools/bpf/bpftool/common.c +++ b/tools/bpf/bpftool/common.c @@ -48,7 +48,6 @@ #include #include #include -#include #include #include @@ -276,7 +275,7 @@ int get_fd_type(int fd) char buf[512]; ssize_t n; - snprintf(path, sizeof(path), "/proc/%d/fd/%d", getpid(), fd); + snprintf(path, sizeof(path), "/proc/self/fd/%d", fd); n = readlink(path, buf, sizeof(buf)); if (n < 0) { @@ -304,7 +303,7 @@ char *get_fdinfo(int fd, const char *key) ssize_t n; FILE *fdi; - snprintf(path, sizeof(path), "/proc/%d/fdinfo/%d", getpid(), fd); + snprintf(path, sizeof(path), "/proc/self/fdinfo/%d", fd); fdi = fopen(path, "r"); if (!fdi) { diff --git a/tools/bpf/bpftool/jit_disasm.c b/tools/bpf/bpftool/jit_disasm.c index b2ed5ee1af5fe..545a92471c336 100644 --- a/tools/bpf/bpftool/jit_disasm.c +++ b/tools/bpf/bpftool/jit_disasm.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include @@ -28,20 +27,12 @@ static void get_exec_path(char *tpath, size_t size) { + const char *path = "/proc/self/exe"; ssize_t len; - char *path; - - snprintf(tpath, size, "/proc/%d/exe", (int) getpid()); - tpath[size - 1] = 0; - - path = strdup(tpath); - assert(path); len = readlink(path, tpath, size - 1); assert(len > 0); tpath[len] = 0; - - free(path); } static int oper_count; From cad4977344b35ea116ec5fefe91a76b1dfa113f5 Mon Sep 17 00:00:00 2001 From: Quentin Monnet Date: Fri, 30 Nov 2018 16:25:45 +0000 Subject: [PATCH 03/51] tools: bpftool: fix bash completion for bpftool prog (attach|detach) Fix bash completion for "bpftool prog (attach|detach) PROG TYPE MAP" so that the list of indices proposed for MAP are map indices, and not PROG indices. Also use variables for map and prog reference types ("id", "pinned", and "tag" for programs). Fixes: b7d3826c2ed6 ("bpf: bpftool, add support for attaching programs to maps") Signed-off-by: Quentin Monnet Reviewed-by: Jakub Kicinski Signed-off-by: Alexei Starovoitov --- tools/bpf/bpftool/bash-completion/bpftool | 73 +++++++++++++++-------- 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/tools/bpf/bpftool/bash-completion/bpftool b/tools/bpf/bpftool/bash-completion/bpftool index 45c2db257d2bf..b7e6c4f25ad1e 100644 --- a/tools/bpf/bpftool/bash-completion/bpftool +++ b/tools/bpf/bpftool/bash-completion/bpftool @@ -243,16 +243,20 @@ _bpftool() # Completion depends on object and command in use case $object in prog) - if [[ $command != "load" && $command != "loadall" ]]; then - case $prev in - id) - _bpftool_get_prog_ids - return 0 - ;; - esac - fi + # Complete id, only for subcommands that use prog (but no map) ids + case $command in + show|list|dump|pin) + case $prev in + id) + _bpftool_get_prog_ids + return 0 + ;; + esac + ;; + esac local PROG_TYPE='id pinned tag' + local MAP_TYPE='id pinned' case $command in show|list) [[ $prev != "$command" ]] && return 0 @@ -293,22 +297,43 @@ _bpftool() return 0 ;; attach|detach) - if [[ ${#words[@]} == 7 ]]; then - COMPREPLY=( $( compgen -W "id pinned" -- "$cur" ) ) - return 0 - fi - - if [[ ${#words[@]} == 6 ]]; then - COMPREPLY=( $( compgen -W "msg_verdict skb_verdict \ - skb_parse flow_dissector" -- "$cur" ) ) - return 0 - fi - - if [[ $prev == "$command" ]]; then - COMPREPLY=( $( compgen -W "id pinned" -- "$cur" ) ) - return 0 - fi - return 0 + case $cword in + 3) + COMPREPLY=( $( compgen -W "$PROG_TYPE" -- "$cur" ) ) + return 0 + ;; + 4) + case $prev in + id) + _bpftool_get_prog_ids + ;; + pinned) + _filedir + ;; + esac + return 0 + ;; + 5) + COMPREPLY=( $( compgen -W 'msg_verdict skb_verdict \ + skb_parse flow_dissector' -- "$cur" ) ) + return 0 + ;; + 6) + COMPREPLY=( $( compgen -W "$MAP_TYPE" -- "$cur" ) ) + return 0 + ;; + 7) + case $prev in + id) + _bpftool_get_map_ids + ;; + pinned) + _filedir + ;; + esac + return 0 + ;; + esac ;; load|loadall) local obj From 4f0fc6f571d29837bc09aa8c51591f330f17bbf0 Mon Sep 17 00:00:00 2001 From: Quentin Monnet Date: Fri, 30 Nov 2018 16:25:46 +0000 Subject: [PATCH 04/51] tools: bpftool: fix bash completion for new map types (queue and stack) Commit 197c2dac74e4 ("bpf: Add BPF_MAP_TYPE_QUEUE and BPF_MAP_TYPE_STACK to bpftool-map") added support for queue and stack eBPF map types in bpftool map handling. Let's update the bash completion accordingly. Fixes: 197c2dac74e4 ("bpf: Add BPF_MAP_TYPE_QUEUE and BPF_MAP_TYPE_STACK to bpftool-map") Signed-off-by: Quentin Monnet Signed-off-by: Alexei Starovoitov --- tools/bpf/bpftool/bash-completion/bpftool | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bpf/bpftool/bash-completion/bpftool b/tools/bpf/bpftool/bash-completion/bpftool index b7e6c4f25ad1e..9a60080f085f1 100644 --- a/tools/bpf/bpftool/bash-completion/bpftool +++ b/tools/bpf/bpftool/bash-completion/bpftool @@ -436,7 +436,7 @@ _bpftool() lru_percpu_hash lpm_trie array_of_maps \ hash_of_maps devmap sockmap cpumap xskmap \ sockhash cgroup_storage reuseport_sockarray \ - percpu_cgroup_storage' -- \ + percpu_cgroup_storage queue stack' -- \ "$cur" ) ) return 0 ;; From 73f0b9db53836a68c40691fad1467ed44a3123e8 Mon Sep 17 00:00:00 2001 From: Quentin Monnet Date: Fri, 30 Nov 2018 16:25:47 +0000 Subject: [PATCH 05/51] tools: bpftool: mark offloaded programs more explicitly in plain output In bpftool (plain) output for "bpftool prog show" or "bpftool map show", an offloaded BPF object is simply denoted with "dev ifname", which is not really explicit. Change it with something that clearly shows the program is offloaded. While at it also add an additional space, as done between other information fields. Example output, before: # bpftool prog show 1337: xdp tag a04f5eef06a7f555 dev foo loaded_at 2018-10-19T16:40:36+0100 uid 0 xlated 16B not jited memlock 4096B After: # bpftool prog show 1337: xdp tag a04f5eef06a7f555 offloaded_to foo loaded_at 2018-10-19T16:40:36+0100 uid 0 xlated 16B not jited memlock 4096B Suggested-by: Daniel Borkmann Signed-off-by: Quentin Monnet Acked-by: Jakub Kicinski Signed-off-by: Alexei Starovoitov --- tools/bpf/bpftool/common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c index 4349b6683ca85..172d3761d9abe 100644 --- a/tools/bpf/bpftool/common.c +++ b/tools/bpf/bpftool/common.c @@ -604,7 +604,7 @@ void print_dev_plain(__u32 ifindex, __u64 ns_dev, __u64 ns_inode) if (!ifindex) return; - printf(" dev "); + printf(" offloaded_to "); if (ifindex_to_name_ns(ifindex, ns_dev, ns_inode, name)) printf("%s", name); else From 99a44bef587082fbbaddf6aaa1e823ebbb629339 Mon Sep 17 00:00:00 2001 From: Quentin Monnet Date: Fri, 30 Nov 2018 16:25:48 +0000 Subject: [PATCH 06/51] tools: bpftool: add owner_prog_type and owner_jited to bpftool output For prog array maps, the type of the owner program, and the JIT-ed state of that program, are available from the file descriptor information under /proc. Add them to "bpftool map show" output. Example output: # bpftool map show 158225: prog_array name jmp_table flags 0x0 key 4B value 4B max_entries 8 memlock 4096B owner_prog_type flow_dissector owner jited # bpftool --json --pretty map show [{ "id": 1337, "type": "prog_array", "name": "jmp_table", "flags": 0, "bytes_key": 4, "bytes_value": 4, "max_entries": 8, "bytes_memlock": 4096, "owner_prog_type": "flow_dissector", "owner_jited": true } ] As we move the table used for associating names to program types, complete it with the missing types (lwt_seg6local and sk_reuseport). Also add missing types to the help message for "bpftool prog" (sk_reuseport and flow_dissector). Suggested-by: Daniel Borkmann Signed-off-by: Quentin Monnet Acked-by: Jakub Kicinski Signed-off-by: Alexei Starovoitov --- tools/bpf/bpftool/main.h | 26 +++++++++++++++++++++ tools/bpf/bpftool/map.c | 50 ++++++++++++++++++++++++++++++++++++++-- tools/bpf/bpftool/prog.c | 25 +------------------- 3 files changed, 75 insertions(+), 26 deletions(-) diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h index 7431669fae0af..2761981669c82 100644 --- a/tools/bpf/bpftool/main.h +++ b/tools/bpf/bpftool/main.h @@ -78,6 +78,32 @@ #define HELP_SPEC_MAP \ "MAP := { id MAP_ID | pinned FILE }" +static const char * const prog_type_name[] = { + [BPF_PROG_TYPE_UNSPEC] = "unspec", + [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter", + [BPF_PROG_TYPE_KPROBE] = "kprobe", + [BPF_PROG_TYPE_SCHED_CLS] = "sched_cls", + [BPF_PROG_TYPE_SCHED_ACT] = "sched_act", + [BPF_PROG_TYPE_TRACEPOINT] = "tracepoint", + [BPF_PROG_TYPE_XDP] = "xdp", + [BPF_PROG_TYPE_PERF_EVENT] = "perf_event", + [BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb", + [BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock", + [BPF_PROG_TYPE_LWT_IN] = "lwt_in", + [BPF_PROG_TYPE_LWT_OUT] = "lwt_out", + [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit", + [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops", + [BPF_PROG_TYPE_SK_SKB] = "sk_skb", + [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device", + [BPF_PROG_TYPE_SK_MSG] = "sk_msg", + [BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", + [BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr", + [BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local", + [BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2", + [BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport", + [BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector", +}; + enum bpf_obj_type { BPF_OBJ_UNKNOWN, BPF_OBJ_PROG, diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c index 3850f8d657030..8469ea6cf1c82 100644 --- a/tools/bpf/bpftool/map.c +++ b/tools/bpf/bpftool/map.c @@ -487,7 +487,6 @@ static int show_map_close_json(int fd, struct bpf_map_info *info) char *memlock; memlock = get_fdinfo(fd, "memlock"); - close(fd); jsonw_start_object(json_wtr); @@ -514,6 +513,30 @@ static int show_map_close_json(int fd, struct bpf_map_info *info) jsonw_int_field(json_wtr, "bytes_memlock", atoi(memlock)); free(memlock); + if (info->type == BPF_MAP_TYPE_PROG_ARRAY) { + char *owner_prog_type = get_fdinfo(fd, "owner_prog_type"); + char *owner_jited = get_fdinfo(fd, "owner_jited"); + + if (owner_prog_type) { + unsigned int prog_type = atoi(owner_prog_type); + + if (prog_type < ARRAY_SIZE(prog_type_name)) + jsonw_string_field(json_wtr, "owner_prog_type", + prog_type_name[prog_type]); + else + jsonw_uint_field(json_wtr, "owner_prog_type", + prog_type); + } + if (atoi(owner_jited)) + jsonw_bool_field(json_wtr, "owner_jited", true); + else + jsonw_bool_field(json_wtr, "owner_jited", false); + + free(owner_prog_type); + free(owner_jited); + } + close(fd); + if (!hash_empty(map_table.table)) { struct pinned_obj *obj; @@ -536,7 +559,6 @@ static int show_map_close_plain(int fd, struct bpf_map_info *info) char *memlock; memlock = get_fdinfo(fd, "memlock"); - close(fd); printf("%u: ", info->id); if (info->type < ARRAY_SIZE(map_type_name)) @@ -557,6 +579,30 @@ static int show_map_close_plain(int fd, struct bpf_map_info *info) printf(" memlock %sB", memlock); free(memlock); + if (info->type == BPF_MAP_TYPE_PROG_ARRAY) { + char *owner_prog_type = get_fdinfo(fd, "owner_prog_type"); + char *owner_jited = get_fdinfo(fd, "owner_jited"); + + printf("\n\t"); + if (owner_prog_type) { + unsigned int prog_type = atoi(owner_prog_type); + + if (prog_type < ARRAY_SIZE(prog_type_name)) + printf("owner_prog_type %s ", + prog_type_name[prog_type]); + else + printf("owner_prog_type %d ", prog_type); + } + if (atoi(owner_jited)) + printf("owner jited"); + else + printf("owner not jited"); + + free(owner_prog_type); + free(owner_jited); + } + close(fd); + printf("\n"); if (!hash_empty(map_table.table)) { struct pinned_obj *obj; diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 4e04699a2350f..56db61c5a91fe 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -54,30 +54,6 @@ #include "main.h" #include "xlated_dumper.h" -static const char * const prog_type_name[] = { - [BPF_PROG_TYPE_UNSPEC] = "unspec", - [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter", - [BPF_PROG_TYPE_KPROBE] = "kprobe", - [BPF_PROG_TYPE_SCHED_CLS] = "sched_cls", - [BPF_PROG_TYPE_SCHED_ACT] = "sched_act", - [BPF_PROG_TYPE_TRACEPOINT] = "tracepoint", - [BPF_PROG_TYPE_XDP] = "xdp", - [BPF_PROG_TYPE_PERF_EVENT] = "perf_event", - [BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb", - [BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock", - [BPF_PROG_TYPE_LWT_IN] = "lwt_in", - [BPF_PROG_TYPE_LWT_OUT] = "lwt_out", - [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit", - [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops", - [BPF_PROG_TYPE_SK_SKB] = "sk_skb", - [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device", - [BPF_PROG_TYPE_SK_MSG] = "sk_msg", - [BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", - [BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr", - [BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2", - [BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector", -}; - static const char * const attach_type_strings[] = { [BPF_SK_SKB_STREAM_PARSER] = "stream_parser", [BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict", @@ -1172,6 +1148,7 @@ static int do_help(int argc, char **argv) " tracepoint | raw_tracepoint | xdp | perf_event | cgroup/skb |\n" " cgroup/sock | cgroup/dev | lwt_in | lwt_out | lwt_xmit |\n" " lwt_seg6local | sockops | sk_skb | sk_msg | lirc_mode2 |\n" + " sk_reuseport | flow_dissector |\n" " cgroup/bind4 | cgroup/bind6 | cgroup/post_bind4 |\n" " cgroup/post_bind6 | cgroup/connect4 | cgroup/connect6 |\n" " cgroup/sendmsg4 | cgroup/sendmsg6 }\n" From 88945f460603ad8909b556c67a9229bb23188d41 Mon Sep 17 00:00:00 2001 From: David Miller Date: Wed, 28 Nov 2018 22:33:53 -0800 Subject: [PATCH 07/51] bpf: Fix verifier log string check for bad alignment. The message got changed a lot time ago. This was responsible for 36 test case failures on sparc64. Fixes: f1174f77b50c ("bpf/verifier: rework value tracking") Signed-off-by: David S. Miller Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/test_verifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index 17021d2b6bfed..5eace1f606fb4 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -14287,7 +14287,7 @@ static void do_test_single(struct bpf_test *test, bool unpriv, reject_from_alignment = fd_prog < 0 && (test->flags & F_NEEDS_EFFICIENT_UNALIGNED_ACCESS) && - strstr(bpf_vlog, "Unknown alignment."); + strstr(bpf_vlog, "misaligned"); #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS if (reject_from_alignment) { printf("FAIL\nFailed due to alignment despite having efficient unaligned access: '%s'!\n", From e9ee9efc0d176512cdce9d27ff8549d7ffa2bfcd Mon Sep 17 00:00:00 2001 From: David Miller Date: Fri, 30 Nov 2018 21:08:14 -0800 Subject: [PATCH 08/51] bpf: Add BPF_F_ANY_ALIGNMENT. Often we want to write tests cases that check things like bad context offset accesses. And one way to do this is to use an odd offset on, for example, a 32-bit load. This unfortunately triggers the alignment checks first on platforms that do not set CONFIG_EFFICIENT_UNALIGNED_ACCESS. So the test case see the alignment failure rather than what it was testing for. It is often not completely possible to respect the original intention of the test, or even test the same exact thing, while solving the alignment issue. Another option could have been to check the alignment after the context and other validations are performed by the verifier, but that is a non-trivial change to the verifier. Signed-off-by: David S. Miller Signed-off-by: Alexei Starovoitov --- include/uapi/linux/bpf.h | 14 ++++++++++++++ kernel/bpf/syscall.c | 7 ++++++- kernel/bpf/verifier.c | 2 ++ tools/include/uapi/linux/bpf.h | 14 ++++++++++++++ tools/lib/bpf/bpf.c | 8 ++++---- tools/lib/bpf/bpf.h | 2 +- tools/testing/selftests/bpf/test_align.c | 4 ++-- tools/testing/selftests/bpf/test_verifier.c | 3 ++- 8 files changed, 45 insertions(+), 9 deletions(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 597afdbc1ab93..8050caea74955 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -232,6 +232,20 @@ enum bpf_attach_type { */ #define BPF_F_STRICT_ALIGNMENT (1U << 0) +/* If BPF_F_ANY_ALIGNMENT is used in BPF_PROF_LOAD command, the + * verifier will allow any alignment whatsoever. On platforms + * with strict alignment requirements for loads ands stores (such + * as sparc and mips) the verifier validates that all loads and + * stores provably follow this requirement. This flag turns that + * checking and enforcement off. + * + * It is mostly used for testing when we want to validate the + * context and memory access aspects of the verifier, but because + * of an unaligned access the alignment check would trigger before + * the one we are interested in. + */ +#define BPF_F_ANY_ALIGNMENT (1U << 1) + /* when bpf_ldimm64->src_reg == BPF_PSEUDO_MAP_FD, bpf_ldimm64->imm == fd */ #define BPF_PSEUDO_MAP_FD 1 diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 85cbeec06e50d..f9554d9a14e18 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1452,9 +1452,14 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) if (CHECK_ATTR(BPF_PROG_LOAD)) return -EINVAL; - if (attr->prog_flags & ~BPF_F_STRICT_ALIGNMENT) + if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | BPF_F_ANY_ALIGNMENT)) return -EINVAL; + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && + (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && + !capable(CAP_SYS_ADMIN)) + return -EPERM; + /* copy eBPF program license from user space */ if (strncpy_from_user(license, u64_to_user_ptr(attr->license), sizeof(license) - 1) < 0) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 9584438fa2cc1..71988337ac14e 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6505,6 +6505,8 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) env->strict_alignment = true; + if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) + env->strict_alignment = false; ret = replace_map_fd_with_map_ptr(env); if (ret < 0) diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 597afdbc1ab93..8050caea74955 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -232,6 +232,20 @@ enum bpf_attach_type { */ #define BPF_F_STRICT_ALIGNMENT (1U << 0) +/* If BPF_F_ANY_ALIGNMENT is used in BPF_PROF_LOAD command, the + * verifier will allow any alignment whatsoever. On platforms + * with strict alignment requirements for loads ands stores (such + * as sparc and mips) the verifier validates that all loads and + * stores provably follow this requirement. This flag turns that + * checking and enforcement off. + * + * It is mostly used for testing when we want to validate the + * context and memory access aspects of the verifier, but because + * of an unaligned access the alignment check would trigger before + * the one we are interested in. + */ +#define BPF_F_ANY_ALIGNMENT (1U << 1) + /* when bpf_ldimm64->src_reg == BPF_PSEUDO_MAP_FD, bpf_ldimm64->imm == fd */ #define BPF_PSEUDO_MAP_FD 1 diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index ce18221945900..c19226cccf39f 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -279,9 +279,9 @@ int bpf_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, } int bpf_verify_program(enum bpf_prog_type type, const struct bpf_insn *insns, - size_t insns_cnt, int strict_alignment, - const char *license, __u32 kern_version, - char *log_buf, size_t log_buf_sz, int log_level) + size_t insns_cnt, __u32 prog_flags, const char *license, + __u32 kern_version, char *log_buf, size_t log_buf_sz, + int log_level) { union bpf_attr attr; @@ -295,7 +295,7 @@ int bpf_verify_program(enum bpf_prog_type type, const struct bpf_insn *insns, attr.log_level = log_level; log_buf[0] = 0; attr.kern_version = kern_version; - attr.prog_flags = strict_alignment ? BPF_F_STRICT_ALIGNMENT : 0; + attr.prog_flags = prog_flags; return sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); } diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 09e8bbe111d4c..60392b70587ca 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -98,7 +98,7 @@ LIBBPF_API int bpf_load_program(enum bpf_prog_type type, char *log_buf, size_t log_buf_sz); LIBBPF_API int bpf_verify_program(enum bpf_prog_type type, const struct bpf_insn *insns, - size_t insns_cnt, int strict_alignment, + size_t insns_cnt, __u32 prog_flags, const char *license, __u32 kern_version, char *log_buf, size_t log_buf_sz, int log_level); diff --git a/tools/testing/selftests/bpf/test_align.c b/tools/testing/selftests/bpf/test_align.c index 5f377ec53f2f8..3c789d03b629d 100644 --- a/tools/testing/selftests/bpf/test_align.c +++ b/tools/testing/selftests/bpf/test_align.c @@ -620,8 +620,8 @@ static int do_test_single(struct bpf_align_test *test) prog_len = probe_filter_length(prog); fd_prog = bpf_verify_program(prog_type ? : BPF_PROG_TYPE_SOCKET_FILTER, - prog, prog_len, 1, "GPL", 0, - bpf_vlog, sizeof(bpf_vlog), 2); + prog, prog_len, BPF_F_STRICT_ALIGNMENT, + "GPL", 0, bpf_vlog, sizeof(bpf_vlog), 2); if (fd_prog < 0 && test->result != REJECT) { printf("Failed to load program.\n"); printf("%s", bpf_vlog); diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index 5eace1f606fb4..78e779c358693 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -14275,7 +14275,8 @@ static void do_test_single(struct bpf_test *test, bool unpriv, prog_len = probe_filter_length(prog); fd_prog = bpf_verify_program(prog_type, prog, prog_len, - test->flags & F_LOAD_WITH_STRICT_ALIGNMENT, + test->flags & F_LOAD_WITH_STRICT_ALIGNMENT ? + BPF_F_STRICT_ALIGNMENT : 0, "GPL", 0, bpf_vlog, sizeof(bpf_vlog), 1); expected_ret = unpriv && test->result_unpriv != UNDEF ? From c7665702d3208b77b8e00f0699b6b88241b04360 Mon Sep 17 00:00:00 2001 From: David Miller Date: Fri, 30 Nov 2018 21:08:26 -0800 Subject: [PATCH 09/51] bpf: Adjust F_NEEDS_EFFICIENT_UNALIGNED_ACCESS handling in test_verifier.c Make it set the flag argument to bpf_verify_program() which will relax the alignment restrictions. Now all such test cases will go properly through the verifier even on inefficient unaligned access architectures. On inefficient unaligned access architectures do not try to run such programs, instead mark the test case as passing but annotate the result similarly to how it is done now in the presence of this flag. So, we get complete full coverage for all REJECT test cases, and at least verifier level coverage for ACCEPT test cases. Signed-off-by: David S. Miller Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/test_verifier.c | 42 ++++++++++++--------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index 78e779c358693..1d1775faaf14d 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -14257,13 +14257,14 @@ static int set_admin(bool admin) static void do_test_single(struct bpf_test *test, bool unpriv, int *passes, int *errors) { - int fd_prog, expected_ret, reject_from_alignment; + int fd_prog, expected_ret, alignment_prevented_execution; int prog_len, prog_type = test->prog_type; struct bpf_insn *prog = test->insns; int map_fds[MAX_NR_MAPS]; const char *expected_err; uint32_t expected_val; uint32_t retval; + __u32 pflags; int i, err; for (i = 0; i < MAX_NR_MAPS; i++) @@ -14274,9 +14275,12 @@ static void do_test_single(struct bpf_test *test, bool unpriv, do_test_fixup(test, prog_type, prog, map_fds); prog_len = probe_filter_length(prog); - fd_prog = bpf_verify_program(prog_type, prog, prog_len, - test->flags & F_LOAD_WITH_STRICT_ALIGNMENT ? - BPF_F_STRICT_ALIGNMENT : 0, + pflags = 0; + if (test->flags & F_LOAD_WITH_STRICT_ALIGNMENT) + pflags |= BPF_F_STRICT_ALIGNMENT; + if (test->flags & F_NEEDS_EFFICIENT_UNALIGNED_ACCESS) + pflags |= BPF_F_ANY_ALIGNMENT; + fd_prog = bpf_verify_program(prog_type, prog, prog_len, pflags, "GPL", 0, bpf_vlog, sizeof(bpf_vlog), 1); expected_ret = unpriv && test->result_unpriv != UNDEF ? @@ -14286,28 +14290,27 @@ static void do_test_single(struct bpf_test *test, bool unpriv, expected_val = unpriv && test->retval_unpriv ? test->retval_unpriv : test->retval; - reject_from_alignment = fd_prog < 0 && - (test->flags & F_NEEDS_EFFICIENT_UNALIGNED_ACCESS) && - strstr(bpf_vlog, "misaligned"); -#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS - if (reject_from_alignment) { - printf("FAIL\nFailed due to alignment despite having efficient unaligned access: '%s'!\n", - strerror(errno)); - goto fail_log; - } -#endif + alignment_prevented_execution = 0; + if (expected_ret == ACCEPT) { - if (fd_prog < 0 && !reject_from_alignment) { + if (fd_prog < 0) { printf("FAIL\nFailed to load prog '%s'!\n", strerror(errno)); goto fail_log; } +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS + if (fd_prog >= 0 && + (test->flags & F_NEEDS_EFFICIENT_UNALIGNED_ACCESS)) { + alignment_prevented_execution = 1; + goto test_ok; + } +#endif } else { if (fd_prog >= 0) { printf("FAIL\nUnexpected success to load!\n"); goto fail_log; } - if (!strstr(bpf_vlog, expected_err) && !reject_from_alignment) { + if (!strstr(bpf_vlog, expected_err)) { printf("FAIL\nUnexpected error message!\n\tEXP: %s\n\tRES: %s\n", expected_err, bpf_vlog); goto fail_log; @@ -14335,9 +14338,12 @@ static void do_test_single(struct bpf_test *test, bool unpriv, goto fail_log; } } +#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS +test_ok: +#endif (*passes)++; - printf("OK%s\n", reject_from_alignment ? - " (NOTE: reject due to unknown alignment)" : ""); + printf("OK%s\n", alignment_prevented_execution ? + " (NOTE: not executed due to unknown alignment)" : ""); close_fds: close(fd_prog); for (i = 0; i < MAX_NR_MAPS; i++) From 2acc5fd5b8c25df0de7f3c8b8e385f5c6f8202ec Mon Sep 17 00:00:00 2001 From: David Miller Date: Fri, 30 Nov 2018 21:08:39 -0800 Subject: [PATCH 10/51] bpf: Make more use of 'any' alignment in test_verifier.c Use F_NEEDS_EFFICIENT_UNALIGNED_ACCESS in more tests where the expected result is REJECT. Signed-off-by: David S. Miller Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/test_verifier.c | 44 +++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index 1d1775faaf14d..6bfd3242ea433 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -1823,6 +1823,7 @@ static struct bpf_test tests[] = { .errstr = "invalid bpf_context access", .result = REJECT, .prog_type = BPF_PROG_TYPE_SK_MSG, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "direct packet read for SK_MSG", @@ -2215,6 +2216,8 @@ static struct bpf_test tests[] = { }, .errstr = "invalid bpf_context access", .result = REJECT, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "check cb access: half, wrong type", @@ -3281,6 +3284,7 @@ static struct bpf_test tests[] = { .result = REJECT, .errstr = "R0 invalid mem access 'inv'", .prog_type = BPF_PROG_TYPE_SCHED_CLS, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "raw_stack: skb_load_bytes, spilled regs corruption 2", @@ -3311,6 +3315,7 @@ static struct bpf_test tests[] = { .result = REJECT, .errstr = "R3 invalid mem access 'inv'", .prog_type = BPF_PROG_TYPE_SCHED_CLS, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "raw_stack: skb_load_bytes, spilled regs + data", @@ -3810,6 +3815,7 @@ static struct bpf_test tests[] = { .errstr = "R2 invalid mem access 'inv'", .result = REJECT, .prog_type = BPF_PROG_TYPE_SCHED_CLS, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "direct packet access: test16 (arith on data_end)", @@ -3993,6 +3999,7 @@ static struct bpf_test tests[] = { .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = REJECT, .errstr = "invalid access to packet, off=0 size=8, R5(id=1,off=0,r=0)", + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "direct packet access: test24 (x += pkt_ptr, 5)", @@ -5149,6 +5156,7 @@ static struct bpf_test tests[] = { .result = REJECT, .errstr = "invalid access to map value, value_size=64 off=-2 size=4", .prog_type = BPF_PROG_TYPE_CGROUP_SKB, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "invalid cgroup storage access 5", @@ -5265,6 +5273,7 @@ static struct bpf_test tests[] = { .result = REJECT, .errstr = "invalid access to map value, value_size=64 off=-2 size=4", .prog_type = BPF_PROG_TYPE_CGROUP_SKB, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "invalid per-cpu cgroup storage access 5", @@ -7206,6 +7215,7 @@ static struct bpf_test tests[] = { .errstr = "invalid mem access 'inv'", .result = REJECT, .result_unpriv = REJECT, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "map element value illegal alu op, 5", @@ -7228,6 +7238,7 @@ static struct bpf_test tests[] = { .fixup_map_hash_48b = { 3 }, .errstr = "R0 invalid mem access 'inv'", .result = REJECT, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "map element value is preserved across register spilling", @@ -9720,6 +9731,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_end > pkt_data', good access", @@ -9758,6 +9770,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_end > pkt_data', bad access 2", @@ -9776,6 +9789,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_data' < pkt_end, good access", @@ -9814,6 +9828,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_data' < pkt_end, bad access 2", @@ -9832,6 +9847,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_end < pkt_data', good access", @@ -9886,6 +9902,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_data' >= pkt_end, good access", @@ -9922,6 +9939,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_data' >= pkt_end, bad access 2", @@ -9997,6 +10015,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_data' <= pkt_end, good access", @@ -10053,6 +10072,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_end <= pkt_data', good access", @@ -10089,6 +10109,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_end <= pkt_data', bad access 2", @@ -10162,6 +10183,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_data > pkt_meta', good access", @@ -10200,6 +10222,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_data > pkt_meta', bad access 2", @@ -10218,6 +10241,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_meta' < pkt_data, good access", @@ -10256,6 +10280,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_meta' < pkt_data, bad access 2", @@ -10274,6 +10299,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_data < pkt_meta', good access", @@ -10328,6 +10354,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_meta' >= pkt_data, good access", @@ -10364,6 +10391,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_meta' >= pkt_data, bad access 2", @@ -10439,6 +10467,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_meta' <= pkt_data, good access", @@ -10495,6 +10524,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_data <= pkt_meta', good access", @@ -10531,6 +10561,7 @@ static struct bpf_test tests[] = { .errstr = "R1 offset is outside of the packet", .result = REJECT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_data <= pkt_meta', bad access 2", @@ -10635,6 +10666,7 @@ static struct bpf_test tests[] = { }, .result = REJECT, .errstr = "dereference of modified ctx ptr", + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "check deducing bounds from const, 8", @@ -10648,6 +10680,7 @@ static struct bpf_test tests[] = { }, .result = REJECT, .errstr = "dereference of modified ctx ptr", + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "check deducing bounds from const, 9", @@ -11122,6 +11155,7 @@ static struct bpf_test tests[] = { .result = REJECT, .errstr = "R6 invalid mem access 'inv'", .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "calls: two calls with args", @@ -11987,6 +12021,7 @@ static struct bpf_test tests[] = { .fixup_map_hash_8b = { 12, 22 }, .result = REJECT, .errstr = "invalid access to map value, value_size=8 off=2 size=8", + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "calls: two calls that receive map_value via arg=ptr_stack_of_caller. test2", @@ -12130,6 +12165,7 @@ static struct bpf_test tests[] = { .fixup_map_hash_8b = { 12, 22 }, .result = REJECT, .errstr = "invalid access to map value, value_size=8 off=2 size=8", + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "calls: two calls that receive map_value_ptr_or_null via arg. test1", @@ -12301,6 +12337,7 @@ static struct bpf_test tests[] = { .result = ACCEPT, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .retval = POINTER_VALUE, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "calls: pkt_ptr spill into caller stack 2", @@ -12332,6 +12369,7 @@ static struct bpf_test tests[] = { .prog_type = BPF_PROG_TYPE_SCHED_CLS, .errstr = "invalid access to packet", .result = REJECT, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "calls: pkt_ptr spill into caller stack 3", @@ -12434,6 +12472,7 @@ static struct bpf_test tests[] = { .prog_type = BPF_PROG_TYPE_SCHED_CLS, .errstr = "same insn cannot be used with different", .result = REJECT, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "calls: pkt_ptr spill into caller stack 6", @@ -12469,6 +12508,7 @@ static struct bpf_test tests[] = { .prog_type = BPF_PROG_TYPE_SCHED_CLS, .errstr = "R4 invalid mem access", .result = REJECT, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "calls: pkt_ptr spill into caller stack 7", @@ -12503,6 +12543,7 @@ static struct bpf_test tests[] = { .prog_type = BPF_PROG_TYPE_SCHED_CLS, .errstr = "R4 invalid mem access", .result = REJECT, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "calls: pkt_ptr spill into caller stack 8", @@ -12584,6 +12625,7 @@ static struct bpf_test tests[] = { .prog_type = BPF_PROG_TYPE_SCHED_CLS, .errstr = "invalid access to packet", .result = REJECT, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "calls: caller stack init to zero or map_value_or_null", @@ -12949,6 +12991,7 @@ static struct bpf_test tests[] = { .result = REJECT, .errstr = "BPF_XADD stores into R2 pkt is not allowed", .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "xadd/w check whether src/dst got mangled, 1", @@ -13435,6 +13478,7 @@ static struct bpf_test tests[] = { .prog_type = BPF_PROG_TYPE_SCHED_CLS, .errstr = "Unreleased reference", .result = REJECT, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "reference tracking: alloc, check, free in both subbranches", From 0a68632488aa0129ed530af9ae9e8573f5650812 Mon Sep 17 00:00:00 2001 From: David Miller Date: Fri, 30 Nov 2018 21:08:50 -0800 Subject: [PATCH 11/51] bpf: Apply F_NEEDS_EFFICIENT_UNALIGNED_ACCESS to more ACCEPT test cases. If a testcase has alignment problems but is expected to be ACCEPT, verify it using F_NEEDS_EFFICIENT_UNALIGNED_ACCESS too. Maybe in the future if we add some architecture specific code to elide the unaligned memory access warnings during the test, we can execute these as well. Signed-off-by: David S. Miller Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/test_verifier.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index 6bfd3242ea433..c3b038f26ecef 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -3918,6 +3918,7 @@ static struct bpf_test tests[] = { }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = ACCEPT, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "direct packet access: test21 (x += pkt_ptr, 2)", @@ -3943,6 +3944,7 @@ static struct bpf_test tests[] = { }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = ACCEPT, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "direct packet access: test22 (x += pkt_ptr, 3)", @@ -3973,6 +3975,7 @@ static struct bpf_test tests[] = { }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = ACCEPT, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "direct packet access: test23 (x += pkt_ptr, 4)", @@ -4025,6 +4028,7 @@ static struct bpf_test tests[] = { }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = ACCEPT, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "direct packet access: test25 (marking on <, good access)", @@ -7732,6 +7736,7 @@ static struct bpf_test tests[] = { .result = ACCEPT, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .retval = 0 /* csum_diff of 64-byte packet */, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "helper access to variable memory: size = 0 not allowed on NULL (!ARG_PTR_TO_MEM_OR_NULL)", @@ -9694,6 +9699,7 @@ static struct bpf_test tests[] = { }, .result = ACCEPT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_data' > pkt_end, bad access 1", @@ -9865,6 +9871,7 @@ static struct bpf_test tests[] = { }, .result = ACCEPT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_end < pkt_data', bad access 1", @@ -9977,6 +9984,7 @@ static struct bpf_test tests[] = { }, .result = ACCEPT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_end >= pkt_data', bad access 1", @@ -10034,6 +10042,7 @@ static struct bpf_test tests[] = { }, .result = ACCEPT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_data' <= pkt_end, bad access 1", @@ -10146,6 +10155,7 @@ static struct bpf_test tests[] = { }, .result = ACCEPT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_meta' > pkt_data, bad access 1", @@ -10317,6 +10327,7 @@ static struct bpf_test tests[] = { }, .result = ACCEPT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_data < pkt_meta', bad access 1", @@ -10429,6 +10440,7 @@ static struct bpf_test tests[] = { }, .result = ACCEPT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_data >= pkt_meta', bad access 1", @@ -10486,6 +10498,7 @@ static struct bpf_test tests[] = { }, .result = ACCEPT, .prog_type = BPF_PROG_TYPE_XDP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "XDP pkt read, pkt_meta' <= pkt_data, bad access 1", @@ -12405,6 +12418,7 @@ static struct bpf_test tests[] = { .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = ACCEPT, .retval = 1, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "calls: pkt_ptr spill into caller stack 4", @@ -12439,6 +12453,7 @@ static struct bpf_test tests[] = { .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = ACCEPT, .retval = 1, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "calls: pkt_ptr spill into caller stack 5", @@ -12584,6 +12599,7 @@ static struct bpf_test tests[] = { }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = ACCEPT, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "calls: pkt_ptr spill into caller stack 9", @@ -13507,6 +13523,7 @@ static struct bpf_test tests[] = { }, .prog_type = BPF_PROG_TYPE_SCHED_CLS, .result = ACCEPT, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, { "reference tracking in call: free reference in subprog", From d606ee5c1d9a9bc4d1bba3c36fae615447f9851c Mon Sep 17 00:00:00 2001 From: Matteo Croce Date: Sat, 1 Dec 2018 01:23:05 +0100 Subject: [PATCH 12/51] samples: bpf: improve xdp1 example Store only the total packet count for every protocol, instead of the whole per-cpu array. Use bpf_map_get_next_key() to iterate the map, instead of looking up all the protocols. Signed-off-by: Matteo Croce Signed-off-by: Alexei Starovoitov --- samples/bpf/xdp1_user.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/samples/bpf/xdp1_user.c b/samples/bpf/xdp1_user.c index b02c531510ed9..4f3d824fc044c 100644 --- a/samples/bpf/xdp1_user.c +++ b/samples/bpf/xdp1_user.c @@ -34,26 +34,24 @@ static void int_exit(int sig) static void poll_stats(int map_fd, int interval) { unsigned int nr_cpus = bpf_num_possible_cpus(); - const unsigned int nr_keys = 256; - __u64 values[nr_cpus], prev[nr_keys][nr_cpus]; - __u32 key; + __u64 values[nr_cpus], prev[UINT8_MAX] = { 0 }; int i; - memset(prev, 0, sizeof(prev)); - while (1) { + __u32 key = UINT32_MAX; + sleep(interval); - for (key = 0; key < nr_keys; key++) { + while (bpf_map_get_next_key(map_fd, &key, &key) != -1) { __u64 sum = 0; assert(bpf_map_lookup_elem(map_fd, &key, values) == 0); for (i = 0; i < nr_cpus; i++) - sum += (values[i] - prev[key][i]); - if (sum) + sum += values[i]; + if (sum > prev[key]) printf("proto %u: %10llu pkt/s\n", - key, sum / interval); - memcpy(prev[key], values, sizeof(values)); + key, (sum - prev[key]) / interval); + prev[key] = sum; } } } From dc378a1ab5b6009629a3f8410f8b3030ac6b14c9 Mon Sep 17 00:00:00 2001 From: Matteo Croce Date: Sat, 1 Dec 2018 01:23:06 +0100 Subject: [PATCH 13/51] samples: bpf: get ifindex from ifname Find the ifindex with if_nametoindex() instead of requiring the numeric ifindex. Signed-off-by: Matteo Croce Signed-off-by: Alexei Starovoitov --- samples/bpf/xdp1_user.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/samples/bpf/xdp1_user.c b/samples/bpf/xdp1_user.c index 4f3d824fc044c..0a197f86ac43a 100644 --- a/samples/bpf/xdp1_user.c +++ b/samples/bpf/xdp1_user.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "bpf_util.h" #include "bpf/bpf.h" @@ -59,7 +60,7 @@ static void poll_stats(int map_fd, int interval) static void usage(const char *prog) { fprintf(stderr, - "usage: %s [OPTS] IFINDEX\n\n" + "usage: %s [OPTS] IFACE\n\n" "OPTS:\n" " -S use skb-mode\n" " -N enforce native mode\n", @@ -102,7 +103,11 @@ int main(int argc, char **argv) return 1; } - ifindex = strtoul(argv[optind], NULL, 0); + ifindex = if_nametoindex(argv[1]); + if (!ifindex) { + perror("if_nametoindex"); + return 1; + } snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]); prog_load_attr.file = filename; From 5482e9a93c83839f94e75db712e6837e6a39962c Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Sat, 1 Dec 2018 17:08:44 -0800 Subject: [PATCH 14/51] bpf: Fix memleak in aux->func_info and aux->btf The aux->func_info and aux->btf are leaked in the error out cases during bpf_prog_load(). This patch fixes it. Fixes: ba64e7d85252 ("bpf: btf: support proper non-jit func info") Cc: Yonghong Song Signed-off-by: Martin KaFai Lau Acked-by: Yonghong Song Signed-off-by: Alexei Starovoitov --- kernel/bpf/syscall.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index f9554d9a14e18..4445d0d084d88 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1560,6 +1560,8 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) return err; free_used_maps: + kvfree(prog->aux->func_info); + btf_put(prog->aux->btf); bpf_prog_kallsyms_del_subprogs(prog); free_used_maps(prog->aux); free_prog: From de94b651eef38b44adc20057ab5eb6d4ca672169 Mon Sep 17 00:00:00 2001 From: Andrey Ignatov Date: Sun, 2 Dec 2018 13:02:15 -0800 Subject: [PATCH 15/51] libbpf: Fix license in README.rst The whole libbpf is licensed as (LGPL-2.1 OR BSD-2-Clause). I missed it while adding README.rst. Fix it and use same license as all other files in libbpf do. Since I'm the only author of README.rst so far, no others' permissions should be needed. Fixes: 76d1b894c515 ("libbpf: Document API and ABI conventions") Signed-off-by: Andrey Ignatov Acked-by: Song Liu Signed-off-by: Daniel Borkmann --- tools/lib/bpf/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/lib/bpf/README.rst b/tools/lib/bpf/README.rst index 2ced9e061c4b5..056f383107226 100644 --- a/tools/lib/bpf/README.rst +++ b/tools/lib/bpf/README.rst @@ -1,4 +1,4 @@ -.. SPDX-License-Identifier: GPL-2.0 +.. SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) libbpf API naming convention ============================ From e3da08d057002f9d0831949d51666c3e15dc6b29 Mon Sep 17 00:00:00 2001 From: Petar Penkov Date: Sun, 2 Dec 2018 20:18:19 -0500 Subject: [PATCH 16/51] bpf: allow BPF read access to qdisc pkt_len The pkt_len field in qdisc_skb_cb stores the skb length as it will appear on the wire after segmentation. For byte accounting, this value is more accurate than skb->len. It is computed on entry to the TC layer, so only valid there. Allow read access to this field from BPF tc classifier and action programs. The implementation is analogous to tc_classid, aside from restricting to read access. To distinguish it from skb->len and self-describe export as wire_len. Changes v1->v2 - Rename pkt_len to wire_len Signed-off-by: Petar Penkov Signed-off-by: Vlad Dumitrescu Signed-off-by: Willem de Bruijn Acked-by: Song Liu Signed-off-by: Daniel Borkmann --- include/uapi/linux/bpf.h | 1 + net/core/filter.c | 16 +++++++++++ tools/include/uapi/linux/bpf.h | 1 + tools/testing/selftests/bpf/test_verifier.c | 32 +++++++++++++++++++++ 4 files changed, 50 insertions(+) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 8050caea74955..0183b8e70a9e6 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -2497,6 +2497,7 @@ struct __sk_buff { __u32 data_meta; struct bpf_flow_keys *flow_keys; __u64 tstamp; + __u32 wire_len; }; struct bpf_tunnel_key { diff --git a/net/core/filter.c b/net/core/filter.c index bd0df75dc7b6e..3d54af4c363d8 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -5773,6 +5773,7 @@ static bool sk_filter_is_valid_access(int off, int size, case bpf_ctx_range(struct __sk_buff, flow_keys): case bpf_ctx_range_till(struct __sk_buff, family, local_port): case bpf_ctx_range(struct __sk_buff, tstamp): + case bpf_ctx_range(struct __sk_buff, wire_len): return false; } @@ -5797,6 +5798,7 @@ static bool cg_skb_is_valid_access(int off, int size, case bpf_ctx_range(struct __sk_buff, tc_classid): case bpf_ctx_range(struct __sk_buff, data_meta): case bpf_ctx_range(struct __sk_buff, flow_keys): + case bpf_ctx_range(struct __sk_buff, wire_len): return false; case bpf_ctx_range(struct __sk_buff, data): case bpf_ctx_range(struct __sk_buff, data_end): @@ -5843,6 +5845,7 @@ static bool lwt_is_valid_access(int off, int size, case bpf_ctx_range(struct __sk_buff, data_meta): case bpf_ctx_range(struct __sk_buff, flow_keys): case bpf_ctx_range(struct __sk_buff, tstamp): + case bpf_ctx_range(struct __sk_buff, wire_len): return false; } @@ -6273,6 +6276,7 @@ static bool sk_skb_is_valid_access(int off, int size, case bpf_ctx_range(struct __sk_buff, data_meta): case bpf_ctx_range(struct __sk_buff, flow_keys): case bpf_ctx_range(struct __sk_buff, tstamp): + case bpf_ctx_range(struct __sk_buff, wire_len): return false; } @@ -6360,6 +6364,7 @@ static bool flow_dissector_is_valid_access(int off, int size, case bpf_ctx_range(struct __sk_buff, data_meta): case bpf_ctx_range_till(struct __sk_buff, family, local_port): case bpf_ctx_range(struct __sk_buff, tstamp): + case bpf_ctx_range(struct __sk_buff, wire_len): return false; } @@ -6685,6 +6690,17 @@ static u32 bpf_convert_ctx_access(enum bpf_access_type type, bpf_target_off(struct sk_buff, tstamp, 8, target_size)); + break; + + case offsetof(struct __sk_buff, wire_len): + BUILD_BUG_ON(FIELD_SIZEOF(struct qdisc_skb_cb, pkt_len) != 4); + + off = si->off; + off -= offsetof(struct __sk_buff, wire_len); + off += offsetof(struct sk_buff, cb); + off += offsetof(struct qdisc_skb_cb, pkt_len); + *target_size = 4; + *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, off); } return insn - insn_buf; diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 8050caea74955..0183b8e70a9e6 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -2497,6 +2497,7 @@ struct __sk_buff { __u32 data_meta; struct bpf_flow_keys *flow_keys; __u64 tstamp; + __u32 wire_len; }; struct bpf_tunnel_key { diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index c3b038f26ecef..b4b4a3f936396 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -14033,6 +14033,38 @@ static struct bpf_test tests[] = { .result_unpriv = REJECT, .result = ACCEPT, }, + { + "check wire_len is not readable by sockets", + .insns = { + BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, + offsetof(struct __sk_buff, wire_len)), + BPF_EXIT_INSN(), + }, + .errstr = "invalid bpf_context access", + .result = REJECT, + }, + { + "check wire_len is readable by tc classifier", + .insns = { + BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, + offsetof(struct __sk_buff, wire_len)), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_SCHED_CLS, + .result = ACCEPT, + }, + { + "check wire_len is not writable by tc classifier", + .insns = { + BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_1, + offsetof(struct __sk_buff, wire_len)), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_SCHED_CLS, + .errstr = "invalid bpf_context access", + .errstr_unpriv = "R1 leaks addr", + .result = REJECT, + }, }; static int probe_filter_length(const struct bpf_insn *fp) From 90b1023f68c78b9b85e364250155776c8e421176 Mon Sep 17 00:00:00 2001 From: Quentin Monnet Date: Mon, 3 Dec 2018 12:13:35 +0000 Subject: [PATCH 17/51] bpf: fix documentation for eBPF helpers The missing indentation on the "Return" sections for bpf_map_pop_elem() and bpf_map_peek_elem() helpers break RST and man pages generation. This patch fixes them, and moves the description of those two helpers towards the end of the list (even though they are somehow related to the three first helpers for maps, the man page explicitly states that the helpers are sorted in chronological order). While at it, bring other minor formatting edits for eBPF helpers documentation: mostly blank lines removal, RST formatting, or other small nits for consistency. Signed-off-by: Quentin Monnet Reviewed-by: Jakub Kicinski Signed-off-by: Daniel Borkmann --- include/uapi/linux/bpf.h | 90 ++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 0183b8e70a9e6..572eb2d427687 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -496,18 +496,6 @@ union bpf_attr { * Return * 0 on success, or a negative error in case of failure. * - * int bpf_map_pop_elem(struct bpf_map *map, void *value) - * Description - * Pop an element from *map*. - * Return - * 0 on success, or a negative error in case of failure. - * - * int bpf_map_peek_elem(struct bpf_map *map, void *value) - * Description - * Get an element from *map* without removing it. - * Return - * 0 on success, or a negative error in case of failure. - * * int bpf_probe_read(void *dst, u32 size, const void *src) * Description * For tracing programs, safely attempt to read *size* bytes from @@ -1931,9 +1919,9 @@ union bpf_attr { * is set to metric from route (IPv4/IPv6 only), and ifindex * is set to the device index of the nexthop from the FIB lookup. * - * *plen* argument is the size of the passed in struct. - * *flags* argument can be a combination of one or more of the - * following values: + * *plen* argument is the size of the passed in struct. + * *flags* argument can be a combination of one or more of the + * following values: * * **BPF_FIB_LOOKUP_DIRECT** * Do a direct table lookup vs full lookup using FIB @@ -1942,9 +1930,9 @@ union bpf_attr { * Perform lookup from an egress perspective (default is * ingress). * - * *ctx* is either **struct xdp_md** for XDP programs or - * **struct sk_buff** tc cls_act programs. - * Return + * *ctx* is either **struct xdp_md** for XDP programs or + * **struct sk_buff** tc cls_act programs. + * Return * * < 0 if any input argument is invalid * * 0 on success (packet is forwarded, nexthop neighbor exists) * * > 0 one of **BPF_FIB_LKUP_RET_** codes explaining why the @@ -2089,8 +2077,8 @@ union bpf_attr { * translated to a keycode using the rc keymap, and reported as * an input key down event. After a period a key up event is * generated. This period can be extended by calling either - * **bpf_rc_keydown** () again with the same values, or calling - * **bpf_rc_repeat** (). + * **bpf_rc_keydown**\ () again with the same values, or calling + * **bpf_rc_repeat**\ (). * * Some protocols include a toggle bit, in case the button was * released and pressed again between consecutive scancodes. @@ -2173,21 +2161,22 @@ union bpf_attr { * The *flags* meaning is specific for each map type, * and has to be 0 for cgroup local storage. * - * Depending on the bpf program type, a local storage area - * can be shared between multiple instances of the bpf program, + * Depending on the BPF program type, a local storage area + * can be shared between multiple instances of the BPF program, * running simultaneously. * * A user should care about the synchronization by himself. - * For example, by using the BPF_STX_XADD instruction to alter + * For example, by using the **BPF_STX_XADD** instruction to alter * the shared data. * Return - * Pointer to the local storage area. + * A pointer to the local storage area. * * int bpf_sk_select_reuseport(struct sk_reuseport_md *reuse, struct bpf_map *map, void *key, u64 flags) * Description - * Select a SO_REUSEPORT sk from a BPF_MAP_TYPE_REUSEPORT_ARRAY map - * It checks the selected sk is matching the incoming - * request in the skb. + * Select a **SO_REUSEPORT** socket from a + * **BPF_MAP_TYPE_REUSEPORT_ARRAY** *map*. + * It checks the selected socket is matching the incoming + * request in the socket buffer. * Return * 0 on success, or a negative error in case of failure. * @@ -2195,7 +2184,7 @@ union bpf_attr { * Description * Look for TCP socket matching *tuple*, optionally in a child * network namespace *netns*. The return value must be checked, - * and if non-NULL, released via **bpf_sk_release**\ (). + * and if non-**NULL**, released via **bpf_sk_release**\ (). * * The *ctx* should point to the context of the program, such as * the skb or socket (depending on the hook in use). This is used @@ -2221,15 +2210,15 @@ union bpf_attr { * This helper is available only if the kernel was compiled with * **CONFIG_NET** configuration option. * Return - * Pointer to *struct bpf_sock*, or NULL in case of failure. - * For sockets with reuseport option, *struct bpf_sock* - * return is from reuse->socks[] using hash of the packet. + * A pointer to *struct bpf_sock*, or **NULL** in case of failure. + * For sockets with reuseport option, **struct bpf_sock** + * return is from **reuse->socks**\ [] using hash of the packet. * * struct bpf_sock *bpf_sk_lookup_udp(void *ctx, struct bpf_sock_tuple *tuple, u32 tuple_size, u32 netns, u64 flags) * Description * Look for UDP socket matching *tuple*, optionally in a child * network namespace *netns*. The return value must be checked, - * and if non-NULL, released via **bpf_sk_release**\ (). + * and if non-**NULL**, released via **bpf_sk_release**\ (). * * The *ctx* should point to the context of the program, such as * the skb or socket (depending on the hook in use). This is used @@ -2255,46 +2244,57 @@ union bpf_attr { * This helper is available only if the kernel was compiled with * **CONFIG_NET** configuration option. * Return - * Pointer to *struct bpf_sock*, or NULL in case of failure. - * For sockets with reuseport option, *struct bpf_sock* - * return is from reuse->socks[] using hash of the packet. + * A pointer to **struct bpf_sock**, or **NULL** in case of + * failure. For sockets with reuseport option, **struct bpf_sock** + * return is from **reuse->socks**\ [] using hash of the packet. * - * int bpf_sk_release(struct bpf_sock *sk) + * int bpf_sk_release(struct bpf_sock *sock) * Description - * Release the reference held by *sock*. *sock* must be a non-NULL - * pointer that was returned from bpf_sk_lookup_xxx\ (). + * Release the reference held by *sock*. *sock* must be a + * non-**NULL** pointer that was returned from + * **bpf_sk_lookup_xxx**\ (). * Return * 0 on success, or a negative error in case of failure. * + * int bpf_map_pop_elem(struct bpf_map *map, void *value) + * Description + * Pop an element from *map*. + * Return + * 0 on success, or a negative error in case of failure. + * + * int bpf_map_peek_elem(struct bpf_map *map, void *value) + * Description + * Get an element from *map* without removing it. + * Return + * 0 on success, or a negative error in case of failure. + * * int bpf_msg_push_data(struct sk_buff *skb, u32 start, u32 len, u64 flags) * Description - * For socket policies, insert *len* bytes into msg at offset + * For socket policies, insert *len* bytes into *msg* at offset * *start*. * * If a program of type **BPF_PROG_TYPE_SK_MSG** is run on a - * *msg* it may want to insert metadata or options into the msg. + * *msg* it may want to insert metadata or options into the *msg*. * This can later be read and used by any of the lower layer BPF * hooks. * * This helper may fail if under memory pressure (a malloc * fails) in these cases BPF programs will get an appropriate * error and BPF programs will need to handle them. - * * Return * 0 on success, or a negative error in case of failure. * * int bpf_msg_pop_data(struct sk_msg_buff *msg, u32 start, u32 pop, u64 flags) - * Description + * Description * Will remove *pop* bytes from a *msg* starting at byte *start*. * This may result in **ENOMEM** errors under certain situations if * an allocation and copy are required due to a full ring buffer. * However, the helper will try to avoid doing the allocation * if possible. Other errors can occur if input parameters are - * invalid either due to *start* byte not being valid part of msg + * invalid either due to *start* byte not being valid part of *msg* * payload and/or *pop* value being to large. - * * Return - * 0 on success, or a negative erro in case of failure. + * 0 on success, or a negative error in case of failure. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ From d59dd69d5576d699d7d3f5da0b4738c3a36d0133 Mon Sep 17 00:00:00 2001 From: "Daniel T. Lee" Date: Mon, 3 Dec 2018 19:39:30 +0900 Subject: [PATCH 18/51] samples: bpf: fix: seg fault with NULL pointer arg When NULL pointer accidentally passed to write_kprobe_events, due to strlen(NULL), segmentation fault happens. Changed code returns -1 to deal with this situation. Bug issued with Smatch, static analysis. Signed-off-by: Daniel T. Lee Acked-by: Song Liu Signed-off-by: Daniel Borkmann --- samples/bpf/bpf_load.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/samples/bpf/bpf_load.c b/samples/bpf/bpf_load.c index 434ea34a59549..eae7b635343d1 100644 --- a/samples/bpf/bpf_load.c +++ b/samples/bpf/bpf_load.c @@ -58,7 +58,9 @@ static int write_kprobe_events(const char *val) { int fd, ret, flags; - if ((val != NULL) && (val[0] == '\0')) + if (val == NULL) + return -1; + else if (val[0] == '\0') flags = O_WRONLY | O_TRUNC; else flags = O_WRONLY | O_APPEND; From b5a36b1e1b138285ea0df34bf96c759e1e30fafd Mon Sep 17 00:00:00 2001 From: Lorenz Bauer Date: Mon, 3 Dec 2018 11:31:23 +0000 Subject: [PATCH 19/51] bpf: respect size hint to BPF_PROG_TEST_RUN if present Use data_size_out as a size hint when copying test output to user space. ENOSPC is returned if the output buffer is too small. Callers which so far did not set data_size_out are not affected. Signed-off-by: Lorenz Bauer Signed-off-by: Alexei Starovoitov --- include/uapi/linux/bpf.h | 7 +++++-- net/bpf/test_run.c | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 572eb2d427687..c8e1eeee2c5ff 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -374,8 +374,11 @@ union bpf_attr { struct { /* anonymous struct used by BPF_PROG_TEST_RUN command */ __u32 prog_fd; __u32 retval; - __u32 data_size_in; - __u32 data_size_out; + __u32 data_size_in; /* input: len of data_in */ + __u32 data_size_out; /* input/output: len of data_out + * returns ENOSPC if data_out + * is too small. + */ __aligned_u64 data_in; __aligned_u64 data_out; __u32 repeat; diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index c89c22c49015f..7663e6a57280b 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -74,8 +74,18 @@ static int bpf_test_finish(const union bpf_attr *kattr, { void __user *data_out = u64_to_user_ptr(kattr->test.data_out); int err = -EFAULT; + u32 copy_size = size; + + /* Clamp copy if the user has provided a size hint, but copy the full + * buffer if not to retain old behaviour. + */ + if (kattr->test.data_size_out && + copy_size > kattr->test.data_size_out) { + copy_size = kattr->test.data_size_out; + err = -ENOSPC; + } - if (data_out && copy_to_user(data_out, data, size)) + if (data_out && copy_to_user(data_out, data, copy_size)) goto out; if (copy_to_user(&uattr->test.data_size_out, &size, sizeof(size))) goto out; @@ -83,7 +93,8 @@ static int bpf_test_finish(const union bpf_attr *kattr, goto out; if (copy_to_user(&uattr->test.duration, &duration, sizeof(duration))) goto out; - err = 0; + if (err != -ENOSPC) + err = 0; out: return err; } From 2587a974f184ff1cd8103de25b6354a3a4da7132 Mon Sep 17 00:00:00 2001 From: Lorenz Bauer Date: Mon, 3 Dec 2018 11:31:24 +0000 Subject: [PATCH 20/51] tools: sync uapi/linux/bpf.h Pull changes from "bpf: respect size hint to BPF_PROG_TEST_RUN if present". Signed-off-by: Lorenz Bauer Signed-off-by: Alexei Starovoitov --- tools/include/uapi/linux/bpf.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 0183b8e70a9e6..64262890feb23 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -374,8 +374,11 @@ union bpf_attr { struct { /* anonymous struct used by BPF_PROG_TEST_RUN command */ __u32 prog_fd; __u32 retval; - __u32 data_size_in; - __u32 data_size_out; + __u32 data_size_in; /* input: len of data_in */ + __u32 data_size_out; /* input/output: len of data_out + * returns ENOSPC if data_out + * is too small. + */ __aligned_u64 data_in; __aligned_u64 data_out; __u32 repeat; From 64a975913b311e5192a76cd10721d4565c9addd1 Mon Sep 17 00:00:00 2001 From: Lorenz Bauer Date: Mon, 3 Dec 2018 11:31:25 +0000 Subject: [PATCH 21/51] libbpf: add bpf_prog_test_run_xattr Add a new function, which encourages safe usage of the test interface. bpf_prog_test_run continues to work as before, but should be considered unsafe. Signed-off-by: Lorenz Bauer Signed-off-by: Alexei Starovoitov --- tools/lib/bpf/bpf.c | 23 +++++++++++++++++++++++ tools/lib/bpf/bpf.h | 19 +++++++++++++++++++ tools/lib/bpf/libbpf.map | 1 + 3 files changed, 43 insertions(+) diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index c19226cccf39f..5c3be06bf0ddf 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -463,6 +463,29 @@ int bpf_prog_test_run(int prog_fd, int repeat, void *data, __u32 size, return ret; } +int bpf_prog_test_run_xattr(struct bpf_prog_test_run_attr *test_attr) +{ + union bpf_attr attr; + int ret; + + if (!test_attr->data_out && test_attr->data_size_out > 0) + return -EINVAL; + + bzero(&attr, sizeof(attr)); + attr.test.prog_fd = test_attr->prog_fd; + attr.test.data_in = ptr_to_u64(test_attr->data_in); + attr.test.data_out = ptr_to_u64(test_attr->data_out); + attr.test.data_size_in = test_attr->data_size_in; + attr.test.data_size_out = test_attr->data_size_out; + attr.test.repeat = test_attr->repeat; + + ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, sizeof(attr)); + test_attr->data_size_out = attr.test.data_size_out; + test_attr->retval = attr.test.retval; + test_attr->duration = attr.test.duration; + return ret; +} + int bpf_prog_get_next_id(__u32 start_id, __u32 *next_id) { union bpf_attr attr; diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 60392b70587ca..098e6f793b765 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -118,6 +118,25 @@ LIBBPF_API int bpf_prog_attach(int prog_fd, int attachable_fd, LIBBPF_API int bpf_prog_detach(int attachable_fd, enum bpf_attach_type type); LIBBPF_API int bpf_prog_detach2(int prog_fd, int attachable_fd, enum bpf_attach_type type); + +struct bpf_prog_test_run_attr { + int prog_fd; + int repeat; + const void *data_in; + __u32 data_size_in; + void *data_out; /* optional */ + __u32 data_size_out; /* in: max length of data_out + * out: length of data_out */ + __u32 retval; /* out: return code of the BPF program */ + __u32 duration; /* out: average per repetition in ns */ +}; + +LIBBPF_API int bpf_prog_test_run_xattr(struct bpf_prog_test_run_attr *test_attr); + +/* + * bpf_prog_test_run does not check that data_out is large enough. Consider + * using bpf_prog_test_run_xattr instead. + */ LIBBPF_API int bpf_prog_test_run(int prog_fd, int repeat, void *data, __u32 size, void *data_out, __u32 *size_out, __u32 *retval, __u32 *duration); diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 4fb29f6d7a80b..8deff22d61bba 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -65,6 +65,7 @@ LIBBPF_0.0.1 { bpf_prog_load_xattr; bpf_prog_query; bpf_prog_test_run; + bpf_prog_test_run_xattr; bpf_program__fd; bpf_program__is_kprobe; bpf_program__is_perf_event; From df47fc376df8338fdc9b27a5c9162a08f6e6157c Mon Sep 17 00:00:00 2001 From: Lorenz Bauer Date: Mon, 3 Dec 2018 11:31:26 +0000 Subject: [PATCH 22/51] selftests: add a test for bpf_prog_test_run_xattr Make sure that bpf_prog_test_run_xattr returns the correct length and that the kernel respects the output size hint. Also check that errno indicates ENOSPC if there is a short output buffer given. Signed-off-by: Lorenz Bauer Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/test_progs.c | 55 +++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 1c57abbe09455..26f1fdf3e2bfa 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -70,7 +70,7 @@ static struct { .tcp.urg_ptr = 123, }; -#define CHECK(condition, tag, format...) ({ \ +#define _CHECK(condition, tag, duration, format...) ({ \ int __ret = !!(condition); \ if (__ret) { \ error_cnt++; \ @@ -83,6 +83,11 @@ static struct { __ret; \ }) +#define CHECK(condition, tag, format...) \ + _CHECK(condition, tag, duration, format) +#define CHECK_ATTR(condition, tag, format...) \ + _CHECK(condition, tag, tattr.duration, format) + static int bpf_find_map(const char *test, struct bpf_object *obj, const char *name) { @@ -124,6 +129,53 @@ static void test_pkt_access(void) bpf_object__close(obj); } +static void test_prog_run_xattr(void) +{ + const char *file = "./test_pkt_access.o"; + struct bpf_object *obj; + char buf[10]; + int err; + struct bpf_prog_test_run_attr tattr = { + .repeat = 1, + .data_in = &pkt_v4, + .data_size_in = sizeof(pkt_v4), + .data_out = buf, + .data_size_out = 5, + }; + + err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, + &tattr.prog_fd); + if (CHECK_ATTR(err, "load", "err %d errno %d\n", err, errno)) + return; + + memset(buf, 0, sizeof(buf)); + + err = bpf_prog_test_run_xattr(&tattr); + CHECK_ATTR(err != -1 || errno != ENOSPC || tattr.retval, "run", + "err %d errno %d retval %d\n", err, errno, tattr.retval); + + CHECK_ATTR(tattr.data_size_out != sizeof(pkt_v4), "data_size_out", + "incorrect output size, want %lu have %u\n", + sizeof(pkt_v4), tattr.data_size_out); + + CHECK_ATTR(buf[5] != 0, "overflow", + "BPF_PROG_TEST_RUN ignored size hint\n"); + + tattr.data_out = NULL; + tattr.data_size_out = 0; + errno = 0; + + err = bpf_prog_test_run_xattr(&tattr); + CHECK_ATTR(err || errno || tattr.retval, "run_no_output", + "err %d errno %d retval %d\n", err, errno, tattr.retval); + + tattr.data_size_out = 1; + err = bpf_prog_test_run_xattr(&tattr); + CHECK_ATTR(err != -EINVAL, "run_wrong_size_out", "err %d\n", err); + + bpf_object__close(obj); +} + static void test_xdp(void) { struct vip key4 = {.protocol = 6, .family = AF_INET}; @@ -1837,6 +1889,7 @@ int main(void) jit_enabled = is_jit_enabled(); test_pkt_access(); + test_prog_run_xattr(); test_xdp(); test_xdp_adjust_tail(); test_l4lb_all(); From dc002bb62f10c5905420f8b8a7d5ec0da567fc82 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 23 Nov 2018 23:18:03 +0100 Subject: [PATCH 23/51] bpf: add __weak hook for allocating executable memory By default, BPF uses module_alloc() to allocate executable memory, but this is not necessary on all arches and potentially undesirable on some of them. So break out the module_alloc() and module_memfree() calls into __weak functions to allow them to be overridden in arch code. Signed-off-by: Ard Biesheuvel Signed-off-by: Daniel Borkmann --- kernel/bpf/core.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index f93ed667546ff..86817ab204e8e 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -623,6 +623,16 @@ static void bpf_jit_uncharge_modmem(u32 pages) atomic_long_sub(pages, &bpf_jit_current); } +void *__weak bpf_jit_alloc_exec(unsigned long size) +{ + return module_alloc(size); +} + +void __weak bpf_jit_free_exec(void *addr) +{ + module_memfree(addr); +} + struct bpf_binary_header * bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, unsigned int alignment, @@ -640,7 +650,7 @@ bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, if (bpf_jit_charge_modmem(pages)) return NULL; - hdr = module_alloc(size); + hdr = bpf_jit_alloc_exec(size); if (!hdr) { bpf_jit_uncharge_modmem(pages); return NULL; @@ -664,7 +674,7 @@ void bpf_jit_binary_free(struct bpf_binary_header *hdr) { u32 pages = hdr->pages; - module_memfree(hdr); + bpf_jit_free_exec(hdr); bpf_jit_uncharge_modmem(pages); } From 91fc957c9b1d6c55168df15a397cbd1af16bc00f Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 23 Nov 2018 23:18:04 +0100 Subject: [PATCH 24/51] arm64/bpf: don't allocate BPF JIT programs in module memory The arm64 module region is a 128 MB region that is kept close to the core kernel, in order to ensure that relative branches are always in range. So using the same region for programs that do not have this restriction is wasteful, and preferably avoided. Now that the core BPF JIT code permits the alloc/free routines to be overridden, implement them by vmalloc()/vfree() calls from a dedicated 128 MB region set aside for BPF programs. This ensures that BPF programs are still in branching range of each other, which is something the JIT currently depends upon (and is not guaranteed when using module_alloc() on KASLR kernels like we do currently). It also ensures that placement of BPF programs does not correlate with the placement of the core kernel or modules, making it less likely that leaking the former will reveal the latter. This also solves an issue under KASAN, where shadow memory is needlessly allocated for all BPF programs (which don't require KASAN shadow pages since they are not KASAN instrumented) Signed-off-by: Ard Biesheuvel Acked-by: Will Deacon Signed-off-by: Daniel Borkmann --- arch/arm64/include/asm/memory.h | 5 ++++- arch/arm64/net/bpf_jit_comp.c | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index b96442960aead..ee20fc63899c5 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -62,8 +62,11 @@ #define PAGE_OFFSET (UL(0xffffffffffffffff) - \ (UL(1) << (VA_BITS - 1)) + 1) #define KIMAGE_VADDR (MODULES_END) +#define BPF_JIT_REGION_START (VA_START + KASAN_SHADOW_SIZE) +#define BPF_JIT_REGION_SIZE (SZ_128M) +#define BPF_JIT_REGION_END (BPF_JIT_REGION_START + BPF_JIT_REGION_SIZE) #define MODULES_END (MODULES_VADDR + MODULES_VSIZE) -#define MODULES_VADDR (VA_START + KASAN_SHADOW_SIZE) +#define MODULES_VADDR (BPF_JIT_REGION_END) #define MODULES_VSIZE (SZ_128M) #define VMEMMAP_START (PAGE_OFFSET - VMEMMAP_SIZE) #define PCI_IO_END (VMEMMAP_START - SZ_2M) diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index b872859240234..0a7371a861394 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -943,3 +943,16 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) tmp : orig_prog); return prog; } + +void *bpf_jit_alloc_exec(unsigned long size) +{ + return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START, + BPF_JIT_REGION_END, GFP_KERNEL, + PAGE_KERNEL_EXEC, 0, NUMA_NO_NODE, + __builtin_return_address(0)); +} + +void bpf_jit_free_exec(void *addr) +{ + return vfree(addr); +} From 30da46b5dc3a9a14db11706d841440e28b12bb53 Mon Sep 17 00:00:00 2001 From: Quentin Monnet Date: Wed, 5 Dec 2018 10:28:24 +0000 Subject: [PATCH 25/51] tools: bpftool: add a command to dump the trace pipe BPF programs can use the bpf_trace_printk() helper to print debug information into the trace pipe. Add a subcommand "bpftool prog tracelog" to simply dump this pipe to the console. This is for a good part copied from iproute2, where the feature is available with "tc exec bpf dbg". Changes include dumping pipe content to stdout instead of stderr and adding JSON support (content is dumped as an array of strings, one per line read from the pipe). This version is dual-licensed, with Daniel's permission. Cc: Daniel Borkmann Suggested-by: Daniel Borkmann Signed-off-by: Quentin Monnet Reviewed-by: Jakub Kicinski Signed-off-by: Daniel Borkmann --- .../bpftool/Documentation/bpftool-prog.rst | 13 +- tools/bpf/bpftool/bash-completion/bpftool | 5 +- tools/bpf/bpftool/main.h | 1 + tools/bpf/bpftool/prog.c | 4 +- tools/bpf/bpftool/tracelog.c | 157 ++++++++++++++++++ 5 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 tools/bpf/bpftool/tracelog.c diff --git a/tools/bpf/bpftool/Documentation/bpftool-prog.rst b/tools/bpf/bpftool/Documentation/bpftool-prog.rst index ab36e920e5521..5524b6dccd856 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-prog.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-prog.rst @@ -26,8 +26,9 @@ MAP COMMANDS | **bpftool** **prog dump jited** *PROG* [{**file** *FILE* | **opcodes**}] | **bpftool** **prog pin** *PROG* *FILE* | **bpftool** **prog { load | loadall }** *OBJ* *PATH* [**type** *TYPE*] [**map** {**idx** *IDX* | **name** *NAME*} *MAP*] [**dev** *NAME*] -| **bpftool** **prog attach** *PROG* *ATTACH_TYPE* [*MAP*] -| **bpftool** **prog detach** *PROG* *ATTACH_TYPE* [*MAP*] +| **bpftool** **prog attach** *PROG* *ATTACH_TYPE* [*MAP*] +| **bpftool** **prog detach** *PROG* *ATTACH_TYPE* [*MAP*] +| **bpftool** **prog tracelog** | **bpftool** **prog help** | | *MAP* := { **id** *MAP_ID* | **pinned** *FILE* } @@ -117,6 +118,14 @@ DESCRIPTION parameter, with the exception of *flow_dissector* which is detached from the current networking name space. + **bpftool prog tracelog** + Dump the trace pipe of the system to the console (stdout). + Hit to stop printing. BPF programs can write to this + trace pipe at runtime with the **bpf_trace_printk()** helper. + This should be used only for debugging purposes. For + streaming data from BPF programs to user space, one can use + perf events (see also **bpftool-map**\ (8)). + **bpftool prog help** Print short help message. diff --git a/tools/bpf/bpftool/bash-completion/bpftool b/tools/bpf/bpftool/bash-completion/bpftool index 9a60080f085f1..44c189ba072a3 100644 --- a/tools/bpf/bpftool/bash-completion/bpftool +++ b/tools/bpf/bpftool/bash-completion/bpftool @@ -398,10 +398,13 @@ _bpftool() ;; esac ;; + tracelog) + return 0 + ;; *) [[ $prev == $object ]] && \ COMPREPLY=( $( compgen -W 'dump help pin attach detach load \ - show list' -- "$cur" ) ) + show list tracelog' -- "$cur" ) ) ;; esac ;; diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h index 2761981669c82..0be0dd8f467ff 100644 --- a/tools/bpf/bpftool/main.h +++ b/tools/bpf/bpftool/main.h @@ -167,6 +167,7 @@ int do_event_pipe(int argc, char **argv); int do_cgroup(int argc, char **arg); int do_perf(int argc, char **arg); int do_net(int argc, char **arg); +int do_tracelog(int argc, char **arg); int parse_u32_arg(int *argc, char ***argv, __u32 *val, const char *what); int prog_parse_fd(int *argc, char ***argv); diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 56db61c5a91fe..54c8dbf05c9cb 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -1140,6 +1140,7 @@ static int do_help(int argc, char **argv) " [pinmaps MAP_DIR]\n" " %s %s attach PROG ATTACH_TYPE [MAP]\n" " %s %s detach PROG ATTACH_TYPE [MAP]\n" + " %s %s tracelog\n" " %s %s help\n" "\n" " " HELP_SPEC_MAP "\n" @@ -1158,7 +1159,7 @@ static int do_help(int argc, char **argv) "", bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2], - bin_name, argv[-2], bin_name, argv[-2]); + bin_name, argv[-2], bin_name, argv[-2], bin_name, argv[-2]); return 0; } @@ -1173,6 +1174,7 @@ static const struct cmd cmds[] = { { "loadall", do_loadall }, { "attach", do_attach }, { "detach", do_detach }, + { "tracelog", do_tracelog }, { 0 } }; diff --git a/tools/bpf/bpftool/tracelog.c b/tools/bpf/bpftool/tracelog.c new file mode 100644 index 0000000000000..1fa8e513f5906 --- /dev/null +++ b/tools/bpf/bpftool/tracelog.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright (c) 2015-2017 Daniel Borkmann */ +/* Copyright (c) 2018 Netronome Systems, Inc. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "main.h" + +#ifndef TRACEFS_MAGIC +# define TRACEFS_MAGIC 0x74726163 +#endif + +#define _textify(x) #x +#define textify(x) _textify(x) + +FILE *trace_pipe_fd; +char *buff; + +static int validate_tracefs_mnt(const char *mnt, unsigned long magic) +{ + struct statfs st_fs; + + if (statfs(mnt, &st_fs) < 0) + return -ENOENT; + if ((unsigned long)st_fs.f_type != magic) + return -ENOENT; + + return 0; +} + +static bool +find_tracefs_mnt_single(unsigned long magic, char *mnt, const char *mntpt) +{ + size_t src_len; + + if (validate_tracefs_mnt(mntpt, magic)) + return false; + + src_len = strlen(mntpt); + if (src_len + 1 >= PATH_MAX) { + p_err("tracefs mount point name too long"); + return false; + } + + strcpy(mnt, mntpt); + return true; +} + +static bool find_tracefs_pipe(char *mnt) +{ + static const char * const known_mnts[] = { + "/sys/kernel/debug/tracing", + "/sys/kernel/tracing", + "/tracing", + "/trace", + }; + const char *pipe_name = "/trace_pipe"; + const char *fstype = "tracefs"; + char type[100], format[32]; + const char * const *ptr; + bool found = false; + FILE *fp; + + for (ptr = known_mnts; ptr < known_mnts + ARRAY_SIZE(known_mnts); ptr++) + if (find_tracefs_mnt_single(TRACEFS_MAGIC, mnt, *ptr)) + goto exit_found; + + fp = fopen("/proc/mounts", "r"); + if (!fp) + return false; + + /* Allow room for NULL terminating byte and pipe file name */ + snprintf(format, sizeof(format), "%%*s %%%zds %%99s %%*s %%*d %%*d\\n", + PATH_MAX - strlen(pipe_name) - 1); + while (fscanf(fp, format, mnt, type) == 2) + if (strcmp(type, fstype) == 0) { + found = true; + break; + } + fclose(fp); + + /* The string from fscanf() might be truncated, check mnt is valid */ + if (!found || validate_tracefs_mnt(mnt, TRACEFS_MAGIC)) + return false; + +exit_found: + strcat(mnt, pipe_name); + return true; +} + +static void exit_tracelog(int signum) +{ + fclose(trace_pipe_fd); + free(buff); + + if (json_output) { + jsonw_end_array(json_wtr); + jsonw_destroy(&json_wtr); + } + + exit(0); +} + +int do_tracelog(int argc, char **argv) +{ + const struct sigaction act = { + .sa_handler = exit_tracelog + }; + char trace_pipe[PATH_MAX]; + bool found_trace_pipe; + size_t buff_len = 0; + + if (json_output) + jsonw_start_array(json_wtr); + + found_trace_pipe = find_tracefs_pipe(trace_pipe); + if (!found_trace_pipe) { + p_err("could not find trace pipe, tracefs not mounted?"); + return -1; + } + + trace_pipe_fd = fopen(trace_pipe, "r"); + if (!trace_pipe_fd) { + p_err("could not open trace pipe: %s", strerror(errno)); + return -1; + } + + sigaction(SIGHUP, &act, NULL); + sigaction(SIGINT, &act, NULL); + sigaction(SIGTERM, &act, NULL); + while (1) { + ssize_t ret; + + ret = getline(&buff, &buff_len, trace_pipe_fd); + if (ret <= 0) { + p_err("failed to read content from trace pipe: %s", + strerror(errno)); + break; + } + if (json_output) + jsonw_string(json_wtr, buff); + else + printf("%s", buff); + } + + fclose(trace_pipe_fd); + free(buff); + return -1; +} From 7337224fc150b3b762190425399ac0e8dee380d1 Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Wed, 5 Dec 2018 17:35:43 -0800 Subject: [PATCH 26/51] bpf: Improve the info.func_info and info.func_info_rec_size behavior 1) When bpf_dump_raw_ok() == false and the kernel can provide >=1 func_info to the userspace, the current behavior is setting the info.func_info_cnt to 0 instead of setting info.func_info to 0. It is different from the behavior in jited_func_lens/nr_jited_func_lens, jited_ksyms/nr_jited_ksyms...etc. This patch fixes it. (i.e. set func_info to 0 instead of func_info_cnt to 0 when bpf_dump_raw_ok() == false). 2) When the userspace passed in info.func_info_cnt == 0, the kernel will set the expected func_info size back to the info.func_info_rec_size. It is a way for the userspace to learn the kernel expected func_info_rec_size introduced in commit 838e96904ff3 ("bpf: Introduce bpf_func_info"). An exception is the kernel expected size is not set when func_info is not available for a bpf_prog. This makes the returned info.func_info_rec_size has different values depending on the returned value of info.func_info_cnt. This patch sets the kernel expected size to info.func_info_rec_size independent of the info.func_info_cnt. 3) The current logic only rejects invalid func_info_rec_size if func_info_cnt is non zero. This patch also rejects invalid nonzero info.func_info_rec_size and not equal to the kernel expected size. 4) Set info.btf_id as long as prog->aux->btf != NULL. That will setup the later copy_to_user() codes look the same as others which then easier to understand and maintain. prog->aux->btf is not NULL only if prog->aux->func_info_cnt > 0. Breaking up info.btf_id from prog->aux->func_info_cnt is needed for the later line info patch anyway. A similar change is made to bpf_get_prog_name(). Fixes: 838e96904ff3 ("bpf: Introduce bpf_func_info") Signed-off-by: Martin KaFai Lau Acked-by: Yonghong Song Signed-off-by: Alexei Starovoitov --- kernel/bpf/core.c | 2 +- kernel/bpf/syscall.c | 46 +++++++++++++++++++------------------------- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 86817ab204e8e..628b3970a49b3 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -410,7 +410,7 @@ static void bpf_get_prog_name(const struct bpf_prog *prog, char *sym) sym = bin2hex(sym, prog->tag, sizeof(prog->tag)); /* prog->aux->name will be ignored if full btf name is available */ - if (prog->aux->btf) { + if (prog->aux->func_info_cnt) { type = btf_type_by_id(prog->aux->btf, prog->aux->func_info[prog->aux->func_idx].type_id); func_name = btf_name_by_offset(prog->aux->btf, type->name_off); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 4445d0d084d88..aa05aa38f4a88 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2083,6 +2083,12 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, return -EFAULT; } + if ((info.func_info_cnt || info.func_info_rec_size) && + info.func_info_rec_size != sizeof(struct bpf_func_info)) + return -EINVAL; + + info.func_info_rec_size = sizeof(struct bpf_func_info); + if (!capable(CAP_SYS_ADMIN)) { info.jited_prog_len = 0; info.xlated_prog_len = 0; @@ -2226,35 +2232,23 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, } } - if (prog->aux->btf) { - u32 krec_size = sizeof(struct bpf_func_info); - u32 ucnt, urec_size; - + if (prog->aux->btf) info.btf_id = btf_id(prog->aux->btf); - ucnt = info.func_info_cnt; - info.func_info_cnt = prog->aux->func_info_cnt; - urec_size = info.func_info_rec_size; - info.func_info_rec_size = krec_size; - if (ucnt) { - /* expect passed-in urec_size is what the kernel expects */ - if (urec_size != info.func_info_rec_size) - return -EINVAL; - - if (bpf_dump_raw_ok()) { - char __user *user_finfo; - - user_finfo = u64_to_user_ptr(info.func_info); - ucnt = min_t(u32, info.func_info_cnt, ucnt); - if (copy_to_user(user_finfo, prog->aux->func_info, - krec_size * ucnt)) - return -EFAULT; - } else { - info.func_info_cnt = 0; - } + ulen = info.func_info_cnt; + info.func_info_cnt = prog->aux->func_info_cnt; + if (info.func_info_cnt && ulen) { + if (bpf_dump_raw_ok()) { + char __user *user_finfo; + + user_finfo = u64_to_user_ptr(info.func_info); + ulen = min_t(u32, info.func_info_cnt, ulen); + if (copy_to_user(user_finfo, prog->aux->func_info, + info.func_info_rec_size * ulen)) + return -EFAULT; + } else { + info.func_info = 0; } - } else { - info.func_info_cnt = 0; } done: From d30d42e08c76cb9323ec6121190eb026b07f773b Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Wed, 5 Dec 2018 17:35:44 -0800 Subject: [PATCH 27/51] bpf: Change insn_offset to insn_off in bpf_func_info The later patch will introduce "struct bpf_line_info" which has member "line_off" and "file_off" referring back to the string section in btf. The line_"off" and file_"off" are more consistent to the naming convention in btf.h that means "offset" (e.g. name_off in "struct btf_type"). The to-be-added "struct bpf_line_info" also has another member, "insn_off" which is the same as the "insn_offset" in "struct bpf_func_info". Hence, this patch renames "insn_offset" to "insn_off" for "struct bpf_func_info". Signed-off-by: Martin KaFai Lau Acked-by: Yonghong Song Signed-off-by: Alexei Starovoitov --- include/uapi/linux/bpf.h | 2 +- kernel/bpf/verifier.c | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index c8e1eeee2c5ff..a84fd232d9342 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -2991,7 +2991,7 @@ struct bpf_flow_keys { }; struct bpf_func_info { - __u32 insn_offset; + __u32 insn_off; __u32 type_id; }; diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 71988337ac14e..7658c61c1a884 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4707,24 +4707,24 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env, goto free_btf; } - /* check insn_offset */ + /* check insn_off */ if (i == 0) { - if (krecord[i].insn_offset) { + if (krecord[i].insn_off) { verbose(env, - "nonzero insn_offset %u for the first func info record", - krecord[i].insn_offset); + "nonzero insn_off %u for the first func info record", + krecord[i].insn_off); ret = -EINVAL; goto free_btf; } - } else if (krecord[i].insn_offset <= prev_offset) { + } else if (krecord[i].insn_off <= prev_offset) { verbose(env, "same or smaller insn offset (%u) than previous func info record (%u)", - krecord[i].insn_offset, prev_offset); + krecord[i].insn_off, prev_offset); ret = -EINVAL; goto free_btf; } - if (env->subprog_info[i].start != krecord[i].insn_offset) { + if (env->subprog_info[i].start != krecord[i].insn_off) { verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); ret = -EINVAL; goto free_btf; @@ -4739,7 +4739,7 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env, goto free_btf; } - prev_offset = krecord[i].insn_offset; + prev_offset = krecord[i].insn_off; urecord += urec_size; } @@ -4762,7 +4762,7 @@ static void adjust_btf_func(struct bpf_verifier_env *env) return; for (i = 0; i < env->subprog_cnt; i++) - env->prog->aux->func_info[i].insn_offset = env->subprog_info[i].start; + env->prog->aux->func_info[i].insn_off = env->subprog_info[i].start; } /* check %cur's range satisfies %old's */ From 555249df711b146a2898949ac80503be29fe7340 Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Wed, 5 Dec 2018 17:35:45 -0800 Subject: [PATCH 28/51] bpf: tools: Sync uapi bpf.h for the name changes in bpf_func_info This patch sync the name changes in bpf_func_info to the tools/. Signed-off-by: Martin KaFai Lau Acked-by: Yonghong Song Signed-off-by: Alexei Starovoitov --- tools/include/uapi/linux/bpf.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 64262890feb23..16263e8827fcd 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -2991,7 +2991,7 @@ struct bpf_flow_keys { }; struct bpf_func_info { - __u32 insn_offset; + __u32 insn_off; __u32 type_id; }; From 84ecc1f98ca7ce28ede9bc5cc70a557fdfa09caa Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Wed, 5 Dec 2018 17:35:47 -0800 Subject: [PATCH 29/51] bpf: Expect !info.func_info and insn_off name changes in test_btf/libbpf/bpftool Similar to info.jited_*, info.func_info could be 0 if bpf_dump_raw_ok() == false. This patch makes changes to test_btf and bpftool to expect info.func_info could be 0. This patch also makes the needed changes for s/insn_offset/insn_off/. Signed-off-by: Martin KaFai Lau Acked-by: Yonghong Song Signed-off-by: Alexei Starovoitov --- tools/bpf/bpftool/prog.c | 7 +++++++ tools/bpf/bpftool/xlated_dumper.c | 4 ++-- tools/lib/bpf/btf.c | 12 ++++++------ tools/testing/selftests/bpf/test_btf.c | 8 +++++++- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 54c8dbf05c9cb..a9a51123454c3 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -589,6 +589,13 @@ static int do_dump(int argc, char **argv) goto err_free; } + if (func_info && !info.func_info) { + /* kernel.kptr_restrict is set. No func_info available. */ + free(func_info); + func_info = NULL; + finfo_cnt = 0; + } + if ((member_len == &info.jited_prog_len && info.jited_prog_insns == 0) || (member_len == &info.xlated_prog_len && diff --git a/tools/bpf/bpftool/xlated_dumper.c b/tools/bpf/bpftool/xlated_dumper.c index e06ac0286a750..131ecd1755330 100644 --- a/tools/bpf/bpftool/xlated_dumper.c +++ b/tools/bpf/bpftool/xlated_dumper.c @@ -261,7 +261,7 @@ void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len, jsonw_start_object(json_wtr); if (btf && record) { - if (record->insn_offset == i) { + if (record->insn_off == i) { btf_dumper_type_only(btf, record->type_id, func_sig, sizeof(func_sig)); @@ -330,7 +330,7 @@ void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len, } if (btf && record) { - if (record->insn_offset == i) { + if (record->insn_off == i) { btf_dumper_type_only(btf, record->type_id, func_sig, sizeof(func_sig)); diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index c2d641f3e16ee..85d6446cf8325 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -45,7 +45,7 @@ struct btf_ext { /* The minimum bpf_func_info checked by the loader */ struct bpf_func_info_min { - __u32 insn_offset; + __u32 insn_off; __u32 type_id; }; @@ -670,7 +670,7 @@ int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext, memcpy(data, sinfo->data, records_len); - /* adjust the insn_offset, the data in .BTF.ext is + /* adjust the insn_off, the data in .BTF.ext is * the actual byte offset, and the kernel expects * the offset in term of bpf_insn. * @@ -681,7 +681,7 @@ int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext, struct bpf_func_info_min *record; record = data + i * record_size; - record->insn_offset /= sizeof(struct bpf_insn); + record->insn_off /= sizeof(struct bpf_insn); } *func_info = data; @@ -722,15 +722,15 @@ int btf_ext__reloc(struct btf *btf, struct btf_ext *btf_ext, return -ENOMEM; memcpy(data + existing_flen, sinfo->data, records_len); - /* adjust insn_offset only, the rest data will be passed + /* adjust insn_off only, the rest data will be passed * to the kernel. */ for (i = 0; i < sinfo->num_func_info; i++) { struct bpf_func_info_min *record; record = data + existing_flen + i * record_size; - record->insn_offset = - record->insn_offset / sizeof(struct bpf_insn) + + record->insn_off = + record->insn_off / sizeof(struct bpf_insn) + insns_cnt; } *func_info = data; diff --git a/tools/testing/selftests/bpf/test_btf.c b/tools/testing/selftests/bpf/test_btf.c index bae7308b7ec51..ff0952ea757a4 100644 --- a/tools/testing/selftests/bpf/test_btf.c +++ b/tools/testing/selftests/bpf/test_btf.c @@ -3156,7 +3156,7 @@ static struct btf_func_type_test { }, { - .descr = "func_type (Incorrect bpf_func_info.insn_offset)", + .descr = "func_type (Incorrect bpf_func_info.insn_off)", .raw_types = { BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ BTF_TYPE_INT_ENC(NAME_TBD, 0, 0, 32, 4), /* [2] */ @@ -3303,6 +3303,12 @@ static int do_test_func_type(int test_num) goto done; } + if (CHECK(!info.func_info, + "info.func_info == 0. kernel.kptr_restrict is set?")) { + err = -1; + goto done; + } + finfo = func_info; for (i = 0; i < 2; i++) { if (CHECK(finfo->type_id != test->func_info[i][1], From 17f6c83fb5ebf7db4fcc94a5be4c22d5a7bfe428 Mon Sep 17 00:00:00 2001 From: Jiong Wang Date: Mon, 3 Dec 2018 17:27:54 -0500 Subject: [PATCH 30/51] mips: bpf: fix encoding bug for mm_srlv32_op For micro-mips, srlv inside POOL32A encoding space should use 0x50 sub-opcode, NOT 0x90. Some early version ISA doc describes the encoding as 0x90 for both srlv and srav, this looks to me was a typo. I checked Binutils libopcode implementation which is using 0x50 for srlv and 0x90 for srav. v1->v2: - Keep mm_srlv32_op sorted by value. Fixes: f31318fdf324 ("MIPS: uasm: Add srlv uasm instruction") Cc: Markos Chandras Cc: Paul Burton Cc: linux-mips@vger.kernel.org Acked-by: Jakub Kicinski Acked-by: Song Liu Signed-off-by: Jiong Wang Signed-off-by: Alexei Starovoitov --- arch/mips/include/uapi/asm/inst.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/include/uapi/asm/inst.h b/arch/mips/include/uapi/asm/inst.h index c05dcf5ab414e..273ef58f4d435 100644 --- a/arch/mips/include/uapi/asm/inst.h +++ b/arch/mips/include/uapi/asm/inst.h @@ -369,8 +369,8 @@ enum mm_32a_minor_op { mm_ext_op = 0x02c, mm_pool32axf_op = 0x03c, mm_srl32_op = 0x040, + mm_srlv32_op = 0x050, mm_sra_op = 0x080, - mm_srlv32_op = 0x090, mm_rotr_op = 0x0c0, mm_lwxs_op = 0x118, mm_addu32_op = 0x150, From ee94b90c8acaa593b627f5f3fe93e076e7779f5c Mon Sep 17 00:00:00 2001 From: Jiong Wang Date: Wed, 5 Dec 2018 13:52:30 -0500 Subject: [PATCH 31/51] mips: bpf: implement jitting of BPF_ALU | BPF_ARSH | BPF_X Jitting of BPF_K is supported already, but not BPF_X. This patch complete the support for the latter on both MIPS and microMIPS. Cc: Paul Burton Cc: linux-mips@vger.kernel.org Acked-by: Paul Burton Signed-off-by: Jiong Wang Signed-off-by: Alexei Starovoitov --- arch/mips/include/asm/uasm.h | 1 + arch/mips/include/uapi/asm/inst.h | 1 + arch/mips/mm/uasm-micromips.c | 1 + arch/mips/mm/uasm-mips.c | 1 + arch/mips/mm/uasm.c | 9 +++++---- arch/mips/net/ebpf_jit.c | 4 ++++ 6 files changed, 13 insertions(+), 4 deletions(-) diff --git a/arch/mips/include/asm/uasm.h b/arch/mips/include/asm/uasm.h index 59dae37f6b8dd..b1990dd75f274 100644 --- a/arch/mips/include/asm/uasm.h +++ b/arch/mips/include/asm/uasm.h @@ -157,6 +157,7 @@ Ip_u2u1s3(_slti); Ip_u2u1s3(_sltiu); Ip_u3u1u2(_sltu); Ip_u2u1u3(_sra); +Ip_u3u2u1(_srav); Ip_u2u1u3(_srl); Ip_u3u2u1(_srlv); Ip_u3u1u2(_subu); diff --git a/arch/mips/include/uapi/asm/inst.h b/arch/mips/include/uapi/asm/inst.h index 273ef58f4d435..40fbb5dd66dfb 100644 --- a/arch/mips/include/uapi/asm/inst.h +++ b/arch/mips/include/uapi/asm/inst.h @@ -371,6 +371,7 @@ enum mm_32a_minor_op { mm_srl32_op = 0x040, mm_srlv32_op = 0x050, mm_sra_op = 0x080, + mm_srav_op = 0x090, mm_rotr_op = 0x0c0, mm_lwxs_op = 0x118, mm_addu32_op = 0x150, diff --git a/arch/mips/mm/uasm-micromips.c b/arch/mips/mm/uasm-micromips.c index 24e5b0d068995..75ef90486fe6c 100644 --- a/arch/mips/mm/uasm-micromips.c +++ b/arch/mips/mm/uasm-micromips.c @@ -104,6 +104,7 @@ static const struct insn insn_table_MM[insn_invalid] = { [insn_sltiu] = {M(mm_sltiu32_op, 0, 0, 0, 0, 0), RT | RS | SIMM}, [insn_sltu] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_sltu_op), RT | RS | RD}, [insn_sra] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_sra_op), RT | RS | RD}, + [insn_srav] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_srav_op), RT | RS | RD}, [insn_srl] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_srl32_op), RT | RS | RD}, [insn_srlv] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_srlv32_op), RT | RS | RD}, [insn_rotr] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_rotr_op), RT | RS | RD}, diff --git a/arch/mips/mm/uasm-mips.c b/arch/mips/mm/uasm-mips.c index 60ceb93c71a08..6abe40fc413dd 100644 --- a/arch/mips/mm/uasm-mips.c +++ b/arch/mips/mm/uasm-mips.c @@ -171,6 +171,7 @@ static const struct insn insn_table[insn_invalid] = { [insn_sltiu] = {M(sltiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM}, [insn_sltu] = {M(spec_op, 0, 0, 0, 0, sltu_op), RS | RT | RD}, [insn_sra] = {M(spec_op, 0, 0, 0, 0, sra_op), RT | RD | RE}, + [insn_srav] = {M(spec_op, 0, 0, 0, 0, srav_op), RS | RT | RD}, [insn_srl] = {M(spec_op, 0, 0, 0, 0, srl_op), RT | RD | RE}, [insn_srlv] = {M(spec_op, 0, 0, 0, 0, srlv_op), RS | RT | RD}, [insn_subu] = {M(spec_op, 0, 0, 0, 0, subu_op), RS | RT | RD}, diff --git a/arch/mips/mm/uasm.c b/arch/mips/mm/uasm.c index 57570c0649b46..45b6264ff308f 100644 --- a/arch/mips/mm/uasm.c +++ b/arch/mips/mm/uasm.c @@ -61,10 +61,10 @@ enum opcode { insn_mthc0, insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_nor, insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb, insn_sc, insn_scd, insn_sd, insn_sh, insn_sll, insn_sllv, - insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra, insn_srl, - insn_srlv, insn_subu, insn_sw, insn_sync, insn_syscall, insn_tlbp, - insn_tlbr, insn_tlbwi, insn_tlbwr, insn_wait, insn_wsbh, insn_xor, - insn_xori, insn_yield, + insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra, insn_srav, + insn_srl, insn_srlv, insn_subu, insn_sw, insn_sync, insn_syscall, + insn_tlbp, insn_tlbr, insn_tlbwi, insn_tlbwr, insn_wait, insn_wsbh, + insn_xor, insn_xori, insn_yield, insn_invalid /* insn_invalid must be last */ }; @@ -353,6 +353,7 @@ I_u2u1s3(_slti) I_u2u1s3(_sltiu) I_u3u1u2(_sltu) I_u2u1u3(_sra) +I_u3u2u1(_srav) I_u2u1u3(_srl) I_u3u2u1(_srlv) I_u2u1u3(_rotr) diff --git a/arch/mips/net/ebpf_jit.c b/arch/mips/net/ebpf_jit.c index aeb7b1b0f2024..b16710a8a9e7a 100644 --- a/arch/mips/net/ebpf_jit.c +++ b/arch/mips/net/ebpf_jit.c @@ -854,6 +854,7 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, case BPF_ALU | BPF_MOD | BPF_X: /* ALU_REG */ case BPF_ALU | BPF_LSH | BPF_X: /* ALU_REG */ case BPF_ALU | BPF_RSH | BPF_X: /* ALU_REG */ + case BPF_ALU | BPF_ARSH | BPF_X: /* ALU_REG */ src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp); dst = ebpf_to_mips_reg(ctx, insn, dst_reg); if (src < 0 || dst < 0) @@ -913,6 +914,9 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, case BPF_RSH: emit_instr(ctx, srlv, dst, dst, src); break; + case BPF_ARSH: + emit_instr(ctx, srav, dst, dst, src); + break; default: pr_err("ALU_REG NOT HANDLED\n"); return -EINVAL; From 44cf43c04bb5f7c688608ff4136b13f2a8a7a129 Mon Sep 17 00:00:00 2001 From: Jiong Wang Date: Wed, 5 Dec 2018 13:52:31 -0500 Subject: [PATCH 32/51] ppc: bpf: implement jitting of BPF_ALU | BPF_ARSH | BPF_* This patch implements code-gen for BPF_ALU | BPF_ARSH | BPF_*. Cc: Naveen N. Rao Cc: Sandipan Das Signed-off-by: Jiong Wang Reviewed-by: Sandipan Das Signed-off-by: Alexei Starovoitov --- arch/powerpc/include/asm/ppc-opcode.h | 2 ++ arch/powerpc/net/bpf_jit.h | 4 ++++ arch/powerpc/net/bpf_jit_comp64.c | 6 ++++++ 3 files changed, 12 insertions(+) diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h index a6e9e314c7077..901459226eca3 100644 --- a/arch/powerpc/include/asm/ppc-opcode.h +++ b/arch/powerpc/include/asm/ppc-opcode.h @@ -342,6 +342,8 @@ #define PPC_INST_SLW 0x7c000030 #define PPC_INST_SLD 0x7c000036 #define PPC_INST_SRW 0x7c000430 +#define PPC_INST_SRAW 0x7c000630 +#define PPC_INST_SRAWI 0x7c000670 #define PPC_INST_SRD 0x7c000436 #define PPC_INST_SRAD 0x7c000634 #define PPC_INST_SRADI 0x7c000674 diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h index 47fc6660845d3..c2d5192ed64fa 100644 --- a/arch/powerpc/net/bpf_jit.h +++ b/arch/powerpc/net/bpf_jit.h @@ -152,6 +152,10 @@ ___PPC_RS(a) | ___PPC_RB(s)) #define PPC_SRW(d, a, s) EMIT(PPC_INST_SRW | ___PPC_RA(d) | \ ___PPC_RS(a) | ___PPC_RB(s)) +#define PPC_SRAW(d, a, s) EMIT(PPC_INST_SRAW | ___PPC_RA(d) | \ + ___PPC_RS(a) | ___PPC_RB(s)) +#define PPC_SRAWI(d, a, i) EMIT(PPC_INST_SRAWI | ___PPC_RA(d) | \ + ___PPC_RS(a) | __PPC_SH(i)) #define PPC_SRD(d, a, s) EMIT(PPC_INST_SRD | ___PPC_RA(d) | \ ___PPC_RS(a) | ___PPC_RB(s)) #define PPC_SRAD(d, a, s) EMIT(PPC_INST_SRAD | ___PPC_RA(d) | \ diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 17482f5de3e26..7dc81877057d8 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -529,9 +529,15 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, if (imm != 0) PPC_SRDI(dst_reg, dst_reg, imm); break; + case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */ + PPC_SRAW(dst_reg, dst_reg, src_reg); + goto bpf_alu32_trunc; case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */ PPC_SRAD(dst_reg, dst_reg, src_reg); break; + case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */ + PPC_SRAWI(dst_reg, dst_reg, imm); + goto bpf_alu32_trunc; case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */ if (imm != 0) PPC_SRADI(dst_reg, dst_reg, imm); From f860203b010ab11995b3073a96cc0d04e520129e Mon Sep 17 00:00:00 2001 From: Jiong Wang Date: Wed, 5 Dec 2018 13:52:32 -0500 Subject: [PATCH 33/51] s390: bpf: implement jitting of BPF_ALU | BPF_ARSH | BPF_* This patch implements code-gen for BPF_ALU | BPF_ARSH | BPF_*. Cc: Martin Schwidefsky Cc: Heiko Carstens Signed-off-by: Jiong Wang Signed-off-by: Alexei Starovoitov --- arch/s390/net/bpf_jit_comp.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index d7052cbe984f8..3ff758eeb71d2 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -821,10 +821,22 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i /* * BPF_ARSH */ + case BPF_ALU | BPF_ARSH | BPF_X: /* ((s32) dst) >>= src */ + /* sra %dst,%dst,0(%src) */ + EMIT4_DISP(0x8a000000, dst_reg, src_reg, 0); + EMIT_ZERO(dst_reg); + break; case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */ /* srag %dst,%dst,0(%src) */ EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0); break; + case BPF_ALU | BPF_ARSH | BPF_K: /* ((s32) dst >> imm */ + if (imm == 0) + break; + /* sra %dst,imm(%r0) */ + EMIT4_DISP(0x8a000000, dst_reg, REG_0, imm); + EMIT_ZERO(dst_reg); + break; case BPF_ALU64 | BPF_ARSH | BPF_K: /* ((s64) dst) >>= imm */ if (imm == 0) break; From 84708c1386770e957347d847522a4aac22ea04dc Mon Sep 17 00:00:00 2001 From: Jiong Wang Date: Wed, 5 Dec 2018 13:52:33 -0500 Subject: [PATCH 34/51] nfp: bpf: implement jitting of BPF_ALU | BPF_ARSH | BPF_* BPF_X support needs indirect shift mode, please see code comments for details. Reviewed-by: Jakub Kicinski Signed-off-by: Jiong Wang Signed-off-by: Alexei Starovoitov --- drivers/net/ethernet/netronome/nfp/bpf/jit.c | 45 ++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/drivers/net/ethernet/netronome/nfp/bpf/jit.c b/drivers/net/ethernet/netronome/nfp/bpf/jit.c index 97d33bb4d84d7..662cbc21d9093 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/jit.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/jit.c @@ -2382,6 +2382,49 @@ static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) return 0; } +static int __ashr_imm(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt) +{ + /* Set signedness bit (MSB of result). */ + emit_alu(nfp_prog, reg_none(), reg_a(dst), ALU_OP_OR, reg_imm(0)); + emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR, reg_b(dst), + SHF_SC_R_SHF, shift_amt); + wrp_immed(nfp_prog, reg_both(dst + 1), 0); + + return 0; +} + +static int ashr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) +{ + const struct bpf_insn *insn = &meta->insn; + u64 umin, umax; + u8 dst, src; + + dst = insn->dst_reg * 2; + umin = meta->umin_src; + umax = meta->umax_src; + if (umin == umax) + return __ashr_imm(nfp_prog, dst, umin); + + src = insn->src_reg * 2; + /* NOTE: the first insn will set both indirect shift amount (source A) + * and signedness bit (MSB of result). + */ + emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst)); + emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR, + reg_b(dst), SHF_SC_R_SHF); + wrp_immed(nfp_prog, reg_both(dst + 1), 0); + + return 0; +} + +static int ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) +{ + const struct bpf_insn *insn = &meta->insn; + u8 dst = insn->dst_reg * 2; + + return __ashr_imm(nfp_prog, dst, insn->imm); +} + static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) { const struct bpf_insn *insn = &meta->insn; @@ -3286,6 +3329,8 @@ static const instr_cb_t instr_cb[256] = { [BPF_ALU | BPF_DIV | BPF_K] = div_imm, [BPF_ALU | BPF_NEG] = neg_reg, [BPF_ALU | BPF_LSH | BPF_K] = shl_imm, + [BPF_ALU | BPF_ARSH | BPF_X] = ashr_reg, + [BPF_ALU | BPF_ARSH | BPF_K] = ashr_imm, [BPF_ALU | BPF_END | BPF_X] = end_reg32, [BPF_LD | BPF_IMM | BPF_DW] = imm_ld8, [BPF_LD | BPF_ABS | BPF_B] = data_ld1, From 2dc6b100f928aac8d7532bf7112d3f8d3f952bad Mon Sep 17 00:00:00 2001 From: Jiong Wang Date: Wed, 5 Dec 2018 13:52:34 -0500 Subject: [PATCH 35/51] bpf: interpreter support BPF_ALU | BPF_ARSH This patch implements interpreting BPF_ALU | BPF_ARSH. Do arithmetic right shift on low 32-bit sub-register, and zero the high 32 bits. Reviewed-by: Jakub Kicinski Signed-off-by: Jiong Wang Signed-off-by: Alexei Starovoitov --- kernel/bpf/core.c | 52 +++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 628b3970a49b3..a5b223ef71310 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -933,32 +933,34 @@ EXPORT_SYMBOL_GPL(__bpf_call_base); #define BPF_INSN_MAP(INSN_2, INSN_3) \ /* 32 bit ALU operations. */ \ /* Register based. */ \ - INSN_3(ALU, ADD, X), \ - INSN_3(ALU, SUB, X), \ - INSN_3(ALU, AND, X), \ - INSN_3(ALU, OR, X), \ - INSN_3(ALU, LSH, X), \ - INSN_3(ALU, RSH, X), \ - INSN_3(ALU, XOR, X), \ - INSN_3(ALU, MUL, X), \ - INSN_3(ALU, MOV, X), \ - INSN_3(ALU, DIV, X), \ - INSN_3(ALU, MOD, X), \ + INSN_3(ALU, ADD, X), \ + INSN_3(ALU, SUB, X), \ + INSN_3(ALU, AND, X), \ + INSN_3(ALU, OR, X), \ + INSN_3(ALU, LSH, X), \ + INSN_3(ALU, RSH, X), \ + INSN_3(ALU, XOR, X), \ + INSN_3(ALU, MUL, X), \ + INSN_3(ALU, MOV, X), \ + INSN_3(ALU, ARSH, X), \ + INSN_3(ALU, DIV, X), \ + INSN_3(ALU, MOD, X), \ INSN_2(ALU, NEG), \ INSN_3(ALU, END, TO_BE), \ INSN_3(ALU, END, TO_LE), \ /* Immediate based. */ \ - INSN_3(ALU, ADD, K), \ - INSN_3(ALU, SUB, K), \ - INSN_3(ALU, AND, K), \ - INSN_3(ALU, OR, K), \ - INSN_3(ALU, LSH, K), \ - INSN_3(ALU, RSH, K), \ - INSN_3(ALU, XOR, K), \ - INSN_3(ALU, MUL, K), \ - INSN_3(ALU, MOV, K), \ - INSN_3(ALU, DIV, K), \ - INSN_3(ALU, MOD, K), \ + INSN_3(ALU, ADD, K), \ + INSN_3(ALU, SUB, K), \ + INSN_3(ALU, AND, K), \ + INSN_3(ALU, OR, K), \ + INSN_3(ALU, LSH, K), \ + INSN_3(ALU, RSH, K), \ + INSN_3(ALU, XOR, K), \ + INSN_3(ALU, MUL, K), \ + INSN_3(ALU, MOV, K), \ + INSN_3(ALU, ARSH, K), \ + INSN_3(ALU, DIV, K), \ + INSN_3(ALU, MOD, K), \ /* 64 bit ALU operations. */ \ /* Register based. */ \ INSN_3(ALU64, ADD, X), \ @@ -1137,6 +1139,12 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn, u64 *stack) DST = (u64) (u32) insn[0].imm | ((u64) (u32) insn[1].imm) << 32; insn++; CONT; + ALU_ARSH_X: + DST = (u64) (u32) ((*(s32 *) &DST) >> SRC); + CONT; + ALU_ARSH_K: + DST = (u64) (u32) ((*(s32 *) &DST) >> IMM); + CONT; ALU64_ARSH_X: (*(s64 *) &DST) >>= SRC; CONT; From c49f7dbd4f9c2c49df7fc0f5b50c1350ee7e01ee Mon Sep 17 00:00:00 2001 From: Jiong Wang Date: Wed, 5 Dec 2018 13:52:35 -0500 Subject: [PATCH 36/51] bpf: verifier remove the rejection on BPF_ALU | BPF_ARSH This patch remove the rejection on BPF_ALU | BPF_ARSH as we have supported them on interpreter and all JIT back-ends Reviewed-by: Jakub Kicinski Signed-off-by: Jiong Wang Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7658c61c1a884..2752d35ad0738 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3649,11 +3649,6 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) return -EINVAL; } - if (opcode == BPF_ARSH && BPF_CLASS(insn->code) != BPF_ALU64) { - verbose(env, "BPF_ARSH not supported for 32 bit ALU\n"); - return -EINVAL; - } - if ((opcode == BPF_LSH || opcode == BPF_RSH || opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) { int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; From c099f3f413f354ecdd518e4be6a232592e91d5b9 Mon Sep 17 00:00:00 2001 From: Jiong Wang Date: Wed, 5 Dec 2018 13:52:36 -0500 Subject: [PATCH 37/51] selftests: bpf: update testcases for BPF_ALU | BPF_ARSH "arsh32 on imm" and "arsh32 on reg" now are accepted. Also added two new testcases to make sure arsh32 won't be treated as arsh64 during interpretation or JIT code-gen for which case the high bits will be moved into low halve that the testcases could catch them. Reviewed-by: Jakub Kicinski Signed-off-by: Jiong Wang Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/test_verifier.c | 29 ++++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index b4b4a3f936396..36ce58b4933eb 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -721,8 +721,18 @@ static struct bpf_test tests[] = { BPF_ALU32_IMM(BPF_ARSH, BPF_REG_0, 5), BPF_EXIT_INSN(), }, - .result = REJECT, - .errstr = "unknown opcode c4", + .result = ACCEPT, + .retval = 0, + }, + { + "arsh32 on imm 2", + .insns = { + BPF_LD_IMM64(BPF_REG_0, 0x1122334485667788), + BPF_ALU32_IMM(BPF_ARSH, BPF_REG_0, 7), + BPF_EXIT_INSN(), + }, + .result = ACCEPT, + .retval = -16069393, }, { "arsh32 on reg", @@ -732,8 +742,19 @@ static struct bpf_test tests[] = { BPF_ALU32_REG(BPF_ARSH, BPF_REG_0, BPF_REG_1), BPF_EXIT_INSN(), }, - .result = REJECT, - .errstr = "unknown opcode cc", + .result = ACCEPT, + .retval = 0, + }, + { + "arsh32 on reg 2", + .insns = { + BPF_LD_IMM64(BPF_REG_0, 0xffff55667788), + BPF_MOV64_IMM(BPF_REG_1, 15), + BPF_ALU32_REG(BPF_ARSH, BPF_REG_0, BPF_REG_1), + BPF_EXIT_INSN(), + }, + .result = ACCEPT, + .retval = 43724, }, { "arsh64 on imm", From c454a46b5efd8eff8880e88ece2976e60a26bf35 Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Fri, 7 Dec 2018 16:42:25 -0800 Subject: [PATCH 38/51] bpf: Add bpf_line_info support This patch adds bpf_line_info support. It accepts an array of bpf_line_info objects during BPF_PROG_LOAD. The "line_info", "line_info_cnt" and "line_info_rec_size" are added to the "union bpf_attr". The "line_info_rec_size" makes bpf_line_info extensible in the future. The new "check_btf_line()" ensures the userspace line_info is valid for the kernel to use. When the verifier is translating/patching the bpf_prog (through "bpf_patch_insn_single()"), the line_infos' insn_off is also adjusted by the newly added "bpf_adj_linfo()". If the bpf_prog is jited, this patch also provides the jited addrs (in aux->jited_linfo) for the corresponding line_info.insn_off. "bpf_prog_fill_jited_linfo()" is added to fill the aux->jited_linfo. It is currently called by the x86 jit. Other jits can also use "bpf_prog_fill_jited_linfo()" and it will be done in the followup patches. In the future, if it deemed necessary, a particular jit could also provide its own "bpf_prog_fill_jited_linfo()" implementation. A few "*line_info*" fields are added to the bpf_prog_info such that the user can get the xlated line_info back (i.e. the line_info with its insn_off reflecting the translated prog). The jited_line_info is available if the prog is jited. It is an array of __u64. If the prog is not jited, jited_line_info_cnt is 0. The verifier's verbose log with line_info will be done in a follow up patch. Signed-off-by: Martin KaFai Lau Acked-by: Yonghong Song Signed-off-by: Alexei Starovoitov --- arch/x86/net/bpf_jit_comp.c | 2 + include/linux/bpf.h | 21 ++++ include/linux/bpf_verifier.h | 1 + include/linux/btf.h | 1 + include/linux/filter.h | 7 ++ include/uapi/linux/bpf.h | 19 ++++ kernel/bpf/btf.c | 2 +- kernel/bpf/core.c | 118 ++++++++++++++++++++- kernel/bpf/syscall.c | 83 +++++++++++++-- kernel/bpf/verifier.c | 198 ++++++++++++++++++++++++++++++----- 10 files changed, 419 insertions(+), 33 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 2580cd2e98b17..5542303c43d9c 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1181,6 +1181,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) } if (!image || !prog->is_func || extra_pass) { + if (image) + bpf_prog_fill_jited_linfo(prog, addrs); out_addrs: kfree(addrs); kfree(jit_data); diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e82b7039fc66d..0c992b86eb2c4 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -319,7 +319,28 @@ struct bpf_prog_aux { struct bpf_prog_offload *offload; struct btf *btf; struct bpf_func_info *func_info; + /* bpf_line_info loaded from userspace. linfo->insn_off + * has the xlated insn offset. + * Both the main and sub prog share the same linfo. + * The subprog can access its first linfo by + * using the linfo_idx. + */ + struct bpf_line_info *linfo; + /* jited_linfo is the jited addr of the linfo. It has a + * one to one mapping to linfo: + * jited_linfo[i] is the jited addr for the linfo[i]->insn_off. + * Both the main and sub prog share the same jited_linfo. + * The subprog can access its first jited_linfo by + * using the linfo_idx. + */ + void **jited_linfo; u32 func_info_cnt; + u32 nr_linfo; + /* subprog can use linfo_idx to access its first linfo and + * jited_linfo. + * main prog always has linfo_idx == 0 + */ + u32 linfo_idx; union { struct work_struct work; struct rcu_head rcu; diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 11f5df1092d9b..c736945be7c5e 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -203,6 +203,7 @@ static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log) struct bpf_subprog_info { u32 start; /* insn idx of function entry point */ + u32 linfo_idx; /* The idx to the main_prog->aux->linfo */ u16 stack_depth; /* max. stack depth used by this function */ }; diff --git a/include/linux/btf.h b/include/linux/btf.h index 8c2199b5d2503..b98405a563832 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -46,6 +46,7 @@ void btf_type_seq_show(const struct btf *btf, u32 type_id, void *obj, struct seq_file *m); int btf_get_fd_by_id(u32 id); u32 btf_id(const struct btf *btf); +bool btf_name_offset_valid(const struct btf *btf, u32 offset); #ifdef CONFIG_BPF_SYSCALL const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id); diff --git a/include/linux/filter.h b/include/linux/filter.h index d16deead65c60..29f21f9d7f68f 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -718,6 +718,13 @@ void bpf_prog_free(struct bpf_prog *fp); bool bpf_opcode_in_insntable(u8 code); +void bpf_prog_free_linfo(struct bpf_prog *prog); +void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, + const u32 *insn_to_jit_off); +int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog); +void bpf_prog_free_jited_linfo(struct bpf_prog *prog); +void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog); + struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags); struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, gfp_t gfp_extra_flags); diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index a84fd232d9342..7a66db8d15d56 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -356,6 +356,9 @@ union bpf_attr { __u32 func_info_rec_size; /* userspace bpf_func_info size */ __aligned_u64 func_info; /* func info */ __u32 func_info_cnt; /* number of bpf_func_info records */ + __u32 line_info_rec_size; /* userspace bpf_line_info size */ + __aligned_u64 line_info; /* line info */ + __u32 line_info_cnt; /* number of bpf_line_info records */ }; struct { /* anonymous struct used by BPF_OBJ_* commands */ @@ -2679,6 +2682,12 @@ struct bpf_prog_info { __u32 func_info_rec_size; __aligned_u64 func_info; __u32 func_info_cnt; + __u32 line_info_cnt; + __aligned_u64 line_info; + __aligned_u64 jited_line_info; + __u32 jited_line_info_cnt; + __u32 line_info_rec_size; + __u32 jited_line_info_rec_size; } __attribute__((aligned(8))); struct bpf_map_info { @@ -2995,4 +3004,14 @@ struct bpf_func_info { __u32 type_id; }; +#define BPF_LINE_INFO_LINE_NUM(line_col) ((line_col) >> 10) +#define BPF_LINE_INFO_LINE_COL(line_col) ((line_col) & 0x3ff) + +struct bpf_line_info { + __u32 insn_off; + __u32 file_name_off; + __u32 line_off; + __u32 line_col; +}; + #endif /* _UAPI__LINUX_BPF_H__ */ diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index a09b2f94ab25b..e0a827f95e19b 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -444,7 +444,7 @@ static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t) return kind_ops[BTF_INFO_KIND(t->info)]; } -static bool btf_name_offset_valid(const struct btf *btf, u32 offset) +bool btf_name_offset_valid(const struct btf *btf, u32 offset) { return BTF_STR_OFFSET_VALID(offset) && offset < btf->hdr.str_len; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index a5b223ef71310..5cdd8da0e7f21 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -105,6 +105,91 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) } EXPORT_SYMBOL_GPL(bpf_prog_alloc); +int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) +{ + if (!prog->aux->nr_linfo || !prog->jit_requested) + return 0; + + prog->aux->jited_linfo = kcalloc(prog->aux->nr_linfo, + sizeof(*prog->aux->jited_linfo), + GFP_KERNEL | __GFP_NOWARN); + if (!prog->aux->jited_linfo) + return -ENOMEM; + + return 0; +} + +void bpf_prog_free_jited_linfo(struct bpf_prog *prog) +{ + kfree(prog->aux->jited_linfo); + prog->aux->jited_linfo = NULL; +} + +void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog) +{ + if (prog->aux->jited_linfo && !prog->aux->jited_linfo[0]) + bpf_prog_free_jited_linfo(prog); +} + +/* The jit engine is responsible to provide an array + * for insn_off to the jited_off mapping (insn_to_jit_off). + * + * The idx to this array is the insn_off. Hence, the insn_off + * here is relative to the prog itself instead of the main prog. + * This array has one entry for each xlated bpf insn. + * + * jited_off is the byte off to the last byte of the jited insn. + * + * Hence, with + * insn_start: + * The first bpf insn off of the prog. The insn off + * here is relative to the main prog. + * e.g. if prog is a subprog, insn_start > 0 + * linfo_idx: + * The prog's idx to prog->aux->linfo and jited_linfo + * + * jited_linfo[linfo_idx] = prog->bpf_func + * + * For i > linfo_idx, + * + * jited_linfo[i] = prog->bpf_func + + * insn_to_jit_off[linfo[i].insn_off - insn_start - 1] + */ +void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, + const u32 *insn_to_jit_off) +{ + u32 linfo_idx, insn_start, insn_end, nr_linfo, i; + const struct bpf_line_info *linfo; + void **jited_linfo; + + if (!prog->aux->jited_linfo) + /* Userspace did not provide linfo */ + return; + + linfo_idx = prog->aux->linfo_idx; + linfo = &prog->aux->linfo[linfo_idx]; + insn_start = linfo[0].insn_off; + insn_end = insn_start + prog->len; + + jited_linfo = &prog->aux->jited_linfo[linfo_idx]; + jited_linfo[0] = prog->bpf_func; + + nr_linfo = prog->aux->nr_linfo - linfo_idx; + + for (i = 1; i < nr_linfo && linfo[i].insn_off < insn_end; i++) + /* The verifier ensures that linfo[i].insn_off is + * strictly increasing + */ + jited_linfo[i] = prog->bpf_func + + insn_to_jit_off[linfo[i].insn_off - insn_start - 1]; +} + +void bpf_prog_free_linfo(struct bpf_prog *prog) +{ + bpf_prog_free_jited_linfo(prog); + kvfree(prog->aux->linfo); +} + struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, gfp_t gfp_extra_flags) { @@ -294,6 +379,26 @@ static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, u32 delta, return ret; } +static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta) +{ + struct bpf_line_info *linfo; + u32 i, nr_linfo; + + nr_linfo = prog->aux->nr_linfo; + if (!nr_linfo || !delta) + return; + + linfo = prog->aux->linfo; + + for (i = 0; i < nr_linfo; i++) + if (off < linfo[i].insn_off) + break; + + /* Push all off < linfo[i].insn_off by delta */ + for (; i < nr_linfo; i++) + linfo[i].insn_off += delta; +} + struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, const struct bpf_insn *patch, u32 len) { @@ -349,6 +454,8 @@ struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, */ BUG_ON(bpf_adj_branches(prog_adj, off, insn_delta, false)); + bpf_adj_linfo(prog_adj, off, insn_delta); + return prog_adj; } @@ -1591,13 +1698,20 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) * be JITed, but falls back to the interpreter. */ if (!bpf_prog_is_dev_bound(fp->aux)) { + *err = bpf_prog_alloc_jited_linfo(fp); + if (*err) + return fp; + fp = bpf_int_jit_compile(fp); -#ifdef CONFIG_BPF_JIT_ALWAYS_ON if (!fp->jited) { + bpf_prog_free_jited_linfo(fp); +#ifdef CONFIG_BPF_JIT_ALWAYS_ON *err = -ENOTSUPP; return fp; - } #endif + } else { + bpf_prog_free_unused_jited_linfo(fp); + } } else { *err = bpf_prog_offload_compile(fp); if (*err) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index aa05aa38f4a88..19c88cff78803 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1215,6 +1215,7 @@ static void __bpf_prog_put(struct bpf_prog *prog, bool do_idr_lock) bpf_prog_kallsyms_del_all(prog); btf_put(prog->aux->btf); kvfree(prog->aux->func_info); + bpf_prog_free_linfo(prog); call_rcu(&prog->aux->rcu, __bpf_prog_put_rcu); } @@ -1439,7 +1440,7 @@ bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type, } /* last field in 'union bpf_attr' used by this command */ -#define BPF_PROG_LOAD_LAST_FIELD func_info_cnt +#define BPF_PROG_LOAD_LAST_FIELD line_info_cnt static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) { @@ -1560,6 +1561,7 @@ static int bpf_prog_load(union bpf_attr *attr, union bpf_attr __user *uattr) return err; free_used_maps: + bpf_prog_free_linfo(prog); kvfree(prog->aux->func_info); btf_put(prog->aux->btf); bpf_prog_kallsyms_del_subprogs(prog); @@ -2041,6 +2043,37 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog) return insns; } +static int set_info_rec_size(struct bpf_prog_info *info) +{ + /* + * Ensure info.*_rec_size is the same as kernel expected size + * + * or + * + * Only allow zero *_rec_size if both _rec_size and _cnt are + * zero. In this case, the kernel will set the expected + * _rec_size back to the info. + */ + + if ((info->func_info_cnt || info->func_info_rec_size) && + info->func_info_rec_size != sizeof(struct bpf_func_info)) + return -EINVAL; + + if ((info->line_info_cnt || info->line_info_rec_size) && + info->line_info_rec_size != sizeof(struct bpf_line_info)) + return -EINVAL; + + if ((info->jited_line_info_cnt || info->jited_line_info_rec_size) && + info->jited_line_info_rec_size != sizeof(__u64)) + return -EINVAL; + + info->func_info_rec_size = sizeof(struct bpf_func_info); + info->line_info_rec_size = sizeof(struct bpf_line_info); + info->jited_line_info_rec_size = sizeof(__u64); + + return 0; +} + static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, const union bpf_attr *attr, union bpf_attr __user *uattr) @@ -2083,11 +2116,9 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, return -EFAULT; } - if ((info.func_info_cnt || info.func_info_rec_size) && - info.func_info_rec_size != sizeof(struct bpf_func_info)) - return -EINVAL; - - info.func_info_rec_size = sizeof(struct bpf_func_info); + err = set_info_rec_size(&info); + if (err) + return err; if (!capable(CAP_SYS_ADMIN)) { info.jited_prog_len = 0; @@ -2095,6 +2126,8 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, info.nr_jited_ksyms = 0; info.nr_jited_func_lens = 0; info.func_info_cnt = 0; + info.line_info_cnt = 0; + info.jited_line_info_cnt = 0; goto done; } @@ -2251,6 +2284,44 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, } } + ulen = info.line_info_cnt; + info.line_info_cnt = prog->aux->nr_linfo; + if (info.line_info_cnt && ulen) { + if (bpf_dump_raw_ok()) { + __u8 __user *user_linfo; + + user_linfo = u64_to_user_ptr(info.line_info); + ulen = min_t(u32, info.line_info_cnt, ulen); + if (copy_to_user(user_linfo, prog->aux->linfo, + info.line_info_rec_size * ulen)) + return -EFAULT; + } else { + info.line_info = 0; + } + } + + ulen = info.jited_line_info_cnt; + if (prog->aux->jited_linfo) + info.jited_line_info_cnt = prog->aux->nr_linfo; + else + info.jited_line_info_cnt = 0; + if (info.jited_line_info_cnt && ulen) { + if (bpf_dump_raw_ok()) { + __u64 __user *user_linfo; + u32 i; + + user_linfo = u64_to_user_ptr(info.jited_line_info); + ulen = min_t(u32, info.jited_line_info_cnt, ulen); + for (i = 0; i < ulen; i++) { + if (put_user((__u64)(long)prog->aux->jited_linfo[i], + &user_linfo[i])) + return -EFAULT; + } + } else { + info.jited_line_info = 0; + } + } + done: if (copy_to_user(uinfo, &info, info_len) || put_user(info_len, &uattr->info.info_len)) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2752d35ad0738..9d25506bd55a2 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4640,15 +4640,17 @@ static int check_cfg(struct bpf_verifier_env *env) #define MIN_BPF_FUNCINFO_SIZE 8 #define MAX_FUNCINFO_REC_SIZE 252 -static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env, - union bpf_attr *attr, union bpf_attr __user *uattr) +static int check_btf_func(struct bpf_verifier_env *env, + const union bpf_attr *attr, + union bpf_attr __user *uattr) { u32 i, nfuncs, urec_size, min_size, prev_offset; u32 krec_size = sizeof(struct bpf_func_info); - struct bpf_func_info *krecord = NULL; + struct bpf_func_info *krecord; const struct btf_type *type; + struct bpf_prog *prog; + const struct btf *btf; void __user *urecord; - struct btf *btf; int ret = 0; nfuncs = attr->func_info_cnt; @@ -4668,20 +4670,15 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env, return -EINVAL; } - btf = btf_get_by_fd(attr->prog_btf_fd); - if (IS_ERR(btf)) { - verbose(env, "unable to get btf from fd\n"); - return PTR_ERR(btf); - } + prog = env->prog; + btf = prog->aux->btf; urecord = u64_to_user_ptr(attr->func_info); min_size = min_t(u32, krec_size, urec_size); krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); - if (!krecord) { - ret = -ENOMEM; - goto free_btf; - } + if (!krecord) + return -ENOMEM; for (i = 0; i < nfuncs; i++) { ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); @@ -4694,12 +4691,12 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env, if (put_user(min_size, &uattr->func_info_rec_size)) ret = -EFAULT; } - goto free_btf; + goto err_free; } if (copy_from_user(&krecord[i], urecord, min_size)) { ret = -EFAULT; - goto free_btf; + goto err_free; } /* check insn_off */ @@ -4709,20 +4706,20 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env, "nonzero insn_off %u for the first func info record", krecord[i].insn_off); ret = -EINVAL; - goto free_btf; + goto err_free; } } else if (krecord[i].insn_off <= prev_offset) { verbose(env, "same or smaller insn offset (%u) than previous func info record (%u)", krecord[i].insn_off, prev_offset); ret = -EINVAL; - goto free_btf; + goto err_free; } if (env->subprog_info[i].start != krecord[i].insn_off) { verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); ret = -EINVAL; - goto free_btf; + goto err_free; } /* check type_id */ @@ -4731,20 +4728,18 @@ static int check_btf_func(struct bpf_prog *prog, struct bpf_verifier_env *env, verbose(env, "invalid type id %d in func info", krecord[i].type_id); ret = -EINVAL; - goto free_btf; + goto err_free; } prev_offset = krecord[i].insn_off; urecord += urec_size; } - prog->aux->btf = btf; prog->aux->func_info = krecord; prog->aux->func_info_cnt = nfuncs; return 0; -free_btf: - btf_put(btf); +err_free: kvfree(krecord); return ret; } @@ -4760,6 +4755,150 @@ static void adjust_btf_func(struct bpf_verifier_env *env) env->prog->aux->func_info[i].insn_off = env->subprog_info[i].start; } +#define MIN_BPF_LINEINFO_SIZE (offsetof(struct bpf_line_info, line_col) + \ + sizeof(((struct bpf_line_info *)(0))->line_col)) +#define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE + +static int check_btf_line(struct bpf_verifier_env *env, + const union bpf_attr *attr, + union bpf_attr __user *uattr) +{ + u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0; + struct bpf_subprog_info *sub; + struct bpf_line_info *linfo; + struct bpf_prog *prog; + const struct btf *btf; + void __user *ulinfo; + int err; + + nr_linfo = attr->line_info_cnt; + if (!nr_linfo) + return 0; + + rec_size = attr->line_info_rec_size; + if (rec_size < MIN_BPF_LINEINFO_SIZE || + rec_size > MAX_LINEINFO_REC_SIZE || + rec_size & (sizeof(u32) - 1)) + return -EINVAL; + + /* Need to zero it in case the userspace may + * pass in a smaller bpf_line_info object. + */ + linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info), + GFP_KERNEL | __GFP_NOWARN); + if (!linfo) + return -ENOMEM; + + prog = env->prog; + btf = prog->aux->btf; + + s = 0; + sub = env->subprog_info; + ulinfo = u64_to_user_ptr(attr->line_info); + expected_size = sizeof(struct bpf_line_info); + ncopy = min_t(u32, expected_size, rec_size); + for (i = 0; i < nr_linfo; i++) { + err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size); + if (err) { + if (err == -E2BIG) { + verbose(env, "nonzero tailing record in line_info"); + if (put_user(expected_size, + &uattr->line_info_rec_size)) + err = -EFAULT; + } + goto err_free; + } + + if (copy_from_user(&linfo[i], ulinfo, ncopy)) { + err = -EFAULT; + goto err_free; + } + + /* + * Check insn_off to ensure + * 1) strictly increasing AND + * 2) bounded by prog->len + * + * The linfo[0].insn_off == 0 check logically falls into + * the later "missing bpf_line_info for func..." case + * because the first linfo[0].insn_off must be the + * first sub also and the first sub must have + * subprog_info[0].start == 0. + */ + if ((i && linfo[i].insn_off <= prev_offset) || + linfo[i].insn_off >= prog->len) { + verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", + i, linfo[i].insn_off, prev_offset, + prog->len); + err = -EINVAL; + goto err_free; + } + + if (!btf_name_offset_valid(btf, linfo[i].line_off) || + !btf_name_offset_valid(btf, linfo[i].file_name_off)) { + verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); + err = -EINVAL; + goto err_free; + } + + if (s != env->subprog_cnt) { + if (linfo[i].insn_off == sub[s].start) { + sub[s].linfo_idx = i; + s++; + } else if (sub[s].start < linfo[i].insn_off) { + verbose(env, "missing bpf_line_info for func#%u\n", s); + err = -EINVAL; + goto err_free; + } + } + + prev_offset = linfo[i].insn_off; + ulinfo += rec_size; + } + + if (s != env->subprog_cnt) { + verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", + env->subprog_cnt - s, s); + err = -EINVAL; + goto err_free; + } + + prog->aux->linfo = linfo; + prog->aux->nr_linfo = nr_linfo; + + return 0; + +err_free: + kvfree(linfo); + return err; +} + +static int check_btf_info(struct bpf_verifier_env *env, + const union bpf_attr *attr, + union bpf_attr __user *uattr) +{ + struct btf *btf; + int err; + + if (!attr->func_info_cnt && !attr->line_info_cnt) + return 0; + + btf = btf_get_by_fd(attr->prog_btf_fd); + if (IS_ERR(btf)) + return PTR_ERR(btf); + env->prog->aux->btf = btf; + + err = check_btf_func(env, attr, uattr); + if (err) + return err; + + err = check_btf_line(env, attr, uattr); + if (err) + return err; + + return 0; +} + /* check %cur's range satisfies %old's */ static bool range_within(struct bpf_reg_state *old, struct bpf_reg_state *cur) @@ -6004,7 +6143,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) int i, j, subprog_start, subprog_end = 0, len, subprog; struct bpf_insn *insn; void *old_bpf_func; - int err = -ENOMEM; + int err; if (env->subprog_cnt <= 1) return 0; @@ -6035,6 +6174,11 @@ static int jit_subprogs(struct bpf_verifier_env *env) insn->imm = 1; } + err = bpf_prog_alloc_jited_linfo(prog); + if (err) + goto out_undo_insn; + + err = -ENOMEM; func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); if (!func) goto out_undo_insn; @@ -6065,6 +6209,10 @@ static int jit_subprogs(struct bpf_verifier_env *env) func[i]->aux->name[0] = 'F'; func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; func[i]->jit_requested = 1; + func[i]->aux->linfo = prog->aux->linfo; + func[i]->aux->nr_linfo = prog->aux->nr_linfo; + func[i]->aux->jited_linfo = prog->aux->jited_linfo; + func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; func[i] = bpf_int_jit_compile(func[i]); if (!func[i]->jited) { err = -ENOTSUPP; @@ -6138,6 +6286,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) prog->bpf_func = func[0]->bpf_func; prog->aux->func = func; prog->aux->func_cnt = env->subprog_cnt; + bpf_prog_free_unused_jited_linfo(prog); return 0; out_free: for (i = 0; i < env->subprog_cnt; i++) @@ -6154,6 +6303,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) insn->off = 0; insn->imm = env->insn_aux_data[i].call_imm; } + bpf_prog_free_jited_linfo(prog); return err; } @@ -6526,7 +6676,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, if (ret < 0) goto skip_full_check; - ret = check_btf_func(env->prog, env, attr, uattr); + ret = check_btf_info(env, attr, uattr); if (ret < 0) goto skip_full_check; From ee491d8dbe496c510fa06a786460638d78428147 Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Fri, 7 Dec 2018 16:42:26 -0800 Subject: [PATCH 39/51] bpf: tools: Sync uapi bpf.h Sync uapi bpf.h to tools/include/uapi/linux for the new bpf_line_info. Signed-off-by: Martin KaFai Lau Acked-by: Yonghong Song Signed-off-by: Alexei Starovoitov --- tools/include/uapi/linux/bpf.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 16263e8827fcd..7973c28b24a02 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -356,6 +356,9 @@ union bpf_attr { __u32 func_info_rec_size; /* userspace bpf_func_info size */ __aligned_u64 func_info; /* func info */ __u32 func_info_cnt; /* number of bpf_func_info records */ + __u32 line_info_rec_size; /* userspace bpf_line_info size */ + __aligned_u64 line_info; /* line info */ + __u32 line_info_cnt; /* number of bpf_line_info records */ }; struct { /* anonymous struct used by BPF_OBJ_* commands */ @@ -2679,6 +2682,12 @@ struct bpf_prog_info { __u32 func_info_rec_size; __aligned_u64 func_info; __u32 func_info_cnt; + __u32 line_info_cnt; + __aligned_u64 line_info; + __aligned_u64 jited_line_info; + __u32 jited_line_info_cnt; + __u32 line_info_rec_size; + __u32 jited_line_info_rec_size; } __attribute__((aligned(8))); struct bpf_map_info { @@ -2995,4 +3004,14 @@ struct bpf_func_info { __u32 type_id; }; +#define BPF_LINE_INFO_LINE_NUM(line_col) ((line_col) >> 10) +#define BPF_LINE_INFO_LINE_COL(line_col) ((line_col) & 0x3ff) + +struct bpf_line_info { + __u32 insn_off; + __u32 file_name_off; + __u32 line_off; + __u32 line_col; +}; + #endif /* _UAPI__LINUX_BPF_H__ */ From 05687352c600930058ca4c68048302502e07a572 Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Fri, 7 Dec 2018 16:42:27 -0800 Subject: [PATCH 40/51] bpf: Refactor and bug fix in test_func_type in test_btf.c 1) bpf_load_program_xattr() is absorbing the EBIG error which makes testing this case impossible. It is replaced with a direct syscall(__NR_bpf, BPF_PROG_LOAD,...). 2) The test_func_type() is renamed to test_info_raw() to prepare for the new line_info test in the next patch. 3) The bpf_obj_get_info_by_fd() testing for func_info is refactored to test_get_finfo(). A new test_get_linfo() will be added in the next patch for testing line_info purpose. 4) The test->func_info_cnt is checked instead of a static value "2". 5) Remove unnecessary "\n" in error message. 6) Adding back info_raw_test_num to the cmd arg such that a specific test case can be tested, like all other existing tests. 7) Fix a bug in handling expected_prog_load_failure. A test could pass even if prog_fd != -1 while expected_prog_load_failure is true. 8) The min rec_size check should be < 8 instead of < 4. Fixes: 4798c4ba3ba9 ("tools/bpf: extends test_btf to test load/retrieve func_type info") Signed-off-by: Martin KaFai Lau Acked-by: Yonghong Song Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/test_btf.c | 211 +++++++++++++++---------- 1 file changed, 125 insertions(+), 86 deletions(-) diff --git a/tools/testing/selftests/bpf/test_btf.c b/tools/testing/selftests/bpf/test_btf.c index ff0952ea757a4..8d5777c89620b 100644 --- a/tools/testing/selftests/bpf/test_btf.c +++ b/tools/testing/selftests/bpf/test_btf.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -114,12 +115,13 @@ static struct args { unsigned int raw_test_num; unsigned int file_test_num; unsigned int get_info_test_num; + unsigned int info_raw_test_num; bool raw_test; bool file_test; bool get_info_test; bool pprint_test; bool always_log; - bool func_type_test; + bool info_raw_test; } args; static char btf_log_buf[BTF_LOG_BUF_SIZE]; @@ -3051,7 +3053,7 @@ static int test_pprint(void) return err; } -static struct btf_func_type_test { +static struct prog_info_raw_test { const char *descr; const char *str_sec; __u32 raw_types[MAX_NR_RAW_TYPES]; @@ -3062,7 +3064,7 @@ static struct btf_func_type_test { __u32 func_info_rec_size; __u32 func_info_cnt; bool expected_prog_load_failure; -} func_type_test[] = { +} info_raw_tests[] = { { .descr = "func_type (main func + one sub)", .raw_types = { @@ -3198,90 +3200,44 @@ static size_t probe_prog_length(const struct bpf_insn *fp) return len + 1; } -static int do_test_func_type(int test_num) +static int test_get_finfo(const struct prog_info_raw_test *test, + int prog_fd) { - const struct btf_func_type_test *test = &func_type_test[test_num]; - unsigned int raw_btf_size, info_len, rec_size; - int i, btf_fd = -1, prog_fd = -1, err = 0; - struct bpf_load_program_attr attr = {}; - void *raw_btf, *func_info = NULL; struct bpf_prog_info info = {}; struct bpf_func_info *finfo; - - fprintf(stderr, "%s......", test->descr); - raw_btf = btf_raw_create(&hdr_tmpl, test->raw_types, - test->str_sec, test->str_sec_size, - &raw_btf_size); - - if (!raw_btf) - return -1; - - *btf_log_buf = '\0'; - btf_fd = bpf_load_btf(raw_btf, raw_btf_size, - btf_log_buf, BTF_LOG_BUF_SIZE, - args.always_log); - free(raw_btf); - - if (CHECK(btf_fd == -1, "invalid btf_fd errno:%d", errno)) { - err = -1; - goto done; - } - - if (*btf_log_buf && args.always_log) - fprintf(stderr, "\n%s", btf_log_buf); - - attr.prog_type = test->prog_type; - attr.insns = test->insns; - attr.insns_cnt = probe_prog_length(attr.insns); - attr.license = "GPL"; - attr.prog_btf_fd = btf_fd; - attr.func_info_rec_size = test->func_info_rec_size; - attr.func_info_cnt = test->func_info_cnt; - attr.func_info = test->func_info; - - *btf_log_buf = '\0'; - prog_fd = bpf_load_program_xattr(&attr, btf_log_buf, - BTF_LOG_BUF_SIZE); - if (test->expected_prog_load_failure && prog_fd == -1) { - err = 0; - goto done; - } - if (CHECK(prog_fd == -1, "invalid prog_id errno:%d", errno)) { - fprintf(stderr, "%s\n", btf_log_buf); - err = -1; - goto done; - } + __u32 info_len, rec_size, i; + void *func_info = NULL; + int err; /* get necessary lens */ info_len = sizeof(struct bpf_prog_info); err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); if (CHECK(err == -1, "invalid get info (1st) errno:%d", errno)) { fprintf(stderr, "%s\n", btf_log_buf); - err = -1; - goto done; + return -1; } - if (CHECK(info.func_info_cnt != 2, - "incorrect info.func_info_cnt (1st) %d\n", + if (CHECK(info.func_info_cnt != test->func_info_cnt, + "incorrect info.func_info_cnt (1st) %d", info.func_info_cnt)) { - err = -1; - goto done; + return -1; } + rec_size = info.func_info_rec_size; - if (CHECK(rec_size < 4, - "incorrect info.func_info_rec_size (1st) %d\n", rec_size)) { - err = -1; - goto done; + if (CHECK(rec_size < 8, + "incorrect info.func_info_rec_size (1st) %d", rec_size)) { + return -1; } + if (!info.func_info_cnt) + return 0; + func_info = malloc(info.func_info_cnt * rec_size); - if (CHECK(!func_info, "out of memory")) { - err = -1; - goto done; - } + if (CHECK(!func_info, "out of memory")) + return -1; /* reset info to only retrieve func_info related data */ memset(&info, 0, sizeof(info)); - info.func_info_cnt = 2; + info.func_info_cnt = test->func_info_cnt; info.func_info_rec_size = rec_size; info.func_info = ptr_to_u64(func_info); err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); @@ -3290,14 +3246,14 @@ static int do_test_func_type(int test_num) err = -1; goto done; } - if (CHECK(info.func_info_cnt != 2, - "incorrect info.func_info_cnt (2nd) %d\n", + if (CHECK(info.func_info_cnt != test->func_info_cnt, + "incorrect info.func_info_cnt (2nd) %d", info.func_info_cnt)) { err = -1; goto done; } - if (CHECK(info.func_info_rec_size != rec_size, - "incorrect info.func_info_rec_size (2nd) %d\n", + if (CHECK(info.func_info_rec_size < 8, + "incorrect info.func_info_rec_size (2nd) %d", info.func_info_rec_size)) { err = -1; goto done; @@ -3310,7 +3266,7 @@ static int do_test_func_type(int test_num) } finfo = func_info; - for (i = 0; i < 2; i++) { + for (i = 0; i < test->func_info_cnt; i++) { if (CHECK(finfo->type_id != test->func_info[i][1], "incorrect func_type %u expected %u", finfo->type_id, test->func_info[i][1])) { @@ -3320,7 +3276,75 @@ static int do_test_func_type(int test_num) finfo = (void *)finfo + rec_size; } + err = 0; + +done: + free(func_info); + return err; +} + +static int do_test_info_raw(unsigned int test_num) +{ + const struct prog_info_raw_test *test = &info_raw_tests[test_num - 1]; + int btf_fd = -1, prog_fd = -1, err = 0; + unsigned int raw_btf_size; + union bpf_attr attr = {}; + void *raw_btf; + + fprintf(stderr, "BTF prog info raw test[%u] (%s): ", test_num, test->descr); + raw_btf = btf_raw_create(&hdr_tmpl, test->raw_types, + test->str_sec, test->str_sec_size, + &raw_btf_size); + + if (!raw_btf) + return -1; + + *btf_log_buf = '\0'; + btf_fd = bpf_load_btf(raw_btf, raw_btf_size, + btf_log_buf, BTF_LOG_BUF_SIZE, + args.always_log); + free(raw_btf); + + if (CHECK(btf_fd == -1, "invalid btf_fd errno:%d", errno)) { + err = -1; + goto done; + } + + if (*btf_log_buf && args.always_log) + fprintf(stderr, "\n%s", btf_log_buf); + *btf_log_buf = '\0'; + + attr.prog_type = test->prog_type; + attr.insns = ptr_to_u64(test->insns); + attr.insn_cnt = probe_prog_length(test->insns); + attr.license = ptr_to_u64("GPL"); + attr.prog_btf_fd = btf_fd; + attr.func_info_rec_size = test->func_info_rec_size; + attr.func_info_cnt = test->func_info_cnt; + attr.func_info = ptr_to_u64(test->func_info); + attr.log_buf = ptr_to_u64(btf_log_buf); + attr.log_size = BTF_LOG_BUF_SIZE; + attr.log_level = 1; + + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, sizeof(attr)); + err = ((prog_fd == -1) != test->expected_prog_load_failure); + if (CHECK(err, "prog_fd:%d expected_prog_load_failure:%u errno:%d", + prog_fd, test->expected_prog_load_failure, errno)) { + err = -1; + goto done; + } + + if (prog_fd == -1) + goto done; + + err = test_get_finfo(test, prog_fd); + if (err) + goto done; + done: + if (!err) + fprintf(stderr, "OK"); + if (*btf_log_buf && (err || args.always_log)) fprintf(stderr, "\n%s", btf_log_buf); @@ -3328,33 +3352,38 @@ static int do_test_func_type(int test_num) close(btf_fd); if (prog_fd != -1) close(prog_fd); - free(func_info); + return err; } -static int test_func_type(void) +static int test_info_raw(void) { unsigned int i; int err = 0; - for (i = 0; i < ARRAY_SIZE(func_type_test); i++) - err |= count_result(do_test_func_type(i)); + if (args.info_raw_test_num) + return count_result(do_test_info_raw(args.info_raw_test_num)); + + for (i = 1; i <= ARRAY_SIZE(info_raw_tests); i++) + err |= count_result(do_test_info_raw(i)); return err; } static void usage(const char *cmd) { - fprintf(stderr, "Usage: %s [-l] [[-r test_num (1 - %zu)] |" - " [-g test_num (1 - %zu)] |" - " [-f test_num (1 - %zu)] | [-p] | [-k] ]\n", + fprintf(stderr, "Usage: %s [-l] [[-r btf_raw_test_num (1 - %zu)] |\n" + "\t[-g btf_get_info_test_num (1 - %zu)] |\n" + "\t[-f btf_file_test_num (1 - %zu)] |\n" + "\t[-k btf_prog_info_raw_test_num (1 - %zu)] |\n" + "\t[-p (pretty print test)]]\n", cmd, ARRAY_SIZE(raw_tests), ARRAY_SIZE(get_info_tests), - ARRAY_SIZE(file_tests)); + ARRAY_SIZE(file_tests), ARRAY_SIZE(info_raw_tests)); } static int parse_args(int argc, char **argv) { - const char *optstr = "lpkf:r:g:"; + const char *optstr = "lpk:f:r:g:"; int opt; while ((opt = getopt(argc, argv, optstr)) != -1) { @@ -3378,7 +3407,8 @@ static int parse_args(int argc, char **argv) args.pprint_test = true; break; case 'k': - args.func_type_test = true; + args.info_raw_test_num = atoi(optarg); + args.info_raw_test = true; break; case 'h': usage(argv[0]); @@ -3413,6 +3443,14 @@ static int parse_args(int argc, char **argv) return -1; } + if (args.info_raw_test_num && + (args.info_raw_test_num < 1 || + args.info_raw_test_num > ARRAY_SIZE(info_raw_tests))) { + fprintf(stderr, "BTF prog info raw test number must be [1 - %zu]\n", + ARRAY_SIZE(info_raw_tests)); + return -1; + } + return 0; } @@ -3445,16 +3483,17 @@ int main(int argc, char **argv) if (args.pprint_test) err |= test_pprint(); - if (args.func_type_test) - err |= test_func_type(); + if (args.info_raw_test) + err |= test_info_raw(); if (args.raw_test || args.get_info_test || args.file_test || - args.pprint_test || args.func_type_test) + args.pprint_test || args.info_raw_test) goto done; err |= test_raw(); err |= test_get_info(); err |= test_file(); + err |= test_info_raw(); done: print_summary(); From 4d6304c763551cdb420201b1b754cc625d56bdd6 Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Fri, 7 Dec 2018 16:42:28 -0800 Subject: [PATCH 41/51] bpf: Add unit tests for bpf_line_info Add unit tests for bpf_line_info for both BPF_PROG_LOAD and BPF_OBJ_GET_INFO_BY_FD. jit enabled: [root@arch-fb-vm1 bpf]# ./test_btf -k 0 BTF prog info raw test[5] (line_info (No subprog)): OK BTF prog info raw test[6] (line_info (No subprog. insn_off >= prog->len)): OK BTF prog info raw test[7] (line_info (No subprog. zero tailing line_info): OK BTF prog info raw test[8] (line_info (No subprog. nonzero tailing line_info)): OK BTF prog info raw test[9] (line_info (subprog)): OK BTF prog info raw test[10] (line_info (subprog + func_info)): OK BTF prog info raw test[11] (line_info (subprog. missing 1st func line info)): OK BTF prog info raw test[12] (line_info (subprog. missing 2nd func line info)): OK BTF prog info raw test[13] (line_info (subprog. unordered insn offset)): OK jit disabled: BTF prog info raw test[5] (line_info (No subprog)): not jited. skipping jited_line_info check. OK BTF prog info raw test[6] (line_info (No subprog. insn_off >= prog->len)): OK BTF prog info raw test[7] (line_info (No subprog. zero tailing line_info): not jited. skipping jited_line_info check. OK BTF prog info raw test[8] (line_info (No subprog. nonzero tailing line_info)): OK BTF prog info raw test[9] (line_info (subprog)): not jited. skipping jited_line_info check. OK BTF prog info raw test[10] (line_info (subprog + func_info)): not jited. skipping jited_line_info check. OK BTF prog info raw test[11] (line_info (subprog. missing 1st func line info)): OK BTF prog info raw test[12] (line_info (subprog. missing 2nd func line info)): OK BTF prog info raw test[13] (line_info (subprog. unordered insn offset)): OK Signed-off-by: Martin KaFai Lau Acked-by: Yonghong Song Signed-off-by: Alexei Starovoitov --- tools/testing/selftests/bpf/test_btf.c | 597 ++++++++++++++++++++++++- 1 file changed, 580 insertions(+), 17 deletions(-) diff --git a/tools/testing/selftests/bpf/test_btf.c b/tools/testing/selftests/bpf/test_btf.c index 8d5777c89620b..7707273736ac6 100644 --- a/tools/testing/selftests/bpf/test_btf.c +++ b/tools/testing/selftests/bpf/test_btf.c @@ -108,7 +108,7 @@ static int __base_pr(const char *format, ...) #define BTF_END_RAW 0xdeadbeef #define NAME_TBD 0xdeadb33f -#define MAX_NR_RAW_TYPES 1024 +#define MAX_NR_RAW_U32 1024 #define BTF_LOG_BUF_SIZE 65535 static struct args { @@ -137,7 +137,7 @@ struct btf_raw_test { const char *str_sec; const char *map_name; const char *err_str; - __u32 raw_types[MAX_NR_RAW_TYPES]; + __u32 raw_types[MAX_NR_RAW_U32]; __u32 str_sec_size; enum bpf_map_type map_type; __u32 key_size; @@ -156,6 +156,9 @@ struct btf_raw_test { int str_len_delta; }; +#define BTF_STR_SEC(str) \ + .str_sec = str, .str_sec_size = sizeof(str) + static struct btf_raw_test raw_tests[] = { /* enum E { * E0, @@ -1858,11 +1861,11 @@ static const char *get_next_str(const char *start, const char *end) return start < end - 1 ? start + 1 : NULL; } -static int get_type_sec_size(const __u32 *raw_types) +static int get_raw_sec_size(const __u32 *raw_types) { int i; - for (i = MAX_NR_RAW_TYPES - 1; + for (i = MAX_NR_RAW_U32 - 1; i >= 0 && raw_types[i] != BTF_END_RAW; i--) ; @@ -1874,7 +1877,8 @@ static void *btf_raw_create(const struct btf_header *hdr, const __u32 *raw_types, const char *str, unsigned int str_sec_size, - unsigned int *btf_size) + unsigned int *btf_size, + const char **ret_next_str) { const char *next_str = str, *end_str = str + str_sec_size; unsigned int size_needed, offset; @@ -1883,7 +1887,7 @@ static void *btf_raw_create(const struct btf_header *hdr, uint32_t *ret_types; void *raw_btf; - type_sec_size = get_type_sec_size(raw_types); + type_sec_size = get_raw_sec_size(raw_types); if (CHECK(type_sec_size < 0, "Cannot get nr_raw_types")) return NULL; @@ -1922,6 +1926,8 @@ static void *btf_raw_create(const struct btf_header *hdr, ret_hdr->str_len = str_sec_size; *btf_size = size_needed; + if (ret_next_str) + *ret_next_str = next_str; return raw_btf; } @@ -1941,7 +1947,7 @@ static int do_test_raw(unsigned int test_num) test->raw_types, test->str_sec, test->str_sec_size, - &raw_btf_size); + &raw_btf_size, NULL); if (!raw_btf) return -1; @@ -2018,7 +2024,7 @@ static int test_raw(void) struct btf_get_info_test { const char *descr; const char *str_sec; - __u32 raw_types[MAX_NR_RAW_TYPES]; + __u32 raw_types[MAX_NR_RAW_U32]; __u32 str_sec_size; int btf_size_delta; int (*special_test)(unsigned int test_num); @@ -2098,7 +2104,7 @@ static int test_big_btf_info(unsigned int test_num) test->raw_types, test->str_sec, test->str_sec_size, - &raw_btf_size); + &raw_btf_size, NULL); if (!raw_btf) return -1; @@ -2182,7 +2188,7 @@ static int test_btf_id(unsigned int test_num) test->raw_types, test->str_sec, test->str_sec_size, - &raw_btf_size); + &raw_btf_size, NULL); if (!raw_btf) return -1; @@ -2320,7 +2326,7 @@ static int do_test_get_info(unsigned int test_num) test->raw_types, test->str_sec, test->str_sec_size, - &raw_btf_size); + &raw_btf_size, NULL); if (!raw_btf) return -1; @@ -2849,7 +2855,7 @@ static int do_test_pprint(void) fprintf(stderr, "%s......", test->descr); raw_btf = btf_raw_create(&hdr_tmpl, test->raw_types, test->str_sec, test->str_sec_size, - &raw_btf_size); + &raw_btf_size, NULL); if (!raw_btf) return -1; @@ -3053,16 +3059,23 @@ static int test_pprint(void) return err; } +#define BPF_LINE_INFO_ENC(insn_off, file_off, line_off, line_num, line_col) \ + (insn_off), (file_off), (line_off), ((line_num) << 10 | ((line_col) & 0x3ff)) + static struct prog_info_raw_test { const char *descr; const char *str_sec; - __u32 raw_types[MAX_NR_RAW_TYPES]; + const char *err_str; + __u32 raw_types[MAX_NR_RAW_U32]; __u32 str_sec_size; struct bpf_insn insns[MAX_INSNS]; __u32 prog_type; __u32 func_info[MAX_SUBPROGS][2]; __u32 func_info_rec_size; __u32 func_info_cnt; + __u32 line_info[MAX_NR_RAW_U32]; + __u32 line_info_rec_size; + __u32 nr_jited_ksyms; bool expected_prog_load_failure; } info_raw_tests[] = { { @@ -3093,6 +3106,7 @@ static struct prog_info_raw_test { .func_info = { {0, 5}, {3, 6} }, .func_info_rec_size = 8, .func_info_cnt = 2, + .line_info = { BTF_END_RAW }, }, { @@ -3123,6 +3137,7 @@ static struct prog_info_raw_test { .func_info = { {0, 5}, {3, 6} }, .func_info_rec_size = 4, .func_info_cnt = 2, + .line_info = { BTF_END_RAW }, .expected_prog_load_failure = true, }, @@ -3154,6 +3169,7 @@ static struct prog_info_raw_test { .func_info = { {0, 5}, {3, 6} }, .func_info_rec_size = 8, .func_info_cnt = 1, + .line_info = { BTF_END_RAW }, .expected_prog_load_failure = true, }, @@ -3185,6 +3201,278 @@ static struct prog_info_raw_test { .func_info = { {0, 5}, {2, 6} }, .func_info_rec_size = 8, .func_info_cnt = 2, + .line_info = { BTF_END_RAW }, + .expected_prog_load_failure = true, +}, + +{ + .descr = "line_info (No subprog)", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0int a=1;\0int b=2;\0return a + b;\0return a + b;"), + .insns = { + BPF_MOV64_IMM(BPF_REG_0, 1), + BPF_MOV64_IMM(BPF_REG_1, 2), + BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_TRACEPOINT, + .func_info_cnt = 0, + .line_info = { + BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10), + BPF_LINE_INFO_ENC(1, 0, NAME_TBD, 2, 9), + BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 3, 8), + BPF_LINE_INFO_ENC(3, 0, NAME_TBD, 4, 7), + BTF_END_RAW, + }, + .line_info_rec_size = sizeof(struct bpf_line_info), + .nr_jited_ksyms = 1, +}, + +{ + .descr = "line_info (No subprog. insn_off >= prog->len)", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0int a=1;\0int b=2;\0return a + b;\0return a + b;"), + .insns = { + BPF_MOV64_IMM(BPF_REG_0, 1), + BPF_MOV64_IMM(BPF_REG_1, 2), + BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_TRACEPOINT, + .func_info_cnt = 0, + .line_info = { + BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10), + BPF_LINE_INFO_ENC(1, 0, NAME_TBD, 2, 9), + BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 3, 8), + BPF_LINE_INFO_ENC(3, 0, NAME_TBD, 4, 7), + BPF_LINE_INFO_ENC(4, 0, 0, 5, 6), + BTF_END_RAW, + }, + .line_info_rec_size = sizeof(struct bpf_line_info), + .nr_jited_ksyms = 1, + .err_str = "line_info[4].insn_off", + .expected_prog_load_failure = true, +}, + +{ + .descr = "line_info (No subprog. zero tailing line_info", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0int a=1;\0int b=2;\0return a + b;\0return a + b;"), + .insns = { + BPF_MOV64_IMM(BPF_REG_0, 1), + BPF_MOV64_IMM(BPF_REG_1, 2), + BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_TRACEPOINT, + .func_info_cnt = 0, + .line_info = { + BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10), 0, + BPF_LINE_INFO_ENC(1, 0, NAME_TBD, 2, 9), 0, + BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 3, 8), 0, + BPF_LINE_INFO_ENC(3, 0, NAME_TBD, 4, 7), 0, + BTF_END_RAW, + }, + .line_info_rec_size = sizeof(struct bpf_line_info) + sizeof(__u32), + .nr_jited_ksyms = 1, +}, + +{ + .descr = "line_info (No subprog. nonzero tailing line_info)", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0int a=1;\0int b=2;\0return a + b;\0return a + b;"), + .insns = { + BPF_MOV64_IMM(BPF_REG_0, 1), + BPF_MOV64_IMM(BPF_REG_1, 2), + BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_TRACEPOINT, + .func_info_cnt = 0, + .line_info = { + BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10), 0, + BPF_LINE_INFO_ENC(1, 0, NAME_TBD, 2, 9), 0, + BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 3, 8), 0, + BPF_LINE_INFO_ENC(3, 0, NAME_TBD, 4, 7), 1, + BTF_END_RAW, + }, + .line_info_rec_size = sizeof(struct bpf_line_info) + sizeof(__u32), + .nr_jited_ksyms = 1, + .err_str = "nonzero tailing record in line_info", + .expected_prog_load_failure = true, +}, + +{ + .descr = "line_info (subprog)", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0int a=1+1;\0return func(a);\0b+=1;\0return b;"), + .insns = { + BPF_MOV64_IMM(BPF_REG_2, 1), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_2), + BPF_CALL_REL(1), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_0, BPF_REG_1), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_TRACEPOINT, + .func_info_cnt = 0, + .line_info = { + BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10), + BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 2, 9), + BPF_LINE_INFO_ENC(5, 0, NAME_TBD, 3, 8), + BPF_LINE_INFO_ENC(7, 0, NAME_TBD, 4, 7), + BTF_END_RAW, + }, + .line_info_rec_size = sizeof(struct bpf_line_info), + .nr_jited_ksyms = 2, +}, + +{ + .descr = "line_info (subprog + func_info)", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_FUNC_PROTO_ENC(1, 1), /* [2] */ + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), + BTF_FUNC_ENC(NAME_TBD, 2), /* [3] */ + BTF_FUNC_ENC(NAME_TBD, 2), /* [4] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0x\0sub\0main\0int a=1+1;\0return func(a);\0b+=1;\0return b;"), + .insns = { + BPF_MOV64_IMM(BPF_REG_2, 1), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_2), + BPF_CALL_REL(1), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_0, BPF_REG_1), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_TRACEPOINT, + .func_info_cnt = 2, + .func_info_rec_size = 8, + .func_info = { {0, 4}, {5, 3} }, + .line_info = { + BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10), + BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 2, 9), + BPF_LINE_INFO_ENC(5, 0, NAME_TBD, 3, 8), + BPF_LINE_INFO_ENC(7, 0, NAME_TBD, 4, 7), + BTF_END_RAW, + }, + .line_info_rec_size = sizeof(struct bpf_line_info), + .nr_jited_ksyms = 2, +}, + +{ + .descr = "line_info (subprog. missing 1st func line info)", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0int a=1+1;\0return func(a);\0b+=1;\0return b;"), + .insns = { + BPF_MOV64_IMM(BPF_REG_2, 1), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_2), + BPF_CALL_REL(1), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_0, BPF_REG_1), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_TRACEPOINT, + .func_info_cnt = 0, + .line_info = { + BPF_LINE_INFO_ENC(1, 0, NAME_TBD, 1, 10), + BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 2, 9), + BPF_LINE_INFO_ENC(5, 0, NAME_TBD, 3, 8), + BPF_LINE_INFO_ENC(7, 0, NAME_TBD, 4, 7), + BTF_END_RAW, + }, + .line_info_rec_size = sizeof(struct bpf_line_info), + .nr_jited_ksyms = 2, + .err_str = "missing bpf_line_info for func#0", + .expected_prog_load_failure = true, +}, + +{ + .descr = "line_info (subprog. missing 2nd func line info)", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0int a=1+1;\0return func(a);\0b+=1;\0return b;"), + .insns = { + BPF_MOV64_IMM(BPF_REG_2, 1), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_2), + BPF_CALL_REL(1), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_0, BPF_REG_1), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_TRACEPOINT, + .func_info_cnt = 0, + .line_info = { + BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10), + BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 2, 9), + BPF_LINE_INFO_ENC(6, 0, NAME_TBD, 3, 8), + BPF_LINE_INFO_ENC(7, 0, NAME_TBD, 4, 7), + BTF_END_RAW, + }, + .line_info_rec_size = sizeof(struct bpf_line_info), + .nr_jited_ksyms = 2, + .err_str = "missing bpf_line_info for func#1", + .expected_prog_load_failure = true, +}, + +{ + .descr = "line_info (subprog. unordered insn offset)", + .raw_types = { + BTF_TYPE_INT_ENC(NAME_TBD, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0int\0int a=1+1;\0return func(a);\0b+=1;\0return b;"), + .insns = { + BPF_MOV64_IMM(BPF_REG_2, 1), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, 1), + BPF_MOV64_REG(BPF_REG_1, BPF_REG_2), + BPF_CALL_REL(1), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_0, BPF_REG_1), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1), + BPF_EXIT_INSN(), + }, + .prog_type = BPF_PROG_TYPE_TRACEPOINT, + .func_info_cnt = 0, + .line_info = { + BPF_LINE_INFO_ENC(0, 0, NAME_TBD, 1, 10), + BPF_LINE_INFO_ENC(5, 0, NAME_TBD, 2, 9), + BPF_LINE_INFO_ENC(2, 0, NAME_TBD, 3, 8), + BPF_LINE_INFO_ENC(7, 0, NAME_TBD, 4, 7), + BTF_END_RAW, + }, + .line_info_rec_size = sizeof(struct bpf_line_info), + .nr_jited_ksyms = 2, + .err_str = "Invalid line_info[2].insn_off", .expected_prog_load_failure = true, }, @@ -3200,6 +3488,46 @@ static size_t probe_prog_length(const struct bpf_insn *fp) return len + 1; } +static __u32 *patch_name_tbd(const __u32 *raw_u32, + const char *str, __u32 str_off, + unsigned int str_sec_size, + unsigned int *ret_size) +{ + int i, raw_u32_size = get_raw_sec_size(raw_u32); + const char *end_str = str + str_sec_size; + const char *next_str = str + str_off; + __u32 *new_u32 = NULL; + + if (raw_u32_size == -1) + return ERR_PTR(-EINVAL); + + if (!raw_u32_size) { + *ret_size = 0; + return NULL; + } + + new_u32 = malloc(raw_u32_size); + if (!new_u32) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < raw_u32_size / sizeof(raw_u32[0]); i++) { + if (raw_u32[i] == NAME_TBD) { + next_str = get_next_str(next_str, end_str); + if (CHECK(!next_str, "Error in getting next_str\n")) { + free(new_u32); + return ERR_PTR(-EINVAL); + } + new_u32[i] = next_str - str; + next_str += strlen(next_str); + } else { + new_u32[i] = raw_u32[i]; + } + } + + *ret_size = raw_u32_size; + return new_u32; +} + static int test_get_finfo(const struct prog_info_raw_test *test, int prog_fd) { @@ -3283,18 +3611,229 @@ static int test_get_finfo(const struct prog_info_raw_test *test, return err; } +static int test_get_linfo(const struct prog_info_raw_test *test, + const void *patched_linfo, + __u32 cnt, int prog_fd) +{ + __u32 i, info_len, nr_jited_ksyms, nr_jited_func_lens; + __u64 *jited_linfo = NULL, *jited_ksyms = NULL; + __u32 rec_size, jited_rec_size, jited_cnt; + struct bpf_line_info *linfo = NULL; + __u32 cur_func_len, ksyms_found; + struct bpf_prog_info info = {}; + __u32 *jited_func_lens = NULL; + __u64 cur_func_ksyms; + int err; + + jited_cnt = cnt; + rec_size = sizeof(*linfo); + jited_rec_size = sizeof(*jited_linfo); + if (test->nr_jited_ksyms) + nr_jited_ksyms = test->nr_jited_ksyms; + else + nr_jited_ksyms = test->func_info_cnt; + nr_jited_func_lens = nr_jited_ksyms; + + info_len = sizeof(struct bpf_prog_info); + err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); + if (CHECK(err == -1, "err:%d errno:%d", err, errno)) { + err = -1; + goto done; + } + + if (!info.jited_prog_len) { + /* prog is not jited */ + jited_cnt = 0; + nr_jited_ksyms = 1; + nr_jited_func_lens = 1; + } + + if (CHECK(info.line_info_cnt != cnt || + info.jited_line_info_cnt != jited_cnt || + info.nr_jited_ksyms != nr_jited_ksyms || + info.nr_jited_func_lens != nr_jited_func_lens || + (!info.line_info_cnt && info.jited_line_info_cnt), + "info: line_info_cnt:%u(expected:%u) jited_line_info_cnt:%u(expected:%u) nr_jited_ksyms:%u(expected:%u) nr_jited_func_lens:%u(expected:%u)", + info.line_info_cnt, cnt, + info.jited_line_info_cnt, jited_cnt, + info.nr_jited_ksyms, nr_jited_ksyms, + info.nr_jited_func_lens, nr_jited_func_lens)) { + err = -1; + goto done; + } + + if (CHECK(info.line_info_rec_size < 16 || + info.jited_line_info_rec_size < 8, + "info: line_info_rec_size:%u(userspace expected:%u) jited_line_info_rec_size:%u(userspace expected:%u)", + info.line_info_rec_size, rec_size, + info.jited_line_info_rec_size, jited_rec_size)) { + err = -1; + goto done; + } + + if (!cnt) + return 0; + + rec_size = info.line_info_rec_size; + jited_rec_size = info.jited_line_info_rec_size; + + memset(&info, 0, sizeof(info)); + + linfo = calloc(cnt, rec_size); + if (CHECK(!linfo, "!linfo")) { + err = -1; + goto done; + } + info.line_info_cnt = cnt; + info.line_info_rec_size = rec_size; + info.line_info = ptr_to_u64(linfo); + + if (jited_cnt) { + jited_linfo = calloc(jited_cnt, jited_rec_size); + jited_ksyms = calloc(nr_jited_ksyms, sizeof(*jited_ksyms)); + jited_func_lens = calloc(nr_jited_func_lens, + sizeof(*jited_func_lens)); + if (CHECK(!jited_linfo || !jited_ksyms || !jited_func_lens, + "jited_linfo:%p jited_ksyms:%p jited_func_lens:%p", + jited_linfo, jited_ksyms, jited_func_lens)) { + err = -1; + goto done; + } + + info.jited_line_info_cnt = jited_cnt; + info.jited_line_info_rec_size = jited_rec_size; + info.jited_line_info = ptr_to_u64(jited_linfo); + info.nr_jited_ksyms = nr_jited_ksyms; + info.jited_ksyms = ptr_to_u64(jited_ksyms); + info.nr_jited_func_lens = nr_jited_func_lens; + info.jited_func_lens = ptr_to_u64(jited_func_lens); + } + + err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); + + /* + * Only recheck the info.*line_info* fields. + * Other fields are not the concern of this test. + */ + if (CHECK(err == -1 || + !info.line_info || + info.line_info_cnt != cnt || + (jited_cnt && !info.jited_line_info) || + info.jited_line_info_cnt != jited_cnt || + info.line_info_rec_size != rec_size || + info.jited_line_info_rec_size != jited_rec_size, + "err:%d errno:%d info: line_info_cnt:%u(expected:%u) jited_line_info_cnt:%u(expected:%u) line_info_rec_size:%u(expected:%u) jited_linfo_rec_size:%u(expected:%u) line_info:%p jited_line_info:%p", + err, errno, + info.line_info_cnt, cnt, + info.jited_line_info_cnt, jited_cnt, + info.line_info_rec_size, rec_size, + info.jited_line_info_rec_size, jited_rec_size, + (void *)(long)info.line_info, + (void *)(long)info.jited_line_info)) { + err = -1; + goto done; + } + + CHECK(linfo[0].insn_off, "linfo[0].insn_off:%u", + linfo[0].insn_off); + for (i = 1; i < cnt; i++) { + const struct bpf_line_info *expected_linfo; + + expected_linfo = patched_linfo + (i * test->line_info_rec_size); + if (CHECK(linfo[i].insn_off <= linfo[i - 1].insn_off, + "linfo[%u].insn_off:%u <= linfo[%u].insn_off:%u", + i, linfo[i].insn_off, + i - 1, linfo[i - 1].insn_off)) { + err = -1; + goto done; + } + if (CHECK(linfo[i].file_name_off != expected_linfo->file_name_off || + linfo[i].line_off != expected_linfo->line_off || + linfo[i].line_col != expected_linfo->line_col, + "linfo[%u] (%u, %u, %u) != (%u, %u, %u)", i, + linfo[i].file_name_off, + linfo[i].line_off, + linfo[i].line_col, + expected_linfo->file_name_off, + expected_linfo->line_off, + expected_linfo->line_col)) { + err = -1; + goto done; + } + } + + if (!jited_cnt) { + fprintf(stderr, "not jited. skipping jited_line_info check. "); + err = 0; + goto done; + } + + if (CHECK(jited_linfo[0] != jited_ksyms[0], + "jited_linfo[0]:%lx != jited_ksyms[0]:%lx", + (long)(jited_linfo[0]), (long)(jited_ksyms[0]))) { + err = -1; + goto done; + } + + ksyms_found = 1; + cur_func_len = jited_func_lens[0]; + cur_func_ksyms = jited_ksyms[0]; + for (i = 1; i < jited_cnt; i++) { + if (ksyms_found < nr_jited_ksyms && + jited_linfo[i] == jited_ksyms[ksyms_found]) { + cur_func_ksyms = jited_ksyms[ksyms_found]; + cur_func_len = jited_ksyms[ksyms_found]; + ksyms_found++; + continue; + } + + if (CHECK(jited_linfo[i] <= jited_linfo[i - 1], + "jited_linfo[%u]:%lx <= jited_linfo[%u]:%lx", + i, (long)jited_linfo[i], + i - 1, (long)(jited_linfo[i - 1]))) { + err = -1; + goto done; + } + + if (CHECK(jited_linfo[i] - cur_func_ksyms > cur_func_len, + "jited_linfo[%u]:%lx - %lx > %u", + i, (long)jited_linfo[i], (long)cur_func_ksyms, + cur_func_len)) { + err = -1; + goto done; + } + } + + if (CHECK(ksyms_found != nr_jited_ksyms, + "ksyms_found:%u != nr_jited_ksyms:%u", + ksyms_found, nr_jited_ksyms)) { + err = -1; + goto done; + } + + err = 0; + +done: + free(linfo); + free(jited_linfo); + free(jited_ksyms); + free(jited_func_lens); + return err; +} + static int do_test_info_raw(unsigned int test_num) { const struct prog_info_raw_test *test = &info_raw_tests[test_num - 1]; + unsigned int raw_btf_size, linfo_str_off, linfo_size; int btf_fd = -1, prog_fd = -1, err = 0; - unsigned int raw_btf_size; + void *raw_btf, *patched_linfo = NULL; + const char *ret_next_str; union bpf_attr attr = {}; - void *raw_btf; fprintf(stderr, "BTF prog info raw test[%u] (%s): ", test_num, test->descr); raw_btf = btf_raw_create(&hdr_tmpl, test->raw_types, test->str_sec, test->str_sec_size, - &raw_btf_size); + &raw_btf_size, &ret_next_str); if (!raw_btf) return -1; @@ -3314,6 +3853,16 @@ static int do_test_info_raw(unsigned int test_num) fprintf(stderr, "\n%s", btf_log_buf); *btf_log_buf = '\0'; + linfo_str_off = ret_next_str - test->str_sec; + patched_linfo = patch_name_tbd(test->line_info, + test->str_sec, linfo_str_off, + test->str_sec_size, &linfo_size); + if (IS_ERR(patched_linfo)) { + fprintf(stderr, "error in creating raw bpf_line_info"); + err = -1; + goto done; + } + attr.prog_type = test->prog_type; attr.insns = ptr_to_u64(test->insns); attr.insn_cnt = probe_prog_length(test->insns); @@ -3325,11 +3874,18 @@ static int do_test_info_raw(unsigned int test_num) attr.log_buf = ptr_to_u64(btf_log_buf); attr.log_size = BTF_LOG_BUF_SIZE; attr.log_level = 1; + if (linfo_size) { + attr.line_info_rec_size = test->line_info_rec_size; + attr.line_info = ptr_to_u64(patched_linfo); + attr.line_info_cnt = linfo_size / attr.line_info_rec_size; + } prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, sizeof(attr)); err = ((prog_fd == -1) != test->expected_prog_load_failure); if (CHECK(err, "prog_fd:%d expected_prog_load_failure:%u errno:%d", - prog_fd, test->expected_prog_load_failure, errno)) { + prog_fd, test->expected_prog_load_failure, errno) || + CHECK(test->err_str && !strstr(btf_log_buf, test->err_str), + "expected err_str:%s", test->err_str)) { err = -1; goto done; } @@ -3341,6 +3897,10 @@ static int do_test_info_raw(unsigned int test_num) if (err) goto done; + err = test_get_linfo(test, patched_linfo, attr.line_info_cnt, prog_fd); + if (err) + goto done; + done: if (!err) fprintf(stderr, "OK"); @@ -3353,6 +3913,9 @@ static int do_test_info_raw(unsigned int test_num) if (prog_fd != -1) close(prog_fd); + if (!IS_ERR(patched_linfo)) + free(patched_linfo); + return err; } From f0187f0b17fad7439f510eff4d65606c9ea1190f Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Fri, 7 Dec 2018 16:42:29 -0800 Subject: [PATCH 42/51] bpf: libbpf: Refactor and bug fix on the bpf_func_info loading logic This patch refactor and fix a bug in the libbpf's bpf_func_info loading logic. The bug fix and refactoring are targeting the same commit 2993e0515bb4 ("tools/bpf: add support to read .BTF.ext sections") which is in the bpf-next branch. 1) In bpf_load_program_xattr(), it should retry when errno == E2BIG regardless of log_buf and log_buf_sz. This patch fixes it. 2) btf_ext__reloc_init() and btf_ext__reloc() are essentially the same except btf_ext__reloc_init() always has insns_cnt == 0. Hence, btf_ext__reloc_init() is removed. btf_ext__reloc() is also renamed to btf_ext__reloc_func_info() to get ready for the line_info support in the next patch. 3) Consolidate func_info section logic from "btf_ext_parse_hdr()", "btf_ext_validate_func_info()" and "btf_ext__new()" to a new function "btf_ext_copy_func_info()" such that similar logic can be reused by the later libbpf's line_info patch. 4) The next line_info patch will store line_info_cnt instead of line_info_len in the bpf_program because the kernel is taking line_info_cnt also. It will save a few "len" to "cnt" conversions and will also save some function args. Hence, this patch also makes bpf_program to store func_info_cnt instead of func_info_len. 5) btf_ext depends on btf. e.g. the func_info's type_id in ".BTF.ext" is not useful when ".BTF" is absent. This patch only init the obj->btf_ext pointer after it has successfully init the obj->btf pointer. This can avoid always checking "obj->btf && obj->btf_ext" together for accessing ".BTF.ext". Checking "obj->btf_ext" alone will do. 6) Move "struct btf_sec_func_info" from btf.h to btf.c. There is no external usage outside btf.c. Fixes: 2993e0515bb4 ("tools/bpf: add support to read .BTF.ext sections") Signed-off-by: Martin KaFai Lau Acked-by: Yonghong Song Signed-off-by: Alexei Starovoitov --- tools/lib/bpf/bpf.c | 7 +- tools/lib/bpf/btf.c | 191 ++++++++++++++++------------------------- tools/lib/bpf/btf.h | 17 +--- tools/lib/bpf/libbpf.c | 139 ++++++++++++++++++++---------- 4 files changed, 177 insertions(+), 177 deletions(-) diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 5c3be06bf0ddf..9fbbc0ed59525 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -205,7 +205,7 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, min(name_len, BPF_OBJ_NAME_LEN - 1)); fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); - if (fd >= 0 || !log_buf || !log_buf_sz) + if (fd >= 0) return fd; /* After bpf_prog_load, the kernel may modify certain attributes @@ -244,10 +244,13 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); - if (fd >= 0 || !log_buf || !log_buf_sz) + if (fd >= 0) goto done; } + if (!log_buf || !log_buf_sz) + goto done; + /* Try again with log */ attr.log_buf = ptr_to_u64(log_buf); attr.log_size = log_buf_sz; diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 85d6446cf8325..aa4fa02b13fcb 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -43,6 +43,13 @@ struct btf_ext { __u32 func_info_len; }; +struct btf_sec_func_info { + __u32 sec_name_off; + __u32 num_func_info; + /* Followed by num_func_info number of bpf func_info records */ + __u8 data[0]; +}; + /* The minimum bpf_func_info checked by the loader */ struct bpf_func_info_min { __u32 insn_off; @@ -479,41 +486,66 @@ int btf__get_from_id(__u32 id, struct btf **btf) return err; } -static int btf_ext_validate_func_info(const void *finfo, __u32 size, - btf_print_fn_t err_log) +static int btf_ext_copy_func_info(struct btf_ext *btf_ext, + __u8 *data, __u32 data_size, + btf_print_fn_t err_log) { - int sec_hdrlen = sizeof(struct btf_sec_func_info); - __u32 size_left, num_records, record_size; + const struct btf_ext_header *hdr = (struct btf_ext_header *)data; const struct btf_sec_func_info *sinfo; - __u64 total_record_size; + __u32 info_left, record_size; + /* The start of the info sec (including the __u32 record_size). */ + const void *info; + + /* data and data_size do not include btf_ext_header from now on */ + data = data + hdr->hdr_len; + data_size -= hdr->hdr_len; + + if (hdr->func_info_off & 0x03) { + elog("BTF.ext func_info section is not aligned to 4 bytes\n"); + return -EINVAL; + } + + if (data_size < hdr->func_info_off || + hdr->func_info_len > data_size - hdr->func_info_off) { + elog("func_info section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n", + hdr->func_info_off, hdr->func_info_len); + return -EINVAL; + } + + info = data + hdr->func_info_off; + info_left = hdr->func_info_len; /* At least a func_info record size */ - if (size < sizeof(__u32)) { + if (info_left < sizeof(__u32)) { elog("BTF.ext func_info record size not found"); return -EINVAL; } - /* The record size needs to meet below minimum standard */ - record_size = *(__u32 *)finfo; + /* The record size needs to meet the minimum standard */ + record_size = *(__u32 *)info; if (record_size < sizeof(struct bpf_func_info_min) || - record_size % sizeof(__u32)) { + record_size & 0x03) { elog("BTF.ext func_info invalid record size"); return -EINVAL; } - sinfo = finfo + sizeof(__u32); - size_left = size - sizeof(__u32); + sinfo = info + sizeof(__u32); + info_left -= sizeof(__u32); /* If no func_info records, return failure now so .BTF.ext * won't be used. */ - if (!size_left) { + if (!info_left) { elog("BTF.ext no func info records"); return -EINVAL; } - while (size_left) { - if (size_left < sec_hdrlen) { + while (info_left) { + unsigned int sec_hdrlen = sizeof(struct btf_sec_func_info); + __u64 total_record_size; + __u32 num_records; + + if (info_left < sec_hdrlen) { elog("BTF.ext func_info header not found"); return -EINVAL; } @@ -526,15 +558,23 @@ static int btf_ext_validate_func_info(const void *finfo, __u32 size, total_record_size = sec_hdrlen + (__u64)num_records * record_size; - if (size_left < total_record_size) { + if (info_left < total_record_size) { elog("incorrect BTF.ext num_func_info"); return -EINVAL; } - size_left -= total_record_size; + info_left -= total_record_size; sinfo = (void *)sinfo + total_record_size; } + btf_ext->func_info_len = hdr->func_info_len - sizeof(__u32); + btf_ext->func_info_rec_size = record_size; + btf_ext->func_info = malloc(btf_ext->func_info_len); + if (!btf_ext->func_info) + return -ENOMEM; + memcpy(btf_ext->func_info, info + sizeof(__u32), + btf_ext->func_info_len); + return 0; } @@ -542,8 +582,6 @@ static int btf_ext_parse_hdr(__u8 *data, __u32 data_size, btf_print_fn_t err_log) { const struct btf_ext_header *hdr = (struct btf_ext_header *)data; - __u32 meta_left, last_func_info_pos; - void *finfo; if (data_size < offsetof(struct btf_ext_header, func_info_off) || data_size < hdr->hdr_len) { @@ -566,34 +604,12 @@ static int btf_ext_parse_hdr(__u8 *data, __u32 data_size, return -ENOTSUP; } - meta_left = data_size - hdr->hdr_len; - if (!meta_left) { + if (data_size == hdr->hdr_len) { elog("BTF.ext has no data\n"); return -EINVAL; } - if (meta_left < hdr->func_info_off) { - elog("Invalid BTF.ext func_info section offset:%u\n", - hdr->func_info_off); - return -EINVAL; - } - - if (hdr->func_info_off & 0x03) { - elog("BTF.ext func_info section is not aligned to 4 bytes\n"); - return -EINVAL; - } - - last_func_info_pos = hdr->hdr_len + hdr->func_info_off + - hdr->func_info_len; - if (last_func_info_pos > data_size) { - elog("Invalid BTF.ext func_info section size:%u\n", - hdr->func_info_len); - return -EINVAL; - } - - finfo = data + hdr->hdr_len + hdr->func_info_off; - return btf_ext_validate_func_info(finfo, hdr->func_info_len, - err_log); + return 0; } void btf_ext__free(struct btf_ext *btf_ext) @@ -607,10 +623,7 @@ void btf_ext__free(struct btf_ext *btf_ext) struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log) { - const struct btf_ext_header *hdr; struct btf_ext *btf_ext; - void *org_fdata, *fdata; - __u32 hdrlen, size_u32; int err; err = btf_ext_parse_hdr(data, size, err_log); @@ -621,81 +634,18 @@ struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log) if (!btf_ext) return ERR_PTR(-ENOMEM); - hdr = (const struct btf_ext_header *)data; - hdrlen = hdr->hdr_len; - size_u32 = sizeof(__u32); - fdata = malloc(hdr->func_info_len - size_u32); - if (!fdata) { - free(btf_ext); - return ERR_PTR(-ENOMEM); + err = btf_ext_copy_func_info(btf_ext, data, size, err_log); + if (err) { + btf_ext__free(btf_ext); + return ERR_PTR(err); } - /* remember record size and copy rest of func_info data */ - org_fdata = data + hdrlen + hdr->func_info_off; - btf_ext->func_info_rec_size = *(__u32 *)org_fdata; - memcpy(fdata, org_fdata + size_u32, hdr->func_info_len - size_u32); - btf_ext->func_info = fdata; - btf_ext->func_info_len = hdr->func_info_len - size_u32; - return btf_ext; } -int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext, - const char *sec_name, void **func_info, - __u32 *func_info_rec_size, __u32 *func_info_len) -{ - __u32 sec_hdrlen = sizeof(struct btf_sec_func_info); - __u32 i, record_size, records_len; - struct btf_sec_func_info *sinfo; - const char *info_sec_name; - __s64 remain_len; - void *data; - - record_size = btf_ext->func_info_rec_size; - sinfo = btf_ext->func_info; - remain_len = btf_ext->func_info_len; - - while (remain_len > 0) { - records_len = sinfo->num_func_info * record_size; - info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off); - if (strcmp(info_sec_name, sec_name)) { - remain_len -= sec_hdrlen + records_len; - sinfo = (void *)sinfo + sec_hdrlen + records_len; - continue; - } - - data = malloc(records_len); - if (!data) - return -ENOMEM; - - memcpy(data, sinfo->data, records_len); - - /* adjust the insn_off, the data in .BTF.ext is - * the actual byte offset, and the kernel expects - * the offset in term of bpf_insn. - * - * adjust the insn offset only, the rest data will - * be passed to kernel. - */ - for (i = 0; i < sinfo->num_func_info; i++) { - struct bpf_func_info_min *record; - - record = data + i * record_size; - record->insn_off /= sizeof(struct bpf_insn); - } - - *func_info = data; - *func_info_len = records_len; - *func_info_rec_size = record_size; - return 0; - } - - return -EINVAL; -} - -int btf_ext__reloc(struct btf *btf, struct btf_ext *btf_ext, - const char *sec_name, __u32 insns_cnt, - void **func_info, __u32 *func_info_len) +int btf_ext__reloc_func_info(struct btf *btf, struct btf_ext *btf_ext, + const char *sec_name, __u32 insns_cnt, + void **func_info, __u32 *cnt) { __u32 sec_hdrlen = sizeof(struct btf_sec_func_info); __u32 i, record_size, existing_flen, records_len; @@ -716,7 +666,7 @@ int btf_ext__reloc(struct btf *btf, struct btf_ext *btf_ext, continue; } - existing_flen = *func_info_len; + existing_flen = (*cnt) * record_size; data = realloc(*func_info, existing_flen + records_len); if (!data) return -ENOMEM; @@ -734,9 +684,14 @@ int btf_ext__reloc(struct btf *btf, struct btf_ext *btf_ext, insns_cnt; } *func_info = data; - *func_info_len = existing_flen + records_len; + *cnt += sinfo->num_func_info; return 0; } - return -EINVAL; + return -ENOENT; +} + +__u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext) +{ + return btf_ext->func_info_rec_size; } diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index 5336b2f37293a..936177a538cd5 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -53,13 +53,6 @@ struct btf_ext_header { __u32 func_info_len; }; -struct btf_sec_func_info { - __u32 sec_name_off; - __u32 num_func_info; - /* Followed by num_func_info number of bpf func_info records */ - __u8 data[0]; -}; - typedef int (*btf_print_fn_t)(const char *, ...) __attribute__((format(printf, 1, 2))); @@ -77,12 +70,10 @@ LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf); struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log); void btf_ext__free(struct btf_ext *btf_ext); -int btf_ext__reloc_init(struct btf *btf, struct btf_ext *btf_ext, - const char *sec_name, void **func_info, - __u32 *func_info_rec_size, __u32 *func_info_len); -int btf_ext__reloc(struct btf *btf, struct btf_ext *btf_ext, - const char *sec_name, __u32 insns_cnt, void **func_info, - __u32 *func_info_len); +int btf_ext__reloc_func_info(struct btf *btf, struct btf_ext *btf_ext, + const char *sec_name, __u32 insns_cnt, + void **func_info, __u32 *func_info_len); +__u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext); #ifdef __cplusplus } /* extern "C" */ diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 59b748ebd15f2..4ea3368bf8037 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -167,7 +167,7 @@ struct bpf_program { int btf_fd; void *func_info; __u32 func_info_rec_size; - __u32 func_info_len; + __u32 func_info_cnt; struct bpf_capabilities *caps; }; @@ -779,6 +779,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) { Elf *elf = obj->efile.elf; GElf_Ehdr *ep = &obj->efile.ehdr; + Elf_Data *btf_ext_data = NULL; Elf_Scn *scn = NULL; int idx = 0, err = 0; @@ -841,14 +842,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) obj->btf = NULL; } } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { - obj->btf_ext = btf_ext__new(data->d_buf, data->d_size, - __pr_debug); - if (IS_ERR(obj->btf_ext)) { - pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", - BTF_EXT_ELF_SEC, - PTR_ERR(obj->btf_ext)); - obj->btf_ext = NULL; - } + btf_ext_data = data; } else if (sh.sh_type == SHT_SYMTAB) { if (obj->efile.symbols) { pr_warning("bpf: multiple SYMTAB in %s\n", @@ -910,6 +904,22 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) pr_warning("Corrupted ELF file: index of strtab invalid\n"); return LIBBPF_ERRNO__FORMAT; } + if (btf_ext_data) { + if (!obj->btf) { + pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", + BTF_EXT_ELF_SEC, BTF_ELF_SEC); + } else { + obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, + btf_ext_data->d_size, + __pr_debug); + if (IS_ERR(obj->btf_ext)) { + pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", + BTF_EXT_ELF_SEC, + PTR_ERR(obj->btf_ext)); + obj->btf_ext = NULL; + } + } + } if (obj->efile.maps_shndx >= 0) { err = bpf_object__init_maps(obj, flags); if (err) @@ -1275,6 +1285,69 @@ bpf_object__create_maps(struct bpf_object *obj) return 0; } +static int +check_btf_ext_reloc_err(struct bpf_program *prog, int err, + void *btf_prog_info, const char *info_name) +{ + if (err != -ENOENT) { + pr_warning("Error in loading %s for sec %s.\n", + info_name, prog->section_name); + return err; + } + + /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ + + if (btf_prog_info) { + /* + * Some info has already been found but has problem + * in the last btf_ext reloc. Must have to error + * out. + */ + pr_warning("Error in relocating %s for sec %s.\n", + info_name, prog->section_name); + return err; + } + + /* + * Have problem loading the very first info. Ignore + * the rest. + */ + pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n", + info_name, prog->section_name, info_name); + return 0; +} + +static int +bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, + const char *section_name, __u32 insn_offset) +{ + int err; + + if (!insn_offset || prog->func_info) { + /* + * !insn_offset => main program + * + * For sub prog, the main program's func_info has to + * be loaded first (i.e. prog->func_info != NULL) + */ + err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, + section_name, insn_offset, + &prog->func_info, + &prog->func_info_cnt); + if (err) + return check_btf_ext_reloc_err(prog, err, + prog->func_info, + "bpf_func_info"); + + prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); + } + + if (!insn_offset) + prog->btf_fd = btf__fd(obj->btf); + + return 0; +} + static int bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, struct reloc_desc *relo) @@ -1306,17 +1379,12 @@ bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, return -ENOMEM; } - if (obj->btf && obj->btf_ext) { - err = btf_ext__reloc(obj->btf, obj->btf_ext, - text->section_name, - prog->insns_cnt, - &prog->func_info, - &prog->func_info_len); - if (err) { - pr_warning("error in btf_ext__reloc for sec %s\n", - text->section_name); + if (obj->btf_ext) { + err = bpf_program_reloc_btf_ext(prog, obj, + text->section_name, + prog->insns_cnt); + if (err) return err; - } } memcpy(new_insn + prog->insns_cnt, text->insns, @@ -1341,18 +1409,11 @@ bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) if (!prog) return 0; - if (obj->btf && obj->btf_ext) { - err = btf_ext__reloc_init(obj->btf, obj->btf_ext, - prog->section_name, - &prog->func_info, - &prog->func_info_rec_size, - &prog->func_info_len); - if (err) { - pr_warning("err in btf_ext__reloc_init for sec %s\n", - prog->section_name); + if (obj->btf_ext) { + err = bpf_program_reloc_btf_ext(prog, obj, + prog->section_name, 0); + if (err) return err; - } - prog->btf_fd = btf__fd(obj->btf); } if (!prog->reloc_desc) @@ -1444,8 +1505,7 @@ static int bpf_object__collect_reloc(struct bpf_object *obj) static int load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, - char *license, __u32 kern_version, int *pfd, - __u32 func_info_cnt) + char *license, __u32 kern_version, int *pfd) { struct bpf_load_program_attr load_attr; char *cp, errmsg[STRERR_BUFSIZE]; @@ -1465,8 +1525,7 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0; load_attr.func_info = prog->func_info; load_attr.func_info_rec_size = prog->func_info_rec_size; - load_attr.func_info_cnt = func_info_cnt; - + load_attr.func_info_cnt = prog->func_info_cnt; if (!load_attr.insns || !load_attr.insns_cnt) return -EINVAL; @@ -1523,14 +1582,8 @@ int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_version) { - __u32 func_info_cnt; int err = 0, fd, i; - if (prog->func_info_len == 0) - func_info_cnt = 0; - else - func_info_cnt = prog->func_info_len / prog->func_info_rec_size; - if (prog->instances.nr < 0 || !prog->instances.fds) { if (prog->preprocessor) { pr_warning("Internal error: can't load program '%s'\n", @@ -1553,8 +1606,7 @@ bpf_program__load(struct bpf_program *prog, prog->section_name, prog->instances.nr); } err = load_program(prog, prog->insns, prog->insns_cnt, - license, kern_version, &fd, - func_info_cnt); + license, kern_version, &fd); if (!err) prog->instances.fds[0] = fd; goto out; @@ -1584,8 +1636,7 @@ bpf_program__load(struct bpf_program *prog, err = load_program(prog, result.new_insn_ptr, result.new_insn_cnt, - license, kern_version, &fd, - func_info_cnt); + license, kern_version, &fd); if (err) { pr_warning("Loading the %dth instance of program '%s' failed\n", From 3d65014146c69bbc4d2947f60dbd722d352cdc46 Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Fri, 7 Dec 2018 16:42:31 -0800 Subject: [PATCH 43/51] bpf: libbpf: Add btf_line_info support to libbpf This patch adds bpf_line_info support to libbpf: 1) Parsing the line_info sec from ".BTF.ext" 2) Relocating the line_info. If the main prog *_info relocation fails, it will ignore the remaining subprog line_info and continue. If the subprog *_info relocation fails, it will bail out. 3) BPF_PROG_LOAD a prog with line_info Signed-off-by: Martin KaFai Lau Acked-by: Yonghong Song Signed-off-by: Alexei Starovoitov --- tools/lib/bpf/bpf.c | 86 +++++++++++------ tools/lib/bpf/bpf.h | 3 + tools/lib/bpf/btf.c | 209 +++++++++++++++++++++++++++++------------ tools/lib/bpf/btf.h | 10 +- tools/lib/bpf/libbpf.c | 20 ++++ 5 files changed, 239 insertions(+), 89 deletions(-) diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 9fbbc0ed59525..3caaa34287744 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -173,11 +173,36 @@ int bpf_create_map_in_map(enum bpf_map_type map_type, const char *name, -1); } +static void * +alloc_zero_tailing_info(const void *orecord, __u32 cnt, + __u32 actual_rec_size, __u32 expected_rec_size) +{ + __u64 info_len = actual_rec_size * cnt; + void *info, *nrecord; + int i; + + info = malloc(info_len); + if (!info) + return NULL; + + /* zero out bytes kernel does not understand */ + nrecord = info; + for (i = 0; i < cnt; i++) { + memcpy(nrecord, orecord, expected_rec_size); + memset(nrecord + expected_rec_size, 0, + actual_rec_size - expected_rec_size); + orecord += actual_rec_size; + nrecord += actual_rec_size; + } + + return info; +} + int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, char *log_buf, size_t log_buf_sz) { + void *finfo = NULL, *linfo = NULL; union bpf_attr attr; - void *finfo = NULL; __u32 name_len; int fd; @@ -201,6 +226,9 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, attr.func_info_rec_size = load_attr->func_info_rec_size; attr.func_info_cnt = load_attr->func_info_cnt; attr.func_info = ptr_to_u64(load_attr->func_info); + attr.line_info_rec_size = load_attr->line_info_rec_size; + attr.line_info_cnt = load_attr->line_info_cnt; + attr.line_info = ptr_to_u64(load_attr->line_info); memcpy(attr.prog_name, load_attr->name, min(name_len, BPF_OBJ_NAME_LEN - 1)); @@ -212,36 +240,35 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, * to give user space a hint how to deal with loading failure. * Check to see whether we can make some changes and load again. */ - if (errno == E2BIG && attr.func_info_cnt && - attr.func_info_rec_size < load_attr->func_info_rec_size) { - __u32 actual_rec_size = load_attr->func_info_rec_size; - __u32 expected_rec_size = attr.func_info_rec_size; - __u32 finfo_cnt = load_attr->func_info_cnt; - __u64 finfo_len = actual_rec_size * finfo_cnt; - const void *orecord; - void *nrecord; - int i; - - finfo = malloc(finfo_len); - if (!finfo) - /* further try with log buffer won't help */ - return fd; - - /* zero out bytes kernel does not understand */ - orecord = load_attr->func_info; - nrecord = finfo; - for (i = 0; i < load_attr->func_info_cnt; i++) { - memcpy(nrecord, orecord, expected_rec_size); - memset(nrecord + expected_rec_size, 0, - actual_rec_size - expected_rec_size); - orecord += actual_rec_size; - nrecord += actual_rec_size; + while (errno == E2BIG && (!finfo || !linfo)) { + if (!finfo && attr.func_info_cnt && + attr.func_info_rec_size < load_attr->func_info_rec_size) { + /* try with corrected func info records */ + finfo = alloc_zero_tailing_info(load_attr->func_info, + load_attr->func_info_cnt, + load_attr->func_info_rec_size, + attr.func_info_rec_size); + if (!finfo) + goto done; + + attr.func_info = ptr_to_u64(finfo); + attr.func_info_rec_size = load_attr->func_info_rec_size; + } else if (!linfo && attr.line_info_cnt && + attr.line_info_rec_size < + load_attr->line_info_rec_size) { + linfo = alloc_zero_tailing_info(load_attr->line_info, + load_attr->line_info_cnt, + load_attr->line_info_rec_size, + attr.line_info_rec_size); + if (!linfo) + goto done; + + attr.line_info = ptr_to_u64(linfo); + attr.line_info_rec_size = load_attr->line_info_rec_size; + } else { + break; } - /* try with corrected func info records */ - attr.func_info = ptr_to_u64(finfo); - attr.func_info_rec_size = load_attr->func_info_rec_size; - fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); if (fd >= 0) @@ -259,6 +286,7 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, fd = sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); done: free(finfo); + free(linfo); return fd; } diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 098e6f793b765..8f09de482839e 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -82,6 +82,9 @@ struct bpf_load_program_attr { __u32 func_info_rec_size; const void *func_info; __u32 func_info_cnt; + __u32 line_info_rec_size; + const void *line_info; + __u32 line_info_cnt; }; /* Flags to direct loading requirements */ diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index aa4fa02b13fcb..d682d3b8f7b9b 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -37,16 +37,26 @@ struct btf { int fd; }; +struct btf_ext_info { + /* + * info points to a deep copy of the individual info section + * (e.g. func_info and line_info) from the .BTF.ext. + * It does not include the __u32 rec_size. + */ + void *info; + __u32 rec_size; + __u32 len; +}; + struct btf_ext { - void *func_info; - __u32 func_info_rec_size; - __u32 func_info_len; + struct btf_ext_info func_info; + struct btf_ext_info line_info; }; -struct btf_sec_func_info { +struct btf_ext_info_sec { __u32 sec_name_off; - __u32 num_func_info; - /* Followed by num_func_info number of bpf func_info records */ + __u32 num_info; + /* Followed by num_info * record_size number of bytes */ __u8 data[0]; }; @@ -56,6 +66,14 @@ struct bpf_func_info_min { __u32 type_id; }; +/* The minimum bpf_line_info checked by the loader */ +struct bpf_line_info_min { + __u32 insn_off; + __u32 file_name_off; + __u32 line_off; + __u32 line_col; +}; + static inline __u64 ptr_to_u64(const void *ptr) { return (__u64) (unsigned long) ptr; @@ -486,12 +504,22 @@ int btf__get_from_id(__u32 id, struct btf **btf) return err; } -static int btf_ext_copy_func_info(struct btf_ext *btf_ext, - __u8 *data, __u32 data_size, - btf_print_fn_t err_log) +struct btf_ext_sec_copy_param { + __u32 off; + __u32 len; + __u32 min_rec_size; + struct btf_ext_info *ext_info; + const char *desc; +}; + +static int btf_ext_copy_info(struct btf_ext *btf_ext, + __u8 *data, __u32 data_size, + struct btf_ext_sec_copy_param *ext_sec, + btf_print_fn_t err_log) { const struct btf_ext_header *hdr = (struct btf_ext_header *)data; - const struct btf_sec_func_info *sinfo; + const struct btf_ext_info_sec *sinfo; + struct btf_ext_info *ext_info; __u32 info_left, record_size; /* The start of the info sec (including the __u32 record_size). */ const void *info; @@ -500,66 +528,69 @@ static int btf_ext_copy_func_info(struct btf_ext *btf_ext, data = data + hdr->hdr_len; data_size -= hdr->hdr_len; - if (hdr->func_info_off & 0x03) { - elog("BTF.ext func_info section is not aligned to 4 bytes\n"); + if (ext_sec->off & 0x03) { + elog(".BTF.ext %s section is not aligned to 4 bytes\n", + ext_sec->desc); return -EINVAL; } - if (data_size < hdr->func_info_off || - hdr->func_info_len > data_size - hdr->func_info_off) { - elog("func_info section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n", - hdr->func_info_off, hdr->func_info_len); + if (data_size < ext_sec->off || + ext_sec->len > data_size - ext_sec->off) { + elog("%s section (off:%u len:%u) is beyond the end of the ELF section .BTF.ext\n", + ext_sec->desc, ext_sec->off, ext_sec->len); return -EINVAL; } - info = data + hdr->func_info_off; - info_left = hdr->func_info_len; + info = data + ext_sec->off; + info_left = ext_sec->len; - /* At least a func_info record size */ + /* At least a record size */ if (info_left < sizeof(__u32)) { - elog("BTF.ext func_info record size not found"); + elog(".BTF.ext %s record size not found\n", ext_sec->desc); return -EINVAL; } /* The record size needs to meet the minimum standard */ record_size = *(__u32 *)info; - if (record_size < sizeof(struct bpf_func_info_min) || + if (record_size < ext_sec->min_rec_size || record_size & 0x03) { - elog("BTF.ext func_info invalid record size"); + elog("%s section in .BTF.ext has invalid record size %u\n", + ext_sec->desc, record_size); return -EINVAL; } sinfo = info + sizeof(__u32); info_left -= sizeof(__u32); - /* If no func_info records, return failure now so .BTF.ext - * won't be used. - */ + /* If no records, return failure now so .BTF.ext won't be used. */ if (!info_left) { - elog("BTF.ext no func info records"); + elog("%s section in .BTF.ext has no records", ext_sec->desc); return -EINVAL; } while (info_left) { - unsigned int sec_hdrlen = sizeof(struct btf_sec_func_info); + unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec); __u64 total_record_size; __u32 num_records; if (info_left < sec_hdrlen) { - elog("BTF.ext func_info header not found"); + elog("%s section header is not found in .BTF.ext\n", + ext_sec->desc); return -EINVAL; } - num_records = sinfo->num_func_info; + num_records = sinfo->num_info; if (num_records == 0) { - elog("incorrect BTF.ext num_func_info"); + elog("%s section has incorrect num_records in .BTF.ext\n", + ext_sec->desc); return -EINVAL; } total_record_size = sec_hdrlen + (__u64)num_records * record_size; if (info_left < total_record_size) { - elog("incorrect BTF.ext num_func_info"); + elog("%s section has incorrect num_records in .BTF.ext\n", + ext_sec->desc); return -EINVAL; } @@ -567,17 +598,49 @@ static int btf_ext_copy_func_info(struct btf_ext *btf_ext, sinfo = (void *)sinfo + total_record_size; } - btf_ext->func_info_len = hdr->func_info_len - sizeof(__u32); - btf_ext->func_info_rec_size = record_size; - btf_ext->func_info = malloc(btf_ext->func_info_len); - if (!btf_ext->func_info) + ext_info = ext_sec->ext_info; + ext_info->len = ext_sec->len - sizeof(__u32); + ext_info->rec_size = record_size; + ext_info->info = malloc(ext_info->len); + if (!ext_info->info) return -ENOMEM; - memcpy(btf_ext->func_info, info + sizeof(__u32), - btf_ext->func_info_len); + memcpy(ext_info->info, info + sizeof(__u32), ext_info->len); return 0; } +static int btf_ext_copy_func_info(struct btf_ext *btf_ext, + __u8 *data, __u32 data_size, + btf_print_fn_t err_log) +{ + const struct btf_ext_header *hdr = (struct btf_ext_header *)data; + struct btf_ext_sec_copy_param param = { + .off = hdr->func_info_off, + .len = hdr->func_info_len, + .min_rec_size = sizeof(struct bpf_func_info_min), + .ext_info = &btf_ext->func_info, + .desc = "func_info" + }; + + return btf_ext_copy_info(btf_ext, data, data_size, ¶m, err_log); +} + +static int btf_ext_copy_line_info(struct btf_ext *btf_ext, + __u8 *data, __u32 data_size, + btf_print_fn_t err_log) +{ + const struct btf_ext_header *hdr = (struct btf_ext_header *)data; + struct btf_ext_sec_copy_param param = { + .off = hdr->line_info_off, + .len = hdr->line_info_len, + .min_rec_size = sizeof(struct bpf_line_info_min), + .ext_info = &btf_ext->line_info, + .desc = "line_info", + }; + + return btf_ext_copy_info(btf_ext, data, data_size, ¶m, err_log); +} + static int btf_ext_parse_hdr(__u8 *data, __u32 data_size, btf_print_fn_t err_log) { @@ -617,7 +680,8 @@ void btf_ext__free(struct btf_ext *btf_ext) if (!btf_ext) return; - free(btf_ext->func_info); + free(btf_ext->func_info.info); + free(btf_ext->line_info.info); free(btf_ext); } @@ -640,25 +704,32 @@ struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log) return ERR_PTR(err); } + err = btf_ext_copy_line_info(btf_ext, data, size, err_log); + if (err) { + btf_ext__free(btf_ext); + return ERR_PTR(err); + } + return btf_ext; } -int btf_ext__reloc_func_info(struct btf *btf, struct btf_ext *btf_ext, - const char *sec_name, __u32 insns_cnt, - void **func_info, __u32 *cnt) +static int btf_ext_reloc_info(const struct btf *btf, + const struct btf_ext_info *ext_info, + const char *sec_name, __u32 insns_cnt, + void **info, __u32 *cnt) { - __u32 sec_hdrlen = sizeof(struct btf_sec_func_info); - __u32 i, record_size, existing_flen, records_len; - struct btf_sec_func_info *sinfo; + __u32 sec_hdrlen = sizeof(struct btf_ext_info_sec); + __u32 i, record_size, existing_len, records_len; + struct btf_ext_info_sec *sinfo; const char *info_sec_name; __u64 remain_len; void *data; - record_size = btf_ext->func_info_rec_size; - sinfo = btf_ext->func_info; - remain_len = btf_ext->func_info_len; + record_size = ext_info->rec_size; + sinfo = ext_info->info; + remain_len = ext_info->len; while (remain_len > 0) { - records_len = sinfo->num_func_info * record_size; + records_len = sinfo->num_info * record_size; info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off); if (strcmp(info_sec_name, sec_name)) { remain_len -= sec_hdrlen + records_len; @@ -666,32 +737,52 @@ int btf_ext__reloc_func_info(struct btf *btf, struct btf_ext *btf_ext, continue; } - existing_flen = (*cnt) * record_size; - data = realloc(*func_info, existing_flen + records_len); + existing_len = (*cnt) * record_size; + data = realloc(*info, existing_len + records_len); if (!data) return -ENOMEM; - memcpy(data + existing_flen, sinfo->data, records_len); + memcpy(data + existing_len, sinfo->data, records_len); /* adjust insn_off only, the rest data will be passed * to the kernel. */ - for (i = 0; i < sinfo->num_func_info; i++) { - struct bpf_func_info_min *record; + for (i = 0; i < sinfo->num_info; i++) { + __u32 *insn_off; - record = data + existing_flen + i * record_size; - record->insn_off = - record->insn_off / sizeof(struct bpf_insn) + + insn_off = data + existing_len + (i * record_size); + *insn_off = *insn_off / sizeof(struct bpf_insn) + insns_cnt; } - *func_info = data; - *cnt += sinfo->num_func_info; + *info = data; + *cnt += sinfo->num_info; return 0; } return -ENOENT; } +int btf_ext__reloc_func_info(const struct btf *btf, const struct btf_ext *btf_ext, + const char *sec_name, __u32 insns_cnt, + void **func_info, __u32 *cnt) +{ + return btf_ext_reloc_info(btf, &btf_ext->func_info, sec_name, + insns_cnt, func_info, cnt); +} + +int btf_ext__reloc_line_info(const struct btf *btf, const struct btf_ext *btf_ext, + const char *sec_name, __u32 insns_cnt, + void **line_info, __u32 *cnt) +{ + return btf_ext_reloc_info(btf, &btf_ext->line_info, sec_name, + insns_cnt, line_info, cnt); +} + __u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext) { - return btf_ext->func_info_rec_size; + return btf_ext->func_info.rec_size; +} + +__u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext) +{ + return btf_ext->line_info.rec_size; } diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index 936177a538cd5..b0610dcdae6bd 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -51,6 +51,8 @@ struct btf_ext_header { /* All offsets are in bytes relative to the end of this header */ __u32 func_info_off; __u32 func_info_len; + __u32 line_info_off; + __u32 line_info_len; }; typedef int (*btf_print_fn_t)(const char *, ...) @@ -70,10 +72,16 @@ LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf); struct btf_ext *btf_ext__new(__u8 *data, __u32 size, btf_print_fn_t err_log); void btf_ext__free(struct btf_ext *btf_ext); -int btf_ext__reloc_func_info(struct btf *btf, struct btf_ext *btf_ext, +int btf_ext__reloc_func_info(const struct btf *btf, + const struct btf_ext *btf_ext, const char *sec_name, __u32 insns_cnt, void **func_info, __u32 *func_info_len); +int btf_ext__reloc_line_info(const struct btf *btf, + const struct btf_ext *btf_ext, + const char *sec_name, __u32 insns_cnt, + void **line_info, __u32 *cnt); __u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext); +__u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext); #ifdef __cplusplus } /* extern "C" */ diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 4ea3368bf8037..e2bc75ee16146 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -170,6 +170,10 @@ struct bpf_program { __u32 func_info_cnt; struct bpf_capabilities *caps; + + void *line_info; + __u32 line_info_rec_size; + __u32 line_info_cnt; }; struct bpf_map { @@ -1342,6 +1346,19 @@ bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); } + if (!insn_offset || prog->line_info) { + err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, + section_name, insn_offset, + &prog->line_info, + &prog->line_info_cnt); + if (err) + return check_btf_ext_reloc_err(prog, err, + prog->line_info, + "bpf_line_info"); + + prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); + } + if (!insn_offset) prog->btf_fd = btf__fd(obj->btf); @@ -1526,6 +1543,9 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, load_attr.func_info = prog->func_info; load_attr.func_info_rec_size = prog->func_info_rec_size; load_attr.func_info_cnt = prog->func_info_cnt; + load_attr.line_info = prog->line_info; + load_attr.line_info_rec_size = prog->line_info_rec_size; + load_attr.line_info_cnt = prog->line_info_cnt; if (!load_attr.insns || !load_attr.insns_cnt) return -EINVAL; From b053b439b72ad152257ecc3f71cfb4c619b0137e Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Fri, 7 Dec 2018 16:42:32 -0800 Subject: [PATCH 44/51] bpf: libbpf: bpftool: Print bpf_line_info during prog dump This patch adds print bpf_line_info function in 'prog dump jitted' and 'prog dump xlated': [root@arch-fb-vm1 bpf]# ~/devshare/fb-kernel/linux/tools/bpf/bpftool/bpftool prog dump jited pinned /sys/fs/bpf/test_btf_haskv [...] int test_long_fname_2(struct dummy_tracepoint_args * arg): bpf_prog_44a040bf25481309_test_long_fname_2: ; static int test_long_fname_2(struct dummy_tracepoint_args *arg) 0: push %rbp 1: mov %rsp,%rbp 4: sub $0x30,%rsp b: sub $0x28,%rbp f: mov %rbx,0x0(%rbp) 13: mov %r13,0x8(%rbp) 17: mov %r14,0x10(%rbp) 1b: mov %r15,0x18(%rbp) 1f: xor %eax,%eax 21: mov %rax,0x20(%rbp) 25: xor %esi,%esi ; int key = 0; 27: mov %esi,-0x4(%rbp) ; if (!arg->sock) 2a: mov 0x8(%rdi),%rdi ; if (!arg->sock) 2e: cmp $0x0,%rdi 32: je 0x0000000000000070 34: mov %rbp,%rsi ; counts = bpf_map_lookup_elem(&btf_map, &key); 37: add $0xfffffffffffffffc,%rsi 3b: movabs $0xffff8881139d7480,%rdi 45: add $0x110,%rdi 4c: mov 0x0(%rsi),%eax 4f: cmp $0x4,%rax 53: jae 0x000000000000005e 55: shl $0x3,%rax 59: add %rdi,%rax 5c: jmp 0x0000000000000060 5e: xor %eax,%eax ; if (!counts) 60: cmp $0x0,%rax 64: je 0x0000000000000070 ; counts->v6++; 66: mov 0x4(%rax),%edi 69: add $0x1,%rdi 6d: mov %edi,0x4(%rax) 70: mov 0x0(%rbp),%rbx 74: mov 0x8(%rbp),%r13 78: mov 0x10(%rbp),%r14 7c: mov 0x18(%rbp),%r15 80: add $0x28,%rbp 84: leaveq 85: retq [...] With linum: [root@arch-fb-vm1 bpf]# ~/devshare/fb-kernel/linux/tools/bpf/bpftool/bpftool prog dump jited pinned /sys/fs/bpf/test_btf_haskv linum int _dummy_tracepoint(struct dummy_tracepoint_args * arg): bpf_prog_b07ccb89267cf242__dummy_tracepoint: ; return test_long_fname_1(arg); [file:/data/users/kafai/fb-kernel/linux/tools/testing/selftests/bpf/test_btf_haskv.c line_num:54 line_col:9] 0: push %rbp 1: mov %rsp,%rbp 4: sub $0x28,%rsp b: sub $0x28,%rbp f: mov %rbx,0x0(%rbp) 13: mov %r13,0x8(%rbp) 17: mov %r14,0x10(%rbp) 1b: mov %r15,0x18(%rbp) 1f: xor %eax,%eax 21: mov %rax,0x20(%rbp) 25: callq 0x000000000000851e ; return test_long_fname_1(arg); [file:/data/users/kafai/fb-kernel/linux/tools/testing/selftests/bpf/test_btf_haskv.c line_num:54 line_col:2] 2a: xor %eax,%eax 2c: mov 0x0(%rbp),%rbx 30: mov 0x8(%rbp),%r13 34: mov 0x10(%rbp),%r14 38: mov 0x18(%rbp),%r15 3c: add $0x28,%rbp 40: leaveq 41: retq [...] Signed-off-by: Martin KaFai Lau Acked-by: Yonghong Song Signed-off-by: Alexei Starovoitov --- .../bpftool/Documentation/bpftool-prog.rst | 16 +- tools/bpf/bpftool/bash-completion/bpftool | 6 +- tools/bpf/bpftool/btf_dumper.c | 64 +++++ tools/bpf/bpftool/jit_disasm.c | 23 +- tools/bpf/bpftool/main.h | 23 +- tools/bpf/bpftool/prog.c | 100 ++++++- tools/bpf/bpftool/xlated_dumper.c | 30 ++- tools/bpf/bpftool/xlated_dumper.h | 7 +- tools/lib/bpf/Build | 2 +- tools/lib/bpf/bpf_prog_linfo.c | 253 ++++++++++++++++++ tools/lib/bpf/libbpf.h | 13 + tools/lib/bpf/libbpf.map | 4 + 12 files changed, 516 insertions(+), 25 deletions(-) create mode 100644 tools/lib/bpf/bpf_prog_linfo.c diff --git a/tools/bpf/bpftool/Documentation/bpftool-prog.rst b/tools/bpf/bpftool/Documentation/bpftool-prog.rst index 5524b6dccd856..7c30731a9b734 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-prog.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-prog.rst @@ -22,8 +22,8 @@ MAP COMMANDS ============= | **bpftool** **prog { show | list }** [*PROG*] -| **bpftool** **prog dump xlated** *PROG* [{**file** *FILE* | **opcodes** | **visual**}] -| **bpftool** **prog dump jited** *PROG* [{**file** *FILE* | **opcodes**}] +| **bpftool** **prog dump xlated** *PROG* [{**file** *FILE* | **opcodes** | **visual** | **linum**}] +| **bpftool** **prog dump jited** *PROG* [{**file** *FILE* | **opcodes** | **linum**}] | **bpftool** **prog pin** *PROG* *FILE* | **bpftool** **prog { load | loadall }** *OBJ* *PATH* [**type** *TYPE*] [**map** {**idx** *IDX* | **name** *NAME*} *MAP*] [**dev** *NAME*] | **bpftool** **prog attach** *PROG* *ATTACH_TYPE* [*MAP*] @@ -56,7 +56,7 @@ DESCRIPTION Output will start with program ID followed by program type and zero or more named attributes (depending on kernel version). - **bpftool prog dump xlated** *PROG* [{ **file** *FILE* | **opcodes** | **visual** }] + **bpftool prog dump xlated** *PROG* [{ **file** *FILE* | **opcodes** | **visual** | **linum** }] Dump eBPF instructions of the program from the kernel. By default, eBPF will be disassembled and printed to standard output in human-readable format. In this case, **opcodes** @@ -69,13 +69,21 @@ DESCRIPTION built instead, and eBPF instructions will be presented with CFG in DOT format, on standard output. - **bpftool prog dump jited** *PROG* [{ **file** *FILE* | **opcodes** }] + If the prog has line_info available, the source line will + be displayed by default. If **linum** is specified, + the filename, line number and line column will also be + displayed on top of the source line. + **bpftool prog dump jited** *PROG* [{ **file** *FILE* | **opcodes** | **linum** }] Dump jited image (host machine code) of the program. If *FILE* is specified image will be written to a file, otherwise it will be disassembled and printed to stdout. **opcodes** controls if raw opcodes will be printed. + If the prog has line_info available, the source line will + be displayed by default. If **linum** is specified, + the filename, line number and line column will also be + displayed on top of the source line. **bpftool prog pin** *PROG* *FILE* Pin program *PROG* as *FILE*. diff --git a/tools/bpf/bpftool/bash-completion/bpftool b/tools/bpf/bpftool/bash-completion/bpftool index 44c189ba072a3..a57febd6abb16 100644 --- a/tools/bpf/bpftool/bash-completion/bpftool +++ b/tools/bpf/bpftool/bash-completion/bpftool @@ -191,7 +191,7 @@ _bpftool() # Deal with simplest keywords case $prev in - help|hex|opcodes|visual) + help|hex|opcodes|visual|linum) return 0 ;; tag) @@ -278,10 +278,10 @@ _bpftool() *) _bpftool_once_attr 'file' if _bpftool_search_list 'xlated'; then - COMPREPLY+=( $( compgen -W 'opcodes visual' -- \ + COMPREPLY+=( $( compgen -W 'opcodes visual linum' -- \ "$cur" ) ) else - COMPREPLY+=( $( compgen -W 'opcodes' -- \ + COMPREPLY+=( $( compgen -W 'opcodes linum' -- \ "$cur" ) ) fi return 0 diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c index c3fd3a7cb7876..dbbf6ece67602 100644 --- a/tools/bpf/bpftool/btf_dumper.c +++ b/tools/bpf/bpftool/btf_dumper.c @@ -385,3 +385,67 @@ void btf_dumper_type_only(const struct btf *btf, __u32 type_id, char *func_sig, if (err < 0) func_sig[0] = '\0'; } + +static const char *ltrim(const char *s) +{ + while (isspace(*s)) + s++; + + return s; +} + +void btf_dump_linfo_plain(const struct btf *btf, + const struct bpf_line_info *linfo, + const char *prefix, bool linum) +{ + const char *line = btf__name_by_offset(btf, linfo->line_off); + + if (!line) + return; + line = ltrim(line); + + if (!prefix) + prefix = ""; + + if (linum) { + const char *file = btf__name_by_offset(btf, linfo->file_name_off); + + /* More forgiving on file because linum option is + * expected to provide more info than the already + * available src line. + */ + if (!file) + file = ""; + + printf("%s%s [file:%s line_num:%u line_col:%u]\n", + prefix, line, file, + BPF_LINE_INFO_LINE_NUM(linfo->line_col), + BPF_LINE_INFO_LINE_COL(linfo->line_col)); + } else { + printf("%s%s\n", prefix, line); + } +} + +void btf_dump_linfo_json(const struct btf *btf, + const struct bpf_line_info *linfo, bool linum) +{ + const char *line = btf__name_by_offset(btf, linfo->line_off); + + if (line) + jsonw_string_field(json_wtr, "src", ltrim(line)); + + if (linum) { + const char *file = btf__name_by_offset(btf, linfo->file_name_off); + + if (file) + jsonw_string_field(json_wtr, "file", file); + + if (BPF_LINE_INFO_LINE_NUM(linfo->line_col)) + jsonw_int_field(json_wtr, "line_num", + BPF_LINE_INFO_LINE_NUM(linfo->line_col)); + + if (BPF_LINE_INFO_LINE_COL(linfo->line_col)) + jsonw_int_field(json_wtr, "line_col", + BPF_LINE_INFO_LINE_COL(linfo->line_col)); + } +} diff --git a/tools/bpf/bpftool/jit_disasm.c b/tools/bpf/bpftool/jit_disasm.c index 545a92471c336..f381f8628ce99 100644 --- a/tools/bpf/bpftool/jit_disasm.c +++ b/tools/bpf/bpftool/jit_disasm.c @@ -21,6 +21,7 @@ #include #include #include +#include #include "json_writer.h" #include "main.h" @@ -68,10 +69,16 @@ static int fprintf_json(void *out, const char *fmt, ...) } void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, - const char *arch, const char *disassembler_options) + const char *arch, const char *disassembler_options, + const struct btf *btf, + const struct bpf_prog_linfo *prog_linfo, + __u64 func_ksym, unsigned int func_idx, + bool linum) { + const struct bpf_line_info *linfo = NULL; disassembler_ftype disassemble; struct disassemble_info info; + unsigned int nr_skip = 0; int count, i, pc = 0; char tpath[PATH_MAX]; bfd *bfdf; @@ -127,12 +134,26 @@ void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, if (json_output) jsonw_start_array(json_wtr); do { + if (prog_linfo) { + linfo = bpf_prog_linfo__lfind_addr_func(prog_linfo, + func_ksym + pc, + func_idx, + nr_skip); + if (linfo) + nr_skip++; + } + if (json_output) { jsonw_start_object(json_wtr); oper_count = 0; + if (linfo) + btf_dump_linfo_json(btf, linfo, linum); jsonw_name(json_wtr, "pc"); jsonw_printf(json_wtr, "\"0x%x\"", pc); } else { + if (linfo) + btf_dump_linfo_plain(btf, linfo, "; ", + linum); printf("%4x:\t", pc); } diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h index 0be0dd8f467ff..d9393abdba782 100644 --- a/tools/bpf/bpftool/main.h +++ b/tools/bpf/bpftool/main.h @@ -138,6 +138,9 @@ struct pinned_obj { struct hlist_node hash; }; +struct btf; +struct bpf_line_info; + int build_pinned_obj_table(struct pinned_obj_table *table, enum bpf_obj_type type); void delete_pinned_obj_table(struct pinned_obj_table *tab); @@ -175,13 +178,23 @@ int map_parse_fd(int *argc, char ***argv); int map_parse_fd_and_info(int *argc, char ***argv, void *info, __u32 *info_len); #ifdef HAVE_LIBBFD_SUPPORT +struct bpf_prog_linfo; void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, - const char *arch, const char *disassembler_options); + const char *arch, const char *disassembler_options, + const struct btf *btf, + const struct bpf_prog_linfo *prog_linfo, + __u64 func_ksym, unsigned int func_idx, + bool linum); int disasm_init(void); #else static inline void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, - const char *arch, const char *disassembler_options) + const char *arch, const char *disassembler_options, + const struct btf *btf, + const struct bpf_prog_linfo *prog_linfo, + __u64 func_ksym, unsigned int func_idx, + bool linum) + { } static inline int disasm_init(void) @@ -217,6 +230,12 @@ int btf_dumper_type(const struct btf_dumper *d, __u32 type_id, void btf_dumper_type_only(const struct btf *btf, __u32 func_type_id, char *func_only, int size); +void btf_dump_linfo_plain(const struct btf *btf, + const struct bpf_line_info *linfo, + const char *prefix, bool linum); +void btf_dump_linfo_json(const struct btf *btf, + const struct bpf_line_info *linfo, bool linum); + struct nlattr; struct ifinfomsg; struct tcmsg; diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index a9a51123454c3..65b921ffd10a1 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -423,24 +423,26 @@ static int do_show(int argc, char **argv) static int do_dump(int argc, char **argv) { + unsigned int finfo_rec_size, linfo_rec_size, jited_linfo_rec_size; + void *func_info = NULL, *linfo = NULL, *jited_linfo = NULL; + unsigned int finfo_cnt, linfo_cnt = 0, jited_linfo_cnt = 0; + struct bpf_prog_linfo *prog_linfo = NULL; unsigned long *func_ksyms = NULL; struct bpf_prog_info info = {}; unsigned int *func_lens = NULL; const char *disasm_opt = NULL; - unsigned int finfo_rec_size; unsigned int nr_func_ksyms; unsigned int nr_func_lens; struct dump_data dd = {}; __u32 len = sizeof(info); struct btf *btf = NULL; - void *func_info = NULL; - unsigned int finfo_cnt; unsigned int buf_size; char *filepath = NULL; bool opcodes = false; bool visual = false; char func_sig[1024]; unsigned char *buf; + bool linum = false; __u32 *member_len; __u64 *member_ptr; ssize_t n; @@ -484,6 +486,9 @@ static int do_dump(int argc, char **argv) } else if (is_prefix(*argv, "visual")) { visual = true; NEXT_ARG(); + } else if (is_prefix(*argv, "linum")) { + linum = true; + NEXT_ARG(); } if (argc) { @@ -543,6 +548,32 @@ static int do_dump(int argc, char **argv) } } + linfo_rec_size = info.line_info_rec_size; + if (info.line_info_cnt && linfo_rec_size && info.btf_id) { + linfo_cnt = info.line_info_cnt; + linfo = malloc(linfo_cnt * linfo_rec_size); + if (!linfo) { + p_err("mem alloc failed"); + close(fd); + goto err_free; + } + } + + jited_linfo_rec_size = info.jited_line_info_rec_size; + if (info.jited_line_info_cnt && + jited_linfo_rec_size && + info.nr_jited_ksyms && + info.nr_jited_func_lens && + info.btf_id) { + jited_linfo_cnt = info.jited_line_info_cnt; + jited_linfo = malloc(jited_linfo_cnt * jited_linfo_rec_size); + if (!jited_linfo) { + p_err("mem alloc failed"); + close(fd); + goto err_free; + } + } + memset(&info, 0, sizeof(info)); *member_ptr = ptr_to_u64(buf); @@ -554,6 +585,13 @@ static int do_dump(int argc, char **argv) info.func_info_cnt = finfo_cnt; info.func_info_rec_size = finfo_rec_size; info.func_info = ptr_to_u64(func_info); + info.line_info_cnt = linfo_cnt; + info.line_info_rec_size = linfo_rec_size; + info.line_info = ptr_to_u64(linfo); + info.jited_line_info_cnt = jited_linfo_cnt; + info.jited_line_info_rec_size = jited_linfo_rec_size; + info.jited_line_info = ptr_to_u64(jited_linfo); + err = bpf_obj_get_info_by_fd(fd, &info, &len); close(fd); @@ -596,6 +634,30 @@ static int do_dump(int argc, char **argv) finfo_cnt = 0; } + if (linfo && info.line_info_cnt != linfo_cnt) { + p_err("incorrect line_info_cnt %u vs. expected %u", + info.line_info_cnt, linfo_cnt); + goto err_free; + } + + if (info.line_info_rec_size != linfo_rec_size) { + p_err("incorrect line_info_rec_size %u vs. expected %u", + info.line_info_rec_size, linfo_rec_size); + goto err_free; + } + + if (jited_linfo && info.jited_line_info_cnt != jited_linfo_cnt) { + p_err("incorrect jited_line_info_cnt %u vs. expected %u", + info.jited_line_info_cnt, jited_linfo_cnt); + goto err_free; + } + + if (info.jited_line_info_rec_size != jited_linfo_rec_size) { + p_err("incorrect jited_line_info_rec_size %u vs. expected %u", + info.jited_line_info_rec_size, jited_linfo_rec_size); + goto err_free; + } + if ((member_len == &info.jited_prog_len && info.jited_prog_insns == 0) || (member_len == &info.xlated_prog_len && @@ -609,6 +671,12 @@ static int do_dump(int argc, char **argv) goto err_free; } + if (linfo_cnt) { + prog_linfo = bpf_prog_linfo__new(&info); + if (!prog_linfo) + p_err("error in processing bpf_line_info. continue without it."); + } + if (filepath) { fd = open(filepath, O_WRONLY | O_CREAT | O_TRUNC, 0600); if (fd < 0) { @@ -690,8 +758,11 @@ static int do_dump(int argc, char **argv) printf("%s:\n", sym_name); } - disasm_print_insn(img, lens[i], opcodes, name, - disasm_opt); + disasm_print_insn(img, lens[i], opcodes, + name, disasm_opt, btf, + prog_linfo, ksyms[i], i, + linum); + img += lens[i]; if (json_output) @@ -704,7 +775,7 @@ static int do_dump(int argc, char **argv) jsonw_end_array(json_wtr); } else { disasm_print_insn(buf, *member_len, opcodes, name, - disasm_opt); + disasm_opt, btf, NULL, 0, 0, false); } } else if (visual) { if (json_output) @@ -718,11 +789,14 @@ static int do_dump(int argc, char **argv) dd.btf = btf; dd.func_info = func_info; dd.finfo_rec_size = finfo_rec_size; + dd.prog_linfo = prog_linfo; if (json_output) - dump_xlated_json(&dd, buf, *member_len, opcodes); + dump_xlated_json(&dd, buf, *member_len, opcodes, + linum); else - dump_xlated_plain(&dd, buf, *member_len, opcodes); + dump_xlated_plain(&dd, buf, *member_len, opcodes, + linum); kernel_syms_destroy(&dd); } @@ -730,6 +804,9 @@ static int do_dump(int argc, char **argv) free(func_ksyms); free(func_lens); free(func_info); + free(linfo); + free(jited_linfo); + bpf_prog_linfo__free(prog_linfo); return 0; err_free: @@ -737,6 +814,9 @@ static int do_dump(int argc, char **argv) free(func_ksyms); free(func_lens); free(func_info); + free(linfo); + free(jited_linfo); + bpf_prog_linfo__free(prog_linfo); return -1; } @@ -1138,8 +1218,8 @@ static int do_help(int argc, char **argv) fprintf(stderr, "Usage: %s %s { show | list } [PROG]\n" - " %s %s dump xlated PROG [{ file FILE | opcodes | visual }]\n" - " %s %s dump jited PROG [{ file FILE | opcodes }]\n" + " %s %s dump xlated PROG [{ file FILE | opcodes | visual | linum }]\n" + " %s %s dump jited PROG [{ file FILE | opcodes | linum }]\n" " %s %s pin PROG FILE\n" " %s %s { load | loadall } OBJ PATH \\\n" " [type TYPE] [dev NAME] \\\n" diff --git a/tools/bpf/bpftool/xlated_dumper.c b/tools/bpf/bpftool/xlated_dumper.c index 131ecd1755330..aef628dcccb6a 100644 --- a/tools/bpf/bpftool/xlated_dumper.c +++ b/tools/bpf/bpftool/xlated_dumper.c @@ -41,6 +41,7 @@ #include #include #include +#include #include "disasm.h" #include "json_writer.h" @@ -234,8 +235,9 @@ static const char *print_imm(void *private_data, } void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len, - bool opcodes) + bool opcodes, bool linum) { + const struct bpf_prog_linfo *prog_linfo = dd->prog_linfo; const struct bpf_insn_cbs cbs = { .cb_print = print_insn_json, .cb_call = print_call, @@ -246,6 +248,7 @@ void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len, struct bpf_insn *insn = buf; struct btf *btf = dd->btf; bool double_insn = false; + unsigned int nr_skip = 0; char func_sig[1024]; unsigned int i; @@ -273,6 +276,16 @@ void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len, } } + if (prog_linfo) { + const struct bpf_line_info *linfo; + + linfo = bpf_prog_linfo__lfind(prog_linfo, i, nr_skip); + if (linfo) { + btf_dump_linfo_json(btf, linfo, linum); + nr_skip++; + } + } + jsonw_name(json_wtr, "disasm"); print_bpf_insn(&cbs, insn + i, true); @@ -307,8 +320,9 @@ void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len, } void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len, - bool opcodes) + bool opcodes, bool linum) { + const struct bpf_prog_linfo *prog_linfo = dd->prog_linfo; const struct bpf_insn_cbs cbs = { .cb_print = print_insn, .cb_call = print_call, @@ -318,6 +332,7 @@ void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len, struct bpf_func_info *record; struct bpf_insn *insn = buf; struct btf *btf = dd->btf; + unsigned int nr_skip = 0; bool double_insn = false; char func_sig[1024]; unsigned int i; @@ -340,6 +355,17 @@ void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len, } } + if (prog_linfo) { + const struct bpf_line_info *linfo; + + linfo = bpf_prog_linfo__lfind(prog_linfo, i, nr_skip); + if (linfo) { + btf_dump_linfo_plain(btf, linfo, "; ", + linum); + nr_skip++; + } + } + double_insn = insn[i].code == (BPF_LD | BPF_IMM | BPF_DW); printf("% 4d: ", i); diff --git a/tools/bpf/bpftool/xlated_dumper.h b/tools/bpf/bpftool/xlated_dumper.h index aec31723e1e5d..a24f89df8cb20 100644 --- a/tools/bpf/bpftool/xlated_dumper.h +++ b/tools/bpf/bpftool/xlated_dumper.h @@ -40,6 +40,8 @@ #define SYM_MAX_NAME 256 +struct bpf_prog_linfo; + struct kernel_sym { unsigned long address; char name[SYM_MAX_NAME]; @@ -54,6 +56,7 @@ struct dump_data { struct btf *btf; void *func_info; __u32 finfo_rec_size; + const struct bpf_prog_linfo *prog_linfo; char scratch_buff[SYM_MAX_NAME + 8]; }; @@ -61,9 +64,9 @@ void kernel_syms_load(struct dump_data *dd); void kernel_syms_destroy(struct dump_data *dd); struct kernel_sym *kernel_syms_search(struct dump_data *dd, unsigned long key); void dump_xlated_json(struct dump_data *dd, void *buf, unsigned int len, - bool opcodes); + bool opcodes, bool linum); void dump_xlated_plain(struct dump_data *dd, void *buf, unsigned int len, - bool opcodes); + bool opcodes, bool linum); void dump_xlated_for_graph(struct dump_data *dd, void *buf, void *buf_end, unsigned int start_index); diff --git a/tools/lib/bpf/Build b/tools/lib/bpf/Build index 7bc31c9050184..197b40f5b5c64 100644 --- a/tools/lib/bpf/Build +++ b/tools/lib/bpf/Build @@ -1 +1 @@ -libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_errno.o str_error.o netlink.o +libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_errno.o str_error.o netlink.o bpf_prog_linfo.o diff --git a/tools/lib/bpf/bpf_prog_linfo.c b/tools/lib/bpf/bpf_prog_linfo.c new file mode 100644 index 0000000000000..b8af65145408c --- /dev/null +++ b/tools/lib/bpf/bpf_prog_linfo.c @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +/* Copyright (c) 2018 Facebook */ + +#include +#include +#include +#include +#include "libbpf.h" + +#ifndef min +#define min(x, y) ((x) < (y) ? (x) : (y)) +#endif + +struct bpf_prog_linfo { + void *raw_linfo; + void *raw_jited_linfo; + __u32 *nr_jited_linfo_per_func; + __u32 *jited_linfo_func_idx; + __u32 nr_linfo; + __u32 nr_jited_func; + __u32 rec_size; + __u32 jited_rec_size; +}; + +static int dissect_jited_func(struct bpf_prog_linfo *prog_linfo, + const __u64 *ksym_func, const __u32 *ksym_len) +{ + __u32 nr_jited_func, nr_linfo; + const void *raw_jited_linfo; + const __u64 *jited_linfo; + __u64 last_jited_linfo; + /* + * Index to raw_jited_linfo: + * i: Index for searching the next ksym_func + * prev_i: Index to the last found ksym_func + */ + __u32 i, prev_i; + __u32 f; /* Index to ksym_func */ + + raw_jited_linfo = prog_linfo->raw_jited_linfo; + jited_linfo = raw_jited_linfo; + if (ksym_func[0] != *jited_linfo) + goto errout; + + prog_linfo->jited_linfo_func_idx[0] = 0; + nr_jited_func = prog_linfo->nr_jited_func; + nr_linfo = prog_linfo->nr_linfo; + + for (prev_i = 0, i = 1, f = 1; + i < nr_linfo && f < nr_jited_func; + i++) { + raw_jited_linfo += prog_linfo->jited_rec_size; + last_jited_linfo = *jited_linfo; + jited_linfo = raw_jited_linfo; + + if (ksym_func[f] == *jited_linfo) { + prog_linfo->jited_linfo_func_idx[f] = i; + + /* Sanity check */ + if (last_jited_linfo - ksym_func[f - 1] + 1 > + ksym_len[f - 1]) + goto errout; + + prog_linfo->nr_jited_linfo_per_func[f - 1] = + i - prev_i; + prev_i = i; + + /* + * The ksym_func[f] is found in jited_linfo. + * Look for the next one. + */ + f++; + } else if (*jited_linfo <= last_jited_linfo) { + /* Ensure the addr is increasing _within_ a func */ + goto errout; + } + } + + if (f != nr_jited_func) + goto errout; + + prog_linfo->nr_jited_linfo_per_func[nr_jited_func - 1] = + nr_linfo - prev_i; + + return 0; + +errout: + return -EINVAL; +} + +void bpf_prog_linfo__free(struct bpf_prog_linfo *prog_linfo) +{ + if (!prog_linfo) + return; + + free(prog_linfo->raw_linfo); + free(prog_linfo->raw_jited_linfo); + free(prog_linfo->nr_jited_linfo_per_func); + free(prog_linfo->jited_linfo_func_idx); + free(prog_linfo); +} + +struct bpf_prog_linfo *bpf_prog_linfo__new(const struct bpf_prog_info *info) +{ + struct bpf_prog_linfo *prog_linfo; + __u32 nr_linfo, nr_jited_func; + + nr_linfo = info->line_info_cnt; + + /* + * Test !info->line_info because the kernel may NULL + * the ptr if kernel.kptr_restrict is set. + */ + if (!nr_linfo || !info->line_info) + return NULL; + + /* + * The min size that bpf_prog_linfo has to access for + * searching purpose. + */ + if (info->line_info_rec_size < + offsetof(struct bpf_line_info, file_name_off)) + return NULL; + + prog_linfo = calloc(1, sizeof(*prog_linfo)); + if (!prog_linfo) + return NULL; + + /* Copy xlated line_info */ + prog_linfo->nr_linfo = nr_linfo; + prog_linfo->rec_size = info->line_info_rec_size; + prog_linfo->raw_linfo = malloc(nr_linfo * prog_linfo->rec_size); + if (!prog_linfo->raw_linfo) + goto err_free; + memcpy(prog_linfo->raw_linfo, (void *)(long)info->line_info, + nr_linfo * prog_linfo->rec_size); + + nr_jited_func = info->nr_jited_ksyms; + if (!nr_jited_func || + !info->jited_line_info || + info->jited_line_info_cnt != nr_linfo || + info->jited_line_info_rec_size < sizeof(__u64) || + info->nr_jited_func_lens != nr_jited_func || + !info->jited_ksyms || + !info->jited_func_lens) + /* Not enough info to provide jited_line_info */ + return prog_linfo; + + /* Copy jited_line_info */ + prog_linfo->nr_jited_func = nr_jited_func; + prog_linfo->jited_rec_size = info->jited_line_info_rec_size; + prog_linfo->raw_jited_linfo = malloc(nr_linfo * + prog_linfo->jited_rec_size); + if (!prog_linfo->raw_jited_linfo) + goto err_free; + memcpy(prog_linfo->raw_jited_linfo, + (void *)(long)info->jited_line_info, + nr_linfo * prog_linfo->jited_rec_size); + + /* Number of jited_line_info per jited func */ + prog_linfo->nr_jited_linfo_per_func = malloc(nr_jited_func * + sizeof(__u32)); + if (!prog_linfo->nr_jited_linfo_per_func) + goto err_free; + + /* + * For each jited func, + * the start idx to the "linfo" and "jited_linfo" array, + */ + prog_linfo->jited_linfo_func_idx = malloc(nr_jited_func * + sizeof(__u32)); + if (!prog_linfo->jited_linfo_func_idx) + goto err_free; + + if (dissect_jited_func(prog_linfo, + (__u64 *)(long)info->jited_ksyms, + (__u32 *)(long)info->jited_func_lens)) + goto err_free; + + return prog_linfo; + +err_free: + bpf_prog_linfo__free(prog_linfo); + return NULL; +} + +const struct bpf_line_info * +bpf_prog_linfo__lfind_addr_func(const struct bpf_prog_linfo *prog_linfo, + __u64 addr, __u32 func_idx, __u32 nr_skip) +{ + __u32 jited_rec_size, rec_size, nr_linfo, start, i; + const void *raw_jited_linfo, *raw_linfo; + const __u64 *jited_linfo; + + if (func_idx >= prog_linfo->nr_jited_func) + return NULL; + + nr_linfo = prog_linfo->nr_jited_linfo_per_func[func_idx]; + if (nr_skip >= nr_linfo) + return NULL; + + start = prog_linfo->jited_linfo_func_idx[func_idx] + nr_skip; + jited_rec_size = prog_linfo->jited_rec_size; + raw_jited_linfo = prog_linfo->raw_jited_linfo + + (start * jited_rec_size); + jited_linfo = raw_jited_linfo; + if (addr < *jited_linfo) + return NULL; + + nr_linfo -= nr_skip; + rec_size = prog_linfo->rec_size; + raw_linfo = prog_linfo->raw_linfo + (start * rec_size); + for (i = 0; i < nr_linfo; i++) { + if (addr < *jited_linfo) + break; + + raw_linfo += rec_size; + raw_jited_linfo += jited_rec_size; + jited_linfo = raw_jited_linfo; + } + + return raw_linfo - rec_size; +} + +const struct bpf_line_info * +bpf_prog_linfo__lfind(const struct bpf_prog_linfo *prog_linfo, + __u32 insn_off, __u32 nr_skip) +{ + const struct bpf_line_info *linfo; + __u32 rec_size, nr_linfo, i; + const void *raw_linfo; + + nr_linfo = prog_linfo->nr_linfo; + if (nr_skip >= nr_linfo) + return NULL; + + rec_size = prog_linfo->rec_size; + raw_linfo = prog_linfo->raw_linfo + (nr_skip * rec_size); + linfo = raw_linfo; + if (insn_off < linfo->insn_off) + return NULL; + + nr_linfo -= nr_skip; + for (i = 0; i < nr_linfo; i++) { + if (insn_off < linfo->insn_off) + break; + + raw_linfo += rec_size; + linfo = raw_linfo; + } + + return raw_linfo - rec_size; +} diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index f30c3d07bb7db..5f68d7b75215c 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -342,6 +342,19 @@ int libbpf_nl_get_qdisc(int sock, unsigned int nl_pid, int ifindex, int libbpf_nl_get_filter(int sock, unsigned int nl_pid, int ifindex, int handle, libbpf_dump_nlmsg_t dump_filter_nlmsg, void *cookie); +struct bpf_prog_linfo; +struct bpf_prog_info; + +LIBBPF_API void bpf_prog_linfo__free(struct bpf_prog_linfo *prog_linfo); +LIBBPF_API struct bpf_prog_linfo * +bpf_prog_linfo__new(const struct bpf_prog_info *info); +LIBBPF_API const struct bpf_line_info * +bpf_prog_linfo__lfind_addr_func(const struct bpf_prog_linfo *prog_linfo, + __u64 addr, __u32 func_idx, __u32 nr_skip); +LIBBPF_API const struct bpf_line_info * +bpf_prog_linfo__lfind(const struct bpf_prog_linfo *prog_linfo, + __u32 insn_off, __u32 nr_skip); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 8deff22d61bba..cd02cd4e2cc35 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -99,6 +99,10 @@ LIBBPF_0.0.1 { bpf_program__unload; bpf_program__unpin; bpf_program__unpin_instance; + bpf_prog_linfo__free; + bpf_prog_linfo__new; + bpf_prog_linfo__lfind_addr_func; + bpf_prog_linfo__lfind; bpf_raw_tracepoint_open; bpf_set_link_xdp_fd; bpf_task_fd_query; From 01d3240a04f4c09392e13c77b54d4423ebce2d72 Mon Sep 17 00:00:00 2001 From: Sean Young Date: Thu, 6 Dec 2018 13:01:03 +0000 Subject: [PATCH 45/51] media: bpf: add bpf function to report mouse movement Some IR remotes have a directional pad or other pointer-like thing that can be used as a mouse. Make it possible to decode these types of IR protocols in BPF. Cc: netdev@vger.kernel.org Signed-off-by: Sean Young Signed-off-by: Alexei Starovoitov --- drivers/media/rc/bpf-lirc.c | 24 +++++++ include/uapi/linux/bpf.h | 17 ++++- tools/include/uapi/linux/bpf.h | 18 ++++- tools/testing/selftests/bpf/bpf_helpers.h | 2 + .../testing/selftests/bpf/test_lirc_mode2.sh | 3 +- .../selftests/bpf/test_lirc_mode2_kern.c | 3 + .../selftests/bpf/test_lirc_mode2_user.c | 65 +++++++++++++------ 7 files changed, 109 insertions(+), 23 deletions(-) diff --git a/drivers/media/rc/bpf-lirc.c b/drivers/media/rc/bpf-lirc.c index 8b97fd1f0ceaf..390a722e62113 100644 --- a/drivers/media/rc/bpf-lirc.c +++ b/drivers/media/rc/bpf-lirc.c @@ -59,6 +59,28 @@ static const struct bpf_func_proto rc_keydown_proto = { .arg4_type = ARG_ANYTHING, }; +BPF_CALL_3(bpf_rc_pointer_rel, u32*, sample, s32, rel_x, s32, rel_y) +{ + struct ir_raw_event_ctrl *ctrl; + + ctrl = container_of(sample, struct ir_raw_event_ctrl, bpf_sample); + + input_report_rel(ctrl->dev->input_dev, REL_X, rel_x); + input_report_rel(ctrl->dev->input_dev, REL_Y, rel_y); + input_sync(ctrl->dev->input_dev); + + return 0; +} + +static const struct bpf_func_proto rc_pointer_rel_proto = { + .func = bpf_rc_pointer_rel, + .gpl_only = true, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_CTX, + .arg2_type = ARG_ANYTHING, + .arg3_type = ARG_ANYTHING, +}; + static const struct bpf_func_proto * lirc_mode2_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { @@ -67,6 +89,8 @@ lirc_mode2_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &rc_repeat_proto; case BPF_FUNC_rc_keydown: return &rc_keydown_proto; + case BPF_FUNC_rc_pointer_rel: + return &rc_pointer_rel_proto; case BPF_FUNC_map_lookup_elem: return &bpf_map_lookup_elem_proto; case BPF_FUNC_map_update_elem: diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 7a66db8d15d56..1bee1135866a0 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -2301,6 +2301,20 @@ union bpf_attr { * payload and/or *pop* value being to large. * Return * 0 on success, or a negative error in case of failure. + * + * int bpf_rc_pointer_rel(void *ctx, s32 rel_x, s32 rel_y) + * Description + * This helper is used in programs implementing IR decoding, to + * report a successfully decoded pointer movement. + * + * The *ctx* should point to the lirc sample as passed into + * the program. + * + * This helper is only available is the kernel was compiled with + * the **CONFIG_BPF_LIRC_MODE2** configuration option set to + * "**y**". + * Return + * 0 */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -2394,7 +2408,8 @@ union bpf_attr { FN(map_pop_elem), \ FN(map_peek_elem), \ FN(msg_push_data), \ - FN(msg_pop_data), + FN(msg_pop_data), \ + FN(rc_pointer_rel), /* integer value in 'imm' field of BPF_CALL instruction selects which helper * function eBPF program intends to call diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 7973c28b24a02..6ad50b6471d36 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -2298,9 +2298,22 @@ union bpf_attr { * if possible. Other errors can occur if input parameters are * invalid either due to *start* byte not being valid part of msg * payload and/or *pop* value being to large. + * Return + * 0 on success, or a negative error in case of failure. + * + * int bpf_rc_pointer_rel(void *ctx, s32 rel_x, s32 rel_y) + * Description + * This helper is used in programs implementing IR decoding, to + * report a successfully decoded pointer movement. + * + * The *ctx* should point to the lirc sample as passed into + * the program. * + * This helper is only available is the kernel was compiled with + * the **CONFIG_BPF_LIRC_MODE2** configuration option set to + * "**y**". * Return - * 0 on success, or a negative erro in case of failure. + * 0 */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -2394,7 +2407,8 @@ union bpf_attr { FN(map_pop_elem), \ FN(map_peek_elem), \ FN(msg_push_data), \ - FN(msg_pop_data), + FN(msg_pop_data), \ + FN(rc_pointer_rel), /* integer value in 'imm' field of BPF_CALL instruction selects which helper * function eBPF program intends to call diff --git a/tools/testing/selftests/bpf/bpf_helpers.h b/tools/testing/selftests/bpf/bpf_helpers.h index 7b69519a09b15..04c060e8f10a5 100644 --- a/tools/testing/selftests/bpf/bpf_helpers.h +++ b/tools/testing/selftests/bpf/bpf_helpers.h @@ -170,6 +170,8 @@ static int (*bpf_skb_vlan_push)(void *ctx, __be16 vlan_proto, __u16 vlan_tci) = (void *) BPF_FUNC_skb_vlan_push; static int (*bpf_skb_vlan_pop)(void *ctx) = (void *) BPF_FUNC_skb_vlan_pop; +static int (*bpf_rc_pointer_rel)(void *ctx, int rel_x, int rel_y) = + (void *) BPF_FUNC_rc_pointer_rel; /* llvm builtin functions that eBPF C program may use to * emit BPF_LD_ABS and BPF_LD_IND instructions diff --git a/tools/testing/selftests/bpf/test_lirc_mode2.sh b/tools/testing/selftests/bpf/test_lirc_mode2.sh index 677686198df34..ec4e15948e406 100755 --- a/tools/testing/selftests/bpf/test_lirc_mode2.sh +++ b/tools/testing/selftests/bpf/test_lirc_mode2.sh @@ -21,13 +21,14 @@ do if grep -q DRV_NAME=rc-loopback $i/uevent then LIRCDEV=$(grep DEVNAME= $i/lirc*/uevent | sed sQDEVNAME=Q/dev/Q) + INPUTDEV=$(grep DEVNAME= $i/input*/event*/uevent | sed sQDEVNAME=Q/dev/Q) fi done if [ -n $LIRCDEV ]; then TYPE=lirc_mode2 - ./test_lirc_mode2_user $LIRCDEV + ./test_lirc_mode2_user $LIRCDEV $INPUTDEV ret=$? if [ $ret -ne 0 ]; then echo -e ${RED}"FAIL: $TYPE"${NC} diff --git a/tools/testing/selftests/bpf/test_lirc_mode2_kern.c b/tools/testing/selftests/bpf/test_lirc_mode2_kern.c index ba26855563a5f..4147130cc3b7b 100644 --- a/tools/testing/selftests/bpf/test_lirc_mode2_kern.c +++ b/tools/testing/selftests/bpf/test_lirc_mode2_kern.c @@ -15,6 +15,9 @@ int bpf_decoder(unsigned int *sample) if (duration & 0x10000) bpf_rc_keydown(sample, 0x40, duration & 0xffff, 0); + if (duration & 0x20000) + bpf_rc_pointer_rel(sample, (duration >> 8) & 0xff, + duration & 0xff); } return 0; diff --git a/tools/testing/selftests/bpf/test_lirc_mode2_user.c b/tools/testing/selftests/bpf/test_lirc_mode2_user.c index d470d63c33dbe..fb5fd6841ef39 100644 --- a/tools/testing/selftests/bpf/test_lirc_mode2_user.c +++ b/tools/testing/selftests/bpf/test_lirc_mode2_user.c @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -47,12 +48,13 @@ int main(int argc, char **argv) { struct bpf_object *obj; - int ret, lircfd, progfd, mode; - int testir = 0x1dead; + int ret, lircfd, progfd, inputfd; + int testir1 = 0x1dead; + int testir2 = 0x20101; u32 prog_ids[10], prog_flags[10], prog_cnt; - if (argc != 2) { - printf("Usage: %s /dev/lircN\n", argv[0]); + if (argc != 3) { + printf("Usage: %s /dev/lircN /dev/input/eventM\n", argv[0]); return 2; } @@ -76,9 +78,9 @@ int main(int argc, char **argv) return 1; } - mode = LIRC_MODE_SCANCODE; - if (ioctl(lircfd, LIRC_SET_REC_MODE, &mode)) { - printf("failed to set rec mode: %m\n"); + inputfd = open(argv[2], O_RDONLY | O_NONBLOCK); + if (inputfd == -1) { + printf("failed to open input device %s: %m\n", argv[1]); return 1; } @@ -102,29 +104,54 @@ int main(int argc, char **argv) } /* Write raw IR */ - ret = write(lircfd, &testir, sizeof(testir)); - if (ret != sizeof(testir)) { + ret = write(lircfd, &testir1, sizeof(testir1)); + if (ret != sizeof(testir1)) { printf("Failed to send test IR message: %m\n"); return 1; } - struct pollfd pfd = { .fd = lircfd, .events = POLLIN }; - struct lirc_scancode lsc; + struct pollfd pfd = { .fd = inputfd, .events = POLLIN }; + struct input_event event; - poll(&pfd, 1, 100); + for (;;) { + poll(&pfd, 1, 100); - /* Read decoded IR */ - ret = read(lircfd, &lsc, sizeof(lsc)); - if (ret != sizeof(lsc)) { - printf("Failed to read decoded IR: %m\n"); - return 1; + /* Read decoded IR */ + ret = read(inputfd, &event, sizeof(event)); + if (ret != sizeof(event)) { + printf("Failed to read decoded IR: %m\n"); + return 1; + } + + if (event.type == EV_MSC && event.code == MSC_SCAN && + event.value == 0xdead) { + break; + } } - if (lsc.scancode != 0xdead || lsc.rc_proto != 64) { - printf("Incorrect scancode decoded\n"); + /* Write raw IR */ + ret = write(lircfd, &testir2, sizeof(testir2)); + if (ret != sizeof(testir2)) { + printf("Failed to send test IR message: %m\n"); return 1; } + for (;;) { + poll(&pfd, 1, 100); + + /* Read decoded IR */ + ret = read(inputfd, &event, sizeof(event)); + if (ret != sizeof(event)) { + printf("Failed to read decoded IR: %m\n"); + return 1; + } + + if (event.type == EV_REL && event.code == REL_Y && + event.value == 1 ) { + break; + } + } + prog_cnt = 10; ret = bpf_prog_query(lircfd, BPF_LIRC_MODE2, 0, prog_flags, prog_ids, &prog_cnt); From e434b8cdf788568ba65a0a0fd9f3cb41f3ca1803 Mon Sep 17 00:00:00 2001 From: Jiong Wang Date: Fri, 7 Dec 2018 12:16:18 -0500 Subject: [PATCH 46/51] bpf: relax verifier restriction on BPF_MOV | BPF_ALU Currently, the destination register is marked as unknown for 32-bit sub-register move (BPF_MOV | BPF_ALU) whenever the source register type is SCALAR_VALUE. This is too conservative that some valid cases will be rejected. Especially, this may turn a constant scalar value into unknown value that could break some assumptions of verifier. For example, test_l4lb_noinline.c has the following C code: struct real_definition *dst 1: if (!get_packet_dst(&dst, &pckt, vip_info, is_ipv6)) 2: return TC_ACT_SHOT; 3: 4: if (dst->flags & F_IPV6) { get_packet_dst is responsible for initializing "dst" into valid pointer and return true (1), otherwise return false (0). The compiled instruction sequence using alu32 will be: 412: (54) (u32) r7 &= (u32) 1 413: (bc) (u32) r0 = (u32) r7 414: (95) exit insn 413, a BPF_MOV | BPF_ALU, however will turn r0 into unknown value even r7 contains SCALAR_VALUE 1. This causes trouble when verifier is walking the code path that hasn't initialized "dst" inside get_packet_dst, for which case 0 is returned and we would then expect verifier concluding line 1 in the above C code pass the "if" check, therefore would skip fall through path starting at line 4. Now, because r0 returned from callee has became unknown value, so verifier won't skip analyzing path starting at line 4 and "dst->flags" requires dereferencing the pointer "dst" which actually hasn't be initialized for this path. This patch relaxed the code marking sub-register move destination. For a SCALAR_VALUE, it is safe to just copy the value from source then truncate it into 32-bit. A unit test also included to demonstrate this issue. This test will fail before this patch. This relaxation could let verifier skipping more paths for conditional comparison against immediate. It also let verifier recording a more accurate/strict value for one register at one state, if this state end up with going through exit without rejection and it is used for state comparison later, then it is possible an inaccurate/permissive value is better. So the real impact on verifier processed insn number is complex. But in all, without this fix, valid program could be rejected. >From real benchmarking on kernel selftests and Cilium bpf tests, there is no impact on processed instruction number when tests ares compiled with default compilation options. There is slightly improvements when they are compiled with -mattr=+alu32 after this patch. Also, test_xdp_noinline/-mattr=+alu32 now passed verification. It is rejected before this fix. Insn processed before/after this patch: default -mattr=+alu32 Kernel selftest === test_xdp.o 371/371 369/369 test_l4lb.o 6345/6345 5623/5623 test_xdp_noinline.o 2971/2971 rejected/2727 test_tcp_estates.o 429/429 430/430 Cilium bpf === bpf_lb-DLB_L3.o: 2085/2085 1685/1687 bpf_lb-DLB_L4.o: 2287/2287 1986/1982 bpf_lb-DUNKNOWN.o: 690/690 622/622 bpf_lxc.o: 95033/95033 N/A bpf_netdev.o: 7245/7245 N/A bpf_overlay.o: 2898/2898 3085/2947 NOTE: - bpf_lxc.o and bpf_netdev.o compiled by -mattr=+alu32 are rejected by verifier due to another issue inside verifier on supporting alu32 binary. - Each cilium bpf program could generate several processed insn number, above number is sum of them. v1->v2: - Restrict the change on SCALAR_VALUE. - Update benchmark numbers on Cilium bpf tests. Signed-off-by: Jiong Wang Signed-off-by: Alexei Starovoitov --- kernel/bpf/verifier.c | 16 ++++++++++++---- tools/testing/selftests/bpf/test_verifier.c | 13 +++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 9d25506bd55a2..2e70b813a1a75 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -3583,12 +3583,15 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) return err; if (BPF_SRC(insn->code) == BPF_X) { + struct bpf_reg_state *src_reg = regs + insn->src_reg; + struct bpf_reg_state *dst_reg = regs + insn->dst_reg; + if (BPF_CLASS(insn->code) == BPF_ALU64) { /* case: R1 = R2 * copy register state to dest reg */ - regs[insn->dst_reg] = regs[insn->src_reg]; - regs[insn->dst_reg].live |= REG_LIVE_WRITTEN; + *dst_reg = *src_reg; + dst_reg->live |= REG_LIVE_WRITTEN; } else { /* R1 = (u32) R2 */ if (is_pointer_value(env, insn->src_reg)) { @@ -3596,9 +3599,14 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) "R%d partial copy of pointer\n", insn->src_reg); return -EACCES; + } else if (src_reg->type == SCALAR_VALUE) { + *dst_reg = *src_reg; + dst_reg->live |= REG_LIVE_WRITTEN; + } else { + mark_reg_unknown(env, regs, + insn->dst_reg); } - mark_reg_unknown(env, regs, insn->dst_reg); - coerce_reg_to_size(®s[insn->dst_reg], 4); + coerce_reg_to_size(dst_reg, 4); } } else { /* case: R = imm diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index 36ce58b4933eb..957e4711c46ce 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -2959,6 +2959,19 @@ static struct bpf_test tests[] = { .result_unpriv = REJECT, .result = ACCEPT, }, + { + "alu32: mov u32 const", + .insns = { + BPF_MOV32_IMM(BPF_REG_7, 0), + BPF_ALU32_IMM(BPF_AND, BPF_REG_7, 1), + BPF_MOV32_REG(BPF_REG_0, BPF_REG_7), + BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 1), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0), + BPF_EXIT_INSN(), + }, + .result = ACCEPT, + .retval = 0, + }, { "unpriv: partial copy of pointer", .insns = { From 10a5ce98539948affbdc28dc0f39a1b6b2307f9d Mon Sep 17 00:00:00 2001 From: Martin KaFai Lau Date: Mon, 10 Dec 2018 10:53:24 -0800 Subject: [PATCH 47/51] bpf: bpftool: Fix newline and p_err issue This patch fixes a few newline issues and also replaces p_err with p_info in prog.c Fixes: b053b439b72a ("bpf: libbpf: bpftool: Print bpf_line_info during prog dump") Cc: Jakub Kicinski Signed-off-by: Martin KaFai Lau Acked-by: Jakub Kicinski Signed-off-by: Alexei Starovoitov --- tools/bpf/bpftool/Documentation/bpftool-prog.rst | 2 ++ tools/bpf/bpftool/main.h | 1 - tools/bpf/bpftool/prog.c | 3 +-- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/bpf/bpftool/Documentation/bpftool-prog.rst b/tools/bpf/bpftool/Documentation/bpftool-prog.rst index 7c30731a9b734..bb1aeb98b6dad 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-prog.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-prog.rst @@ -73,6 +73,7 @@ DESCRIPTION be displayed by default. If **linum** is specified, the filename, line number and line column will also be displayed on top of the source line. + **bpftool prog dump jited** *PROG* [{ **file** *FILE* | **opcodes** | **linum** }] Dump jited image (host machine code) of the program. If *FILE* is specified image will be written to a file, @@ -84,6 +85,7 @@ DESCRIPTION be displayed by default. If **linum** is specified, the filename, line number and line column will also be displayed on top of the source line. + **bpftool prog pin** *PROG* *FILE* Pin program *PROG* as *FILE*. diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h index d9393abdba782..0b37599f8cda4 100644 --- a/tools/bpf/bpftool/main.h +++ b/tools/bpf/bpftool/main.h @@ -194,7 +194,6 @@ void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, const struct bpf_prog_linfo *prog_linfo, __u64 func_ksym, unsigned int func_idx, bool linum) - { } static inline int disasm_init(void) diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 65b921ffd10a1..9a78ebbcea1dc 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -592,7 +592,6 @@ static int do_dump(int argc, char **argv) info.jited_line_info_rec_size = jited_linfo_rec_size; info.jited_line_info = ptr_to_u64(jited_linfo); - err = bpf_obj_get_info_by_fd(fd, &info, &len); close(fd); if (err) { @@ -674,7 +673,7 @@ static int do_dump(int argc, char **argv) if (linfo_cnt) { prog_linfo = bpf_prog_linfo__new(&info); if (!prog_linfo) - p_err("error in processing bpf_line_info. continue without it."); + p_info("error in processing bpf_line_info. continue without it."); } if (filepath) { From 7a5725ddc6e18aab61f647c0996d3b7eb03ff5cb Mon Sep 17 00:00:00 2001 From: Song Liu Date: Mon, 10 Dec 2018 11:17:50 -0800 Subject: [PATCH 48/51] bpf: clean up bpf_prog_get_info_by_fd() info.nr_jited_ksyms and info.nr_jited_func_lens cannot be 0 in these two statements, so we don't need to check them. Signed-off-by: Song Liu Acked-by: Martin KaFai Lau Signed-off-by: Alexei Starovoitov --- kernel/bpf/syscall.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 19c88cff78803..a99a23bf59101 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2209,7 +2209,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, ulen = info.nr_jited_ksyms; info.nr_jited_ksyms = prog->aux->func_cnt ? : 1; - if (info.nr_jited_ksyms && ulen) { + if (ulen) { if (bpf_dump_raw_ok()) { unsigned long ksym_addr; u64 __user *user_ksyms; @@ -2240,7 +2240,7 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, ulen = info.nr_jited_func_lens; info.nr_jited_func_lens = prog->aux->func_cnt ? : 1; - if (info.nr_jited_func_lens && ulen) { + if (ulen) { if (bpf_dump_raw_ok()) { u32 __user *user_lens; u32 func_len, i; From 11d8b82d2222cade12caad2c125f23023777dcbc Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Mon, 10 Dec 2018 14:14:08 -0800 Subject: [PATCH 49/51] bpf: rename *_info_cnt to nr_*_info in bpf_prog_info In uapi bpf.h, currently we have the following fields in the struct bpf_prog_info: __u32 func_info_cnt; __u32 line_info_cnt; __u32 jited_line_info_cnt; The above field names "func_info_cnt" and "line_info_cnt" also appear in union bpf_attr for program loading. The original intention is to keep the names the same between bpf_prog_info and bpf_attr so it will imply what we returned to user space will be the same as what the user space passed to the kernel. Such a naming convention in bpf_prog_info is not consistent with other fields like: __u32 nr_jited_ksyms; __u32 nr_jited_func_lens; This patch made this adjustment so in bpf_prog_info newly introduced *_info_cnt becomes nr_*_info. Acked-by: Song Liu Acked-by: Martin KaFai Lau Signed-off-by: Yonghong Song Signed-off-by: Alexei Starovoitov --- include/uapi/linux/bpf.h | 6 +++--- kernel/bpf/syscall.c | 38 +++++++++++++++++++------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index 1bee1135866a0..f943ed8033099 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -2696,11 +2696,11 @@ struct bpf_prog_info { __u32 btf_id; __u32 func_info_rec_size; __aligned_u64 func_info; - __u32 func_info_cnt; - __u32 line_info_cnt; + __u32 nr_func_info; + __u32 nr_line_info; __aligned_u64 line_info; __aligned_u64 jited_line_info; - __u32 jited_line_info_cnt; + __u32 nr_jited_line_info; __u32 line_info_rec_size; __u32 jited_line_info_rec_size; } __attribute__((aligned(8))); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index a99a23bf59101..5745c78376214 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2055,15 +2055,15 @@ static int set_info_rec_size(struct bpf_prog_info *info) * _rec_size back to the info. */ - if ((info->func_info_cnt || info->func_info_rec_size) && + if ((info->nr_func_info || info->func_info_rec_size) && info->func_info_rec_size != sizeof(struct bpf_func_info)) return -EINVAL; - if ((info->line_info_cnt || info->line_info_rec_size) && + if ((info->nr_line_info || info->line_info_rec_size) && info->line_info_rec_size != sizeof(struct bpf_line_info)) return -EINVAL; - if ((info->jited_line_info_cnt || info->jited_line_info_rec_size) && + if ((info->nr_jited_line_info || info->jited_line_info_rec_size) && info->jited_line_info_rec_size != sizeof(__u64)) return -EINVAL; @@ -2125,9 +2125,9 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, info.xlated_prog_len = 0; info.nr_jited_ksyms = 0; info.nr_jited_func_lens = 0; - info.func_info_cnt = 0; - info.line_info_cnt = 0; - info.jited_line_info_cnt = 0; + info.nr_func_info = 0; + info.nr_line_info = 0; + info.nr_jited_line_info = 0; goto done; } @@ -2268,14 +2268,14 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, if (prog->aux->btf) info.btf_id = btf_id(prog->aux->btf); - ulen = info.func_info_cnt; - info.func_info_cnt = prog->aux->func_info_cnt; - if (info.func_info_cnt && ulen) { + ulen = info.nr_func_info; + info.nr_func_info = prog->aux->func_info_cnt; + if (info.nr_func_info && ulen) { if (bpf_dump_raw_ok()) { char __user *user_finfo; user_finfo = u64_to_user_ptr(info.func_info); - ulen = min_t(u32, info.func_info_cnt, ulen); + ulen = min_t(u32, info.nr_func_info, ulen); if (copy_to_user(user_finfo, prog->aux->func_info, info.func_info_rec_size * ulen)) return -EFAULT; @@ -2284,14 +2284,14 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, } } - ulen = info.line_info_cnt; - info.line_info_cnt = prog->aux->nr_linfo; - if (info.line_info_cnt && ulen) { + ulen = info.nr_line_info; + info.nr_line_info = prog->aux->nr_linfo; + if (info.nr_line_info && ulen) { if (bpf_dump_raw_ok()) { __u8 __user *user_linfo; user_linfo = u64_to_user_ptr(info.line_info); - ulen = min_t(u32, info.line_info_cnt, ulen); + ulen = min_t(u32, info.nr_line_info, ulen); if (copy_to_user(user_linfo, prog->aux->linfo, info.line_info_rec_size * ulen)) return -EFAULT; @@ -2300,18 +2300,18 @@ static int bpf_prog_get_info_by_fd(struct bpf_prog *prog, } } - ulen = info.jited_line_info_cnt; + ulen = info.nr_jited_line_info; if (prog->aux->jited_linfo) - info.jited_line_info_cnt = prog->aux->nr_linfo; + info.nr_jited_line_info = prog->aux->nr_linfo; else - info.jited_line_info_cnt = 0; - if (info.jited_line_info_cnt && ulen) { + info.nr_jited_line_info = 0; + if (info.nr_jited_line_info && ulen) { if (bpf_dump_raw_ok()) { __u64 __user *user_linfo; u32 i; user_linfo = u64_to_user_ptr(info.jited_line_info); - ulen = min_t(u32, info.jited_line_info_cnt, ulen); + ulen = min_t(u32, info.nr_jited_line_info, ulen); for (i = 0; i < ulen; i++) { if (put_user((__u64)(long)prog->aux->jited_linfo[i], &user_linfo[i])) From b4f8623c0cefdfb818f99b3eb3c1e4c7708dd84e Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Mon, 10 Dec 2018 14:14:09 -0800 Subject: [PATCH 50/51] tools/bpf: sync kernel uapi bpf.h to tools directory Sync kernel uapi bpf.h "*_info_cnt => nr_*_info" changes to tools directory. Acked-by: Martin KaFai Lau Signed-off-by: Yonghong Song Signed-off-by: Alexei Starovoitov --- tools/include/uapi/linux/bpf.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 6ad50b6471d36..620ee1f919cf2 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -2695,11 +2695,11 @@ struct bpf_prog_info { __u32 btf_id; __u32 func_info_rec_size; __aligned_u64 func_info; - __u32 func_info_cnt; - __u32 line_info_cnt; + __u32 nr_func_info; + __u32 nr_line_info; __aligned_u64 line_info; __aligned_u64 jited_line_info; - __u32 jited_line_info_cnt; + __u32 nr_jited_line_info; __u32 line_info_rec_size; __u32 jited_line_info_rec_size; } __attribute__((aligned(8))); From cfc542411bd40ff4f8a70b3d061bd6acdfb05629 Mon Sep 17 00:00:00 2001 From: Yonghong Song Date: Mon, 10 Dec 2018 14:14:10 -0800 Subject: [PATCH 51/51] tools/bpf: rename *_info_cnt to nr_*_info Rename all occurances of *_info_cnt field access to nr_*_info in tools directory. The local variables finfo_cnt, linfo_cnt and jited_linfo_cnt in function do_dump() of tools/bpf/bpftool/prog.c are also changed to nr_finfo, nr_linfo and nr_jited_linfo to keep naming convention consistent. Acked-by: Martin KaFai Lau Signed-off-by: Yonghong Song Signed-off-by: Alexei Starovoitov --- tools/bpf/bpftool/prog.c | 48 ++++++++++----------- tools/lib/bpf/bpf_prog_linfo.c | 4 +- tools/testing/selftests/bpf/test_btf.c | 60 +++++++++++++------------- 3 files changed, 56 insertions(+), 56 deletions(-) diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 9a78ebbcea1dc..b73b4e473948d 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -425,7 +425,7 @@ static int do_dump(int argc, char **argv) { unsigned int finfo_rec_size, linfo_rec_size, jited_linfo_rec_size; void *func_info = NULL, *linfo = NULL, *jited_linfo = NULL; - unsigned int finfo_cnt, linfo_cnt = 0, jited_linfo_cnt = 0; + unsigned int nr_finfo, nr_linfo = 0, nr_jited_linfo = 0; struct bpf_prog_linfo *prog_linfo = NULL; unsigned long *func_ksyms = NULL; struct bpf_prog_info info = {}; @@ -537,10 +537,10 @@ static int do_dump(int argc, char **argv) } } - finfo_cnt = info.func_info_cnt; + nr_finfo = info.nr_func_info; finfo_rec_size = info.func_info_rec_size; - if (finfo_cnt && finfo_rec_size) { - func_info = malloc(finfo_cnt * finfo_rec_size); + if (nr_finfo && finfo_rec_size) { + func_info = malloc(nr_finfo * finfo_rec_size); if (!func_info) { p_err("mem alloc failed"); close(fd); @@ -549,9 +549,9 @@ static int do_dump(int argc, char **argv) } linfo_rec_size = info.line_info_rec_size; - if (info.line_info_cnt && linfo_rec_size && info.btf_id) { - linfo_cnt = info.line_info_cnt; - linfo = malloc(linfo_cnt * linfo_rec_size); + if (info.nr_line_info && linfo_rec_size && info.btf_id) { + nr_linfo = info.nr_line_info; + linfo = malloc(nr_linfo * linfo_rec_size); if (!linfo) { p_err("mem alloc failed"); close(fd); @@ -560,13 +560,13 @@ static int do_dump(int argc, char **argv) } jited_linfo_rec_size = info.jited_line_info_rec_size; - if (info.jited_line_info_cnt && + if (info.nr_jited_line_info && jited_linfo_rec_size && info.nr_jited_ksyms && info.nr_jited_func_lens && info.btf_id) { - jited_linfo_cnt = info.jited_line_info_cnt; - jited_linfo = malloc(jited_linfo_cnt * jited_linfo_rec_size); + nr_jited_linfo = info.nr_jited_line_info; + jited_linfo = malloc(nr_jited_linfo * jited_linfo_rec_size); if (!jited_linfo) { p_err("mem alloc failed"); close(fd); @@ -582,13 +582,13 @@ static int do_dump(int argc, char **argv) info.nr_jited_ksyms = nr_func_ksyms; info.jited_func_lens = ptr_to_u64(func_lens); info.nr_jited_func_lens = nr_func_lens; - info.func_info_cnt = finfo_cnt; + info.nr_func_info = nr_finfo; info.func_info_rec_size = finfo_rec_size; info.func_info = ptr_to_u64(func_info); - info.line_info_cnt = linfo_cnt; + info.nr_line_info = nr_linfo; info.line_info_rec_size = linfo_rec_size; info.line_info = ptr_to_u64(linfo); - info.jited_line_info_cnt = jited_linfo_cnt; + info.nr_jited_line_info = nr_jited_linfo; info.jited_line_info_rec_size = jited_linfo_rec_size; info.jited_line_info = ptr_to_u64(jited_linfo); @@ -614,9 +614,9 @@ static int do_dump(int argc, char **argv) goto err_free; } - if (info.func_info_cnt != finfo_cnt) { - p_err("incorrect func_info_cnt %d vs. expected %d", - info.func_info_cnt, finfo_cnt); + if (info.nr_func_info != nr_finfo) { + p_err("incorrect nr_func_info %d vs. expected %d", + info.nr_func_info, nr_finfo); goto err_free; } @@ -630,12 +630,12 @@ static int do_dump(int argc, char **argv) /* kernel.kptr_restrict is set. No func_info available. */ free(func_info); func_info = NULL; - finfo_cnt = 0; + nr_finfo = 0; } - if (linfo && info.line_info_cnt != linfo_cnt) { - p_err("incorrect line_info_cnt %u vs. expected %u", - info.line_info_cnt, linfo_cnt); + if (linfo && info.nr_line_info != nr_linfo) { + p_err("incorrect nr_line_info %u vs. expected %u", + info.nr_line_info, nr_linfo); goto err_free; } @@ -645,9 +645,9 @@ static int do_dump(int argc, char **argv) goto err_free; } - if (jited_linfo && info.jited_line_info_cnt != jited_linfo_cnt) { - p_err("incorrect jited_line_info_cnt %u vs. expected %u", - info.jited_line_info_cnt, jited_linfo_cnt); + if (jited_linfo && info.nr_jited_line_info != nr_jited_linfo) { + p_err("incorrect nr_jited_line_info %u vs. expected %u", + info.nr_jited_line_info, nr_jited_linfo); goto err_free; } @@ -670,7 +670,7 @@ static int do_dump(int argc, char **argv) goto err_free; } - if (linfo_cnt) { + if (nr_linfo) { prog_linfo = bpf_prog_linfo__new(&info); if (!prog_linfo) p_info("error in processing bpf_line_info. continue without it."); diff --git a/tools/lib/bpf/bpf_prog_linfo.c b/tools/lib/bpf/bpf_prog_linfo.c index b8af65145408c..addd6e9971cc0 100644 --- a/tools/lib/bpf/bpf_prog_linfo.c +++ b/tools/lib/bpf/bpf_prog_linfo.c @@ -105,7 +105,7 @@ struct bpf_prog_linfo *bpf_prog_linfo__new(const struct bpf_prog_info *info) struct bpf_prog_linfo *prog_linfo; __u32 nr_linfo, nr_jited_func; - nr_linfo = info->line_info_cnt; + nr_linfo = info->nr_line_info; /* * Test !info->line_info because the kernel may NULL @@ -138,7 +138,7 @@ struct bpf_prog_linfo *bpf_prog_linfo__new(const struct bpf_prog_info *info) nr_jited_func = info->nr_jited_ksyms; if (!nr_jited_func || !info->jited_line_info || - info->jited_line_info_cnt != nr_linfo || + info->nr_jited_line_info != nr_linfo || info->jited_line_info_rec_size < sizeof(__u64) || info->nr_jited_func_lens != nr_jited_func || !info->jited_ksyms || diff --git a/tools/testing/selftests/bpf/test_btf.c b/tools/testing/selftests/bpf/test_btf.c index 7707273736ac6..d4c63316c862b 100644 --- a/tools/testing/selftests/bpf/test_btf.c +++ b/tools/testing/selftests/bpf/test_btf.c @@ -2548,9 +2548,9 @@ static int do_test_file(unsigned int test_num) err = -1; goto done; } - if (CHECK(info.func_info_cnt != 3, - "incorrect info.func_info_cnt (1st) %d", - info.func_info_cnt)) { + if (CHECK(info.nr_func_info != 3, + "incorrect info.nr_func_info (1st) %d", + info.nr_func_info)) { err = -1; goto done; } @@ -2561,7 +2561,7 @@ static int do_test_file(unsigned int test_num) goto done; } - func_info = malloc(info.func_info_cnt * rec_size); + func_info = malloc(info.nr_func_info * rec_size); if (CHECK(!func_info, "out of memory")) { err = -1; goto done; @@ -2569,7 +2569,7 @@ static int do_test_file(unsigned int test_num) /* reset info to only retrieve func_info related data */ memset(&info, 0, sizeof(info)); - info.func_info_cnt = 3; + info.nr_func_info = 3; info.func_info_rec_size = rec_size; info.func_info = ptr_to_u64(func_info); @@ -2580,9 +2580,9 @@ static int do_test_file(unsigned int test_num) err = -1; goto done; } - if (CHECK(info.func_info_cnt != 3, - "incorrect info.func_info_cnt (2nd) %d", - info.func_info_cnt)) { + if (CHECK(info.nr_func_info != 3, + "incorrect info.nr_func_info (2nd) %d", + info.nr_func_info)) { err = -1; goto done; } @@ -3544,9 +3544,9 @@ static int test_get_finfo(const struct prog_info_raw_test *test, fprintf(stderr, "%s\n", btf_log_buf); return -1; } - if (CHECK(info.func_info_cnt != test->func_info_cnt, - "incorrect info.func_info_cnt (1st) %d", - info.func_info_cnt)) { + if (CHECK(info.nr_func_info != test->func_info_cnt, + "incorrect info.nr_func_info (1st) %d", + info.nr_func_info)) { return -1; } @@ -3556,16 +3556,16 @@ static int test_get_finfo(const struct prog_info_raw_test *test, return -1; } - if (!info.func_info_cnt) + if (!info.nr_func_info) return 0; - func_info = malloc(info.func_info_cnt * rec_size); + func_info = malloc(info.nr_func_info * rec_size); if (CHECK(!func_info, "out of memory")) return -1; /* reset info to only retrieve func_info related data */ memset(&info, 0, sizeof(info)); - info.func_info_cnt = test->func_info_cnt; + info.nr_func_info = test->func_info_cnt; info.func_info_rec_size = rec_size; info.func_info = ptr_to_u64(func_info); err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); @@ -3574,9 +3574,9 @@ static int test_get_finfo(const struct prog_info_raw_test *test, err = -1; goto done; } - if (CHECK(info.func_info_cnt != test->func_info_cnt, - "incorrect info.func_info_cnt (2nd) %d", - info.func_info_cnt)) { + if (CHECK(info.nr_func_info != test->func_info_cnt, + "incorrect info.nr_func_info (2nd) %d", + info.nr_func_info)) { err = -1; goto done; } @@ -3648,14 +3648,14 @@ static int test_get_linfo(const struct prog_info_raw_test *test, nr_jited_func_lens = 1; } - if (CHECK(info.line_info_cnt != cnt || - info.jited_line_info_cnt != jited_cnt || + if (CHECK(info.nr_line_info != cnt || + info.nr_jited_line_info != jited_cnt || info.nr_jited_ksyms != nr_jited_ksyms || info.nr_jited_func_lens != nr_jited_func_lens || - (!info.line_info_cnt && info.jited_line_info_cnt), - "info: line_info_cnt:%u(expected:%u) jited_line_info_cnt:%u(expected:%u) nr_jited_ksyms:%u(expected:%u) nr_jited_func_lens:%u(expected:%u)", - info.line_info_cnt, cnt, - info.jited_line_info_cnt, jited_cnt, + (!info.nr_line_info && info.nr_jited_line_info), + "info: nr_line_info:%u(expected:%u) nr_jited_line_info:%u(expected:%u) nr_jited_ksyms:%u(expected:%u) nr_jited_func_lens:%u(expected:%u)", + info.nr_line_info, cnt, + info.nr_jited_line_info, jited_cnt, info.nr_jited_ksyms, nr_jited_ksyms, info.nr_jited_func_lens, nr_jited_func_lens)) { err = -1; @@ -3684,7 +3684,7 @@ static int test_get_linfo(const struct prog_info_raw_test *test, err = -1; goto done; } - info.line_info_cnt = cnt; + info.nr_line_info = cnt; info.line_info_rec_size = rec_size; info.line_info = ptr_to_u64(linfo); @@ -3700,7 +3700,7 @@ static int test_get_linfo(const struct prog_info_raw_test *test, goto done; } - info.jited_line_info_cnt = jited_cnt; + info.nr_jited_line_info = jited_cnt; info.jited_line_info_rec_size = jited_rec_size; info.jited_line_info = ptr_to_u64(jited_linfo); info.nr_jited_ksyms = nr_jited_ksyms; @@ -3717,15 +3717,15 @@ static int test_get_linfo(const struct prog_info_raw_test *test, */ if (CHECK(err == -1 || !info.line_info || - info.line_info_cnt != cnt || + info.nr_line_info != cnt || (jited_cnt && !info.jited_line_info) || - info.jited_line_info_cnt != jited_cnt || + info.nr_jited_line_info != jited_cnt || info.line_info_rec_size != rec_size || info.jited_line_info_rec_size != jited_rec_size, - "err:%d errno:%d info: line_info_cnt:%u(expected:%u) jited_line_info_cnt:%u(expected:%u) line_info_rec_size:%u(expected:%u) jited_linfo_rec_size:%u(expected:%u) line_info:%p jited_line_info:%p", + "err:%d errno:%d info: nr_line_info:%u(expected:%u) nr_jited_line_info:%u(expected:%u) line_info_rec_size:%u(expected:%u) jited_linfo_rec_size:%u(expected:%u) line_info:%p jited_line_info:%p", err, errno, - info.line_info_cnt, cnt, - info.jited_line_info_cnt, jited_cnt, + info.nr_line_info, cnt, + info.nr_jited_line_info, jited_cnt, info.line_info_rec_size, rec_size, info.jited_line_info_rec_size, jited_rec_size, (void *)(long)info.line_info,