Skip to content

Commit

Permalink
Merge pull request #482 from ESWZY/traffic-mgr/base-map
Browse files Browse the repository at this point in the history
Implement kernel abstraction of Service and Pod, and use maps for storage and information transfer
  • Loading branch information
chenamy2017 authored Aug 6, 2023
2 parents cda1f72 + 74f7d19 commit 26d5632
Show file tree
Hide file tree
Showing 10 changed files with 1,565 additions and 33 deletions.
4 changes: 3 additions & 1 deletion eBPF_Supermarket/TrafficManager/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,6 @@ build:
init:
cd bpf/headers/bpf/; ./update.sh
cd bpf/headers/; bpftool btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h
cd bpf; go generate;

clean:
rm main
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
# Install tutorial
# eBPF Traffic Manager

## Ubuntu 22.04
## Install tutorial

### Ubuntu 22.04

```bash
# Install Go
Expand Down Expand Up @@ -31,3 +33,7 @@ sudo rm -rf bpftool/
sudo make init
sudo make
```

## Usage

Developing...
143 changes: 143 additions & 0 deletions eBPF_Supermarket/TrafficManager/bpf/bpf_connect_bpf.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

62 changes: 47 additions & 15 deletions eBPF_Supermarket/TrafficManager/bpf/connect.c
Original file line number Diff line number Diff line change
Expand Up @@ -9,30 +9,62 @@ const __be16 pod_port = 0x901f; // 8080 (0x1f * 256 + 0x144)

static int sock4_forward_entry(struct bpf_sock_addr *ctx)
{
// 0x846F070A; // 10.7.111.132
// 0x0529050A; // 10.5.41.5
// 0x0100007F; // 127.0.0.1
struct lb4_key key = {}, orig_key;
struct lb4_service *svc;
struct lb4_service *backend_slot;
int backend_id = -1;
struct lb4_backend *backend;

__be32 ori_dst_ip = ctx_get_dst_ip(ctx);
__be16 ori_dst_port = ctx_get_dst_port(ctx);
bpf_printk("dest: %08x:%04x", ori_dst_ip, ori_dst_port);
key.address = ori_dst_ip,
key.dport = ori_dst_port,
key.backend_slot = 0,
orig_key = key;

// 1. find service by ip and port
svc = lb4_lookup_service(&key);
if (!svc || svc->count == 0)
return -ENXIO;
bpf_printk("1. Service backend ID (must be zero): %d", svc->backend_id);

// 2. find backend slots from service
key.backend_slot = sock_select_random_slot(svc->count);
bpf_printk("2. select backend from service slot: %d", key.backend_slot);

// 3. lookup backend slot from constructed backend key
backend_slot = lookup_lb4_backend_slot(&key);
if (!backend_slot)
return -ENOENT;
bpf_printk("3. find backend slot: %d", backend_slot->backend_id);
backend_id = backend_slot->backend_id;

// 4. find the info of real backend
backend = lookup_lb4_backend(backend_id);
if (!backend)
return -ENOENT;
bpf_printk("4. real backend: %x", backend->address);

// 5. got the address and port
bpf_printk("verdict: %08x:%04x", backend->address, backend->port);
ctx_set_dst_ip(ctx, backend->address);
ctx_set_dst_port(ctx, backend->port);

print_ip_formatted(ctx->user_ip4);

bpf_printk("original: %08x:%04x", ori_dst_ip, ori_dst_port);
bpf_printk("service: %08x:%04x", service_ip, service_port);
bpf_printk("pod: %08x:%04x", pod_ip, pod_port);

if(ori_dst_ip == service_ip && ori_dst_port == service_port)
{
bpf_printk("redirect to %08x:%04x", pod_ip, pod_port);
ctx_set_ip(ctx, pod_ip);
ctx_set_port(ctx, pod_port);
}
else
{
bpf_printk("skipped, not modified");
}
return 0;
}

SEC("cgroup/connect4")
int sock4_connect(struct bpf_sock_addr *ctx)
{
sock4_forward_entry(ctx);
int ret = sock4_forward_entry(ctx);
if(!ret)
bpf_printk("skipped, not modified");
return SYS_PROCEED;
}

Expand Down
85 changes: 81 additions & 4 deletions eBPF_Supermarket/TrafficManager/bpf/connect.go
Original file line number Diff line number Diff line change
@@ -1,14 +1,33 @@
// Copyright 2023 The LMP Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://github.com/linuxkerneltravel/lmp/blob/develop/LICENSE
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// author: Woa <[email protected]>

package bpf

//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang -target bpf -cflags "-D__TARGET_ARCH_x86" bpf_connect connect.c -- -I./headers

import (
"bufio"
"fmt"
"net"
"os"
"os/signal"
"strings"
"time"

"github.com/cilium/cilium/pkg/loadbalancer"
"github.com/cilium/cilium/pkg/u8proto"
"github.com/cilium/ebpf"
"github.com/cilium/ebpf/link"
"github.com/cilium/ebpf/rlimit"
Expand Down Expand Up @@ -99,6 +118,18 @@ func (p *Programs) Close() {
p.connectCgroup.Close()
}

if p.connectObj.bpf_connectMaps.LB4SERVICES_MAP_V2 != nil {
p.connectObj.bpf_connectMaps.LB4SERVICES_MAP_V2.Unpin()
p.connectObj.bpf_connectMaps.LB4SERVICES_MAP_V2.Close()
fmt.Println("Unpin and close")
}

if p.connectObj.bpf_connectMaps.LB4BACKEND_MAP_V2 != nil {
p.connectObj.bpf_connectMaps.LB4BACKEND_MAP_V2.Unpin()
p.connectObj.bpf_connectMaps.LB4BACKEND_MAP_V2.Close()
fmt.Println("Unpin and close")
}

_ = os.Remove(MapsPinPath)
}

Expand All @@ -109,13 +140,59 @@ func Sample() {
return
}

// set service
serviceIP := "1.1.1.1"
servicePort := 80
backendNumber := 2
svcKey := NewService4Key(net.ParseIP(serviceIP), uint16(servicePort), u8proto.ANY, 0, 0)
// use index 0 to indicate service item
svcValue := NewService4Value(Backend4Key{0}, uint16(backendNumber))
err = progs.connectObj.bpf_connectMaps.LB4SERVICES_MAP_V2.Update(svcKey.ToNetwork(), svcValue.ToNetwork(), ebpf.UpdateAny)
if err != nil {
panic(err)
}

podIp1 := "1.1.1.1"
backendPort1 := 80
backendID1 := 0
slotIndex1 := 1
backendKey1 := Backend4Key{uint32(backendID1)}
backendServiceKey := NewService4Key(net.ParseIP(serviceIP), uint16(servicePort), u8proto.ANY, 0, uint16(slotIndex1))
backendServiceValue := NewService4Value(backendKey1, 0)
err = progs.connectObj.bpf_connectMaps.LB4SERVICES_MAP_V2.Update(backendServiceKey.ToNetwork(), backendServiceValue.ToNetwork(), ebpf.UpdateAny)
if err != nil {
panic(err)
}
backendValue, _ := NewBackend4Value(net.ParseIP(podIp1), uint16(backendPort1), u8proto.ANY, loadbalancer.BackendStateActive)
err = progs.connectObj.bpf_connectMaps.LB4BACKEND_MAP_V2.Update(backendKey1, backendValue.ToNetwork(), ebpf.UpdateAny)
if err != nil {
panic(err)
}

// python3 -m http.server 8888
podIp2 := "127.0.0.1"
backendPort2 := 8888
backendID2 := 1
slotIndex2 := 2
backendKey2 := Backend4Key{uint32(backendID2)}
backendServiceKey2 := NewService4Key(net.ParseIP(serviceIP), uint16(servicePort), u8proto.ANY, 0, uint16(slotIndex2))
backendServiceValue2 := NewService4Value(backendKey2, 0)
err = progs.connectObj.bpf_connectMaps.LB4SERVICES_MAP_V2.Update(backendServiceKey2.ToNetwork(), backendServiceValue2.ToNetwork(), ebpf.UpdateAny)
if err != nil {
panic(err)
}
backendValue2, _ := NewBackend4Value(net.ParseIP(podIp2), uint16(backendPort2), u8proto.ANY, loadbalancer.BackendStateActive)
err = progs.connectObj.bpf_connectMaps.LB4BACKEND_MAP_V2.Update(backendKey2, backendValue2.ToNetwork(), ebpf.UpdateAny)
if err != nil {
panic(err)
}

err = progs.Attach()
if err != nil {
fmt.Println("[ERROR] Attaching failed:", err)
}
defer progs.Close()

c := make(chan os.Signal, 1)
signal.Notify(c)
<-c
time.Sleep(time.Minute)
fmt.Println("[INFO] Time is up...")
}
Loading

0 comments on commit 26d5632

Please sign in to comment.