Skip to content

Commit

Permalink
samples: bpf: Refactor hbm program with libbpf
Browse files Browse the repository at this point in the history 
This commit refactors the existing cgroup programs with libbpf
bpf loader. Since bpf_program__attach doesn't support cgroup program
attachment, this explicitly attaches cgroup bpf program with
bpf_program__attach_cgroup(bpf_prog, cg1).

Also, to change attach_type of bpf program, this uses libbpf's
bpf_program__set_expected_attach_type helper to switch EGRESS to
INGRESS. To keep bpf program attached to the cgroup hierarchy even
after the exit, this commit uses the BPF_LINK_PINNING to pin the link
attachment even after it is closed.

Besides, this program was broken due to the typo of BPF MAP definition.
But this commit solves the problem by fixing this from 'queue_stats' map
struct hvm_queue_stats -> hbm_queue_stats.

Fixes: 36b5d47 ("selftests/bpf: samples/bpf: Split off legacy stuff from bpf_helpers.h")
Signed-off-by: Daniel T. Lee <[email protected]>
Signed-off-by: Andrii Nakryiko <[email protected]>
Link: https://lore.kernel.org/bpf/[email protected]
  • Loading branch information
@DanielTimLee @anakryiko
DanielTimLee authored and anakryiko committed Nov 27, 2020
1 parent fb35581 commit c5815ac
Show file tree
Hide file tree
Showing 5 changed files with 78 additions and 72 deletions.
3 changes: 3 additions & 0 deletions samples/bpf/.gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,6 @@ xdp_tx_iptunnel
xdpsock
xsk_fwd
testfile.img
hbm_out.log
iperf.*
*.out
2 changes: 1 addition & 1 deletion samples/bpf/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ xdp_fwd-objs := xdp_fwd_user.o
task_fd_query-objs := bpf_load.o task_fd_query_user.o $(TRACE_HELPERS)
xdp_sample_pkts-objs := xdp_sample_pkts_user.o $(TRACE_HELPERS)
ibumad-objs := bpf_load.o ibumad_user.o $(TRACE_HELPERS)
hbm-objs := bpf_load.o hbm.o $(CGROUP_HELPERS)
hbm-objs := hbm.o $(CGROUP_HELPERS)

# Tell kbuild to always build the programs
always-y := $(tprogs-y)
Expand Down
32 changes: 14 additions & 18 deletions samples/bpf/do_hbm_test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,16 @@ qdisc=""
flags=""
do_stats=0

BPFFS=/sys/fs/bpf
function config_bpffs () {
if mount | grep $BPFFS > /dev/null; then
echo "bpffs already mounted"
else
echo "bpffs not mounted. Mounting..."
mount -t bpf none $BPFFS
fi
}

function start_hbm () {
rm -f hbm.out
echo "./hbm $dir -n $id -r $rate -t $dur $flags $dbg $prog" > hbm.out
Expand Down Expand Up @@ -192,6 +202,7 @@ processArgs () {
}

processArgs
config_bpffs

if [ $debug_flag -eq 1 ] ; then
rm -f hbm_out.log
Expand All @@ -201,7 +212,7 @@ hbm_pid=$(start_hbm)
usleep 100000

host=`hostname`
cg_base_dir=/sys/fs/cgroup
cg_base_dir=/sys/fs/cgroup/unified
cg_dir="$cg_base_dir/cgroup-test-work-dir/hbm$id"

echo $$ >> $cg_dir/cgroup.procs
Expand Down Expand Up @@ -411,23 +422,8 @@ fi

sleep 1

# Detach any BPF programs that may have lingered
ttx=`bpftool cgroup tree | grep hbm`
v=2
for x in $ttx ; do
if [ "${x:0:36}" == "/sys/fs/cgroup/cgroup-test-work-dir/" ] ; then
cg=$x ; v=0
else
if [ $v -eq 0 ] ; then
id=$x ; v=1
else
if [ $v -eq 1 ] ; then
type=$x ; bpftool cgroup detach $cg $type id $id
v=0
fi
fi
fi
done
# Detach any pinned BPF programs that may have lingered
rm -rf $BPFFS/hbm*

if [ $use_netperf -ne 0 ] ; then
if [ "$server" == "" ] ; then
Expand Down
111 changes: 59 additions & 52 deletions samples/bpf/hbm.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@
#include <bpf/bpf.h>
#include <getopt.h>

#include "bpf_load.h"
#include "bpf_rlimit.h"
#include "cgroup_helpers.h"
#include "hbm.h"
Expand All @@ -70,9 +69,9 @@ static void do_error(char *msg, bool errno_flag);

#define DEBUGFS "/sys/kernel/debug/tracing/"

struct bpf_object *obj;
int bpfprog_fd;
int cgroup_storage_fd;
static struct bpf_program *bpf_prog;
static struct bpf_object *obj;
static int queue_stats_fd;

static void read_trace_pipe2(void)
{
Expand Down Expand Up @@ -121,56 +120,50 @@ static void do_error(char *msg, bool errno_flag)

static int prog_load(char *prog)
{
struct bpf_prog_load_attr prog_load_attr = {
.prog_type = BPF_PROG_TYPE_CGROUP_SKB,
.file = prog,
.expected_attach_type = BPF_CGROUP_INET_EGRESS,
};
int map_fd;
struct bpf_map *map;

int ret = 0;

if (access(prog, O_RDONLY) < 0) {
printf("Error accessing file %s: %s\n", prog, strerror(errno));
obj = bpf_object__open_file(prog, NULL);
if (libbpf_get_error(obj)) {
printf("ERROR: opening BPF object file failed\n");
return 1;
}
if (bpf_prog_load_xattr(&prog_load_attr, &obj, &bpfprog_fd))
ret = 1;
if (!ret) {
map = bpf_object__find_map_by_name(obj, "queue_stats");
map_fd = bpf_map__fd(map);
if (map_fd < 0) {
printf("Map not found: %s\n", strerror(map_fd));
ret = 1;
}

/* load BPF program */
if (bpf_object__load(obj)) {
printf("ERROR: loading BPF object file failed\n");
goto err;
}

if (ret) {
printf("ERROR: bpf_prog_load_xattr failed for: %s\n", prog);
printf(" Output from verifier:\n%s\n------\n", bpf_log_buf);
ret = -1;
} else {
ret = map_fd;
bpf_prog = bpf_object__find_program_by_title(obj, "cgroup_skb/egress");
if (!bpf_prog) {
printf("ERROR: finding a prog in obj file failed\n");
goto err;
}

queue_stats_fd = bpf_object__find_map_fd_by_name(obj, "queue_stats");
if (queue_stats_fd < 0) {
printf("ERROR: finding a map in obj file failed\n");
goto err;
}

return ret;
return 0;

err:
bpf_object__close(obj);
return 1;
}

static int run_bpf_prog(char *prog, int cg_id)
{
int map_fd;
int rc = 0;
struct hbm_queue_stats qstats = {0};
char cg_dir[100], cg_pin_path[100];
struct bpf_link *link = NULL;
int key = 0;
int cg1 = 0;
int type = BPF_CGROUP_INET_EGRESS;
char cg_dir[100];
struct hbm_queue_stats qstats = {0};
int rc = 0;

sprintf(cg_dir, "/hbm%d", cg_id);
map_fd = prog_load(prog);
if (map_fd == -1)
return 1;
rc = prog_load(prog);
if (rc != 0)
return rc;

if (setup_cgroup_environment()) {
printf("ERROR: setting cgroup environment\n");
Expand All @@ -190,16 +183,24 @@ static int run_bpf_prog(char *prog, int cg_id)
qstats.stats = stats_flag ? 1 : 0;
qstats.loopback = loopback_flag ? 1 : 0;
qstats.no_cn = no_cn_flag ? 1 : 0;
if (bpf_map_update_elem(map_fd, &key, &qstats, BPF_ANY)) {
if (bpf_map_update_elem(queue_stats_fd, &key, &qstats, BPF_ANY)) {
printf("ERROR: Could not update map element\n");
goto err;
}

if (!outFlag)
type = BPF_CGROUP_INET_INGRESS;
if (bpf_prog_attach(bpfprog_fd, cg1, type, 0)) {
printf("ERROR: bpf_prog_attach fails!\n");
log_err("Attaching prog");
bpf_program__set_expected_attach_type(bpf_prog, BPF_CGROUP_INET_INGRESS);

link = bpf_program__attach_cgroup(bpf_prog, cg1);
if (libbpf_get_error(link)) {
fprintf(stderr, "ERROR: bpf_program__attach_cgroup failed\n");
goto err;
}

sprintf(cg_pin_path, "/sys/fs/bpf/hbm%d", cg_id);
rc = bpf_link__pin(link, cg_pin_path);
if (rc < 0) {
printf("ERROR: bpf_link__pin failed: %d\n", rc);
goto err;
}

Expand All @@ -213,7 +214,7 @@ static int run_bpf_prog(char *prog, int cg_id)
#define DELTA_RATE_CHECK 10000 /* in us */
#define RATE_THRESHOLD 9500000000 /* 9.5 Gbps */

bpf_map_lookup_elem(map_fd, &key, &qstats);
bpf_map_lookup_elem(queue_stats_fd, &key, &qstats);
if (gettimeofday(&t0, NULL) < 0)
do_error("gettimeofday failed", true);
t_last = t0;
Expand Down Expand Up @@ -242,7 +243,7 @@ static int run_bpf_prog(char *prog, int cg_id)
fclose(fin);
printf(" new_eth_tx_bytes:%llu\n",
new_eth_tx_bytes);
bpf_map_lookup_elem(map_fd, &key, &qstats);
bpf_map_lookup_elem(queue_stats_fd, &key, &qstats);
new_cg_tx_bytes = qstats.bytes_total;
delta_bytes = new_eth_tx_bytes - last_eth_tx_bytes;
last_eth_tx_bytes = new_eth_tx_bytes;
Expand Down Expand Up @@ -289,14 +290,14 @@ static int run_bpf_prog(char *prog, int cg_id)
rate = minRate;
qstats.rate = rate;
}
if (bpf_map_update_elem(map_fd, &key, &qstats, BPF_ANY))
if (bpf_map_update_elem(queue_stats_fd, &key, &qstats, BPF_ANY))
do_error("update map element fails", false);
}
} else {
sleep(dur);
}
// Get stats!
if (stats_flag && bpf_map_lookup_elem(map_fd, &key, &qstats)) {
if (stats_flag && bpf_map_lookup_elem(queue_stats_fd, &key, &qstats)) {
char fname[100];
FILE *fout;

Expand Down Expand Up @@ -394,14 +395,20 @@ static int run_bpf_prog(char *prog, int cg_id)

if (debugFlag)
read_trace_pipe2();
return rc;
goto cleanup;

err:
rc = 1;

if (cg1)
cleanup:
bpf_link__destroy(link);
bpf_object__close(obj);

if (cg1 != -1)
close(cg1);
cleanup_cgroup_environment();

if (rc != 0)
cleanup_cgroup_environment();
return rc;
}

Expand Down
2 changes: 1 addition & 1 deletion samples/bpf/hbm_kern.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__uint(max_entries, 1);
__type(key, u32);
__type(value, struct hvm_queue_stats);
__type(value, struct hbm_queue_stats);
} queue_stats SEC(".maps");

struct hbm_pkt_info {
Expand Down

0 comments on commit c5815ac

Please sign in to comment.