Skip to content

Commit

Permalink
bpf ppc64: Add BPF_PROBE_MEM support for JIT
Browse files Browse the repository at this point in the history 
BPF load instruction with BPF_PROBE_MEM mode can cause a fault
inside kernel. Append exception table for such instructions
within BPF program.

Unlike other archs which uses extable 'fixup' field to pass dest_reg
and nip, BPF exception table on PowerPC follows the generic PowerPC
exception table design, where it populates both fixup and extable
sections within BPF program. fixup section contains two instructions,
first instruction clears dest_reg and 2nd jumps to next instruction
in the BPF code. extable 'insn' field contains relative offset of
the instruction and 'fixup' field contains relative offset of the
fixup entry. Example layout of BPF program with extable present:

             +------------------+
             |                  |
             |                  |
   0x4020 -->| ld   r27,4(r3)   |
             |                  |
             |                  |
   0x40ac -->| lwz  r3,0(r4)    |
             |                  |
             |                  |
             |------------------|
   0x4280 -->| li  r27,0        |  \ fixup entry
             | b   0x4024       |  /
   0x4288 -->| li  r3,0         |
             | b   0x40b0       |
             |------------------|
   0x4290 -->| insn=0xfffffd90  |  \ extable entry
             | fixup=0xffffffec |  /
   0x4298 -->| insn=0xfffffe14  |
             | fixup=0xffffffec |
             +------------------+

   (Addresses shown here are chosen random, not real)

Signed-off-by: Ravi Bangoria <[email protected]>
Signed-off-by: Hari Bathini <[email protected]>
Reviewed-by: Christophe Leroy <[email protected]>
Signed-off-by: Michael Ellerman <[email protected]>
Link: https://lore.kernel.org/r/[email protected]
  • Loading branch information
@mpe
Ravi Bangoria authored and mpe committed Nov 25, 2021
1 parent f15a71b commit 983bdc0
Show file tree
Hide file tree
Showing 4 changed files with 80 additions and 9 deletions.
8 changes: 7 additions & 1 deletion arch/powerpc/net/bpf_jit.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,11 @@ struct codegen_context {
unsigned int idx;
unsigned int stack_size;
int b2p[ARRAY_SIZE(b2p)];
unsigned int exentry_idx;
};

#define BPF_FIXUP_LEN 2 /* Two instructions => 8 bytes */

static inline void bpf_flush_icache(void *start, void *end)
{
smp_wmb(); /* smp write barrier */
Expand All @@ -175,11 +178,14 @@ static inline void bpf_clear_seen_register(struct codegen_context *ctx, int i)

void bpf_jit_emit_func_call_rel(u32 *image, struct codegen_context *ctx, u64 func);
int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
u32 *addrs);
u32 *addrs, int pass);
void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx);
void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx);
void bpf_jit_realloc_regs(struct codegen_context *ctx);

int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx,
int insn_idx, int jmp_off, int dst_reg);

#endif

#endif
66 changes: 60 additions & 6 deletions arch/powerpc/net/bpf_jit_comp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
struct bpf_prog *tmp_fp;
bool bpf_blinded = false;
bool extra_pass = false;
u32 extable_len;
u32 fixup_len;

if (!fp->jit_requested)
return org_fp;
Expand Down Expand Up @@ -131,7 +133,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
image = jit_data->image;
bpf_hdr = jit_data->header;
proglen = jit_data->proglen;
alloclen = proglen + FUNCTION_DESCR_SIZE;
extra_pass = true;
goto skip_init_ctx;
}
Expand All @@ -149,7 +150,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
cgctx.stack_size = round_up(fp->aux->stack_depth, 16);

/* Scouting faux-generate pass 0 */
if (bpf_jit_build_body(fp, 0, &cgctx, addrs)) {
if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0)) {
/* We hit something illegal or unsupported. */
fp = org_fp;
goto out_addrs;
Expand All @@ -162,7 +163,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
*/
if (cgctx.seen & SEEN_TAILCALL) {
cgctx.idx = 0;
if (bpf_jit_build_body(fp, 0, &cgctx, addrs)) {
if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0)) {
fp = org_fp;
goto out_addrs;
}
Expand All @@ -177,15 +178,21 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
bpf_jit_build_prologue(0, &cgctx);
bpf_jit_build_epilogue(0, &cgctx);

fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4;
extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry);

proglen = cgctx.idx * 4;
alloclen = proglen + FUNCTION_DESCR_SIZE;
alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;

bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns);
if (!bpf_hdr) {
fp = org_fp;
goto out_addrs;
}

if (extable_len)
fp->aux->extable = (void *)image + FUNCTION_DESCR_SIZE + proglen + fixup_len;

skip_init_ctx:
code_base = (u32 *)(image + FUNCTION_DESCR_SIZE);

Expand All @@ -210,7 +217,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
/* Now build the prologue, body code & epilogue for real. */
cgctx.idx = 0;
bpf_jit_build_prologue(code_base, &cgctx);
if (bpf_jit_build_body(fp, code_base, &cgctx, addrs)) {
if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, pass)) {
bpf_jit_binary_free(bpf_hdr);
fp = org_fp;
goto out_addrs;
Expand Down Expand Up @@ -238,7 +245,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)

fp->bpf_func = (void *)image;
fp->jited = 1;
fp->jited_len = alloclen;
fp->jited_len = proglen + FUNCTION_DESCR_SIZE;

bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE));
if (!fp->is_func || extra_pass) {
Expand All @@ -262,3 +269,50 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)

return fp;
}

/*
* The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling
* this function, as this only applies to BPF_PROBE_MEM, for now.
*/
int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx,
int insn_idx, int jmp_off, int dst_reg)
{
off_t offset;
unsigned long pc;
struct exception_table_entry *ex;
u32 *fixup;

/* Populate extable entries only in the last pass */
if (pass != 2)
return 0;

if (!fp->aux->extable ||
WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries))
return -EINVAL;

pc = (unsigned long)&image[insn_idx];

fixup = (void *)fp->aux->extable -
(fp->aux->num_exentries * BPF_FIXUP_LEN * 4) +
(ctx->exentry_idx * BPF_FIXUP_LEN * 4);

fixup[0] = PPC_RAW_LI(dst_reg, 0);

fixup[BPF_FIXUP_LEN - 1] =
PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]);

ex = &fp->aux->extable[ctx->exentry_idx];

offset = pc - (long)&ex->insn;
if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
return -ERANGE;
ex->insn = offset;

offset = (long)fixup - (long)&ex->fixup;
if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
return -ERANGE;
ex->fixup = offset;

ctx->exentry_idx++;
return 0;
}
2 changes: 1 addition & 1 deletion arch/powerpc/net/bpf_jit_comp32.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o

/* Assemble the body code between the prologue & epilogue */
int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
u32 *addrs)
u32 *addrs, int pass)
{
const struct bpf_insn *insn = fp->insnsi;
int flen = fp->len;
Expand Down
13 changes: 12 additions & 1 deletion arch/powerpc/net/bpf_jit_comp64.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ asm (

/* Assemble the body code between the prologue & epilogue */
int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx,
u32 *addrs)
u32 *addrs, int pass)
{
enum stf_barrier_type stf_barrier = stf_barrier_type_get();
const struct bpf_insn *insn = fp->insnsi;
Expand Down Expand Up @@ -779,12 +779,16 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
*/
/* dst = *(u8 *)(ul) (src + off) */
case BPF_LDX | BPF_MEM | BPF_B:
case BPF_LDX | BPF_PROBE_MEM | BPF_B:
/* dst = *(u16 *)(ul) (src + off) */
case BPF_LDX | BPF_MEM | BPF_H:
case BPF_LDX | BPF_PROBE_MEM | BPF_H:
/* dst = *(u32 *)(ul) (src + off) */
case BPF_LDX | BPF_MEM | BPF_W:
case BPF_LDX | BPF_PROBE_MEM | BPF_W:
/* dst = *(u64 *)(ul) (src + off) */
case BPF_LDX | BPF_MEM | BPF_DW:
case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
switch (size) {
case BPF_B:
EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off));
Expand All @@ -802,6 +806,13 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *

if (size != BPF_DW && insn_is_zext(&insn[i + 1]))
addrs[++i] = ctx->idx * 4;

if (BPF_MODE(code) == BPF_PROBE_MEM) {
ret = bpf_add_extable_entry(fp, image, pass, ctx, ctx->idx - 1,
4, dst_reg);
if (ret)
return ret;
}
break;

/*
Expand Down

0 comments on commit 983bdc0

Please sign in to comment.