forked from torvalds/linux
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
powerpc/ebpf/jit: Implement JIT compiler for extended BPF
PPC64 eBPF JIT compiler. Enable with: echo 1 > /proc/sys/net/core/bpf_jit_enable or echo 2 > /proc/sys/net/core/bpf_jit_enable ... to see the generated JIT code. This can further be processed with tools/net/bpf_jit_disasm. With CONFIG_TEST_BPF=m and 'modprobe test_bpf': test_bpf: Summary: 305 PASSED, 0 FAILED, [297/297 JIT'ed] ... on both ppc64 BE and LE. The details of the approach are documented through various comments in the code. Acked-by: Alexei Starovoitov <[email protected]> Signed-off-by: Naveen N. Rao <[email protected]> Signed-off-by: Michael Ellerman <[email protected]>
- Loading branch information
Showing
8 changed files
with
1,315 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,8 @@ | ||
# | ||
# Arch-specific network modules | ||
# | ||
ifeq ($(CONFIG_PPC64),y) | ||
obj-$(CONFIG_BPF_JIT) += bpf_jit_asm64.o bpf_jit_comp64.o | ||
else | ||
obj-$(CONFIG_BPF_JIT) += bpf_jit_asm.o bpf_jit_comp.o | ||
endif |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,6 +2,7 @@ | |
* bpf_jit.h: BPF JIT compiler for PPC | ||
* | ||
* Copyright 2011 Matt Evans <[email protected]>, IBM Corporation | ||
* 2016 Naveen N. Rao <[email protected]> | ||
* | ||
* This program is free software; you can redistribute it and/or | ||
* modify it under the terms of the GNU General Public License | ||
|
@@ -13,7 +14,9 @@ | |
|
||
#ifndef __ASSEMBLY__ | ||
|
||
#ifdef CONFIG_PPC64 | ||
#include <asm/types.h> | ||
|
||
#ifdef PPC64_ELF_ABI_v1 | ||
#define FUNCTION_DESCR_SIZE 24 | ||
#else | ||
#define FUNCTION_DESCR_SIZE 0 | ||
|
@@ -52,6 +55,10 @@ | |
___PPC_RA(base) | IMM_L(i)) | ||
#define PPC_STWU(r, base, i) EMIT(PPC_INST_STWU | ___PPC_RS(r) | \ | ||
___PPC_RA(base) | IMM_L(i)) | ||
#define PPC_STH(r, base, i) EMIT(PPC_INST_STH | ___PPC_RS(r) | \ | ||
___PPC_RA(base) | IMM_L(i)) | ||
#define PPC_STB(r, base, i) EMIT(PPC_INST_STB | ___PPC_RS(r) | \ | ||
___PPC_RA(base) | IMM_L(i)) | ||
|
||
#define PPC_LBZ(r, base, i) EMIT(PPC_INST_LBZ | ___PPC_RT(r) | \ | ||
___PPC_RA(base) | IMM_L(i)) | ||
|
@@ -63,6 +70,19 @@ | |
___PPC_RA(base) | IMM_L(i)) | ||
#define PPC_LHBRX(r, base, b) EMIT(PPC_INST_LHBRX | ___PPC_RT(r) | \ | ||
___PPC_RA(base) | ___PPC_RB(b)) | ||
#define PPC_LDBRX(r, base, b) EMIT(PPC_INST_LDBRX | ___PPC_RT(r) | \ | ||
___PPC_RA(base) | ___PPC_RB(b)) | ||
|
||
#define PPC_BPF_LDARX(t, a, b, eh) EMIT(PPC_INST_LDARX | ___PPC_RT(t) | \ | ||
___PPC_RA(a) | ___PPC_RB(b) | \ | ||
__PPC_EH(eh)) | ||
#define PPC_BPF_LWARX(t, a, b, eh) EMIT(PPC_INST_LWARX | ___PPC_RT(t) | \ | ||
___PPC_RA(a) | ___PPC_RB(b) | \ | ||
__PPC_EH(eh)) | ||
#define PPC_BPF_STWCX(s, a, b) EMIT(PPC_INST_STWCX | ___PPC_RS(s) | \ | ||
___PPC_RA(a) | ___PPC_RB(b)) | ||
#define PPC_BPF_STDCX(s, a, b) EMIT(PPC_INST_STDCX | ___PPC_RS(s) | \ | ||
___PPC_RA(a) | ___PPC_RB(b)) | ||
|
||
#ifdef CONFIG_PPC64 | ||
#define PPC_BPF_LL(r, base, i) do { PPC_LD(r, base, i); } while(0) | ||
|
@@ -76,14 +96,23 @@ | |
|
||
#define PPC_CMPWI(a, i) EMIT(PPC_INST_CMPWI | ___PPC_RA(a) | IMM_L(i)) | ||
#define PPC_CMPDI(a, i) EMIT(PPC_INST_CMPDI | ___PPC_RA(a) | IMM_L(i)) | ||
#define PPC_CMPW(a, b) EMIT(PPC_INST_CMPW | ___PPC_RA(a) | \ | ||
___PPC_RB(b)) | ||
#define PPC_CMPD(a, b) EMIT(PPC_INST_CMPD | ___PPC_RA(a) | \ | ||
___PPC_RB(b)) | ||
#define PPC_CMPLWI(a, i) EMIT(PPC_INST_CMPLWI | ___PPC_RA(a) | IMM_L(i)) | ||
#define PPC_CMPLDI(a, i) EMIT(PPC_INST_CMPLDI | ___PPC_RA(a) | IMM_L(i)) | ||
#define PPC_CMPLW(a, b) EMIT(PPC_INST_CMPLW | ___PPC_RA(a) | \ | ||
___PPC_RB(b)) | ||
#define PPC_CMPLD(a, b) EMIT(PPC_INST_CMPLD | ___PPC_RA(a) | \ | ||
___PPC_RB(b)) | ||
|
||
#define PPC_SUB(d, a, b) EMIT(PPC_INST_SUB | ___PPC_RT(d) | \ | ||
___PPC_RB(a) | ___PPC_RA(b)) | ||
#define PPC_ADD(d, a, b) EMIT(PPC_INST_ADD | ___PPC_RT(d) | \ | ||
___PPC_RA(a) | ___PPC_RB(b)) | ||
#define PPC_MULD(d, a, b) EMIT(PPC_INST_MULLD | ___PPC_RT(d) | \ | ||
___PPC_RA(a) | ___PPC_RB(b)) | ||
#define PPC_MULW(d, a, b) EMIT(PPC_INST_MULLW | ___PPC_RT(d) | \ | ||
___PPC_RA(a) | ___PPC_RB(b)) | ||
#define PPC_MULHWU(d, a, b) EMIT(PPC_INST_MULHWU | ___PPC_RT(d) | \ | ||
|
@@ -92,6 +121,8 @@ | |
___PPC_RA(a) | IMM_L(i)) | ||
#define PPC_DIVWU(d, a, b) EMIT(PPC_INST_DIVWU | ___PPC_RT(d) | \ | ||
___PPC_RA(a) | ___PPC_RB(b)) | ||
#define PPC_DIVD(d, a, b) EMIT(PPC_INST_DIVD | ___PPC_RT(d) | \ | ||
___PPC_RA(a) | ___PPC_RB(b)) | ||
#define PPC_AND(d, a, b) EMIT(PPC_INST_AND | ___PPC_RA(d) | \ | ||
___PPC_RS(a) | ___PPC_RB(b)) | ||
#define PPC_ANDI(d, a, i) EMIT(PPC_INST_ANDI | ___PPC_RA(d) | \ | ||
|
@@ -100,6 +131,7 @@ | |
___PPC_RS(a) | ___PPC_RB(b)) | ||
#define PPC_OR(d, a, b) EMIT(PPC_INST_OR | ___PPC_RA(d) | \ | ||
___PPC_RS(a) | ___PPC_RB(b)) | ||
#define PPC_MR(d, a) PPC_OR(d, a, a) | ||
#define PPC_ORI(d, a, i) EMIT(PPC_INST_ORI | ___PPC_RA(d) | \ | ||
___PPC_RS(a) | IMM_L(i)) | ||
#define PPC_ORIS(d, a, i) EMIT(PPC_INST_ORIS | ___PPC_RA(d) | \ | ||
|
@@ -110,13 +142,30 @@ | |
___PPC_RS(a) | IMM_L(i)) | ||
#define PPC_XORIS(d, a, i) EMIT(PPC_INST_XORIS | ___PPC_RA(d) | \ | ||
___PPC_RS(a) | IMM_L(i)) | ||
#define PPC_EXTSW(d, a) EMIT(PPC_INST_EXTSW | ___PPC_RA(d) | \ | ||
___PPC_RS(a)) | ||
#define PPC_SLW(d, a, s) EMIT(PPC_INST_SLW | ___PPC_RA(d) | \ | ||
___PPC_RS(a) | ___PPC_RB(s)) | ||
#define PPC_SLD(d, a, s) EMIT(PPC_INST_SLD | ___PPC_RA(d) | \ | ||
___PPC_RS(a) | ___PPC_RB(s)) | ||
#define PPC_SRW(d, a, s) EMIT(PPC_INST_SRW | ___PPC_RA(d) | \ | ||
___PPC_RS(a) | ___PPC_RB(s)) | ||
#define PPC_SRD(d, a, s) EMIT(PPC_INST_SRD | ___PPC_RA(d) | \ | ||
___PPC_RS(a) | ___PPC_RB(s)) | ||
#define PPC_SRAD(d, a, s) EMIT(PPC_INST_SRAD | ___PPC_RA(d) | \ | ||
___PPC_RS(a) | ___PPC_RB(s)) | ||
#define PPC_SRADI(d, a, i) EMIT(PPC_INST_SRADI | ___PPC_RA(d) | \ | ||
___PPC_RS(a) | __PPC_SH(i) | \ | ||
(((i) & 0x20) >> 4)) | ||
#define PPC_RLWINM(d, a, i, mb, me) EMIT(PPC_INST_RLWINM | ___PPC_RA(d) | \ | ||
___PPC_RS(a) | __PPC_SH(i) | \ | ||
__PPC_MB(mb) | __PPC_ME(me)) | ||
#define PPC_RLWIMI(d, a, i, mb, me) EMIT(PPC_INST_RLWIMI | ___PPC_RA(d) | \ | ||
___PPC_RS(a) | __PPC_SH(i) | \ | ||
__PPC_MB(mb) | __PPC_ME(me)) | ||
#define PPC_RLDICL(d, a, i, mb) EMIT(PPC_INST_RLDICL | ___PPC_RA(d) | \ | ||
___PPC_RS(a) | __PPC_SH(i) | \ | ||
__PPC_MB64(mb) | (((i) & 0x20) >> 4)) | ||
#define PPC_RLDICR(d, a, i, me) EMIT(PPC_INST_RLDICR | ___PPC_RA(d) | \ | ||
___PPC_RS(a) | __PPC_SH(i) | \ | ||
__PPC_ME64(me) | (((i) & 0x20) >> 4)) | ||
|
@@ -127,6 +176,8 @@ | |
#define PPC_SRWI(d, a, i) PPC_RLWINM(d, a, 32-(i), i, 31) | ||
/* sldi = rldicr Rx, Ry, n, 63-n */ | ||
#define PPC_SLDI(d, a, i) PPC_RLDICR(d, a, i, 63-(i)) | ||
/* sldi = rldicl Rx, Ry, 64-n, n */ | ||
#define PPC_SRDI(d, a, i) PPC_RLDICL(d, a, 64-(i), i) | ||
|
||
#define PPC_NEG(d, a) EMIT(PPC_INST_NEG | ___PPC_RT(d) | ___PPC_RA(a)) | ||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,102 @@ | ||
/* | ||
* bpf_jit64.h: BPF JIT compiler for PPC64 | ||
* | ||
* Copyright 2016 Naveen N. Rao <[email protected]> | ||
* IBM Corporation | ||
* | ||
* This program is free software; you can redistribute it and/or | ||
* modify it under the terms of the GNU General Public License | ||
* as published by the Free Software Foundation; version 2 | ||
* of the License. | ||
*/ | ||
#ifndef _BPF_JIT64_H | ||
#define _BPF_JIT64_H | ||
|
||
#include "bpf_jit.h" | ||
|
||
/* | ||
* Stack layout: | ||
* | ||
* [ prev sp ] <------------- | ||
* [ nv gpr save area ] 8*8 | | ||
* fp (r31) --> [ ebpf stack space ] 512 | | ||
* [ local/tmp var space ] 16 | | ||
* [ frame header ] 32/112 | | ||
* sp (r1) ---> [ stack pointer ] -------------- | ||
*/ | ||
|
||
/* for bpf JIT code internal usage */ | ||
#define BPF_PPC_STACK_LOCALS 16 | ||
/* for gpr non volatile registers BPG_REG_6 to 10, plus skb cache registers */ | ||
#define BPF_PPC_STACK_SAVE (8*8) | ||
/* Ensure this is quadword aligned */ | ||
#define BPF_PPC_STACKFRAME (STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_LOCALS + \ | ||
MAX_BPF_STACK + BPF_PPC_STACK_SAVE) | ||
|
||
#ifndef __ASSEMBLY__ | ||
|
||
/* BPF register usage */ | ||
#define SKB_HLEN_REG (MAX_BPF_REG + 0) | ||
#define SKB_DATA_REG (MAX_BPF_REG + 1) | ||
#define TMP_REG_1 (MAX_BPF_REG + 2) | ||
#define TMP_REG_2 (MAX_BPF_REG + 3) | ||
|
||
/* BPF to ppc register mappings */ | ||
static const int b2p[] = { | ||
/* function return value */ | ||
[BPF_REG_0] = 8, | ||
/* function arguments */ | ||
[BPF_REG_1] = 3, | ||
[BPF_REG_2] = 4, | ||
[BPF_REG_3] = 5, | ||
[BPF_REG_4] = 6, | ||
[BPF_REG_5] = 7, | ||
/* non volatile registers */ | ||
[BPF_REG_6] = 27, | ||
[BPF_REG_7] = 28, | ||
[BPF_REG_8] = 29, | ||
[BPF_REG_9] = 30, | ||
/* frame pointer aka BPF_REG_10 */ | ||
[BPF_REG_FP] = 31, | ||
/* eBPF jit internal registers */ | ||
[SKB_HLEN_REG] = 25, | ||
[SKB_DATA_REG] = 26, | ||
[TMP_REG_1] = 9, | ||
[TMP_REG_2] = 10 | ||
}; | ||
|
||
/* Assembly helpers */ | ||
#define DECLARE_LOAD_FUNC(func) u64 func(u64 r3, u64 r4); \ | ||
u64 func##_negative_offset(u64 r3, u64 r4); \ | ||
u64 func##_positive_offset(u64 r3, u64 r4); | ||
|
||
DECLARE_LOAD_FUNC(sk_load_word); | ||
DECLARE_LOAD_FUNC(sk_load_half); | ||
DECLARE_LOAD_FUNC(sk_load_byte); | ||
|
||
#define CHOOSE_LOAD_FUNC(imm, func) \ | ||
(imm < 0 ? \ | ||
(imm >= SKF_LL_OFF ? func##_negative_offset : func) : \ | ||
func##_positive_offset) | ||
|
||
#define SEEN_FUNC 0x1000 /* might call external helpers */ | ||
#define SEEN_STACK 0x2000 /* uses BPF stack */ | ||
#define SEEN_SKB 0x4000 /* uses sk_buff */ | ||
|
||
struct codegen_context { | ||
/* | ||
* This is used to track register usage as well | ||
* as calls to external helpers. | ||
* - register usage is tracked with corresponding | ||
* bits (r3-r10 and r25-r31) | ||
* - rest of the bits can be used to track other | ||
* things -- for now, we use bits 16 to 23 | ||
* encoded in SEEN_* macros above | ||
*/ | ||
unsigned int seen; | ||
unsigned int idx; | ||
}; | ||
|
||
#endif /* !__ASSEMBLY__ */ | ||
|
||
#endif |
Oops, something went wrong.