break data up into smaller chunks

binder_transaction.byte.c

@@ -6,91 +6,125 @@
 
 char __license[] SEC("license") = "Dual MIT/GPL";
 
-#define MAX_DATA_LENGTH 0x2000
+#define CHUNK_SIZE 0x400
+#define MAX_CHUNKS 16
 
 struct trace_event {
-	u32 pid;
-	u32 uid;
-	u32 code;
-	u32 flags;
-	u64 data_size;
-	u8 data[MAX_DATA_LENGTH];
+    u32 pid;
+    u32 uid;
+    u32 code;
+    u32 flags;
+    u64 data_size;
+    u64 transaction_id;
+    u64 chunk_index;
+    u8 chunk_data[CHUNK_SIZE];
 };
 
 struct {
-	__uint(type, BPF_MAP_TYPE_RINGBUF);
-	__uint(max_entries, 1 << 24);
+    __uint(type, BPF_MAP_TYPE_RINGBUF);
+    __uint(max_entries, 1 << 24);
 } trace_event_map SEC(".maps");
 
 struct trace_config {
     u32 uid;
 };
 
 struct {
-	__uint(type, BPF_MAP_TYPE_HASH);
-	__uint(key_size, sizeof(u32));
-	__uint(value_size, sizeof(struct trace_config));
-	__uint(max_entries, 1);
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(key_size, sizeof(u32));
+    __uint(value_size, sizeof(struct trace_config));
+    __uint(max_entries, 1);
 } trace_config_map SEC(".maps");
 
+static __always_inline u64 get_transaction_id() {
+    u64 id = bpf_ktime_get_ns();
+    return id;
+}
+
 // Force emitting struct event into the ELF.
 const struct trace_event *unused_trace_event __attribute__((unused));
 const struct trace_config *unused_trace_config __attribute__((unused));
 
 SEC("kprobe/binder_transaction")
 int kprobe_binder_transaction(struct pt_regs *ctx) {
 
-	u32 config_key = 0;
+    u32 config_key = 0;
     struct trace_config* conf = bpf_map_lookup_elem(&trace_config_map, &config_key);
-    if (conf == NULL){
-		return 0;
-	} 
-
-	int reply = PT_REGS_PARM4(ctx);
-	if (reply){
-		return 0;
-	}
-
-	struct binder_transaction_data *tr = (struct binder_transaction_data *)PT_REGS_PARM3(ctx);
-	if (!tr){
-		bpf_printk("binder_transaction error: tr is null");
-		return 0;
-	}
-
-	u32 current_uid = bpf_get_current_uid_gid() >> 32;
-	if ((conf->uid != 0) && (conf->uid != current_uid)){
-		return 0;
-	}
-
-	struct trace_event *binder_transaction_event;
-	binder_transaction_event = bpf_ringbuf_reserve(&trace_event_map, sizeof(struct trace_event), 0);
-	if (!binder_transaction_event) {
-		bpf_printk("binder_transaction error: event is null");
+    if (conf == NULL) {
+        return 0;
+    }
+
+    int reply = PT_REGS_PARM4(ctx);
+    if (reply) {
+        return 0;
+    }
+
+    struct binder_transaction_data *tr = (struct binder_transaction_data *)PT_REGS_PARM3(ctx);
+    if (!tr) {
+        bpf_printk("kprobe_binder_transaction error: tr is null");
+        return 0;
+    }
+
+    u32 current_uid = bpf_get_current_uid_gid() >> 32;
+    if ((conf->uid != 0) && (conf->uid != current_uid)) {
+        return 0;
+    }
+
+	u32 current_pid = bpf_get_current_pid_tgid() >> 32;
+    u64 transaction_id = get_transaction_id();
+
+    u32 code;
+    u32 flags;
+    u64 data_size;
+    bpf_probe_read(&code, sizeof(__u32), &(tr->code));
+    bpf_probe_read(&flags, sizeof(__u32), &(tr->flags));
+    bpf_probe_read(&data_size, sizeof(binder_size_t), &(tr->data_size));
+
+    union {
+        struct {
+            binder_uintptr_t buffer;
+            binder_uintptr_t offsets;
+        } ptr;
+        __u8 buf[8];
+    } data;
+    bpf_probe_read(&data, sizeof(data), &(tr->data));
+
+    u32 total_chunks = (data_size + CHUNK_SIZE - 1) / CHUNK_SIZE;
+
+    if (total_chunks > MAX_CHUNKS) {
+        bpf_printk("kprobe_binder_transaction error: data size is too long：%d",data_size);
 		return 0;
-	}
-
-	binder_transaction_event->pid = bpf_get_current_pid_tgid() >> 32;
-	binder_transaction_event->uid = current_uid;
-	bpf_probe_read(&(binder_transaction_event->code), sizeof(__u32), &(tr->code));
-	bpf_probe_read(&(binder_transaction_event->flags), sizeof(__u32), &(tr->flags));
-	bpf_probe_read(&(binder_transaction_event->data_size), sizeof(binder_size_t), &(tr->data_size));
-
-	union {
-		struct {
-			binder_uintptr_t buffer;
-			binder_uintptr_t offsets;
-		} ptr;
-		__u8 buf[8];
-	} data;
-	bpf_probe_read(&data, sizeof(data), &(tr->data));
-	unsigned data_size = binder_transaction_event->data_size;
-	if (data_size > MAX_DATA_LENGTH){
-		bpf_printk("binder_transaction error: data_size=%d,max_data_size=%d", data_size, MAX_DATA_LENGTH);
-	}
-	unsigned probe_read_size = data_size < sizeof(binder_transaction_event->data) ? data_size : sizeof(binder_transaction_event->data);
-	bpf_probe_read_user(&(binder_transaction_event->data), probe_read_size, (void *)data.ptr.buffer);
-
-	bpf_ringbuf_submit(binder_transaction_event, 0);
-
-	return 0;
+    }
+
+    for (u32 i = 0; i < MAX_CHUNKS; i++) {
+
+		if (i >= total_chunks) {
+			return 0;
+    	}
+
+        struct trace_event *binder_transaction_event = bpf_ringbuf_reserve(&trace_event_map, sizeof(struct trace_event), 0);
+
+        if (!binder_transaction_event) {
+            bpf_printk("kprobe_binder_transaction error: failed to reserve ring buffer space");
+            return 0;
+        }
+
+        binder_transaction_event->pid = current_pid;
+        binder_transaction_event->uid = current_uid;
+        binder_transaction_event->code = code;
+        binder_transaction_event->flags = flags;
+        binder_transaction_event->data_size = data_size;
+        binder_transaction_event->transaction_id = transaction_id;
+        binder_transaction_event->chunk_index = i;
+
+        u64 chunk_size = ((i + 1) * CHUNK_SIZE > data_size) ? (data_size - i * CHUNK_SIZE) : CHUNK_SIZE;
+		unsigned probe_read_size = chunk_size < sizeof(binder_transaction_event->chunk_data) ? chunk_size : sizeof(binder_transaction_event->chunk_data);
+        bpf_probe_read_user(binder_transaction_event->chunk_data, probe_read_size, (void *)(data.ptr.buffer + i * CHUNK_SIZE));
+
+		//bpf_printk("kprobe_binder_transaction: transaction_id=%lx,data_size=%d,probe_read_size=%d",transaction_id,data_size,probe_read_size);
+
+        bpf_ringbuf_submit(binder_transaction_event, 0);
+    }
+
+    return 0;
 }

doc/BPF Ring Buffer Chunking.md

@@ -0,0 +1,15 @@
+# BPF Ring Buffer Chunking
+
+在我们的场景里，BPF程序需要将hook到的数据发送到用户空间，BPF perf buffer是一个选择，但它存在一些问题，例如浪费内存、事件顺序无法保证等，所以我们倾向于使用内核5.8引入的另一个新的BPF 数据结构：BPF ring buffer（环形缓冲区），它提供了一套新的reserve/commit API（先预留再提交），内存效率更高，且能保证事件顺序。
+
+可是BPF ring buffer的reserve/commit API有一个限制：BPF 校验器在校验时必须知道预留数据的大小，因此不支持动态大小的事件数据。所以在btrace的v1.0.0版本里我们使用了固定大小的事件结构：
+
+![1717997795515](image/BPFRingBufferChunking/1717997795515.png)
+
+但这样既浪费空间（很多事件里的data远小于0x2000），又不能完整传递数据（有的事件里的data大于0x2000）。
+
+在btrace的v1.1.0版本里，我们进行了改进，在ebpf程序里对大块的data进行分片处理，每个分片带上一个事件唯一标识transaction_id，然后通过BPF ring buffer传送到用户空间，用户空间根据transaction_id来汇聚和重组分片。
+
+但是BPF verifier为了保证 BPF 程序的所有操作必须在有限时间（bounded time）内完成，所以不支持动态次数的循环（5.17内核已经支持 `bpf_loop()`特性，但是我手上的手机和PC都是5.15的内核），所以我们只能在程序里手动限定循环次数（MAX_CHUNKS）：
+
+![1717997982951](image/BPFRingBufferChunking/1717997982951.png)

main.go

@@ -18,7 +18,8 @@ import (
 	"github.com/davecgh/go-spew/spew"
 )
 
-const MAX_DATA_LENGTH = 0x2000
+const ChunkSize = 0x400
+const MaxChunks = 16
 
 type Config struct {
 	PackageName string
@@ -45,6 +46,16 @@ func parseFlags() Config {
 	return config
 }
 
+func mergeChunks(chunks [][]byte) []byte {
+	mergedData := make([]byte, 0, len(chunks)*ChunkSize)
+
+    for _, chunk := range chunks {
+        mergedData = append(mergedData, chunk...)
+    }
+
+    return mergedData
+}
+
 func main() {
 
 	conf := parseFlags()
@@ -132,6 +143,7 @@ func main() {
 
 	// bpfEvent is generated by bpf2go.
 	var event bpfTraceEvent
+	var transactionBuffers = make(map[uint64][][]byte)
 	for {
 		record, err := rd.Read()
 		if err != nil {
@@ -149,39 +161,55 @@ func main() {
 			continue
 		}
 
-		if event.DataSize <= 16 {
-			continue
+		if _, exists := transactionBuffers[event.TransactionId]; !exists {
+			var totalChunks = (event.DataSize + ChunkSize - 1) / ChunkSize
+			transactionBuffers[event.TransactionId] = make([][]byte, totalChunks)
 		}
-		if event.DataSize > MAX_DATA_LENGTH {
-			continue
+		transactionBuffers[event.TransactionId][event.ChunkIndex] = append([]byte(nil), event.ChunkData[:]...)
+
+		var complete = true
+		for _, chunk := range transactionBuffers[event.TransactionId] {
+			if chunk == nil {
+				complete = false
+				break
+			}
 		}
-		parcelData := event.Data[:event.DataSize-1]
+
+		if complete {
+			completeData := mergeChunks(transactionBuffers[event.TransactionId])
+			delete(transactionBuffers, event.TransactionId)
 
-		interfaceToken, err := ExtractInterfaceName(parcelData)
+			if len(completeData) <= 16 {
+				continue
+			}
+			if len(completeData) > ChunkSize*MaxChunks {
+				continue
+			}
 
-		if err != nil {
-			logger.Println("Error parsing parcel:", err)
-			continue
-		}
-
-		//fmt.Println("Interface Token:", interfaceToken)
+			parcelData := completeData[:event.DataSize-1]
 
-		methodName, err := GetMethodName(interfaceToken, int(event.Code))
-		if err != nil {
-			//fmt.Println("Error:", err)
-			methodName = fmt.Sprintf("%d", event.Code)
-		}
+			interfaceToken, err := ExtractInterfaceName(parcelData)
 
-		packageName, err := GetPackageNameByUid(int(event.Uid))
-		if err != nil {
-			//fmt.Println("Error:", err)
-			packageName = fmt.Sprintf("%d", event.Uid)
-		}
-
-		if conf.Args {
-			logger.Printf("(pid:%d, uid:%d, package:%s) -> (interface:%s, method:%s)\n%s", event.Pid, event.Uid, packageName, interfaceToken, methodName, spew.Sdump(parcelData))
-		} else {
-			logger.Printf("(pid:%d, uid:%d, package:%s) -> (interface:%s, method:%s)\n", event.Pid, event.Uid, packageName, interfaceToken, methodName)
+			if err != nil {
+				logger.Println("Error parsing parcel:", err)
+				continue
+			}
+
+			methodName, err := GetMethodName(interfaceToken, int(event.Code))
+			if err != nil {
+				methodName = fmt.Sprintf("%d", event.Code)
+			}
+
+			packageName, err := GetPackageNameByUid(int(event.Uid))
+			if err != nil {
+				packageName = fmt.Sprintf("%d", event.Uid)
+			}
+
+			if conf.Args {
+				logger.Printf("(pid:%d, uid:%d, package:%s) -> (interface:%s, method:%s)\n%s", event.Pid, event.Uid, packageName, interfaceToken, methodName, spew.Sdump(parcelData))
+			} else {
+				logger.Printf("(pid:%d, uid:%d, package:%s) -> (interface:%s, method:%s)\n", event.Pid, event.Uid, packageName, interfaceToken, methodName)
+			}
 		}
 	}
 }