Skip to content

Commit

Permalink
Python 3 compatibility fixes around string handling (iovisor#986)
Browse files Browse the repository at this point in the history
This fixes the bcc module and all the affected tools for issues related to string handling in Python 3. Specifically, when passing Python strings to C libraries they are encoded as ASCII, and when constructing Python strings from C strings, they are decoded first.
  • Loading branch information
r4f4 authored and goldshtn committed Mar 26, 2017
1 parent 6bbf49c commit 78948e4
Show file tree
Hide file tree
Showing 34 changed files with 77 additions and 66 deletions.
7 changes: 4 additions & 3 deletions src/python/bcc/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -759,11 +759,12 @@ def get_user_addresses(name, sym_re):
def get_user_functions_and_addresses(name, sym_re):
addresses = []
def sym_cb(sym_name, addr):
if re.match(sym_re, sym_name):
addresses.append((sym_name, addr))
dname = sym_name.decode()
if re.match(sym_re, dname):
addresses.append((dname, addr))
return 0

res = lib.bcc_foreach_symbol(name, _SYM_CB_TYPE(sym_cb))
res = lib.bcc_foreach_symbol(name.encode('ascii'), _SYM_CB_TYPE(sym_cb))
if res < 0:
raise Exception("Error %d enumerating symbols in %s" % (res, name))
return addresses
Expand Down
8 changes: 5 additions & 3 deletions src/python/bcc/usdt.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,15 +138,17 @@ def __init__(self, pid=None, path=None):
"either a pid or a binary path must be specified")

def enable_probe(self, probe, fn_name):
if lib.bcc_usdt_enable_probe(self.context, probe, fn_name) != 0:
if lib.bcc_usdt_enable_probe(self.context, probe.encode('ascii'),
fn_name.encode('ascii')) != 0:
raise USDTException(
("failed to enable probe '%s'; a possible cause " +
"can be that the probe requires a pid to enable") %
probe
)

def enable_probe_or_bail(self, probe, fn_name):
if lib.bcc_usdt_enable_probe(self.context, probe, fn_name) != 0:
if lib.bcc_usdt_enable_probe(self.context, probe.encode('ascii'),
fn_name.encode('ascii')) != 0:
print(
"""Error attaching USDT probes: the specified pid might not contain the
given language's runtime, or the runtime was not built with the required
Expand Down Expand Up @@ -175,7 +177,7 @@ def _add_probe(probe):
def attach_uprobes(self, bpf):
probes = self.enumerate_active_probes()
for (binpath, fn_name, addr, pid) in probes:
bpf.attach_uprobe(name=binpath, fn_name=fn_name,
bpf.attach_uprobe(name=binpath.decode(), fn_name=fn_name.decode(),
addr=addr, pid=pid)

def enumerate_active_probes(self):
Expand Down
2 changes: 1 addition & 1 deletion tools/bashreadline.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ class Data(ct.Structure):

def print_event(cpu, data, size):
event = ct.cast(data, ct.POINTER(Data)).contents
print("%-9s %-6d %s" % (strftime("%H:%M:%S"), event.pid, event.str))
print("%-9s %-6d %s" % (strftime("%H:%M:%S"), event.pid, event.str.decode()))

b["events"].open_perf_buffer(print_event)
while 1:
Expand Down
3 changes: 2 additions & 1 deletion tools/biosnoop.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,8 @@ def print_event(cpu, data, size):
delta = float(delta) + (event.ts - prev_ts)

print("%-14.9f %-14.14s %-6s %-7s %-2s %-9s %-7s %7.2f" % (
delta / 1000000, event.name, event.pid, event.disk_name, rwflg, val,
delta / 1000000, event.name.decode(), event.pid,
event.disk_name.decode(), rwflg, val,
event.len, float(event.delta) / 1000000))

prev_ts = event.ts
Expand Down
6 changes: 3 additions & 3 deletions tools/biotop.py
Original file line number Diff line number Diff line change
Expand Up @@ -212,9 +212,9 @@ def signal_ignore(signal, frame):

# print line
avg_ms = (float(v.us) / 1000) / v.io
print("%-6d %-16s %1s %-3d %-3d %-8s %5s %7s %6.2f" % (k.pid, k.name,
"W" if k.rwflag else "R", k.major, k.minor, diskname, v.io,
v.bytes / 1024, avg_ms))
print("%-6d %-16s %1s %-3d %-3d %-8s %5s %7s %6.2f" % (k.pid,
k.name.decode(), "W" if k.rwflag else "R", k.major, k.minor,
diskname, v.io, v.bytes / 1024, avg_ms))

line += 1
if line >= maxrows:
Expand Down
3 changes: 2 additions & 1 deletion tools/bitesize.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,4 +72,5 @@
try:
sleep(99999999)
except KeyboardInterrupt:
dist.print_log2_hist("Kbytes", "Process Name")
dist.print_log2_hist("Kbytes", "Process Name",
section_print_fn=bytes.decode)
8 changes: 4 additions & 4 deletions tools/btrfsslower.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,12 +311,12 @@ def print_event(cpu, data, size):

if (csv):
print("%d,%s,%d,%s,%d,%d,%d,%s" % (
event.ts_us, event.task, event.pid, type, event.size,
event.offset, event.delta_us, event.file))
event.ts_us, event.task.decode(), event.pid, type, event.size,
event.offset, event.delta_us, event.file.decode()))
return
print("%-8s %-14.14s %-6s %1s %-7s %-8d %7.2f %s" % (strftime("%H:%M:%S"),
event.task, event.pid, type, event.size, event.offset / 1024,
float(event.delta_us) / 1000, event.file))
event.task.decode(), event.pid, type, event.size, event.offset / 1024,
float(event.delta_us) / 1000, event.file.decode()))

# initialize BPF
b = BPF(text=bpf_text)
Expand Down
2 changes: 1 addition & 1 deletion tools/cachetop.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def get_processes_stats(
counts = bpf.get_table("counts")
stats = defaultdict(lambda: defaultdict(int))
for k, v in counts.items():
stats["%d-%d-%s" % (k.pid, k.uid, k.comm)][k.ip] = v.value
stats["%d-%d-%s" % (k.pid, k.uid, k.comm.decode())][k.ip] = v.value
stats_list = []

for pid, count in sorted(stats.items(), key=lambda stat: stat[0]):
Expand Down
3 changes: 2 additions & 1 deletion tools/capable.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,8 @@ def print_event(cpu, data, size):
else:
name = "?"
print("%-9s %-6d %-6d %-16s %-4d %-20s %d" % (strftime("%H:%M:%S"),
event.uid, event.pid, event.comm, event.cap, name, event.audit))
event.uid, event.pid, event.comm.decode(), event.cap, name,
event.audit))

# loop with callback to print_event
b["events"].open_perf_buffer(print_event)
Expand Down
4 changes: 2 additions & 2 deletions tools/dcsnoop.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,8 @@ class Data(ct.Structure):
def print_event(cpu, data, size):
event = ct.cast(data, ct.POINTER(Data)).contents
print("%-11.6f %-6d %-16s %1s %s" % (
time.time() - start_ts, event.pid, event.comm, mode_s[event.type],
event.filename))
time.time() - start_ts, event.pid, event.comm.decode(),
mode_s[event.type], event.filename.decode()))

# header
print("%-11s %-6s %-16s %1s %s" % ("TIME(s)", "PID", "COMM", "T", "FILE"))
Expand Down
2 changes: 1 addition & 1 deletion tools/ext4dist.py
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@
if args.interval and (not args.notimestamp):
print(strftime("%H:%M:%S:"))

dist.print_log2_hist(label, "operation")
dist.print_log2_hist(label, "operation", section_print_fn=bytes.decode)
dist.clear()

countdown -= 1
Expand Down
8 changes: 4 additions & 4 deletions tools/ext4slower.py
Original file line number Diff line number Diff line change
Expand Up @@ -305,12 +305,12 @@ def print_event(cpu, data, size):

if (csv):
print("%d,%s,%d,%s,%d,%d,%d,%s" % (
event.ts_us, event.task, event.pid, type, event.size,
event.offset, event.delta_us, event.file))
event.ts_us, event.task.decode(), event.pid, type, event.size,
event.offset, event.delta_us, event.file.decode()))
return
print("%-8s %-14.14s %-6s %1s %-7s %-8d %7.2f %s" % (strftime("%H:%M:%S"),
event.task, event.pid, type, event.size, event.offset / 1024,
float(event.delta_us) / 1000, event.file))
event.task.decode(), event.pid, type, event.size, event.offset / 1024,
float(event.delta_us) / 1000, event.file.decode()))

# initialize BPF
b = BPF(text=bpf_text)
Expand Down
2 changes: 1 addition & 1 deletion tools/filelife.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ class Data(ct.Structure):
def print_event(cpu, data, size):
event = ct.cast(data, ct.POINTER(Data)).contents
print("%-8s %-6d %-16s %-7.2f %s" % (strftime("%H:%M:%S"), event.pid,
event.comm, float(event.delta) / 1000, event.fname))
event.comm.decode(), float(event.delta) / 1000, event.fname.decode()))

b["events"].open_perf_buffer(print_event)
while 1:
Expand Down
6 changes: 3 additions & 3 deletions tools/fileslower.py
Original file line number Diff line number Diff line change
Expand Up @@ -235,13 +235,13 @@ def print_event(cpu, data, size):
event = ct.cast(data, ct.POINTER(Data)).contents

ms = float(event.delta_us) / 1000
name = event.name
name = event.name.decode()
if event.name_len > DNAME_INLINE_LEN:
name = name[:-3] + "..."

print("%-8.3f %-14.14s %-6s %1s %-7s %7.2f %s" % (
time.time() - start_ts, event.comm, event.pid, mode_s[event.mode],
event.sz, ms, name))
time.time() - start_ts, event.comm.decode(), event.pid,
mode_s[event.mode], event.sz, ms, name))

b["events"].open_perf_buffer(print_event, page_cnt=64)
while 1:
Expand Down
2 changes: 1 addition & 1 deletion tools/gethostlatency.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ class Data(ct.Structure):
def print_event(cpu, data, size):
event = ct.cast(data, ct.POINTER(Data)).contents
print("%-9s %-6d %-16s %10.2f %s" % (strftime("%H:%M:%S"), event.pid,
event.comm, (event.delta / 1000000), event.host))
event.comm.decode(), (event.delta / 1000000), event.host.decode()))

# loop with callback to print_event
b["events"].open_perf_buffer(print_event)
Expand Down
2 changes: 1 addition & 1 deletion tools/hardirqs.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@
else:
print("%-26s %11s" % ("HARDIRQ", "TOTAL_" + label))
for k, v in sorted(dist.items(), key=lambda dist: dist[1].value):
print("%-26s %11d" % (k.name, v.value / factor))
print("%-26s %11d" % (k.name.decode(), v.value / factor))
dist.clear()

countdown -= 1
Expand Down
2 changes: 1 addition & 1 deletion tools/killsnoop.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ def print_event(cpu, data, size):
return

print("%-9s %-6d %-16s %-4d %-6d %d" % (strftime("%H:%M:%S"),
event.pid, event.comm, event.sig, event.tpid, event.ret))
event.pid, event.comm.decode(), event.sig, event.tpid, event.ret))

# loop with callback to print_event
b["events"].open_perf_buffer(print_event)
Expand Down
4 changes: 2 additions & 2 deletions tools/mdflush.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ class Data(ct.Structure):
# process event
def print_event(cpu, data, size):
event = ct.cast(data, ct.POINTER(Data)).contents
print("%-8s %-6d %-16s %s" % (strftime("%H:%M:%S"), event.pid, event.comm,
event.disk))
print("%-8s %-6d %-16s %s" % (strftime("%H:%M:%S"), event.pid,
event.comm.decode(), event.disk.decode()))

# read events
b["events"].open_perf_buffer(print_event)
Expand Down
2 changes: 1 addition & 1 deletion tools/offcputime.py
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ def signal_ignore(signal, frame):
print(" --")
for addr in user_stack:
print(" %s" % b.sym(addr, k.tgid))
print(" %-16s %s (%d)" % ("-", k.name, k.pid))
print(" %-16s %s (%d)" % ("-", k.name.decode(), k.pid))
print(" %d\n" % v.value)

if missing_stacks > 0:
Expand Down
4 changes: 2 additions & 2 deletions tools/offwaketime.py
Original file line number Diff line number Diff line change
Expand Up @@ -306,7 +306,7 @@ def signal_ignore(signal, frame):

else:
# print wakeup name then stack in reverse order
print(" %-16s %s %s" % ("waker:", k.waker, k.t_pid))
print(" %-16s %s %s" % ("waker:", k.waker.decode(), k.t_pid))
for addr in waker_user_stack:
print(" %s" % b.sym(addr, k.tgid))
if args.delimited:
Expand All @@ -324,7 +324,7 @@ def signal_ignore(signal, frame):
print(" -")
for addr in target_user_stack:
print(" %s" % b.sym(addr, k.tgid))
print(" %-16s %s %s" % ("target:", k.target, k.w_pid))
print(" %-16s %s %s" % ("target:", k.target.decode(), k.w_pid))
print(" %d\n" % v.value)

if missing_stacks > 0:
Expand Down
3 changes: 2 additions & 1 deletion tools/oomkill.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ def print_event(cpu, data, size):
avgline = stats.read().rstrip()
print(("%s Triggered by PID %d (\"%s\"), OOM kill of PID %d (\"%s\")"
", %d pages, loadavg: %s") % (strftime("%H:%M:%S"), event.fpid,
event.fcomm, event.tpid, event.tcomm, event.pages, avgline))
event.fcomm.decode(), event.tpid, event.tcomm.decode(), event.pages,
avgline))

# initialize BPF
b = BPF(text=bpf_text)
Expand Down
2 changes: 1 addition & 1 deletion tools/opensnoop.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ def print_event(cpu, data, size):

print("%-6d %-16s %4d %3d %s" %
(event.id & 0xffffffff if args.tid else event.id >> 32,
event.comm, fd_s, err, event.fname))
event.comm.decode(), fd_s, err, event.fname.decode()))

# loop with callback to print_event
b["events"].open_perf_buffer(print_event, page_cnt=64)
Expand Down
2 changes: 1 addition & 1 deletion tools/profile.py
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ def aksym(addr):
print(" --")
for addr in user_stack:
print(" %s" % b.sym(addr, k.pid))
print(" %-16s %s (%d)" % ("-", k.name, k.pid))
print(" %-16s %s (%d)" % ("-", k.name.decode(), k.pid))
print(" %d\n" % v.value)

# check missing
Expand Down
2 changes: 1 addition & 1 deletion tools/slabratetop.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ def signal_ignore(signal, frame):
line = 0
for k, v in reversed(sorted(counts.items(),
key=lambda counts: counts[1].size)):
print("%-32s %6d %10d" % (k.name, v.count, v.size))
print("%-32s %6d %10d" % (k.name.decode(), v.count, v.size))

line += 1
if line >= maxrows:
Expand Down
5 changes: 3 additions & 2 deletions tools/sslsniff.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,10 +200,11 @@ def print_event(cpu, data, size, rw):
" bytes lost) " + "-" * 5

print("%-12s %-18.9f %-16s %-6d %-6d\n%s\n%s\n%s\n\n" % (rw, time_s,
event.comm,
event.comm.decode(),
event.pid,
event.len,
s_mark, event.v0,
s_mark,
event.v0.decode(),
e_mark))

b["perf_SSL_write"].open_perf_buffer(print_event_write)
Expand Down
2 changes: 1 addition & 1 deletion tools/stacksnoop.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ def print_event(cpu, data, size):

if verbose:
print("%-18.9f %-12.12s %-6d %-3d %s" %
(ts, event.comm, event.pid, cpu, function))
(ts, event.comm.decode(), event.pid, cpu, function))
else:
print("%-18.9f %s" % (ts, function))

Expand Down
4 changes: 2 additions & 2 deletions tools/statsnoop.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,8 +155,8 @@ def print_event(cpu, data, size):
if args.timestamp:
print("%-14.9f" % (float(event.ts_ns - start_ts) / 1000000000), end="")

print("%-6d %-16s %4d %3d %s" % (event.pid, event.comm,
fd_s, err, event.fname))
print("%-6d %-16s %4d %3d %s" % (event.pid, event.comm.decode(),
fd_s, err, event.fname.decode()))

# loop with callback to print_event
b["events"].open_perf_buffer(print_event, page_cnt=64)
Expand Down
9 changes: 5 additions & 4 deletions tools/tcpaccept.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,8 +159,9 @@ def print_ipv4_event(cpu, data, size):
if start_ts == 0:
start_ts = event.ts_us
print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid, event.task,
event.ip, inet_ntop(AF_INET, pack("I", event.daddr)),
print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid,
event.task.decode(), event.ip,
inet_ntop(AF_INET, pack("I", event.daddr)),
inet_ntop(AF_INET, pack("I", event.saddr)), event.lport))

def print_ipv6_event(cpu, data, size):
Expand All @@ -170,8 +171,8 @@ def print_ipv6_event(cpu, data, size):
if start_ts == 0:
start_ts = event.ts_us
print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid, event.task,
event.ip, inet_ntop(AF_INET6, event.daddr),
print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid,
event.task.decode(), event.ip, inet_ntop(AF_INET6, event.daddr),
inet_ntop(AF_INET6, event.saddr), event.lport))

# initialize BPF
Expand Down
7 changes: 4 additions & 3 deletions tools/tcpconnect.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,8 +201,9 @@ def print_ipv4_event(cpu, data, size):
if start_ts == 0:
start_ts = event.ts_us
print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid, event.task,
event.ip, inet_ntop(AF_INET, pack("I", event.saddr)),
print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid,
event.task.decode(), event.ip,
inet_ntop(AF_INET, pack("I", event.saddr)),
inet_ntop(AF_INET, pack("I", event.daddr)), event.dport))

def print_ipv6_event(cpu, data, size):
Expand All @@ -213,7 +214,7 @@ def print_ipv6_event(cpu, data, size):
start_ts = event.ts_us
print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
print("%-6d %-12.12s %-2d %-16s %-16s %-4d" % (event.pid,
event.task, event.ip, inet_ntop(AF_INET6, event.saddr),
event.task.decode(), event.ip, inet_ntop(AF_INET6, event.saddr),
inet_ntop(AF_INET6, event.daddr), event.dport))

# initialize BPF
Expand Down
9 changes: 5 additions & 4 deletions tools/tcpconnlat.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,8 +199,9 @@ def print_ipv4_event(cpu, data, size):
if start_ts == 0:
start_ts = event.ts_us
print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
print("%-6d %-12.12s %-2d %-16s %-16s %-5d %.2f" % (event.pid, event.task,
event.ip, inet_ntop(AF_INET, pack("I", event.saddr)),
print("%-6d %-12.12s %-2d %-16s %-16s %-5d %.2f" % (event.pid,
event.task.decode(), event.ip,
inet_ntop(AF_INET, pack("I", event.saddr)),
inet_ntop(AF_INET, pack("I", event.daddr)), event.dport,
float(event.delta_us) / 1000))

Expand All @@ -211,8 +212,8 @@ def print_ipv6_event(cpu, data, size):
if start_ts == 0:
start_ts = event.ts_us
print("%-9.3f" % ((float(event.ts_us) - start_ts) / 1000000), end="")
print("%-6d %-12.12s %-2d %-16s %-16s %-5d %.2f" % (event.pid, event.task,
event.ip, inet_ntop(AF_INET6, event.saddr),
print("%-6d %-12.12s %-2d %-16s %-16s %-5d %.2f" % (event.pid,
event.task.decode(), event.ip, inet_ntop(AF_INET6, event.saddr),
inet_ntop(AF_INET6, event.daddr), event.dport,
float(event.delta_us) / 1000))

Expand Down
Loading

0 comments on commit 78948e4

Please sign in to comment.