runtime: faster version of findfunc

Use a lookup table to find the function which contains a pc.  It is
faster than the old binary search.  findfunc is used primarily for
stack copying and garbage collection.

benchmark              old ns/op     new ns/op     delta
BenchmarkStackCopy     294746596     255400980     -13.35%

(findfunc is one of several tasks done by stack copy, the findfunc
time itself is about 2.5x faster.)

The lookup table is built at link time.  The table grows the binary
size by about 0.5% of the text segment.

We impose a lower limit of 16 bytes on any function, which should not
have much of an impact.  (The real constraint required is <=256
functions in every 4096 bytes, but 16 bytes/function is easier to
implement.)

Change-Id: Ic315b7a2c83e1f7203cd2a50e5d21a822e18fdca
Reviewed-on: https://go-review.googlesource.com/2097
Reviewed-by: Russ Cox <[email protected]>

src/cmd/ld/data.c

@@ -1315,7 +1315,10 @@ textaddress(void)
 			sub->value += va;
 		if(sym->size == 0 && sym->sub != S)
 			ctxt->cursym = sym;
-		va += sym->size;
+		if(sym->size < MINFUNC)
+			va += MINFUNC; // spacing required for findfunctab
+		else
+			va += sym->size;
 	}
 	sect->len = va - sect->vaddr;
 }

src/cmd/ld/lib.h

@@ -35,6 +35,7 @@
 
 enum {
 	MAXIO		= 8192,
+	MINFUNC		= 16,	// minimum size for a function
 };
 
 typedef struct Segment Segment;
@@ -260,6 +261,7 @@ void	patch(void);
 int	pathchar(void);
 void	pcln(void);
 void	pclntab(void);
+void	findfunctab(void);
 void	putelfsectionsym(LSym* s, int shndx);
 void	putelfsymshndx(vlong sympos, int shndx);
 void	putsymb(LSym *s, char *name, int t, vlong v, vlong size, int ver, LSym *typ);

src/cmd/ld/pcln.c

@@ -242,3 +242,58 @@ pclntab(void)
 	if(debug['v'])
 		Bprint(&bso, "%5.2f pclntab=%lld bytes, funcdata total %lld bytes\n", cputime(), (vlong)ftab->size, (vlong)funcdata_bytes);
 }	
+
+enum {
+	BUCKETSIZE = 256*MINFUNC,
+	SUBBUCKETS = 16,
+};
+
+// findfunctab generates a lookup table to quickly find the containing
+// function for a pc.  See src/runtime/symtab.go:findfunc for details.
+void
+findfunctab(void)
+{
+	LSym *t, *s;
+	int32 idx, bidx, i, j, nbuckets;
+	vlong min, max;
+
+	t = linklookup(ctxt, "runtime.findfunctab", 0);
+	t->type = SRODATA;
+	t->reachable = 1;
+
+	// find min and max address
+	min = ctxt->textp->value;
+	max = 0;
+	for(s = ctxt->textp; s != nil; s = s->next)
+		max = s->value + s->size;
+
+	// allocate table
+	nbuckets = (max-min+BUCKETSIZE-1)/BUCKETSIZE;
+	symgrow(ctxt, t, nbuckets * (4+SUBBUCKETS));
+
+	// fill in table
+	s = ctxt->textp;
+	idx = 0;
+	for(i = 0; i < nbuckets; i++) {
+		// Find first function which overlaps this bucket.
+		// Only do leaf symbols; skip symbols which are just containers (sub != nil but outer == nil).
+		while(s != nil && (s->value+s->size <= min + i * BUCKETSIZE || s->sub != nil && s->outer == nil)) {
+			s = s->next;
+			idx++;
+		}
+		// record this function in bucket header
+		setuint32(ctxt, t, i*(4+SUBBUCKETS), idx);
+		bidx = idx;
+
+		// compute SUBBUCKETS deltas
+		for(j = 0; j < SUBBUCKETS; j++) {
+			while(s != nil && (s->value+s->size <= min + i * BUCKETSIZE + j * (BUCKETSIZE/SUBBUCKETS) || s->sub != nil && s->outer == nil)) {
+				s = s->next;
+				idx++;
+			}
+			if(idx - bidx >= 256)
+				diag("too many functions in a findfunc bucket! %d %s", idx-bidx, s->name);
+			setuint8(ctxt, t, i*(4+SUBBUCKETS)+4+j, idx-bidx);
+		}
+	}
+}

src/cmd/ld/pobj.c

@@ -187,6 +187,7 @@ main(int argc, char *argv[])
 	gentext();		// trampolines, call stubs, etc.
 	textaddress();
 	pclntab();
+	findfunctab();
 	symtab();
 	dodata();
 	address();

src/runtime/symtab.go

@@ -34,14 +34,32 @@ var (
 	ftab      []functab
 	filetab   []uint32
 
-	pclntab, epclntab struct{} // linker symbols
+	pclntab, epclntab, findfunctab struct{} // linker symbols
+
+	minpc, maxpc uintptr
 )
 
 type functab struct {
 	entry   uintptr
 	funcoff uintptr
 }
 
+const minfunc = 16 // minimum function size
+const pcbucketsize = 256*minfunc // size of bucket in the pc->func lookup table
+
+// findfunctab is an array of these structures.
+// Each bucket represents 4096 bytes of the text segment.
+// Each subbucket represents 256 bytes of the text segment.
+// To find a function given a pc, locate the bucket and subbucket for
+// that pc.  Add together the idx and subbucket value to obtain a
+// function index.  Then scan the functab array starting at that
+// index to find the target function.
+// This table uses 20 bytes for every 4096 bytes of code, or ~0.5% overhead.
+type findfuncbucket struct {
+	idx uint32
+	subbuckets [16]byte
+}
+
 func symtabinit() {
 	// See golang.org/s/go12symtab for header: 0xfffffffb,
 	// two zero bytes, a byte giving the PC quantum,
@@ -96,6 +114,9 @@ func symtabinit() {
 	sp.cap = 1
 	sp.len = int(filetab[0])
 	sp.cap = sp.len
+
+	minpc = ftab[0].entry
+	maxpc = ftab[nftab].entry
 }
 
 // FuncForPC returns a *Func describing the function that contains the
@@ -126,32 +147,26 @@ func (f *Func) FileLine(pc uintptr) (file string, line int) {
 }
 
 func findfunc(pc uintptr) *_func {
-	if len(ftab) == 0 {
+	if pc < minpc || pc >= maxpc {
 		return nil
 	}
+	const nsub = uintptr(len(findfuncbucket{}.subbuckets))
 
-	if pc < ftab[0].entry || pc >= ftab[len(ftab)-1].entry {
-		return nil
-	}
+	x := pc - minpc
+	b := x / pcbucketsize
+	i := x % pcbucketsize / (pcbucketsize/nsub)
 
-	// binary search to find func with entry <= pc.
-	lo := 0
-	nf := len(ftab) - 1 // last entry is sentinel
-	for nf > 0 {
-		n := nf / 2
-		f := &ftab[lo+n]
-		if f.entry <= pc && pc < ftab[lo+n+1].entry {
-			return (*_func)(unsafe.Pointer(&pclntable[f.funcoff]))
-		} else if pc < f.entry {
-			nf = n
-		} else {
-			lo += n + 1
-			nf -= n + 1
-		}
+	ffb := (*findfuncbucket)(add(unsafe.Pointer(&findfunctab), b * unsafe.Sizeof(findfuncbucket{})))
+	idx := ffb.idx + uint32(ffb.subbuckets[i])
+	if pc < ftab[idx].entry {
+		throw("findfunc: bad findfunctab entry")
 	}
 
-	throw("findfunc: binary search failed")
-	return nil
+	// linear search to find func with pc >= entry.
+	for ftab[idx+1].entry <= pc {
+		idx++
+	}
+	return (*_func)(unsafe.Pointer(&pclntable[ftab[idx].funcoff]))
 }
 
 func pcvalue(f *_func, off int32, targetpc uintptr, strict bool) int32 {