linker.go

package ebpf

import (
	"bytes"
	"encoding/binary"
	"fmt"

	"github.com/cilium/ebpf/asm"
	"github.com/cilium/ebpf/internal/btf"
)

// The linker is responsible for resolving bpf-to-bpf calls between programs
// within an ELF. Each BPF program must be a self-contained binary blob,
// so when an instruction in one ELF program section wants to jump to
// a function in another, the linker needs to pull in the bytecode
// (and BTF info) of the target function and concatenate the instruction
// streams.
//
// Later on in the pipeline, all call sites are fixed up with relative jumps
// within this newly-created instruction stream to then finally hand off to
// the kernel with BPF_PROG_LOAD.
//
// Each function is denoted by an ELF symbol and the compiler takes care of
// register setup before each jump instruction.

// populateReferences populates all of progs' Instructions and references
// with their full dependency chains including transient dependencies.
func populateReferences(progs map[string]*ProgramSpec) error {
	type props struct {
		insns asm.Instructions
		refs  map[string]*ProgramSpec
	}

	out := make(map[string]props)

	// Resolve and store direct references between all progs.
	if err := findReferences(progs); err != nil {
		return fmt.Errorf("finding references: %w", err)
	}

	// Flatten all progs' instruction streams.
	for name, prog := range progs {
		insns, refs := prog.flatten(nil)

		prop := props{
			insns: insns,
			refs:  refs,
		}

		out[name] = prop
	}

	// Replace all progs' instructions and references
	for name, props := range out {
		progs[name].Instructions = props.insns
		progs[name].references = props.refs
	}

	return nil
}

// findReferences finds bpf-to-bpf calls between progs and populates each
// prog's references field with its direct neighbours.
func findReferences(progs map[string]*ProgramSpec) error {
	// Check all ProgramSpecs in the collection against each other.
	for _, prog := range progs {
		prog.references = make(map[string]*ProgramSpec)

		// Look up call targets in progs and store pointers to their corresponding
		// ProgramSpecs as direct references.
		for refname := range prog.Instructions.FunctionReferences() {
			ref := progs[refname]
			if ref == nil {
				return fmt.Errorf("symbol reference %s not present in progs", refname)
			}
			prog.references[refname] = ref
		}
	}

	return nil
}

// marshalFuncInfos returns the BTF func infos of all progs in order.
func marshalFuncInfos(layout []reference) ([]byte, error) {
	if len(layout) == 0 {
		return nil, nil
	}

	buf := bytes.NewBuffer(make([]byte, 0, binary.Size(&btf.FuncInfo{})*len(layout)))
	for _, sym := range layout {
		if err := sym.spec.BTF.FuncInfo.Marshal(buf, sym.offset); err != nil {
			return nil, fmt.Errorf("marshaling prog %s func info: %w", sym.spec.Name, err)
		}
	}

	return buf.Bytes(), nil
}

// marshalLineInfos returns the BTF line infos of all progs in order.
func marshalLineInfos(layout []reference) ([]byte, error) {
	if len(layout) == 0 {
		return nil, nil
	}

	buf := bytes.NewBuffer(make([]byte, 0, binary.Size(&btf.LineInfo{})*len(layout)))
	for _, sym := range layout {
		if err := sym.spec.BTF.LineInfos.Marshal(buf, sym.offset); err != nil {
			return nil, fmt.Errorf("marshaling prog %s line infos: %w", sym.spec.Name, err)
		}
	}

	return buf.Bytes(), nil
}

func fixupJumpsAndCalls(insns asm.Instructions) error {
	symbolOffsets := make(map[string]asm.RawInstructionOffset)
	iter := insns.Iterate()
	for iter.Next() {
		ins := iter.Ins

		if ins.Symbol == "" {
			continue
		}

		if _, ok := symbolOffsets[ins.Symbol]; ok {
			return fmt.Errorf("duplicate symbol %s", ins.Symbol)
		}

		symbolOffsets[ins.Symbol] = iter.Offset
	}

	iter = insns.Iterate()
	for iter.Next() {
		i := iter.Index
		offset := iter.Offset
		ins := iter.Ins

		if ins.Reference == "" {
			continue
		}

		symOffset, ok := symbolOffsets[ins.Reference]
		switch {
		case ins.IsFunctionReference() && ins.Constant == -1:
			if !ok {
				break
			}

			ins.Constant = int64(symOffset - offset - 1)
			continue

		case ins.OpCode.Class().IsJump() && ins.Offset == -1:
			if !ok {
				break
			}

			ins.Offset = int16(symOffset - offset - 1)
			continue

		case ins.IsLoadFromMap() && ins.MapPtr() == -1:
			return fmt.Errorf("map %s: %w", ins.Reference, errUnsatisfiedReference)
		default:
			// no fixup needed
			continue
		}

		return fmt.Errorf("%s at %d: reference to missing symbol %q", ins.OpCode, i, ins.Reference)
	}

	// fixupBPFCalls replaces bpf_probe_read_{kernel,user}[_str] with bpf_probe_read[_str] on older kernels
	// https://github.com/libbpf/libbpf/blob/master/src/libbpf.c#L6009
	iter = insns.Iterate()
	for iter.Next() {
		ins := iter.Ins
		if !ins.IsBuiltinCall() {
			continue
		}
		switch asm.BuiltinFunc(ins.Constant) {
		case asm.FnProbeReadKernel, asm.FnProbeReadUser:
			if err := haveProbeReadKernel(); err != nil {
				ins.Constant = int64(asm.FnProbeRead)
			}
		case asm.FnProbeReadKernelStr, asm.FnProbeReadUserStr:
			if err := haveProbeReadKernel(); err != nil {
				ins.Constant = int64(asm.FnProbeReadStr)
			}
		}
	}

	return nil
}