// +build linux // Copyright 2016 Cilium Project // Copyright 2016 Sylvain Afchain // Copyright 2016 Kinvolk // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package elf import ( "bytes" "debug/elf" "encoding/binary" "errors" "fmt" "io" "os" "path/filepath" "strings" "syscall" "unsafe" "github.com/iovisor/gobpf/pkg/bpffs" "github.com/iovisor/gobpf/pkg/cpuonline" ) /* #define _GNU_SOURCE #include #include #include #include #include #include #include #include #include #include #include #include #include "include/bpf.h" #include "include/bpf_map.h" #include #include #include typedef struct bpf_map { int fd; bpf_map_def def; } bpf_map; // from https://github.com/safchain/goebpf // Apache License, Version 2.0 extern int bpf_pin_object(int fd, const char *pathname); __u64 ptr_to_u64(void *ptr) { return (__u64) (unsigned long) ptr; } static void bpf_apply_relocation(int fd, struct bpf_insn *insn) { insn->src_reg = BPF_PSEUDO_MAP_FD; insn->imm = fd; } static int bpf_create_map(enum bpf_map_type map_type, int key_size, int value_size, int max_entries) { int ret; union bpf_attr attr; memset(&attr, 0, sizeof(attr)); attr.map_type = map_type; attr.key_size = key_size; attr.value_size = value_size; attr.max_entries = max_entries; ret = syscall(__NR_bpf, BPF_MAP_CREATE, &attr, sizeof(attr)); if (ret < 0 && errno == EPERM) { // When EPERM is returned, two reasons are possible: // 1. user has no permissions for bpf() // 2. user has insufficent rlimit for locked memory // Unfortunately, there is no api to inspect the current usage of locked // mem for the user, so an accurate calculation of how much memory to lock // for this new program is difficult to calculate. As a hack, bump the limit // to unlimited. If program load fails again, return the error. struct rlimit rl = {}; if (getrlimit(RLIMIT_MEMLOCK, &rl) == 0) { rl.rlim_max = RLIM_INFINITY; rl.rlim_cur = rl.rlim_max; if (setrlimit(RLIMIT_MEMLOCK, &rl) == 0) { ret = syscall(__NR_bpf, BPF_MAP_CREATE, &attr, sizeof(attr)); } else { printf("setrlimit() failed with errno=%d\n", errno); return -1; } } } return ret; } void create_bpf_obj_get(const char *pathname, void *attr) { union bpf_attr *ptr_bpf_attr; ptr_bpf_attr = (union bpf_attr *)attr; ptr_bpf_attr->pathname = ptr_to_u64((void *) pathname); } int get_pinned_obj_fd(const char *path) { union bpf_attr attr = {}; create_bpf_obj_get(path, &attr); return syscall(__NR_bpf, BPF_OBJ_GET, &attr, sizeof(attr)); } static bpf_map *bpf_load_map(bpf_map_def *map_def, const char *path) { bpf_map *map; struct stat st; int ret, do_pin = 0; map = calloc(1, sizeof(bpf_map)); if (map == NULL) return NULL; memcpy(&map->def, map_def, sizeof(bpf_map_def)); switch (map_def->pinning) { case 1: // PIN_OBJECT_NS // TODO to be implemented return 0; case 2: // PIN_GLOBAL_NS case 3: // PIN_CUSTOM_NS if (stat(path, &st) == 0) { ret = get_pinned_obj_fd(path); if (ret < 0) { return 0; } map->fd = ret; return map; } do_pin = 1; } map->fd = bpf_create_map(map_def->type, map_def->key_size, map_def->value_size, map_def->max_entries ); if (map->fd < 0) { return 0; } if (do_pin) { ret = bpf_pin_object(map->fd, path); if (ret < 0) { return 0; } } return map; } static int bpf_prog_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, int prog_len, const char *license, int kern_version, char *log_buf, int log_size) { int ret; union bpf_attr attr; memset(&attr, 0, sizeof(attr)); attr.prog_type = prog_type; attr.insn_cnt = prog_len / sizeof(struct bpf_insn); attr.insns = ptr_to_u64((void *) insns); attr.license = ptr_to_u64((void *) license); attr.log_buf = ptr_to_u64(log_buf); attr.log_size = log_size; attr.log_level = 1; attr.kern_version = kern_version; ret = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, sizeof(attr)); if (ret < 0 && errno == EPERM) { // When EPERM is returned, two reasons are possible: // 1. user has no permissions for bpf() // 2. user has insufficent rlimit for locked memory // Unfortunately, there is no api to inspect the current usage of locked // mem for the user, so an accurate calculation of how much memory to lock // for this new program is difficult to calculate. As a hack, bump the limit // to unlimited. If program load fails again, return the error. struct rlimit rl = {}; if (getrlimit(RLIMIT_MEMLOCK, &rl) == 0) { rl.rlim_max = RLIM_INFINITY; rl.rlim_cur = rl.rlim_max; if (setrlimit(RLIMIT_MEMLOCK, &rl) == 0) { ret = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, sizeof(attr)); } else { printf("setrlimit() failed with errno=%d\n", errno); return -1; } } } return ret; } static int bpf_update_element(int fd, void *key, void *value, unsigned long long flags) { union bpf_attr attr = { .map_fd = fd, .key = ptr_to_u64(key), .value = ptr_to_u64(value), .flags = flags, }; return syscall(__NR_bpf, BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr)); } static int perf_event_open_map(int pid, int cpu, int group_fd, unsigned long flags) { struct perf_event_attr attr = {0,}; attr.type = PERF_TYPE_SOFTWARE; attr.sample_type = PERF_SAMPLE_RAW; attr.wakeup_events = 1; attr.size = sizeof(struct perf_event_attr); attr.config = 10; // PERF_COUNT_SW_BPF_OUTPUT return syscall(__NR_perf_event_open, &attr, pid, cpu, group_fd, flags); } */ import "C" const ( useCurrentKernelVersion = 0xFFFFFFFE // Object pin settings should correspond to those of other projects, e.g.: // https://git.kernel.org/pub/scm/linux/kernel/git/shemminger/iproute2.git/tree/include/bpf_elf.h#n25 // Also it should be self-consistent with `elf/include/bpf.h` in the same repository. PIN_NONE = 0 PIN_OBJECT_NS = 1 PIN_GLOBAL_NS = 2 PIN_CUSTOM_NS = 3 ) // Based on https://github.com/safchain/goebpf // Apache License func elfReadLicense(file *elf.File) (string, error) { if lsec := file.Section("license"); lsec != nil { data, err := lsec.Data() if err != nil { return "", err } return string(data), nil } return "", nil } func elfReadVersion(file *elf.File) (uint32, error) { if vsec := file.Section("version"); vsec != nil { data, err := vsec.Data() if err != nil { return 0, err } if len(data) != 4 { return 0, errors.New("version is not a __u32") } version := *(*C.uint32_t)(unsafe.Pointer(&data[0])) return uint32(version), nil } return 0, nil } func createPinPath(path string) (string, error) { if err := bpffs.Mount(); err != nil { return "", err } if err := os.MkdirAll(filepath.Dir(path), syscall.S_IRWXU); err != nil { return "", fmt.Errorf("error creating map directory %q: %v", filepath.Dir(path), err) } return path, nil } func validateMapPath(path string) bool { if !strings.HasPrefix(path, BPFFSPath) { return false } return filepath.Clean(path) == path } func getMapNamespace(mapDef *C.bpf_map_def) string { namespacePtr := &mapDef.namespace[0] return C.GoStringN(namespacePtr, C.int(C.strnlen(namespacePtr, C.BUF_SIZE_MAP_NS))) } func getMapPath(mapDef *C.bpf_map_def, mapName, pinPath string) (string, error) { var mapPath string switch mapDef.pinning { case PIN_OBJECT_NS: return "", fmt.Errorf("not implemented yet") case PIN_GLOBAL_NS: namespace := getMapNamespace(mapDef) if namespace == "" { return "", fmt.Errorf("map %q has empty namespace", mapName) } mapPath = filepath.Join(BPFFSPath, namespace, BPFDirGlobals, mapName) case PIN_CUSTOM_NS: if pinPath == "" { return "", fmt.Errorf("no pin path given for map %q with PIN_CUSTOM_NS", mapName) } mapPath = filepath.Join(BPFFSPath, pinPath) default: // map is not pinned return "", nil } return mapPath, nil } func createMapPath(mapDef *C.bpf_map_def, mapName string, params SectionParams) (string, error) { mapPath, err := getMapPath(mapDef, mapName, params.PinPath) if err != nil || mapPath == "" { return "", err } if !validateMapPath(mapPath) { return "", fmt.Errorf("invalid path %q", mapPath) } return createPinPath(mapPath) } func elfReadMaps(file *elf.File, params map[string]SectionParams) (map[string]*Map, error) { maps := make(map[string]*Map) for _, section := range file.Sections { if !strings.HasPrefix(section.Name, "maps/") { continue } data, err := section.Data() if err != nil { return nil, err } if len(data) != C.sizeof_struct_bpf_map_def { return nil, fmt.Errorf("only one map with size %d bytes allowed per section (check bpf_map_def)", C.sizeof_struct_bpf_map_def) } name := strings.TrimPrefix(section.Name, "maps/") mapDef := (*C.bpf_map_def)(unsafe.Pointer(&data[0])) mapPath, err := createMapPath(mapDef, name, params[section.Name]) if err != nil { return nil, err } mapPathC := C.CString(mapPath) defer C.free(unsafe.Pointer(mapPathC)) cm, err := C.bpf_load_map(mapDef, mapPathC) if cm == nil { return nil, fmt.Errorf("error while loading map %q: %v", section.Name, err) } if oldMap, ok := maps[name]; ok { return nil, fmt.Errorf("duplicate map: %q and %q", oldMap.Name, name) } maps[name] = &Map{ Name: name, m: cm, } } return maps, nil } func (b *Module) relocate(data []byte, rdata []byte) error { var symbol elf.Symbol var offset uint64 symbols, err := b.file.Symbols() if err != nil { return err } br := bytes.NewReader(data) for { switch b.file.Class { case elf.ELFCLASS64: var rel elf.Rel64 err := binary.Read(br, b.file.ByteOrder, &rel) if err != nil { if err == io.EOF { return nil } return err } symNo := rel.Info >> 32 symbol = symbols[symNo-1] offset = rel.Off case elf.ELFCLASS32: var rel elf.Rel32 err := binary.Read(br, b.file.ByteOrder, &rel) if err != nil { if err == io.EOF { return nil } return err } symNo := rel.Info >> 8 symbol = symbols[symNo-1] offset = uint64(rel.Off) default: return errors.New("architecture not supported") } rinsn := (*C.struct_bpf_insn)(unsafe.Pointer(&rdata[offset])) if rinsn.code != (C.BPF_LD | C.BPF_IMM | C.BPF_DW) { symbolSec := b.file.Sections[symbol.Section] return fmt.Errorf("invalid relocation: insn code=%#x, symbol name=%s\nsymbol section: Name=%s, Type=%s, Flags=%s", *(*C.uchar)(unsafe.Pointer(&rinsn.code)), symbol.Name, symbolSec.Name, symbolSec.Type.String(), symbolSec.Flags.String(), ) } symbolSec := b.file.Sections[symbol.Section] if !strings.HasPrefix(symbolSec.Name, "maps/") { return fmt.Errorf("map location not supported: map %q is in section %q instead of \"maps/%s\"", symbol.Name, symbolSec.Name, symbol.Name) } name := strings.TrimPrefix(symbolSec.Name, "maps/") m := b.Map(name) if m == nil { return fmt.Errorf("relocation error, symbol %q not found in section %q", symbol.Name, symbolSec.Name) } C.bpf_apply_relocation(m.m.fd, rinsn) } } type SectionParams struct { PerfRingBufferPageCount int SkipPerfMapInitialization bool PinPath string // path to be pinned, relative to "/sys/fs/bpf" } // Load loads the BPF programs and BPF maps in the module. Each ELF section // can optionally have parameters that changes how it is configured. func (b *Module) Load(parameters map[string]SectionParams) error { if b.fileName != "" { fileReader, err := os.Open(b.fileName) if err != nil { return err } defer fileReader.Close() b.fileReader = fileReader } var err error b.file, err = elf.NewFile(b.fileReader) if err != nil { return err } license, err := elfReadLicense(b.file) if err != nil { return err } lp := unsafe.Pointer(C.CString(license)) defer C.free(lp) version, err := elfReadVersion(b.file) if err != nil { return err } if version == useCurrentKernelVersion { version, err = CurrentKernelVersion() if err != nil { return err } } maps, err := elfReadMaps(b.file, parameters) if err != nil { return err } b.maps = maps processed := make([]bool, len(b.file.Sections)) for i, section := range b.file.Sections { if processed[i] { continue } data, err := section.Data() if err != nil { return err } if len(data) == 0 { continue } if section.Type == elf.SHT_REL { rsection := b.file.Sections[section.Info] processed[i] = true processed[section.Info] = true secName := rsection.Name isKprobe := strings.HasPrefix(secName, "kprobe/") isKretprobe := strings.HasPrefix(secName, "kretprobe/") isCgroupSkb := strings.HasPrefix(secName, "cgroup/skb") isCgroupSock := strings.HasPrefix(secName, "cgroup/sock") isSocketFilter := strings.HasPrefix(secName, "socket") isTracepoint := strings.HasPrefix(secName, "tracepoint/") isSchedCls := strings.HasPrefix(secName, "sched_cls/") isSchedAct := strings.HasPrefix(secName, "sched_act/") var progType uint32 switch { case isKprobe: fallthrough case isKretprobe: progType = uint32(C.BPF_PROG_TYPE_KPROBE) case isCgroupSkb: progType = uint32(C.BPF_PROG_TYPE_CGROUP_SKB) case isCgroupSock: progType = uint32(C.BPF_PROG_TYPE_CGROUP_SOCK) case isSocketFilter: progType = uint32(C.BPF_PROG_TYPE_SOCKET_FILTER) case isTracepoint: progType = uint32(C.BPF_PROG_TYPE_TRACEPOINT) case isSchedCls: progType = uint32(C.BPF_PROG_TYPE_SCHED_CLS) case isSchedAct: progType = uint32(C.BPF_PROG_TYPE_SCHED_ACT) } if isKprobe || isKretprobe || isCgroupSkb || isCgroupSock || isSocketFilter || isTracepoint || isSchedCls || isSchedAct { rdata, err := rsection.Data() if err != nil { return err } if len(rdata) == 0 { continue } err = b.relocate(data, rdata) if err != nil { return err } insns := (*C.struct_bpf_insn)(unsafe.Pointer(&rdata[0])) progFd, err := C.bpf_prog_load(progType, insns, C.int(rsection.Size), (*C.char)(lp), C.int(version), (*C.char)(unsafe.Pointer(&b.log[0])), C.int(len(b.log))) if progFd < 0 { return fmt.Errorf("error while loading %q (%v):\n%s", secName, err, b.log) } switch { case isKprobe: fallthrough case isKretprobe: b.probes[secName] = &Kprobe{ Name: secName, insns: insns, fd: int(progFd), efd: -1, } case isCgroupSkb: fallthrough case isCgroupSock: b.cgroupPrograms[secName] = &CgroupProgram{ Name: secName, insns: insns, fd: int(progFd), } case isSocketFilter: b.socketFilters[secName] = &SocketFilter{ Name: secName, insns: insns, fd: int(progFd), } case isTracepoint: b.tracepointPrograms[secName] = &TracepointProgram{ Name: secName, insns: insns, fd: int(progFd), } case isSchedCls: fallthrough case isSchedAct: b.schedPrograms[secName] = &SchedProgram{ Name: secName, insns: insns, fd: int(progFd), } } } } } for i, section := range b.file.Sections { if processed[i] { continue } secName := section.Name isKprobe := strings.HasPrefix(secName, "kprobe/") isKretprobe := strings.HasPrefix(secName, "kretprobe/") isCgroupSkb := strings.HasPrefix(secName, "cgroup/skb") isCgroupSock := strings.HasPrefix(secName, "cgroup/sock") isSocketFilter := strings.HasPrefix(secName, "socket") isTracepoint := strings.HasPrefix(secName, "tracepoint/") isSchedCls := strings.HasPrefix(secName, "sched_cls/") isSchedAct := strings.HasPrefix(secName, "sched_act/") var progType uint32 switch { case isKprobe: fallthrough case isKretprobe: progType = uint32(C.BPF_PROG_TYPE_KPROBE) case isCgroupSkb: progType = uint32(C.BPF_PROG_TYPE_CGROUP_SKB) case isCgroupSock: progType = uint32(C.BPF_PROG_TYPE_CGROUP_SOCK) case isSocketFilter: progType = uint32(C.BPF_PROG_TYPE_SOCKET_FILTER) case isTracepoint: progType = uint32(C.BPF_PROG_TYPE_TRACEPOINT) case isSchedCls: progType = uint32(C.BPF_PROG_TYPE_SCHED_CLS) case isSchedAct: progType = uint32(C.BPF_PROG_TYPE_SCHED_ACT) } if isKprobe || isKretprobe || isCgroupSkb || isCgroupSock || isSocketFilter || isTracepoint || isSchedCls || isSchedAct { data, err := section.Data() if err != nil { return err } if len(data) == 0 { continue } insns := (*C.struct_bpf_insn)(unsafe.Pointer(&data[0])) progFd, err := C.bpf_prog_load(progType, insns, C.int(section.Size), (*C.char)(lp), C.int(version), (*C.char)(unsafe.Pointer(&b.log[0])), C.int(len(b.log))) if progFd < 0 { return fmt.Errorf("error while loading %q (%v):\n%s", section.Name, err, b.log) } switch { case isKprobe: fallthrough case isKretprobe: b.probes[secName] = &Kprobe{ Name: secName, insns: insns, fd: int(progFd), efd: -1, } case isCgroupSkb: fallthrough case isCgroupSock: b.cgroupPrograms[secName] = &CgroupProgram{ Name: secName, insns: insns, fd: int(progFd), } case isSocketFilter: b.socketFilters[secName] = &SocketFilter{ Name: secName, insns: insns, fd: int(progFd), } case isTracepoint: b.tracepointPrograms[secName] = &TracepointProgram{ Name: secName, insns: insns, fd: int(progFd), } case isSchedCls: fallthrough case isSchedAct: b.schedPrograms[secName] = &SchedProgram{ Name: secName, insns: insns, fd: int(progFd), } } } } return b.initializePerfMaps(parameters) } func (b *Module) initializePerfMaps(parameters map[string]SectionParams) error { for name, m := range b.maps { if m.m != nil && m.m.def._type != C.BPF_MAP_TYPE_PERF_EVENT_ARRAY { continue } pageSize := os.Getpagesize() b.maps[name].pageCount = 8 // reasonable default sectionName := "maps/" + name if params, ok := parameters[sectionName]; ok { if params.SkipPerfMapInitialization { continue } if params.PerfRingBufferPageCount > 0 { if (params.PerfRingBufferPageCount & (params.PerfRingBufferPageCount - 1)) != 0 { return fmt.Errorf("number of pages (%d) must be stricly positive and a power of 2", params.PerfRingBufferPageCount) } b.maps[name].pageCount = params.PerfRingBufferPageCount } } cpus, err := cpuonline.Get() if err != nil { return fmt.Errorf("failed to determine online cpus: %v", err) } for _, cpu := range cpus { cpuC := C.int(cpu) pmuFD, err := C.perf_event_open_map(-1 /* pid */, cpuC /* cpu */, -1 /* group_fd */, C.PERF_FLAG_FD_CLOEXEC) if pmuFD < 0 { return fmt.Errorf("perf_event_open for map error: %v", err) } // mmap mmapSize := pageSize * (b.maps[name].pageCount + 1) base, err := syscall.Mmap(int(pmuFD), 0, mmapSize, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED) if err != nil { return fmt.Errorf("mmap error: %v", err) } // enable _, _, err2 := syscall.Syscall(syscall.SYS_IOCTL, uintptr(pmuFD), C.PERF_EVENT_IOC_ENABLE, 0) if err2 != 0 { return fmt.Errorf("error enabling perf event: %v", err2) } // assign perf fd to map ret, err := C.bpf_update_element(C.int(b.maps[name].m.fd), unsafe.Pointer(&cpuC), unsafe.Pointer(&pmuFD), C.BPF_ANY) if ret != 0 { return fmt.Errorf("cannot assign perf fd to map %q: %v (cpu %d)", name, err, cpuC) } b.maps[name].pmuFDs = append(b.maps[name].pmuFDs, pmuFD) b.maps[name].headers = append(b.maps[name].headers, (*C.struct_perf_event_mmap_page)(unsafe.Pointer(&base[0]))) } } return nil } // Map represents a eBPF map. An eBPF map has to be declared in the // C file. type Map struct { Name string m *C.bpf_map // only for perf maps pmuFDs []C.int headers []*C.struct_perf_event_mmap_page pageCount int } func (b *Module) IterMaps() <-chan *Map { ch := make(chan *Map) go func() { for name := range b.maps { ch <- b.maps[name] } close(ch) }() return ch } func (b *Module) Map(name string) *Map { return b.maps[name] } func (m *Map) Fd() int { return int(m.m.fd) }