Merge pull request #2411 from kinvolk/iaguis/update-tcptracer

Update tcptracer-bpf and re-enable test 311

integration/311_container_to_container_edge_with_ebpf_test.sh

@@ -3,30 +3,26 @@
 # shellcheck disable=SC1091
 . ./config.sh
 
-start_suite "Test short lived connections between containers, with ebpf connection tracking enabled [DISABLED]"
+start_suite "Test short lived connections between containers, with ebpf connection tracking enabled"
 
 weave_on "$HOST1" launch
 scope_on "$HOST1" launch --probe.ebpf.connections=true
+weave_on "$HOST1" run -d --name nginx nginx
+weave_on "$HOST1" run -d --name client alpine /bin/sh -c "while true; do \
+	wget http://nginx.weave.local:80/ -O - >/dev/null || true; \
+	sleep 1; \
+done"
 
-## Test disabled: it is currently flaky
-## https://github.com/weaveworks/scope/issues/2379
+wait_for_containers "$HOST1" 60 nginx client
 
-# weave_on "$HOST1" run -d --name nginx nginx
-# weave_on "$HOST1" run -d --name client alpine /bin/sh -c "while true; do \
-# 	wget http://nginx.weave.local:80/ -O - >/dev/null || true; \
-# 	sleep 1; \
-# done"
-#
-# wait_for_containers "$HOST1" 60 nginx client
-#
-# has_container "$HOST1" nginx
-# has_container "$HOST1" client
-#
-# list_containers "$HOST1"
-# list_connections "$HOST1"
-#
-# has_connection containers "$HOST1" client nginx
-#
-# endpoints_have_ebpf "$HOST1"
+has_container "$HOST1" nginx
+has_container "$HOST1" client
+
+list_containers "$HOST1"
+list_connections "$HOST1"
+
+has_connection containers "$HOST1" client nginx
+
+endpoints_have_ebpf "$HOST1"
 
 scope_end_suite

vendor/github.com/weaveworks/tcptracer-bpf/pkg/tracer/offsetguess.go

@@ -8,6 +8,7 @@ import (
 	"math/rand"
 	"net"
 	"os"
+	"runtime"
 	"strconv"
 	"strings"
 	"syscall"
@@ -227,7 +228,7 @@ func tryCurrentOffset(module *elf.Module, mp *elf.Map, status *tcpTracerStatus,
 // checkAndUpdateCurrentOffset checks the value for the current offset stored
 // in the eBPF map against the expected value, incrementing the offset if it
 // doesn't match, or going to the next field to guess if it does
-func checkAndUpdateCurrentOffset(module *elf.Module, mp *elf.Map, status *tcpTracerStatus, expected *fieldValues) error {
+func checkAndUpdateCurrentOffset(module *elf.Module, mp *elf.Map, status *tcpTracerStatus, expected *fieldValues, maxRetries *int) error {
 	// get the updated map value so we can check if the current offset is
 	// the right one
 	if err := module.LookupElement(mp, unsafe.Pointer(&zero), unsafe.Pointer(status)); err != nil {
@@ -235,7 +236,14 @@ func checkAndUpdateCurrentOffset(module *elf.Module, mp *elf.Map, status *tcpTra
 	}
 
 	if status.state != stateChecked {
-		return fmt.Errorf("invalid guessing state while guessing %v, got %v expected %v", whatString[status.what], stateString[status.state], stateString[stateChecked])
+		if *maxRetries == 0 {
+			return fmt.Errorf("invalid guessing state while guessing %v, got %v expected %v",
+				whatString[status.what], stateString[status.state], stateString[stateChecked])
+		} else {
+			*maxRetries--
+			time.Sleep(10 * time.Millisecond)
+			return nil
+		}
 	}
 
 	switch status.what {
@@ -334,6 +342,11 @@ func guess(b *elf.Module) error {
 
 	mp := b.Map("tcptracer_status")
 
+	// pid & tid must not change during the guessing work: the communication
+	// between ebpf and userspace relies on it
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
 	pidTgid := uint64(os.Getpid())<<32 | uint64(syscall.Gettid())
 
 	status := &tcpTracerStatus{
@@ -370,12 +383,18 @@ func guess(b *elf.Module) error {
 		family: syscall.AF_INET,
 	}
 
+	// if the kretprobe for tcp_v4_connect() is configured with a too-low
+	// maxactive, some kretprobe might be missing. In this case, we detect
+	// it and try again.
+	// See https://github.com/weaveworks/tcptracer-bpf/issues/24
+	var maxRetries int = 100
+
 	for status.state != stateReady {
 		if err := tryCurrentOffset(b, mp, status, expected, stop); err != nil {
 			return err
 		}
 
-		if err := checkAndUpdateCurrentOffset(b, mp, status, expected); err != nil {
+		if err := checkAndUpdateCurrentOffset(b, mp, status, expected, &maxRetries); err != nil {
 			return err
 		}
 

vendor/github.com/weaveworks/tcptracer-bpf/tcptracer-bpf.c

@@ -131,7 +131,11 @@ static int are_offsets_ready_v4(struct tcptracer_status_t *status, struct sock *
 			return 0;
 	}
 
-	if (status->pid_tgid >> 32 != pid >> 32)
+	// Only accept the exact pid & tid. Extraneous connections from other
+	// threads must be ignored here. Userland must take care to generate
+	// connections from the correct thread. In Golang, this can be achieved
+	// with runtime.LockOSThread.
+	if (status->pid_tgid != pid)
 		return 0;
 
 	struct tcptracer_status_t new_status = { };
@@ -235,7 +239,11 @@ static int are_offsets_ready_v6(struct tcptracer_status_t *status, struct sock *
 			return 0;
 	}
 
-	if (status->pid_tgid >> 32 != pid >> 32)
+	// Only accept the exact pid & tid. Extraneous connections from other
+	// threads must be ignored here. Userland must take care to generate
+	// connections from the correct thread. In Golang, this can be achieved
+	// with runtime.LockOSThread.
+	if (status->pid_tgid != pid)
 		return 0;
 
 	struct tcptracer_status_t new_status = { };

vendor/github.com/weaveworks/tcptracer-bpf/tests/tracer.go

@@ -47,6 +47,8 @@ func main() {
 		os.Exit(1)
 	}
 
+	fmt.Printf("Ready\n")
+
 	sig := make(chan os.Signal, 1)
 	signal.Notify(sig, os.Interrupt, os.Kill)
 

vendor/manifest

@@ -1462,7 +1462,7 @@
 			"importpath": "github.com/weaveworks/tcptracer-bpf",
 			"repository": "https://github.com/weaveworks/tcptracer-bpf",
 			"vcs": "git",
-			"revision": "d4a42f2cc89eb19e60113f7e35945a29b97c695f",
+			"revision": "5b0b56d81d81cf9739739d347798a04ba1754b2e",
 			"branch": "master",
 			"notests": true
 		},