Fallback to proc when ebpf timestamps are wrong

probe/endpoint/connection_tracker.go

@@ -88,8 +88,18 @@ func (t *connectionTracker) ReportConnections(rpt *report.Report) {
 	hostNodeID := report.MakeHostNodeID(t.conf.HostID)
 
 	if t.ebpfTracker != nil {
-		t.performEbpfTrack(rpt, hostNodeID)
-		return
+		if !t.ebpfTracker.isDead() {
+			t.performEbpfTrack(rpt, hostNodeID)
+			return
+		}
+		log.Warnf("ebpf tracker died, gently falling back to proc scanning")
+		if t.conf.WalkProc && t.conf.Scanner == nil {
+			t.conf.Scanner = procspy.NewConnectionScanner(t.conf.ProcessCache)
+		}
+		if t.flowWalker == nil {
+			t.flowWalker = newConntrackFlowWalker(t.conf.UseConntrack, t.conf.ProcRoot, t.conf.BufferSize, "--any-nat")
+		}
+		t.ebpfTracker = nil
 	}
 
 	// seenTuples contains information about connections seen by conntrack and it will be passed to the /proc parser

probe/endpoint/ebpf.go

@@ -25,6 +25,7 @@ type eventTracker interface {
 	walkConnections(f func(ebpfConnection))
 	feedInitialConnections(ci procspy.ConnIter, seenTuples map[string]fourTuple, hostNodeID string)
 	isReadyToHandleConnections() bool
+	isDead() bool
 	stop()
 }
 
@@ -99,7 +100,13 @@ var lastTimestampV4 uint64
 
 func tcpEventCbV4(e tracer.TcpV4) {
 	if lastTimestampV4 > e.Timestamp {
-		log.Errorf("ERROR: late event!\n")
+		// A kernel bug can cause the timestamps to be wrong (e.g. on Ubuntu with Linux 4.4.0-47.68)
+		// Upgrading the kernel will fix the problem. For further info see:
+		// https://github.com/iovisor/bcc/issues/790#issuecomment-263704235
+		// https://github.com/weaveworks/scope/issues/2334
+		log.Errorf("tcp tracer received event with timestamp %v even though the last timestamp was %v. Stopping the eBPF tracker.", e.Timestamp, lastTimestampV4)
+		ebpfTracker.dead = true
+		ebpfTracker.stop()
 	}
 
 	lastTimestampV4 = e.Timestamp
@@ -197,6 +204,10 @@ func (t *EbpfTracker) isReadyToHandleConnections() bool {
 	return t.readyToHandleConnections
 }
 
+func (t *EbpfTracker) isDead() bool {
+	return t.dead
+}
+
 func (t *EbpfTracker) stop() {
 	// TODO: implement proper stopping logic
 	//