Merge pull request #3661 from weaveworks/endpoint-tweaks

Reduce leaks in probe endpoint reporter

probe/endpoint/connection_tracker.go

@@ -87,8 +87,8 @@ func (t *connectionTracker) ReportConnections(rpt *report.Report) {
 		ebpfLastFailureTime := t.ebpfLastFailureTime
 		t.ebpfLastFailureTime = time.Now()
 
-		if ebpfLastFailureTime.After(time.Now().Add(-5 * time.Minute)) {
-			// Multiple failures in the last 5 minutes, fall back to proc parsing
+		if ebpfLastFailureTime.After(time.Now().Add(-1 * time.Minute)) {
+			// Multiple failures in the last minute, fall back to proc parsing
 			log.Warnf("ebpf tracker died again, gently falling back to proc scanning")
 			t.useProcfs()
 		} else {

probe/endpoint/conntrack.go

@@ -135,16 +135,20 @@ func (c *conntrackWalker) run() {
 		return
 	}
 
-	defer log.Infof("conntrack exiting")
-
+	periodicRestart := time.After(6 * time.Hour)
 	// Handle conntrack events from netlink socket
 	for {
 		select {
+		case <-periodicRestart:
+			log.Debugf("conntrack periodic restart")
+			return
 		case <-c.quit:
+			log.Infof("conntrack quit signal - exiting")
 			stop()
 			return
 		case f, ok := <-events:
 			if !ok {
+				log.Errorf("conntrack events read failed - exiting")
 				return
 			}
 			if f.Err != nil {

probe/endpoint/ebpf.go

@@ -129,6 +129,7 @@ func newEbpfTracker() (*EbpfTracker, error) {
 		debugBPF = true
 	}
 
+	tracer.TimestampOffset = 200000 // Delay events by 0.2ms to avoid out-of-order reporting
 	tracker := &EbpfTracker{
 		debugBPF: debugBPF,
 	}