From 07e7adbd631e15b13b825f28a8210cc6f94f8591 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Mon, 10 Jul 2017 07:22:12 +0100 Subject: [PATCH 01/11] refactor: make performFlowWalk data flow more obvious --- probe/endpoint/connection_tracker.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/probe/endpoint/connection_tracker.go b/probe/endpoint/connection_tracker.go index 7d6f7ea2c..b7c38f263 100644 --- a/probe/endpoint/connection_tracker.go +++ b/probe/endpoint/connection_tracker.go @@ -91,16 +91,18 @@ func (t *connectionTracker) ReportConnections(rpt *report.Report) { t.useProcfs() } - // seenTuples contains information about connections seen by conntrack and it will be passed to the /proc parser - seenTuples := map[string]fourTuple{} - t.performFlowWalk(rpt, seenTuples) + // seenTuples contains information about connections seen by + // conntrack + seenTuples := t.performFlowWalk(rpt) + if t.conf.WalkProc && t.conf.Scanner != nil { t.performWalkProc(rpt, hostNodeID, seenTuples) } } -func (t *connectionTracker) performFlowWalk(rpt *report.Report, seenTuples map[string]fourTuple) { - // Consult the flowWalker for short-lived connections +// performFlowWalk consults the flowWalker for short-lived connections +func (t *connectionTracker) performFlowWalk(rpt *report.Report) map[string]fourTuple { + seenTuples := map[string]fourTuple{} extraNodeInfo := map[string]string{ Conntracked: "true", } @@ -109,6 +111,7 @@ func (t *connectionTracker) performFlowWalk(rpt *report.Report, seenTuples map[s seenTuples[tuple.key()] = tuple t.addConnection(rpt, tuple, "", extraNodeInfo, extraNodeInfo) }) + return seenTuples } func (t *connectionTracker) performWalkProc(rpt *report.Report, hostNodeID string, seenTuples map[string]fourTuple) error { From 7ea0800f8b70cb5fb6f7ee9b7c09477b296fa498 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Mon, 10 Jul 2017 07:34:20 +0100 Subject: [PATCH 02/11] refactor: extract helper to get initial flows --- probe/endpoint/connection_tracker.go | 30 +++++++++++++++++----------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/probe/endpoint/connection_tracker.go b/probe/endpoint/connection_tracker.go index b7c38f263..242d86bbc 100644 --- a/probe/endpoint/connection_tracker.go +++ b/probe/endpoint/connection_tracker.go @@ -114,6 +114,21 @@ func (t *connectionTracker) performFlowWalk(rpt *report.Report) map[string]fourT return seenTuples } +func (t *connectionTracker) existingFlows() map[string]fourTuple { + seenTuples := map[string]fourTuple{} + if err := IsConntrackSupported(t.conf.ProcRoot); t.conf.UseConntrack && err != nil { + log.Warnf("Not using conntrack: not supported by the kernel: %s", err) + } else if existingFlows, err := existingConnections([]string{"--any-nat"}); err != nil { + log.Errorf("conntrack existingConnections error: %v", err) + } else { + for _, f := range existingFlows { + tuple := flowToTuple(f) + seenTuples[tuple.key()] = tuple + } + } + return seenTuples +} + func (t *connectionTracker) performWalkProc(rpt *report.Report, hostNodeID string, seenTuples map[string]fourTuple) error { conns, err := t.conf.Scanner.Connections() if err != nil { @@ -147,18 +162,9 @@ func (t *connectionTracker) getInitialState() { processCache.Tick() scanner := procspy.NewSyncConnectionScanner(processCache, t.conf.SpyProcs) - seenTuples := map[string]fourTuple{} - // Consult the flowWalker to get the initial state - if err := IsConntrackSupported(t.conf.ProcRoot); t.conf.UseConntrack && err != nil { - log.Warnf("Not using conntrack: not supported by the kernel: %s", err) - } else if existingFlows, err := existingConnections([]string{"--any-nat"}); err != nil { - log.Errorf("conntrack existingConnections error: %v", err) - } else { - for _, f := range existingFlows { - tuple := flowToTuple(f) - seenTuples[tuple.key()] = tuple - } - } + + // Consult conntrack to get the initial state + seenTuples := t.existingFlows() conns, err := scanner.Connections() if err != nil { From 8bd0188537bfe2a24280e768b39bdc69a76101e1 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Mon, 10 Jul 2017 07:37:23 +0100 Subject: [PATCH 03/11] respect UseConntrack setting in ebpf initialisation --- probe/endpoint/connection_tracker.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/probe/endpoint/connection_tracker.go b/probe/endpoint/connection_tracker.go index 242d86bbc..c7bd029b9 100644 --- a/probe/endpoint/connection_tracker.go +++ b/probe/endpoint/connection_tracker.go @@ -116,7 +116,9 @@ func (t *connectionTracker) performFlowWalk(rpt *report.Report) map[string]fourT func (t *connectionTracker) existingFlows() map[string]fourTuple { seenTuples := map[string]fourTuple{} - if err := IsConntrackSupported(t.conf.ProcRoot); t.conf.UseConntrack && err != nil { + if !t.conf.UseConntrack { + // log.Warnf("Not using conntrack: disabled") + } else if err := IsConntrackSupported(t.conf.ProcRoot); err != nil { log.Warnf("Not using conntrack: not supported by the kernel: %s", err) } else if existingFlows, err := existingConnections([]string{"--any-nat"}); err != nil { log.Errorf("conntrack existingConnections error: %v", err) From 8a565406486977b9f514f3826c883b0b3bb0ff6f Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Mon, 10 Jul 2017 09:04:14 +0100 Subject: [PATCH 04/11] refactor: eliminate global var --- probe/endpoint/ebpf.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/probe/endpoint/ebpf.go b/probe/endpoint/ebpf.go index 13e580925..0b619901f 100644 --- a/probe/endpoint/ebpf.go +++ b/probe/endpoint/ebpf.go @@ -40,6 +40,7 @@ type EbpfTracker struct { tracer *tracer.Tracer readyToHandleConnections bool dead bool + lastTimestampV4 uint64 openConnections map[string]ebpfConnection closedConnections []ebpfConnection @@ -99,20 +100,18 @@ func newEbpfTracker() (eventTracker, error) { return tracker, nil } -var lastTimestampV4 uint64 - func (t *EbpfTracker) tcpEventCbV4(e tracer.TcpV4) { - if lastTimestampV4 > e.Timestamp { + if t.lastTimestampV4 > e.Timestamp { // A kernel bug can cause the timestamps to be wrong (e.g. on Ubuntu with Linux 4.4.0-47.68) // Upgrading the kernel will fix the problem. For further info see: // https://github.com/iovisor/bcc/issues/790#issuecomment-263704235 // https://github.com/weaveworks/scope/issues/2334 - log.Errorf("tcp tracer received event with timestamp %v even though the last timestamp was %v. Stopping the eBPF tracker.", e.Timestamp, lastTimestampV4) + log.Errorf("tcp tracer received event with timestamp %v even though the last timestamp was %v. Stopping the eBPF tracker.", e.Timestamp, t.lastTimestampV4) t.dead = true t.stop() } - lastTimestampV4 = e.Timestamp + t.lastTimestampV4 = e.Timestamp if e.Type == tracer.EventFdInstall { t.handleFdInstall(e.Type, int(e.Pid), int(e.Fd)) From ad7b5cdc19a54e7ae9ba7e5bba7e17cffd576032 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Mon, 10 Jul 2017 09:12:45 +0100 Subject: [PATCH 05/11] refactor: remove pointless interface premature abstraction --- probe/endpoint/connection_tracker.go | 2 +- probe/endpoint/ebpf.go | 11 +---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/probe/endpoint/connection_tracker.go b/probe/endpoint/connection_tracker.go index c7bd029b9..511bb280d 100644 --- a/probe/endpoint/connection_tracker.go +++ b/probe/endpoint/connection_tracker.go @@ -27,7 +27,7 @@ type connectionTrackerConfig struct { type connectionTracker struct { conf connectionTrackerConfig flowWalker flowWalker // Interface - ebpfTracker eventTracker + ebpfTracker *EbpfTracker reverseResolver *reverseResolver } diff --git a/probe/endpoint/ebpf.go b/probe/endpoint/ebpf.go index 0b619901f..895c468de 100644 --- a/probe/endpoint/ebpf.go +++ b/probe/endpoint/ebpf.go @@ -24,15 +24,6 @@ type ebpfConnection struct { pid int } -type eventTracker interface { - handleConnection(ev tracer.EventType, tuple fourTuple, pid int, networkNamespace string) - walkConnections(f func(ebpfConnection)) - feedInitialConnections(ci procspy.ConnIter, seenTuples map[string]fourTuple, processesWaitingInAccept []int, hostNodeID string) - isReadyToHandleConnections() bool - isDead() bool - stop() -} - // EbpfTracker contains the sets of open and closed TCP connections. // Closed connections are kept in the `closedConnections` slice for one iteration of `walkConnections`. type EbpfTracker struct { @@ -80,7 +71,7 @@ func isKernelSupported() error { return nil } -func newEbpfTracker() (eventTracker, error) { +func newEbpfTracker() (*EbpfTracker, error) { if err := isKernelSupported(); err != nil { return nil, fmt.Errorf("kernel not supported: %v", err) } From 3baeb3d2382116f5ef75436a1584dd659f518d23 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Mon, 10 Jul 2017 11:05:44 +0100 Subject: [PATCH 06/11] refactor: use fourTuple as map key instead of string --- probe/endpoint/ebpf.go | 16 ++++++++-------- probe/endpoint/ebpf_test.go | 20 ++++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/probe/endpoint/ebpf.go b/probe/endpoint/ebpf.go index 895c468de..dd8119269 100644 --- a/probe/endpoint/ebpf.go +++ b/probe/endpoint/ebpf.go @@ -33,7 +33,7 @@ type EbpfTracker struct { dead bool lastTimestampV4 uint64 - openConnections map[string]ebpfConnection + openConnections map[fourTuple]ebpfConnection closedConnections []ebpfConnection } @@ -77,7 +77,7 @@ func newEbpfTracker() (*EbpfTracker, error) { } tracker := &EbpfTracker{ - openConnections: map[string]ebpfConnection{}, + openConnections: map[fourTuple]ebpfConnection{}, } tracer, err := tracer.NewTracer(tracker.tcpEventCbV4, tracker.tcpEventCbV6, tracker.lostCb) @@ -183,7 +183,7 @@ func (t *EbpfTracker) handleFdInstall(ev tracer.EventType, pid int, fd int) { pid: pid, networkNamespace: netns, } - t.openConnections[tuple.String()] = conn + t.openConnections[tuple] = conn if !process.IsProcInAccept("/proc", strconv.Itoa(pid)) { t.tracer.RemoveFdInstallWatcher(uint32(pid)) } @@ -208,7 +208,7 @@ func (t *EbpfTracker) handleConnection(ev tracer.EventType, tuple fourTuple, pid pid: pid, networkNamespace: networkNamespace, } - t.openConnections[tuple.String()] = conn + t.openConnections[tuple] = conn case tracer.EventAccept: conn := ebpfConnection{ incoming: true, @@ -216,13 +216,13 @@ func (t *EbpfTracker) handleConnection(ev tracer.EventType, tuple fourTuple, pid pid: pid, networkNamespace: networkNamespace, } - t.openConnections[tuple.String()] = conn + t.openConnections[tuple] = conn case tracer.EventClose: - if deadConn, ok := t.openConnections[tuple.String()]; ok { - delete(t.openConnections, tuple.String()) + if deadConn, ok := t.openConnections[tuple]; ok { + delete(t.openConnections, tuple) t.closedConnections = append(t.closedConnections, deadConn) } else { - log.Debugf("EbpfTracker: unmatched close event: %s pid=%d netns=%s", tuple.String(), pid, networkNamespace) + log.Debugf("EbpfTracker: unmatched close event: %s pid=%d netns=%s", tuple, pid, networkNamespace) } default: log.Debugf("EbpfTracker: unknown event: %s (%d)", ev, ev) diff --git a/probe/endpoint/ebpf_test.go b/probe/endpoint/ebpf_test.go index e22d00acd..24d96aab5 100644 --- a/probe/endpoint/ebpf_test.go +++ b/probe/endpoint/ebpf_test.go @@ -96,37 +96,37 @@ func TestHandleConnection(t *testing.T) { readyToHandleConnections: true, dead: false, - openConnections: map[string]ebpfConnection{}, + openConnections: map[fourTuple]ebpfConnection{}, closedConnections: []ebpfConnection{}, } tuple := fourTuple{IPv4ConnectEvent.SAddr.String(), IPv4ConnectEvent.DAddr.String(), uint16(IPv4ConnectEvent.SPort), uint16(IPv4ConnectEvent.DPort)} mockEbpfTracker.handleConnection(IPv4ConnectEvent.Type, tuple, int(IPv4ConnectEvent.Pid), strconv.FormatUint(uint64(IPv4ConnectEvent.NetNS), 10)) - if !reflect.DeepEqual(mockEbpfTracker.openConnections[tuple.String()], IPv4ConnectEbpfConnection) { + if !reflect.DeepEqual(mockEbpfTracker.openConnections[tuple], IPv4ConnectEbpfConnection) { t.Errorf("Connection mismatch connect event\nTarget connection:%v\nParsed connection:%v", - IPv4ConnectEbpfConnection, mockEbpfTracker.openConnections[tuple.String()]) + IPv4ConnectEbpfConnection, mockEbpfTracker.openConnections[tuple]) } tuple = fourTuple{IPv4ConnectCloseEvent.SAddr.String(), IPv4ConnectCloseEvent.DAddr.String(), uint16(IPv4ConnectCloseEvent.SPort), uint16(IPv4ConnectCloseEvent.DPort)} mockEbpfTracker.handleConnection(IPv4ConnectCloseEvent.Type, tuple, int(IPv4ConnectCloseEvent.Pid), strconv.FormatUint(uint64(IPv4ConnectCloseEvent.NetNS), 10)) if len(mockEbpfTracker.openConnections) != 0 { t.Errorf("Connection mismatch close event\nConnection to close:%v", - mockEbpfTracker.openConnections[tuple.String()]) + mockEbpfTracker.openConnections[tuple]) } mockEbpfTracker = &EbpfTracker{ readyToHandleConnections: true, dead: false, - openConnections: map[string]ebpfConnection{}, + openConnections: map[fourTuple]ebpfConnection{}, closedConnections: []ebpfConnection{}, } tuple = fourTuple{IPv4AcceptEvent.SAddr.String(), IPv4AcceptEvent.DAddr.String(), uint16(IPv4AcceptEvent.SPort), uint16(IPv4AcceptEvent.DPort)} mockEbpfTracker.handleConnection(IPv4AcceptEvent.Type, tuple, int(IPv4AcceptEvent.Pid), strconv.FormatUint(uint64(IPv4AcceptEvent.NetNS), 10)) - if !reflect.DeepEqual(mockEbpfTracker.openConnections[tuple.String()], IPv4AcceptEbpfConnection) { + if !reflect.DeepEqual(mockEbpfTracker.openConnections[tuple], IPv4AcceptEbpfConnection) { t.Errorf("Connection mismatch connect event\nTarget connection:%v\nParsed connection:%v", - IPv4AcceptEbpfConnection, mockEbpfTracker.openConnections[tuple.String()]) + IPv4AcceptEbpfConnection, mockEbpfTracker.openConnections[tuple]) } tuple = fourTuple{IPv4AcceptCloseEvent.SAddr.String(), IPv4AcceptCloseEvent.DAddr.String(), uint16(IPv4AcceptCloseEvent.SPort), uint16(IPv4AcceptCloseEvent.DPort)} @@ -158,8 +158,8 @@ func TestWalkConnections(t *testing.T) { mockEbpfTracker := &EbpfTracker{ readyToHandleConnections: true, dead: false, - openConnections: map[string]ebpfConnection{ - activeTuple.String(): { + openConnections: map[fourTuple]ebpfConnection{ + activeTuple: { tuple: activeTuple, networkNamespace: "12345", incoming: true, @@ -207,7 +207,7 @@ func TestInvalidTimeStampDead(t *testing.T) { mockEbpfTracker := &EbpfTracker{ readyToHandleConnections: true, dead: false, - openConnections: map[string]ebpfConnection{}, + openConnections: map[fourTuple]ebpfConnection{}, } event.Timestamp = 0 mockEbpfTracker.tcpEventCbV4(event) From e2cbe7ac261405429cb42a575eaac7eafe49887f Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Mon, 10 Jul 2017 15:09:19 +0100 Subject: [PATCH 07/11] refactor: a bit of inlining --- probe/endpoint/ebpf.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/probe/endpoint/ebpf.go b/probe/endpoint/ebpf.go index dd8119269..b6faa704b 100644 --- a/probe/endpoint/ebpf.go +++ b/probe/endpoint/ebpf.go @@ -177,13 +177,12 @@ func (t *EbpfTracker) handleFdInstall(ev tracer.EventType, pid int, fd int) { if !ok { return } - conn := ebpfConnection{ + t.openConnections[tuple] = ebpfConnection{ incoming: true, tuple: tuple, pid: pid, networkNamespace: netns, } - t.openConnections[tuple] = conn if !process.IsProcInAccept("/proc", strconv.Itoa(pid)) { t.tracer.RemoveFdInstallWatcher(uint32(pid)) } @@ -202,21 +201,19 @@ func (t *EbpfTracker) handleConnection(ev tracer.EventType, tuple fourTuple, pid switch ev { case tracer.EventConnect: - conn := ebpfConnection{ + t.openConnections[tuple] = ebpfConnection{ incoming: false, tuple: tuple, pid: pid, networkNamespace: networkNamespace, } - t.openConnections[tuple] = conn case tracer.EventAccept: - conn := ebpfConnection{ + t.openConnections[tuple] = ebpfConnection{ incoming: true, tuple: tuple, pid: pid, networkNamespace: networkNamespace, } - t.openConnections[tuple] = conn case tracer.EventClose: if deadConn, ok := t.openConnections[tuple]; ok { delete(t.openConnections, tuple) From 3883d8f1af1454260472a712d54aab806a7b5ecd Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Mon, 10 Jul 2017 15:31:11 +0100 Subject: [PATCH 08/11] fix a minor leak in ebfp fdinstall_pids table when we got an fd install event but the pid was dead by time we processed it, we would fail to remove the watcher for that pid from the fdinstall_pids table. This is a minor, and bounded, leak, since the table only contains pids that were alive when we initialized ebpf. And this change only plugs that leak very partially, since we will never remove pids that die while sitting in accept(). --- probe/endpoint/ebpf.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/probe/endpoint/ebpf.go b/probe/endpoint/ebpf.go index b6faa704b..7b1708952 100644 --- a/probe/endpoint/ebpf.go +++ b/probe/endpoint/ebpf.go @@ -172,6 +172,9 @@ func tupleFromPidFd(pid int, fd int) (tuple fourTuple, netns string, ok bool) { } func (t *EbpfTracker) handleFdInstall(ev tracer.EventType, pid int, fd int) { + if !process.IsProcInAccept("/proc", strconv.Itoa(pid)) { + t.tracer.RemoveFdInstallWatcher(uint32(pid)) + } tuple, netns, ok := tupleFromPidFd(pid, fd) log.Debugf("EbpfTracker: got fd-install event: pid=%d fd=%d -> tuple=%s netns=%s ok=%v", pid, fd, tuple, netns, ok) if !ok { @@ -183,9 +186,6 @@ func (t *EbpfTracker) handleFdInstall(ev tracer.EventType, pid int, fd int) { pid: pid, networkNamespace: netns, } - if !process.IsProcInAccept("/proc", strconv.Itoa(pid)) { - t.tracer.RemoveFdInstallWatcher(uint32(pid)) - } } func (t *EbpfTracker) handleConnection(ev tracer.EventType, tuple fourTuple, pid int, networkNamespace string) { From 15215d0c2c80527f89863f89cd7007ada1288a2f Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Mon, 10 Jul 2017 15:41:00 +0100 Subject: [PATCH 09/11] prevent concurrent map access in ebpf fd install event handler which presumably could cause havoc --- probe/endpoint/ebpf.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/probe/endpoint/ebpf.go b/probe/endpoint/ebpf.go index 7b1708952..f69d48df1 100644 --- a/probe/endpoint/ebpf.go +++ b/probe/endpoint/ebpf.go @@ -180,6 +180,10 @@ func (t *EbpfTracker) handleFdInstall(ev tracer.EventType, pid int, fd int) { if !ok { return } + + t.Lock() + defer t.Unlock() + t.openConnections[tuple] = ebpfConnection{ incoming: true, tuple: tuple, From cf6353327a4bb3f5e9b5916f214d86dba8a581a2 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Mon, 10 Jul 2017 15:54:34 +0100 Subject: [PATCH 10/11] eliminate race in ebpf initialization We were enabling event processing before feeding in the initial connections, which results in a non-deterministic outcome. --- probe/endpoint/ebpf.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/probe/endpoint/ebpf.go b/probe/endpoint/ebpf.go index f69d48df1..9e9765f2a 100644 --- a/probe/endpoint/ebpf.go +++ b/probe/endpoint/ebpf.go @@ -246,15 +246,19 @@ func (t *EbpfTracker) walkConnections(f func(ebpfConnection)) { } func (t *EbpfTracker) feedInitialConnections(conns procspy.ConnIter, seenTuples map[string]fourTuple, processesWaitingInAccept []int, hostNodeID string) { - t.readyToHandleConnections = true + t.Lock() for conn := conns.Next(); conn != nil; conn = conns.Next() { tuple, namespaceID, incoming := connectionTuple(conn, seenTuples) - if incoming { - t.handleConnection(tracer.EventAccept, tuple, int(conn.Proc.PID), namespaceID) - } else { - t.handleConnection(tracer.EventConnect, tuple, int(conn.Proc.PID), namespaceID) + t.openConnections[tuple] = ebpfConnection{ + incoming: incoming, + tuple: tuple, + pid: int(conn.Proc.PID), + networkNamespace: namespaceID, } } + t.readyToHandleConnections = true + t.Unlock() + for _, p := range processesWaitingInAccept { t.tracer.AddFdInstallWatcher(uint32(p)) log.Debugf("EbpfTracker: install fd-install watcher: pid=%d", p) From d568c50ec4595a75db17425423828d6e7c9f7c30 Mon Sep 17 00:00:00 2001 From: Matthias Radestock Date: Mon, 10 Jul 2017 20:46:55 +0100 Subject: [PATCH 11/11] make EbpfTracker.dead go-routine-safe and .stop() idempotent Without synchronisation, the isDead() call might return a stale value, delaying deadness detection potentially indefinitely. Without the guards / idempotence in .stop(), invoking stop() more than once could cause a panic, since tracer.Stop() closes a channel (which panics on a closed channel). Multiple stop() invocations are rare, but not impossible. --- probe/endpoint/ebpf.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/probe/endpoint/ebpf.go b/probe/endpoint/ebpf.go index 9e9765f2a..f9d57dcec 100644 --- a/probe/endpoint/ebpf.go +++ b/probe/endpoint/ebpf.go @@ -98,7 +98,6 @@ func (t *EbpfTracker) tcpEventCbV4(e tracer.TcpV4) { // https://github.com/iovisor/bcc/issues/790#issuecomment-263704235 // https://github.com/weaveworks/scope/issues/2334 log.Errorf("tcp tracer received event with timestamp %v even though the last timestamp was %v. Stopping the eBPF tracker.", e.Timestamp, t.lastTimestampV4) - t.dead = true t.stop() } @@ -118,7 +117,6 @@ func (t *EbpfTracker) tcpEventCbV6(e tracer.TcpV6) { func (t *EbpfTracker) lostCb(count uint64) { log.Errorf("tcp tracer lost %d events. Stopping the eBPF tracker", count) - t.dead = true t.stop() } @@ -270,12 +268,17 @@ func (t *EbpfTracker) isReadyToHandleConnections() bool { } func (t *EbpfTracker) isDead() bool { + t.Lock() + defer t.Unlock() return t.dead } func (t *EbpfTracker) stop() { - if t.tracer != nil { + t.Lock() + alreadyDead := t.dead + t.dead = true + t.Unlock() + if !alreadyDead && t.tracer != nil { t.tracer.Stop() } - t.dead = true }