From ce904fc56cb0cb564b182084ecedefaa56b92682 Mon Sep 17 00:00:00 2001 From: Michael Schubert Date: Thu, 9 Mar 2017 16:17:52 +0100 Subject: [PATCH 1/4] Remove redundant arg from `newEbpfTracker` --- probe/endpoint/connection_tracker.go | 2 +- probe/endpoint/ebpf.go | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/probe/endpoint/connection_tracker.go b/probe/endpoint/connection_tracker.go index 3479c318b..14bf7665b 100644 --- a/probe/endpoint/connection_tracker.go +++ b/probe/endpoint/connection_tracker.go @@ -42,7 +42,7 @@ func newConnectionTracker(conf connectionTrackerConfig) connectionTracker { } } // When ebpf will be active by default, check if it starts correctly otherwise fallback to flowWalk - et, err := newEbpfTracker(conf.UseEbpfConn) + et, err := newEbpfTracker() if err != nil { // TODO: fallback to flowWalker, when ebpf is enabled by default log.Errorf("Error setting up the ebpfTracker, connections will not be reported: %s", err) diff --git a/probe/endpoint/ebpf.go b/probe/endpoint/ebpf.go index bed984d42..07ce10f05 100644 --- a/probe/endpoint/ebpf.go +++ b/probe/endpoint/ebpf.go @@ -1,7 +1,6 @@ package endpoint import ( - "errors" "fmt" "regexp" "strconv" @@ -77,11 +76,7 @@ func isKernelSupported() error { return nil } -func newEbpfTracker(useEbpfConn bool) (eventTracker, error) { - if !useEbpfConn { - return nil, errors.New("ebpf tracker not enabled") - } - +func newEbpfTracker() (eventTracker, error) { if err := isKernelSupported(); err != nil { return nil, fmt.Errorf("kernel not supported: %v", err) } From 5f2ba891a4aa09709d223f3cf89ecfd9c53e6a02 Mon Sep 17 00:00:00 2001 From: Michael Schubert Date: Fri, 10 Mar 2017 10:35:10 +0100 Subject: [PATCH 2/4] endpoint/reporter: only stop scanner if not nil --- probe/endpoint/reporter.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/probe/endpoint/reporter.go b/probe/endpoint/reporter.go index 7f0301be3..d627208cb 100644 --- a/probe/endpoint/reporter.go +++ b/probe/endpoint/reporter.go @@ -83,7 +83,9 @@ func (Reporter) Name() string { return "Endpoint" } func (r *Reporter) Stop() { r.connectionTracker.Stop() r.natMapper.stop() - r.conf.Scanner.Stop() + if r.conf.Scanner != nil { + r.conf.Scanner.Stop() + } } // Report implements Reporter. From 22ae6c45a01f9a8d412439328ddec7fc9adb4c0e Mon Sep 17 00:00:00 2001 From: Michael Schubert Date: Fri, 10 Mar 2017 08:22:58 +0100 Subject: [PATCH 3/4] Implement ebpf proc fallback --- probe/endpoint/connection_tracker.go | 54 ++++++++++++++-------------- probe/endpoint/reporter.go | 3 ++ prog/probe.go | 8 +---- 3 files changed, 30 insertions(+), 35 deletions(-) diff --git a/probe/endpoint/connection_tracker.go b/probe/endpoint/connection_tracker.go index 14bf7665b..9e753180d 100644 --- a/probe/endpoint/connection_tracker.go +++ b/probe/endpoint/connection_tracker.go @@ -19,6 +19,7 @@ type connectionTrackerConfig struct { UseEbpfConn bool ProcRoot string BufferSize int + ProcessCache *process.CachingWalker Scanner procspy.ConnectionScanner DNSSnooper *DNSSnooper } @@ -28,43 +29,36 @@ type connectionTracker struct { flowWalker flowWalker // Interface ebpfTracker eventTracker reverseResolver *reverseResolver - processCache *process.CachingWalker +} + +func newProcfsConnectionTracker(conf connectionTrackerConfig) connectionTracker { + if conf.WalkProc && conf.Scanner == nil { + conf.Scanner = procspy.NewConnectionScanner(conf.ProcessCache) + } + return connectionTracker{ + conf: conf, + flowWalker: newConntrackFlowWalker(conf.UseConntrack, conf.ProcRoot, conf.BufferSize, "--any-nat"), + ebpfTracker: nil, + reverseResolver: newReverseResolver(), + } } func newConnectionTracker(conf connectionTrackerConfig) connectionTracker { if !conf.UseEbpfConn { - // ebpf OFF, use flowWalker - return connectionTracker{ - conf: conf, - flowWalker: newConntrackFlowWalker(conf.UseConntrack, conf.ProcRoot, conf.BufferSize, "--any-nat"), - ebpfTracker: nil, - reverseResolver: newReverseResolver(), - } + // ebpf off, use proc scanning for connection tracking + return newProcfsConnectionTracker(conf) } - // When ebpf will be active by default, check if it starts correctly otherwise fallback to flowWalk et, err := newEbpfTracker() if err != nil { - // TODO: fallback to flowWalker, when ebpf is enabled by default - log.Errorf("Error setting up the ebpfTracker, connections will not be reported: %s", err) - noopConnectionTracker := connectionTracker{ - conf: conf, - flowWalker: nil, - ebpfTracker: nil, - reverseResolver: nil, - } - return noopConnectionTracker + // ebpf failed, fallback to proc scanning for connection tracking + log.Warnf("Error setting up the eBPF tracker, falling back to proc scanning: %v", err) + return newProcfsConnectionTracker(conf) } - - var processCache *process.CachingWalker - processCache = process.NewCachingWalker(process.NewWalker(conf.ProcRoot)) - processCache.Tick() - ct := connectionTracker{ conf: conf, flowWalker: nil, ebpfTracker: et, reverseResolver: newReverseResolver(), - processCache: processCache, } go ct.getInitialState() return ct @@ -89,8 +83,7 @@ func flowToTuple(f flow) (ft fourTuple) { return ft } -// ReportConnections calls trackers accordingly to the configuration. -// When ebpf is enabled, only performEbpfTrack() is called +// ReportConnections calls trackers according to the configuration. func (t *connectionTracker) ReportConnections(rpt *report.Report) { hostNodeID := report.MakeHostNodeID(t.conf.HostID) @@ -164,9 +157,14 @@ func (t *connectionTracker) performWalkProc(rpt *report.Report, hostNodeID strin return nil } +// getInitialState runs conntrack and proc parsing synchronously only +// once to initialize ebpfTracker func (t *connectionTracker) getInitialState() { - scanner := procspy.NewSyncConnectionScanner(t.processCache) - // Run conntrack and proc parsing synchronously only once to initialize ebpfTracker + var processCache *process.CachingWalker + processCache = process.NewCachingWalker(process.NewWalker(t.conf.ProcRoot)) + processCache.Tick() + + scanner := procspy.NewSyncConnectionScanner(processCache) seenTuples := map[string]fourTuple{} // Consult the flowWalker to get the initial state if err := IsConntrackSupported(t.conf.ProcRoot); t.conf.UseConntrack && err != nil { diff --git a/probe/endpoint/reporter.go b/probe/endpoint/reporter.go index d627208cb..87251cc13 100644 --- a/probe/endpoint/reporter.go +++ b/probe/endpoint/reporter.go @@ -5,6 +5,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/weaveworks/scope/probe/endpoint/procspy" + "github.com/weaveworks/scope/probe/process" "github.com/weaveworks/scope/report" ) @@ -29,6 +30,7 @@ type ReporterConfig struct { UseEbpfConn bool ProcRoot string BufferSize int + ProcessCache *process.CachingWalker Scanner procspy.ConnectionScanner DNSSnooper *DNSSnooper } @@ -69,6 +71,7 @@ func NewReporter(conf ReporterConfig) *Reporter { UseEbpfConn: conf.UseEbpfConn, ProcRoot: conf.ProcRoot, BufferSize: conf.BufferSize, + ProcessCache: conf.ProcessCache, Scanner: conf.Scanner, DNSSnooper: conf.DNSSnooper, }), diff --git a/prog/probe.go b/prog/probe.go index cb74cf70c..4a2d430d3 100644 --- a/prog/probe.go +++ b/prog/probe.go @@ -27,7 +27,6 @@ import ( "github.com/weaveworks/scope/probe/controls" "github.com/weaveworks/scope/probe/docker" "github.com/weaveworks/scope/probe/endpoint" - "github.com/weaveworks/scope/probe/endpoint/procspy" "github.com/weaveworks/scope/probe/host" "github.com/weaveworks/scope/probe/kubernetes" "github.com/weaveworks/scope/probe/overlay" @@ -158,13 +157,8 @@ func probeMain(flags probeFlags, targets []appclient.Target) { p.AddTagger(probe.NewTopologyTagger(), host.NewTagger(hostID)) var processCache *process.CachingWalker - var scanner procspy.ConnectionScanner if flags.procEnabled { processCache = process.NewCachingWalker(process.NewWalker(flags.procRoot)) - // The eBPF tracker finds connections itself and does not need the connection scanner - if !flags.useEbpfConn { - scanner = procspy.NewConnectionScanner(processCache) - } p.AddTicker(processCache) p.AddReporter(process.NewReporter(processCache, hostID, process.GetDeltaTotalJiffies, flags.noCommandLineArguments)) } @@ -185,7 +179,7 @@ func probeMain(flags probeFlags, targets []appclient.Target) { UseEbpfConn: flags.useEbpfConn, ProcRoot: flags.procRoot, BufferSize: flags.conntrackBufferSize, - Scanner: scanner, + ProcessCache: processCache, DNSSnooper: dnsSnooper, }) defer endpointReporter.Stop() From 21047dc77f504f54d5a7def44c1c37c3a7711daa Mon Sep 17 00:00:00 2001 From: Michael Schubert Date: Mon, 13 Mar 2017 11:18:41 +0100 Subject: [PATCH 4/4] Add 313_container_to_container_edge_with_ebpf_proc_fallback_test.sh --- ...ainer_edge_with_ebpf_proc_fallback_test.sh | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100755 integration/313_container_to_container_edge_with_ebpf_proc_fallback_test.sh diff --git a/integration/313_container_to_container_edge_with_ebpf_proc_fallback_test.sh b/integration/313_container_to_container_edge_with_ebpf_proc_fallback_test.sh new file mode 100755 index 000000000..7bdfc9bd2 --- /dev/null +++ b/integration/313_container_to_container_edge_with_ebpf_proc_fallback_test.sh @@ -0,0 +1,31 @@ +#! /bin/bash + +# shellcheck disable=SC1091 +. ./config.sh + +start_suite "Test short lived connections between containers, with ebpf proc fallback" + +weave_on "$HOST1" launch +# Manually start scope in order to set +# `WEAVESCOPE_DOCKER_ARGS="-v /tmp:/sys/kernel/debug/tracing:ro"` +# to make ebpf fail and test the proc fallback. +DOCKER_HOST=tcp://${HOST1}:${DOCKER_PORT} CHECKPOINT_DISABLE=true \ + WEAVESCOPE_DOCKER_ARGS="-v /tmp:/sys/kernel/debug/tracing:ro" \ + "${SCOPE}" launch --probe.ebpf.connections=true +weave_on "$HOST1" run -d --name nginx nginx +weave_on "$HOST1" run -d --name client alpine /bin/sh -c "while true; do \ + wget http://nginx.weave.local:80/ -O - >/dev/null || true; \ + sleep 1; \ +done" + +wait_for_containers "$HOST1" 60 nginx client + +has_container "$HOST1" nginx +has_container "$HOST1" client +has_connection containers "$HOST1" client nginx + +# Save stdout for debugging output +exec 3>&1 +assert_raises "docker_on $HOST1 logs weavescope 2>&1 | grep 'Error setting up the eBPF tracker, falling back to proc scanning' || (docker_on $HOST1 logs weavescope 2>&3 ; false)" + +scope_end_suite