github/weave-scope
1
0
Fork 0
mirror of https://github.com/weaveworks/scope.git synced 2026-07-17 20:41:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

performance: in connection tracker, hold IP addresses in binary rather than strings

Browse Source 
This is more compact, and saves effort converting to and from the string format.
This commit is contained in:
Bryan Boreham
2019-09-25 19:58:11 +00:00
parent ca420b07aa
commit 2941850a75
6 changed files with 86 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
probe/endpoint/connection_tracker.go
View File

@@ -3,6 +3,7 @@
package endpoint
import (
"net"
"strconv"
"time"
@@ -43,22 +44,11 @@ func newConnectionTracker(conf ReporterConfig) connectionTracker {
}
func flowToTuple(f conntrack.Conn) (ft fourTuple) {
ft = fourTuple{
f.Orig.Src.String(),
f.Orig.Dst.String(),
uint16(f.Orig.SrcPort),
uint16(f.Orig.DstPort),
if f.Orig.Dst.Equal(f.Reply.Src) {
return makeFourTuple(f.Orig.Src, f.Orig.Dst, uint16(f.Orig.SrcPort), uint16(f.Orig.DstPort))
}
// Handle DNAT-ed connections in the initial state
if !f.Orig.Dst.Equal(f.Reply.Src) {
ft = fourTuple{
f.Reply.Dst.String(),
f.Reply.Src.String(),
uint16(f.Reply.DstPort),
uint16(f.Reply.SrcPort),
}
}
return ft
return makeFourTuple(f.Orig.Dst, f.Orig.Src, uint16(f.Orig.DstPort), uint16(f.Orig.SrcPort))
}
func (t *connectionTracker) useProcfs() {
@@ -110,7 +100,7 @@ func (t *connectionTracker) ReportConnections(rpt *report.Report) {
t.flowWalker.walkFlows(func(f conntrack.Conn, alive bool) {
tuple := flowToTuple(f)
seenTuples[tuple.key()] = tuple
t.addConnection(rpt, false, tuple, "", nil, nil)
t.addConnection(rpt, false, tuple, 0, nil, nil)
})
if t.conf.WalkProc && t.conf.Scanner != nil {
@@ -200,24 +190,25 @@ func (t *connectionTracker) performEbpfTrack(rpt *report.Report, hostNodeID stri
return nil
}
func (t *connectionTracker) addConnection(rpt *report.Report, incoming bool, ft fourTuple, namespaceID string, extraFromNode, extraToNode map[string]string) {
func (t *connectionTracker) addConnection(rpt *report.Report, incoming bool, ft fourTuple, namespaceID uint64, extraFromNode, extraToNode map[string]string) {
if incoming {
ft = reverse(ft)
extraFromNode, extraToNode = extraToNode, extraFromNode
}
var (
fromNode = t.makeEndpointNode(namespaceID, ft.fromAddr, ft.fromPort, extraFromNode)
toNode = t.makeEndpointNode(namespaceID, ft.toAddr, ft.toPort, extraToNode)
fromAddr = net.IP(ft.fromAddr[:])
fromNode = t.makeEndpointNode(namespaceID, fromAddr, ft.fromPort, extraFromNode)
toAddr = net.IP(ft.toAddr[:])
toNode = t.makeEndpointNode(namespaceID, toAddr, ft.toPort, extraToNode)
)
rpt.Endpoint.AddNode(fromNode.WithAdjacent(toNode.ID))
rpt.Endpoint.AddNode(toNode)
t.addDNS(rpt, ft.fromAddr)
t.addDNS(rpt, ft.toAddr)
t.addDNS(rpt, fromAddr.String())
t.addDNS(rpt, toAddr.String())
}
func (t *connectionTracker) makeEndpointNode(namespaceID string, addr string, port uint16, extra map[string]string) report.Node {
portStr := strconv.Itoa(int(port))
node := report.MakeNodeWith(report.MakeEndpointNodeID(t.conf.HostID, namespaceID, addr, portStr), nil)
func (t *connectionTracker) makeEndpointNode(namespaceID uint64, addr net.IP, port uint16, extra map[string]string) report.Node {
node := report.MakeNodeWith(report.MakeEndpointNodeIDB(t.conf.HostID, namespaceID, addr, port), nil)
if extra != nil {
node = node.WithLatests(extra)
}
@@ -249,22 +240,14 @@ func (t *connectionTracker) Stop() error {
return nil
}
func connectionTuple(conn *procspy.Connection, seenTuples map[string]fourTuple) (fourTuple, string, bool) {
namespaceID := ""
tuple := fourTuple{
conn.LocalAddress.String(),
conn.RemoteAddress.String(),
conn.LocalPort,
conn.RemotePort,
}
if conn.Proc.NetNamespaceID > 0 {
namespaceID = strconv.FormatUint(conn.Proc.NetNamespaceID, 10)
}
func connectionTuple(conn *procspy.Connection, seenTuples map[string]fourTuple) (fourTuple, uint64, bool) {
tuple := makeFourTuple(conn.LocalAddress, conn.RemoteAddress, conn.LocalPort, conn.RemotePort)
// If we've already seen this connection, we should know the direction
// (or have already figured it out), so we normalize and use the
// canonical direction. Otherwise, we can use a port-heuristic to guess
// the direction.
canonical, ok := seenTuples[tuple.key()]
return tuple, namespaceID, (ok && canonical != tuple) || (!ok && tuple.fromPort < tuple.toPort)
incoming := (ok && canonical != tuple) || (!ok && tuple.fromPort < tuple.toPort)
return tuple, conn.Proc.NetNamespaceID, incoming
}

28
probe/endpoint/ebpf.go
View File

@@ -25,7 +25,7 @@ import (
// An ebpfConnection represents a TCP connection
type ebpfConnection struct {
tuple fourTuple
networkNamespace string
networkNamespace uint64
incoming bool
pid int
}
@@ -172,8 +172,8 @@ func (t *EbpfTracker) TCPEventV4(e tracer.TcpV4) {
if e.Type == tracer.EventFdInstall {
t.handleFdInstall(e.Type, int(e.Pid), int(e.Fd))
} else {
tuple := fourTuple{e.SAddr.String(), e.DAddr.String(), e.SPort, e.DPort}
t.handleConnection(e.Type, tuple, int(e.Pid), strconv.Itoa(int(e.NetNS)))
tuple := makeFourTuple(e.SAddr, e.DAddr, e.SPort, e.DPort)
t.handleConnection(e.Type, tuple, int(e.Pid), uint64(e.NetNS))
}
}
@@ -198,7 +198,7 @@ func (t *EbpfTracker) LostV6(count uint64) {
// TODO: IPv6 not supported in Scope
}
func tupleFromPidFd(pid int, fd int) (tuple fourTuple, netns string, ok bool) {
func tupleFromPidFd(pid int, fd int) (tuple fourTuple, netns uint64, ok bool) {
// read /proc/$pid/ns/net
//
// probe/endpoint/procspy/proc_linux.go supports Linux < 3.8 but we
@@ -206,21 +206,20 @@ func tupleFromPidFd(pid int, fd int) (tuple fourTuple, netns string, ok bool) {
netnsIno, err := procspy.ReadNetnsFromPID(pid)
if err != nil {
log.Debugf("netns proc file for pid %d disappeared before we could read it: %v", pid, err)
return fourTuple{}, "", false
return fourTuple{}, 0, false
}
netns = fmt.Sprintf("%d", netnsIno)
// find /proc/$pid/fd/$fd's ino
fdFilename := fmt.Sprintf("/proc/%d/fd/%d", pid, fd)
var statFdFile syscall.Stat_t
if err := fs.Stat(fdFilename, &statFdFile); err != nil {
log.Debugf("proc file %q disappeared before we could read it", fdFilename)
return fourTuple{}, "", false
return fourTuple{}, 0, false
}
if statFdFile.Mode&syscall.S_IFMT != syscall.S_IFSOCK {
log.Errorf("file %q is not a socket", fdFilename)
return fourTuple{}, "", false
return fourTuple{}, 0, false
}
ino := statFdFile.Ino
@@ -228,7 +227,7 @@ func tupleFromPidFd(pid int, fd int) (tuple fourTuple, netns string, ok bool) {
buf := bytes.NewBuffer(make([]byte, 0, 5000))
if _, err := procspy.ReadTCPFiles(pid, buf); err != nil {
log.Debugf("TCP proc file for pid %d disappeared before we could read it: %v", pid, err)
return fourTuple{}, "", false
return fourTuple{}, 0, false
}
// find /proc/$pid/fd/$fd's ino in /proc/pid/net/tcp
@@ -240,11 +239,12 @@ func tupleFromPidFd(pid int, fd int) (tuple fourTuple, netns string, ok bool) {
break
}
if n.Inode == ino {
return fourTuple{n.LocalAddress.String(), n.RemoteAddress.String(), n.LocalPort, n.RemotePort}, netns, true
tuple := makeFourTuple(n.LocalAddress, n.RemoteAddress, n.LocalPort, n.RemotePort)
return tuple, netnsIno, true
}
}
return fourTuple{}, "", false
return fourTuple{}, 0, false
}
func (t *EbpfTracker) handleFdInstall(ev tracer.EventType, pid int, fd int) {
@@ -252,7 +252,7 @@ func (t *EbpfTracker) handleFdInstall(ev tracer.EventType, pid int, fd int) {
t.tracer.RemoveFdInstallWatcher(uint32(pid))
}
tuple, netns, ok := tupleFromPidFd(pid, fd)
log.Debugf("EbpfTracker: got fd-install event: pid=%d fd=%d -> tuple=%s netns=%s ok=%v", pid, fd, tuple, netns, ok)
log.Debugf("EbpfTracker: got fd-install event: pid=%d fd=%d -> tuple=%s netns=%v ok=%v", pid, fd, tuple, netns, ok)
if !ok {
return
}
@@ -268,7 +268,7 @@ func (t *EbpfTracker) handleFdInstall(ev tracer.EventType, pid int, fd int) {
}
}
func (t *EbpfTracker) handleConnection(ev tracer.EventType, tuple fourTuple, pid int, networkNamespace string) {
func (t *EbpfTracker) handleConnection(ev tracer.EventType, tuple fourTuple, pid int, networkNamespace uint64) {
t.Lock()
defer t.Unlock()
@@ -298,7 +298,7 @@ func (t *EbpfTracker) handleConnection(ev tracer.EventType, tuple fourTuple, pid
delete(t.openConnections, tuple)
t.closedConnections = append(t.closedConnections, deadConn)
} else {
log.Debugf("EbpfTracker: unmatched close event: %s pid=%d netns=%s", tuple, pid, networkNamespace)
log.Debugf("EbpfTracker: unmatched close event: %s pid=%d netns=%v", tuple, pid, networkNamespace)
}
default:
log.Debugf("EbpfTracker: unknown event: %s (%d)", ev, ev)

60
probe/endpoint/ebpf_test.go
View File

@@ -5,7 +5,6 @@ package endpoint
import (
"net"
"reflect"
"strconv"
"testing"
"time"
@@ -27,8 +26,10 @@ func TestHandleConnection(t *testing.T) {
var (
ServerPid uint32 = 42
ClientPid uint32 = 43
ServerIP = net.IP("127.0.0.1")
ClientIP = net.IP("127.0.0.2")
ServerAddr = [net.IPv4len]byte{127, 0, 0, 1}
ServerIP = net.IP(ServerAddr[:])
ClientAddr = [net.IPv4len]byte{127, 0, 0, 2}
ClientIP = net.IP(ClientAddr[:])
ServerPort uint16 = 12345
ClientPort uint16 = 6789
NetNS uint32 = 123456789
@@ -47,12 +48,12 @@ func TestHandleConnection(t *testing.T) {
IPv4ConnectEbpfConnection = ebpfConnection{
tuple: fourTuple{
fromAddr: ClientIP.String(),
toAddr: ServerIP.String(),
fromAddr: ClientAddr,
toAddr: ServerAddr,
fromPort: ClientPort,
toPort: ServerPort,
},
networkNamespace: strconv.Itoa(int(NetNS)),
networkNamespace: uint64(NetNS),
incoming: false,
pid: int(ClientPid),
}
@@ -83,12 +84,12 @@ func TestHandleConnection(t *testing.T) {
IPv4AcceptEbpfConnection = ebpfConnection{
tuple: fourTuple{
fromAddr: ServerIP.String(),
toAddr: ClientIP.String(),
fromAddr: ServerAddr,
toAddr: ClientAddr,
fromPort: ServerPort,
toPort: ClientPort,
},
networkNamespace: strconv.Itoa(int(NetNS)),
networkNamespace: uint64(NetNS),
incoming: true,
pid: int(ServerPid),
}
@@ -108,15 +109,15 @@ func TestHandleConnection(t *testing.T) {
mockEbpfTracker := newMockEbpfTracker()
tuple := fourTuple{IPv4ConnectEvent.SAddr.String(), IPv4ConnectEvent.DAddr.String(), uint16(IPv4ConnectEvent.SPort), uint16(IPv4ConnectEvent.DPort)}
mockEbpfTracker.handleConnection(IPv4ConnectEvent.Type, tuple, int(IPv4ConnectEvent.Pid), strconv.FormatUint(uint64(IPv4ConnectEvent.NetNS), 10))
tuple := fourTuple{ClientAddr, ServerAddr, uint16(IPv4ConnectEvent.SPort), uint16(IPv4ConnectEvent.DPort)}
mockEbpfTracker.handleConnection(IPv4ConnectEvent.Type, tuple, int(IPv4ConnectEvent.Pid), uint64(NetNS))
if !reflect.DeepEqual(mockEbpfTracker.openConnections[tuple], IPv4ConnectEbpfConnection) {
t.Errorf("Connection mismatch connect event\nTarget connection:%v\nParsed connection:%v",
IPv4ConnectEbpfConnection, mockEbpfTracker.openConnections[tuple])
}
tuple = fourTuple{IPv4ConnectCloseEvent.SAddr.String(), IPv4ConnectCloseEvent.DAddr.String(), uint16(IPv4ConnectCloseEvent.SPort), uint16(IPv4ConnectCloseEvent.DPort)}
mockEbpfTracker.handleConnection(IPv4ConnectCloseEvent.Type, tuple, int(IPv4ConnectCloseEvent.Pid), strconv.FormatUint(uint64(IPv4ConnectCloseEvent.NetNS), 10))
tuple = fourTuple{ClientAddr, ServerAddr, uint16(IPv4ConnectCloseEvent.SPort), uint16(IPv4ConnectCloseEvent.DPort)}
mockEbpfTracker.handleConnection(IPv4ConnectCloseEvent.Type, tuple, int(IPv4ConnectCloseEvent.Pid), uint64(NetNS))
if len(mockEbpfTracker.openConnections) != 0 {
t.Errorf("Connection mismatch close event\nConnection to close:%v",
mockEbpfTracker.openConnections[tuple])
@@ -124,15 +125,15 @@ func TestHandleConnection(t *testing.T) {
mockEbpfTracker = newMockEbpfTracker()
tuple = fourTuple{IPv4AcceptEvent.SAddr.String(), IPv4AcceptEvent.DAddr.String(), uint16(IPv4AcceptEvent.SPort), uint16(IPv4AcceptEvent.DPort)}
mockEbpfTracker.handleConnection(IPv4AcceptEvent.Type, tuple, int(IPv4AcceptEvent.Pid), strconv.FormatUint(uint64(IPv4AcceptEvent.NetNS), 10))
tuple = fourTuple{ServerAddr, ClientAddr, uint16(IPv4AcceptEvent.SPort), uint16(IPv4AcceptEvent.DPort)}
mockEbpfTracker.handleConnection(IPv4AcceptEvent.Type, tuple, int(IPv4AcceptEvent.Pid), uint64(NetNS))
if !reflect.DeepEqual(mockEbpfTracker.openConnections[tuple], IPv4AcceptEbpfConnection) {
t.Errorf("Connection mismatch connect event\nTarget connection:%v\nParsed connection:%v",
IPv4AcceptEbpfConnection, mockEbpfTracker.openConnections[tuple])
}
tuple = fourTuple{IPv4AcceptCloseEvent.SAddr.String(), IPv4AcceptCloseEvent.DAddr.String(), uint16(IPv4AcceptCloseEvent.SPort), uint16(IPv4AcceptCloseEvent.DPort)}
mockEbpfTracker.handleConnection(IPv4AcceptCloseEvent.Type, tuple, int(IPv4AcceptCloseEvent.Pid), strconv.FormatUint(uint64(IPv4AcceptCloseEvent.NetNS), 10))
tuple = fourTuple{ServerAddr, ClientAddr, uint16(IPv4AcceptCloseEvent.SPort), uint16(IPv4AcceptCloseEvent.DPort)}
mockEbpfTracker.handleConnection(IPv4AcceptCloseEvent.Type, tuple, int(IPv4AcceptCloseEvent.Pid), uint64(NetNS))
if len(mockEbpfTracker.openConnections) != 0 {
t.Errorf("Connection mismatch close event\nConnection to close:%v",
@@ -142,32 +143,21 @@ func TestHandleConnection(t *testing.T) {
func TestWalkConnections(t *testing.T) {
var (
cnt int
activeTuple = fourTuple{
fromAddr: "",
toAddr: "",
fromPort: 0,
toPort: 0,
}
inactiveTuple = fourTuple{
fromAddr: "",
toAddr: "",
fromPort: 0,
toPort: 0,
}
cnt int
activeTuple = fourTuple{}
inactiveTuple = fourTuple{}
)
mockEbpfTracker := newMockEbpfTracker()
mockEbpfTracker.openConnections[activeTuple] = ebpfConnection{
tuple: activeTuple,
networkNamespace: "12345",
networkNamespace: 12345,
incoming: true,
pid: 0,
}
mockEbpfTracker.closedConnections = append(mockEbpfTracker.closedConnections,
ebpfConnection{
tuple: inactiveTuple,
networkNamespace: "12345",
networkNamespace: 12345,
incoming: false,
pid: 0,
})
@@ -183,8 +173,8 @@ func TestInvalidTimeStampDead(t *testing.T) {
var (
cnt int
ClientPid uint32 = 43
ServerIP = net.IP("127.0.0.1")
ClientIP = net.IP("127.0.0.2")
ServerIP = net.ParseIP("127.0.0.1")
ClientIP = net.ParseIP("127.0.0.2")
ServerPort uint16 = 12345
ClientPort uint16 = 6789
NetNS uint32 = 123456789

10
probe/endpoint/four_tuple.go
View File

@@ -2,6 +2,7 @@ package endpoint
import (
"fmt"
"net"
"sort"
"strings"
)
@@ -10,10 +11,17 @@ import (
// active tells whether the connection belongs to an activeFlow (see
// conntrack.go)
type fourTuple struct {
fromAddr, toAddr string
fromAddr, toAddr [net.IPv4len]byte
fromPort, toPort uint16
}
func makeFourTuple(fromAddr, toAddr net.IP, fromPort, toPort uint16) fourTuple {
tuple := fourTuple{fromPort: fromPort, toPort: toPort}
copy(tuple.fromAddr[:], fromAddr.To4())
copy(tuple.toAddr[:], toAddr.To4())
return tuple
}
func (t fourTuple) String() string {
return fmt.Sprintf("%s:%d-%s:%d", t.fromAddr, t.fromPort, t.toAddr, t.toPort)
}

7
probe/endpoint/nat.go
View File

@@ -4,7 +4,6 @@ package endpoint
import (
"net"
"strconv"
"github.com/typetypetype/conntrack"
@@ -57,10 +56,8 @@ func (n natMapper) applyNAT(rpt report.Report, scope string) {
n.flowWalker.walkFlows(func(f conntrack.Conn, _ bool) {
mapping := toMapping(f)
realEndpointPort := strconv.Itoa(int(mapping.originalPort))
copyEndpointPort := strconv.Itoa(int(mapping.rewrittenPort))
realEndpointID := report.MakeEndpointNodeID(scope, "", mapping.originalIP.String(), realEndpointPort)
copyEndpointID := report.MakeEndpointNodeID(scope, "", mapping.rewrittenIP.String(), copyEndpointPort)
realEndpointID := report.MakeEndpointNodeIDB(scope, 0, mapping.originalIP, mapping.originalPort)
copyEndpointID := report.MakeEndpointNodeIDB(scope, 0, mapping.rewrittenIP, mapping.rewrittenPort)
node, ok := rpt.Endpoint.Nodes[realEndpointID]
if !ok {

23
report/id.go
View File

@@ -2,6 +2,7 @@ package report
import (
"net"
"strconv"
"strings"
)
@@ -29,25 +30,37 @@ const (
// MakeEndpointNodeID produces an endpoint node ID from its composite parts.
func MakeEndpointNodeID(hostID, namespaceID, address, port string) string {
return makeAddressID(hostID, namespaceID, address) + ScopeDelim + port
addressIP := net.ParseIP(address)
return makeAddressID(hostID, namespaceID, address, addressIP) + ScopeDelim + port
}
// MakeEndpointNodeIDB produces an endpoint node ID from its composite parts in binary, not strings.
func MakeEndpointNodeIDB(hostID string, namespaceID uint64, addressIP net.IP, port uint16) string {
namespace := strconv.FormatUint(namespaceID, 10)
return makeAddressID(hostID, namespace, addressIP.String(), addressIP) + ScopeDelim + strconv.Itoa(int(port))
}
// MakeAddressNodeID produces an address node ID from its composite parts.
func MakeAddressNodeID(hostID, address string) string {
return makeAddressID(hostID, "", address)
addressIP := net.ParseIP(address)
return makeAddressID(hostID, "", address, addressIP)
}
func makeAddressID(hostID, namespaceID, address string) string {
// MakeAddressNodeIDB produces an address node ID from its composite parts, in binary not string.
func MakeAddressNodeIDB(hostID string, addressIP net.IP) string {
return makeAddressID(hostID, "", addressIP.String(), addressIP)
}
func makeAddressID(hostID, namespaceID, address string, addressIP net.IP) string {
var scope string
// Loopback addresses and addresses explicitly marked as local get
// scoped by hostID
// Loopback addresses are also scoped by the networking
// namespace if available, since they can clash.
addressIP := net.ParseIP(address)
if addressIP != nil && LocalNetworks.Contains(addressIP) {
scope = hostID
} else if IsLoopback(address) {
} else if addressIP != nil && addressIP.IsLoopback() {
scope = hostID
if namespaceID != "" {
scope += "-" + namespaceID