mirror of
https://github.com/weaveworks/scope.git
synced 2026-03-05 03:01:11 +00:00
Merge pull request #381 from weaveworks/356-ftrace
[WIP] First working ftrace prototype.
This commit is contained in:
Tom Wilkie
204
experimental/ftrace/ftrace.go
Normal file
204
experimental/ftrace/ftrace.go
Normal file
@@ -0,0 +1,204 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
perms = 0777
|
||||
)
|
||||
|
||||
var (
|
||||
lineMatcher = regexp.MustCompile(`^\s*[a-z\-]+\-(\d+)\s+\[(\d{3})] (?:\.|1){4} ([\d\.]+): (.*)$`)
|
||||
enterMatcher = regexp.MustCompile(`^([\w_]+)\((.*)\)$`)
|
||||
argMatcher = regexp.MustCompile(`(\w+): (\w+)`)
|
||||
exitMatcher = regexp.MustCompile(`^([\w_]+) -> (\w+)$`)
|
||||
)
|
||||
|
||||
// Ftrace is a tracer using ftrace...
|
||||
type Ftrace struct {
|
||||
ftraceRoot string
|
||||
root string
|
||||
outstanding map[int]*syscall // map from pid (readlly tid) to outstanding syscall
|
||||
}
|
||||
|
||||
type syscall struct {
|
||||
pid int
|
||||
cpu int
|
||||
ts float64
|
||||
name string
|
||||
args map[string]string
|
||||
returnCode int64
|
||||
}
|
||||
|
||||
func findDebugFS() (string, error) {
|
||||
contents, err := ioutil.ReadFile("/proc/mounts")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
scanner := bufio.NewScanner(bytes.NewBuffer(contents))
|
||||
for scanner.Scan() {
|
||||
fields := strings.Fields(scanner.Text())
|
||||
if len(fields) < 3 {
|
||||
continue
|
||||
}
|
||||
if fields[2] == "debugfs" {
|
||||
return fields[1], nil
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "", fmt.Errorf("Not found")
|
||||
}
|
||||
|
||||
// NewFtracer constucts a new Ftrace instance.
|
||||
func NewFtracer() (*Ftrace, error) {
|
||||
root, err := findDebugFS()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scopeRoot := path.Join(root, "tracing", "instances", "scope")
|
||||
if err := os.Mkdir(scopeRoot, perms); err != nil && os.IsExist(err) {
|
||||
if err := os.Remove(scopeRoot); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Mkdir(scopeRoot, perms); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Ftrace{
|
||||
ftraceRoot: root,
|
||||
root: scopeRoot,
|
||||
outstanding: map[int]*syscall{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (f *Ftrace) destroy() error {
|
||||
return os.Remove(f.root)
|
||||
}
|
||||
|
||||
func (f *Ftrace) enableTracing() error {
|
||||
// need to enable tracing at root to get trace_pipe to block in my instance. Weird
|
||||
if err := ioutil.WriteFile(path.Join(f.ftraceRoot, "tracing", "tracing_on"), []byte("1"), perms); err != nil {
|
||||
return err
|
||||
}
|
||||
return ioutil.WriteFile(path.Join(f.root, "tracing_on"), []byte("1"), perms)
|
||||
}
|
||||
|
||||
func (f *Ftrace) disableTracing() error {
|
||||
if err := ioutil.WriteFile(path.Join(f.root, "tracing_on"), []byte("0"), perms); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return ioutil.WriteFile(path.Join(f.ftraceRoot, "tracing", "tracing_on"), []byte("1"), perms)
|
||||
}
|
||||
|
||||
func (f *Ftrace) enableEvent(class, event string) error {
|
||||
return ioutil.WriteFile(path.Join(f.root, "events", class, event, "enable"), []byte("1"), perms)
|
||||
}
|
||||
|
||||
func mustAtoi(a string) int {
|
||||
i, err := strconv.Atoi(a)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func mustAtof(a string) float64 {
|
||||
i, err := strconv.ParseFloat(a, 64)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func (f *Ftrace) events(out chan<- *syscall) {
|
||||
file, err := os.Open(path.Join(f.root, "trace_pipe"))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
matches := lineMatcher.FindStringSubmatch(scanner.Text())
|
||||
if matches == nil {
|
||||
continue
|
||||
}
|
||||
pid := mustAtoi(matches[1])
|
||||
log := matches[4]
|
||||
|
||||
if enterMatches := enterMatcher.FindStringSubmatch(log); enterMatches != nil {
|
||||
name := enterMatches[1]
|
||||
args := map[string]string{}
|
||||
for _, arg := range argMatcher.FindAllStringSubmatch(enterMatches[2], -1) {
|
||||
args[arg[1]] = arg[2]
|
||||
}
|
||||
|
||||
s := &syscall{
|
||||
pid: pid,
|
||||
cpu: mustAtoi(matches[2]),
|
||||
ts: mustAtof(matches[3]),
|
||||
name: strings.TrimPrefix(name, "sys_"),
|
||||
args: args,
|
||||
}
|
||||
|
||||
f.outstanding[pid] = s
|
||||
|
||||
} else if exitMatches := exitMatcher.FindStringSubmatch(log); exitMatches != nil {
|
||||
s, ok := f.outstanding[pid]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
delete(f.outstanding, pid)
|
||||
returnCode, err := strconv.ParseUint(exitMatches[2], 0, 64)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
s.returnCode = int64(returnCode)
|
||||
out <- s
|
||||
} else {
|
||||
fmt.Printf("Unmatched: %s", log)
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Ftrace) start() error {
|
||||
for _, e := range []struct{ class, event string }{
|
||||
{"syscalls", "sys_enter_connect"},
|
||||
{"syscalls", "sys_exit_connect"},
|
||||
{"syscalls", "sys_enter_accept"},
|
||||
{"syscalls", "sys_exit_accept"},
|
||||
{"syscalls", "sys_enter_accept4"},
|
||||
{"syscalls", "sys_exit_accept4"},
|
||||
{"syscalls", "sys_enter_close"},
|
||||
{"syscalls", "sys_exit_close"},
|
||||
} {
|
||||
if err := f.enableEvent(e.class, e.event); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return f.enableTracing()
|
||||
}
|
||||
|
||||
func (f *Ftrace) stop() error {
|
||||
defer f.destroy()
|
||||
return f.disableTracing()
|
||||
}
|
||||
117
experimental/ftrace/main.go
Normal file
117
experimental/ftrace/main.go
Normal file
@@ -0,0 +1,117 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/bluele/gcache"
|
||||
)
|
||||
|
||||
const cacheSize = 500
|
||||
|
||||
// On every connect and accept, we lookup the local addr
|
||||
// As this is expensive, we cache the result
|
||||
var fdAddrCache = gcache.New(cacheSize).LRU().Expiration(15 * time.Second).Build()
|
||||
|
||||
type fdCacheKey struct {
|
||||
pid int
|
||||
fd int
|
||||
}
|
||||
type fdCacheValue struct {
|
||||
addr uint32
|
||||
port uint16
|
||||
}
|
||||
|
||||
func getCachedLocalAddr(pid, fd int) (uint32, uint16, error) {
|
||||
key := fdCacheKey{pid, fd}
|
||||
val, err := fdAddrCache.Get(key)
|
||||
if val != nil {
|
||||
return val.(fdCacheValue).addr, val.(fdCacheValue).port, nil
|
||||
}
|
||||
|
||||
addr, port, err := getLocalAddr(pid, fd)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
fdAddrCache.Set(key, fdCacheValue{addr, port})
|
||||
return addr, port, nil
|
||||
}
|
||||
|
||||
// On every connect or accept, we cache the syscall that caused
|
||||
// it for matching with a connection from conntrack
|
||||
var syscallCache = gcache.New(cacheSize).LRU().Expiration(15 * time.Second).Build()
|
||||
|
||||
type syscallCacheKey struct {
|
||||
localAddr uint32
|
||||
localPort uint16
|
||||
}
|
||||
type syscallCacheValue *syscall
|
||||
|
||||
// One ever conntrack connection, we cache it by local addr, port to match with
|
||||
// a future syscall
|
||||
var conntrackCache = gcache.New(cacheSize).LRU().Expiration(15 * time.Second).Build()
|
||||
|
||||
type conntrackCacheKey syscallCacheKey
|
||||
|
||||
// And keep a list of successfully matched connection, for us to close out
|
||||
// when we get the close syscall
|
||||
|
||||
func main() {
|
||||
ftrace, err := NewFtracer()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
ftrace.start()
|
||||
defer ftrace.stop()
|
||||
|
||||
syscalls := make(chan *syscall, 100)
|
||||
go ftrace.events(syscalls)
|
||||
|
||||
onConnection := func(s *syscall) {
|
||||
fdStr, ok := s.args["fd"]
|
||||
if !ok {
|
||||
panic("no pid")
|
||||
}
|
||||
fd64, err := strconv.ParseInt(fdStr, 32, 16)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fd := int(fd64)
|
||||
|
||||
addr, port, err := getCachedLocalAddr(s.pid, fd)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to get local addr for pid=%d fd=%d: %v\n", s.pid, fd, err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("%+v %d %d\n", s, addr, port)
|
||||
syscallCache.Set(syscallCacheKey{addr, port}, s)
|
||||
}
|
||||
|
||||
onAccept := func(s *syscall) {
|
||||
|
||||
}
|
||||
|
||||
onClose := func(s *syscall) {
|
||||
|
||||
}
|
||||
|
||||
fmt.Println("Started")
|
||||
|
||||
for {
|
||||
select {
|
||||
case s := <-syscalls:
|
||||
|
||||
switch s.name {
|
||||
case "connect":
|
||||
onConnection(s)
|
||||
case "accept", "accept4":
|
||||
onAccept(s)
|
||||
case "close":
|
||||
onClose(s)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
94
experimental/ftrace/proc.go
Normal file
94
experimental/ftrace/proc.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
const (
|
||||
socketPattern = `^socket:\[(\d+)\]$`
|
||||
tcpPattern = `^\s*(?P<fd>\d+): (?P<localaddr>[A-F0-9]{8}):(?P<localport>[A-F0-9]{4}) ` +
|
||||
`(?P<remoteaddr>[A-F0-9]{8}):(?P<remoteport>[A-F0-9]{4}) (?:[A-F0-9]{2}) (?:[A-F0-9]{8}):(?:[A-F0-9]{8}) ` +
|
||||
`(?:[A-F0-9]{2}):(?:[A-F0-9]{8}) (?:[A-F0-9]{8}) \s+(?:\d+) \s+(?:\d+) (?P<inode>\d+)`
|
||||
)
|
||||
|
||||
var (
|
||||
socketRegex = regexp.MustCompile(socketPattern)
|
||||
tcpRegexp = regexp.MustCompile(tcpPattern)
|
||||
)
|
||||
|
||||
func getLocalAddr(pid, fd int) (addr uint32, port uint16, err error) {
|
||||
var (
|
||||
socket string
|
||||
match []string
|
||||
inode int
|
||||
tcpFile *os.File
|
||||
scanner *bufio.Scanner
|
||||
candidate int
|
||||
port64 int64
|
||||
addr64 int64
|
||||
)
|
||||
|
||||
socket, err = os.Readlink(fmt.Sprintf("/proc/%d/fd/%d", pid, fd))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
match = socketRegex.FindStringSubmatch(socket)
|
||||
if match == nil {
|
||||
err = fmt.Errorf("Fd %d not a socket", fd)
|
||||
return
|
||||
}
|
||||
|
||||
inode, err = strconv.Atoi(match[1])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
tcpFile, err = os.Open(fmt.Sprintf("/proc/%d/net/tcp", pid))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer tcpFile.Close()
|
||||
|
||||
scanner = bufio.NewScanner(tcpFile)
|
||||
for scanner.Scan() {
|
||||
match = tcpRegexp.FindStringSubmatch(scanner.Text())
|
||||
if match == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
candidate, err = strconv.Atoi(match[6])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if candidate != inode {
|
||||
continue
|
||||
}
|
||||
|
||||
addr64, err = strconv.ParseInt(match[2], 16, 32)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
addr = uint32(addr64)
|
||||
|
||||
// use a 32 bit int for target, at the result is a uint16
|
||||
port64, err = strconv.ParseInt(match[3], 16, 32)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
port = uint16(port64)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err = scanner.Err(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
err = fmt.Errorf("Fd %d not found for proc %d", fd, pid)
|
||||
return
|
||||
}
|
||||
Reference in New Issue
Block a user