From e09774ba5a5b3f58cc24144701f5c88a775557bf Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Thu, 14 May 2015 15:42:21 +0000 Subject: [PATCH 01/23] Prototype to trace incoming and outgoing connections from containers. --- .gitignore | 2 + experimental/tracer/.gitignore | 3 + experimental/tracer/Makefile | 14 ++ experimental/tracer/README.md | 9 + experimental/tracer/main/Dockerfile | 6 + experimental/tracer/main/http.go | 79 +++++++ experimental/tracer/main/main.go | 54 +++++ experimental/tracer/main/store.go | 108 +++++++++ experimental/tracer/ptrace/fd.go | 165 ++++++++++++++ experimental/tracer/ptrace/process.go | 99 ++++++++ experimental/tracer/ptrace/ptracer.go | 314 ++++++++++++++++++++++++++ experimental/tracer/ptrace/thread.go | 253 +++++++++++++++++++++ experimental/tracer/tracer | 50 ++++ experimental/tracer/ui/index.html | 98 ++++++++ experimental/tracer/ui/sprintf.min.js | 4 + probe/docker/container.go | 6 + 16 files changed, 1264 insertions(+) create mode 100644 experimental/tracer/.gitignore create mode 100644 experimental/tracer/Makefile create mode 100644 experimental/tracer/README.md create mode 100644 experimental/tracer/main/Dockerfile create mode 100644 experimental/tracer/main/http.go create mode 100644 experimental/tracer/main/main.go create mode 100644 experimental/tracer/main/store.go create mode 100644 experimental/tracer/ptrace/fd.go create mode 100644 experimental/tracer/ptrace/process.go create mode 100644 experimental/tracer/ptrace/ptracer.go create mode 100644 experimental/tracer/ptrace/thread.go create mode 100755 experimental/tracer/tracer create mode 100644 experimental/tracer/ui/index.html create mode 100644 experimental/tracer/ui/sprintf.min.js diff --git a/.gitignore b/.gitignore index 427d5ddaf..9dcf0e17b 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,8 @@ experimental/genreport/genreport experimental/graphviz/graphviz experimental/oneshot/oneshot experimental/_integration/_integration +experimental/tracer/main/main +experimental/tracer/tracer.tar *sublime-project *sublime-workspace *npm-debug.log diff --git a/experimental/tracer/.gitignore b/experimental/tracer/.gitignore new file mode 100644 index 000000000..681fc6b1d --- /dev/null +++ b/experimental/tracer/.gitignore @@ -0,0 +1,3 @@ +main/main +tracer.tar +main/static.go diff --git a/experimental/tracer/Makefile b/experimental/tracer/Makefile new file mode 100644 index 000000000..8310d7067 --- /dev/null +++ b/experimental/tracer/Makefile @@ -0,0 +1,14 @@ +tracer.tar: main/main main/Dockerfile + docker build -t weaveworks/tracer main/ + docker save weaveworks/tracer:latest >$@ + +main/main: main/*.go main/static.go ptrace/*.go + go get -tags netgo ./$(@D) + go build -ldflags "-extldflags \"-static\" -X main.version $(SCOPE_VERSION)" -tags netgo -o $@ ./$(@D) + +main/static.go: ui/* + esc -o main/static.go -prefix ui ui + +clean: + go clean ./.. + rm -f main/static.go tracer.tar main/main diff --git a/experimental/tracer/README.md b/experimental/tracer/README.md new file mode 100644 index 000000000..7a077cee8 --- /dev/null +++ b/experimental/tracer/README.md @@ -0,0 +1,9 @@ +Run tracer: +- make +- ./tracer.sh start + +TODO: +- need to stich traces together +- deal with persistant connections +- make it work for goroutines +- test with jvm based app diff --git a/experimental/tracer/main/Dockerfile b/experimental/tracer/main/Dockerfile new file mode 100644 index 000000000..c9555cb51 --- /dev/null +++ b/experimental/tracer/main/Dockerfile @@ -0,0 +1,6 @@ +FROM gliderlabs/alpine +MAINTAINER Weaveworks Inc +WORKDIR /home/weave +COPY ./main /home/weave/ +EXPOSE 4050 +ENTRYPOINT ["/home/weave/main"] diff --git a/experimental/tracer/main/http.go b/experimental/tracer/main/http.go new file mode 100644 index 000000000..414de7d4c --- /dev/null +++ b/experimental/tracer/main/http.go @@ -0,0 +1,79 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "strconv" + + "github.com/gorilla/mux" + dockerClient "github.com/fsouza/go-dockerclient" + + "github.com/weaveworks/scope/probe/docker" +) + +func respondWith(w http.ResponseWriter, code int, response interface{}) { + w.Header().Set("Content-Type", "application/json") + w.Header().Add("Cache-Control", "no-cache") + w.WriteHeader(code) + if err := json.NewEncoder(w).Encode(response); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + log.Printf("Error handling http request: %v", err.Error()) + } +} + +func (t *tracer) http(port int) { + router := mux.NewRouter() + + router.Methods("GET").Path("/containers").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + containers := []*dockerClient.Container{} + t.docker.WalkContainers(func(container docker.Container) { + containers = append(containers, container.Container()) + }) + + respondWith(w, http.StatusOK, containers) + }) + + router.Methods("GET").Path("/pid").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + respondWith(w, http.StatusOK, t.ptrace.AttachedPIDs()) + }) + + router.Methods("POST").Path("/pid/{pid:\\d+}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + pid, err := strconv.Atoi(mux.Vars(r)["pid"]) + if err != nil { + respondWith(w, http.StatusBadRequest, err.Error()) + return + } + + t.ptrace.TraceProcess(pid) + w.WriteHeader(204) + }) + + router.Methods("DELETE").Path("/pid/{pid:\\d+}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + pid, err := strconv.Atoi(mux.Vars(r)["pid"]) + if err != nil { + respondWith(w, http.StatusBadRequest, err.Error()) + return + } + + t.ptrace.StopTracing(pid) + w.WriteHeader(204) + }) + + router.Methods("GET").Path("/trace").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + respondWith(w, http.StatusOK, t.store.Traces()) + }) + + router.Methods("GET").PathPrefix("/").Handler(http.FileServer(FS(false))) // everything else is static + + log.Printf("Launching HTTP API on port %d", port) + srv := &http.Server{ + Addr: fmt.Sprintf(":%d", port), + Handler: router, + } + + if err := srv.ListenAndServe(); err != nil { + log.Printf("Unable to create http listener: %v", err) + } +} diff --git a/experimental/tracer/main/main.go b/experimental/tracer/main/main.go new file mode 100644 index 000000000..2d6675245 --- /dev/null +++ b/experimental/tracer/main/main.go @@ -0,0 +1,54 @@ +package main + +import ( + "log" + _ "net/http/pprof" + "os" + "os/signal" + "runtime" + "syscall" + "time" + + "github.com/weaveworks/scope/experimental/tracer/ptrace" + "github.com/weaveworks/scope/probe/docker" +) + +const ( + procRoot = "/proc" + pollInterval = 10 * time.Second +) + +type tracer struct { + ptrace ptrace.PTracer + store *store + docker docker.Registry +} + +func main() { + dockerRegistry, err := docker.NewRegistry(pollInterval) + if err != nil { + log.Fatalf("Could start docker watcher: %v", err) + } + + tracer := tracer{ + ptrace: ptrace.NewPTracer(), + store: newStore(), + docker: dockerRegistry, + } + go tracer.http(6060) + handleSignals() +} + +func handleSignals() { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGQUIT) + buf := make([]byte, 1<<20) + for { + sig := <-sigs + switch sig { + case syscall.SIGQUIT: + stacklen := runtime.Stack(buf, true) + log.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf[:stacklen]) + } + } +} diff --git a/experimental/tracer/main/store.go b/experimental/tracer/main/store.go new file mode 100644 index 000000000..beb2875b5 --- /dev/null +++ b/experimental/tracer/main/store.go @@ -0,0 +1,108 @@ +package main + +import ( + "math/rand" + "sync" + + "github.com/msackman/skiplist" + + "github.com/weaveworks/scope/experimental/tracer/ptrace" +) + +const epsilon = int64(5) + +// Traces are indexed by from addr, from port, and start time. +type key struct { + fromAddr uint32 + fromPort uint16 + startTime int64 +} + +type trace struct { + pid int + root *ptrace.Fd + children []*trace +} + +type store struct { + sync.RWMutex + traces *skiplist.SkipList +} + +func newKey(fd *ptrace.Fd) key { + var fromAddr uint32 + for _, b := range fd.FromAddr.To4() { + fromAddr <<= 8 + fromAddr |= uint32(b) + } + + return key{fromAddr, fd.FromPort, fd.Start} +} + +func (l key) LessThan(other skiplist.Comparable) bool { + r := other.(key) + return l.fromAddr < r.fromAddr && l.fromPort < r.fromPort && l.startTime < r.startTime +} + +func (l key) Equal(other skiplist.Comparable) bool { + r := other.(key) + if l.fromAddr != r.fromAddr || l.fromPort != r.fromPort { + return false + } + + diff := l.startTime - r.startTime + return -epsilon < diff && diff < epsilon +} + +func newStore() *store { + return &store{traces: skiplist.New(rand.New(rand.NewSource(0)))} +} + +func (s *store) RecordConnection(pid int, connection *ptrace.Fd) { + s.Lock() + defer s.Unlock() + + newTrace := &trace{pid: pid, root: connection} + newTraceKey := newKey(connection) + + // First, see if this new conneciton is a child of an existing connection. + // This indicates we have a parent connection to attach to. + // If not, insert this connection. + if parentNode := s.traces.Get(newTraceKey); parentNode != nil { + parentNode.Remove() + parentTrace := parentNode.Value.(*trace) + parentTrace.children = append(parentTrace.children, newTrace) + } else { + s.traces.Insert(newTraceKey, newTrace) + } + + // Next, see if we already know about the child connections + // If not, insert each of our children. + for _, childConnection := range connection.Children { + childTraceKey := newKey(childConnection) + + if childNode := s.traces.Get(childTraceKey); childNode != nil { + childNode.Remove() + childTrace := childNode.Value.(*trace) + newTrace.children = append(newTrace.children, childTrace) + } else { + s.traces.Insert(childTraceKey, newTrace) + } + } +} + +func (s *store) Traces() []*trace { + s.RLock() + defer s.RUnlock() + + var traces []*trace + var cur = s.traces.First() + for { + traces = append(traces, cur.Value.(*trace)) + cur = cur.Next() + if cur == nil { + break + } + } + return traces +} diff --git a/experimental/tracer/ptrace/fd.go b/experimental/tracer/ptrace/fd.go new file mode 100644 index 000000000..05138f4d0 --- /dev/null +++ b/experimental/tracer/ptrace/fd.go @@ -0,0 +1,165 @@ +package ptrace + +import ( + "bufio" + "encoding/hex" + "fmt" + "net" + "os" + "regexp" + "strconv" + "time" +) + +const ( + listening = iota + incoming + outgoing +) + +const ( + socketPattern = `^socket:\[(\d+)\]$` + tcpPattern = `^\s*(?P\d+): (?P[A-F0-9]{8}):(?P[A-F0-9]{4}) ` + + `(?P[A-F0-9]{8}):(?P[A-F0-9]{4}) (?:[A-F0-9]{2}) (?:[A-F0-9]{8}):(?:[A-F0-9]{8}) ` + + `(?:[A-F0-9]{2}):(?:[A-F0-9]{8}) (?:[A-F0-9]{8}) \s+(?:\d+) \s+(?:\d+) (?P\d+)` +) + +var ( + socketRegex = regexp.MustCompile(socketPattern) + tcpRegexp = regexp.MustCompile(tcpPattern) +) + +// Fd represents a connect and subsequent connections caused by it. +type Fd struct { + direction int + fd int + + Start int64 + stop int64 + sent int64 + received int64 + + FromAddr net.IP + FromPort uint16 + toAddr net.IP + toPort uint16 + + // Fds are connections, and can have a causal-link to other Fds + Children []*Fd +} + +func getLocalAddr(pid, fd int) (addr net.IP, port uint16, err error) { + var ( + socket string + match []string + inode int + tcpFile *os.File + scanner *bufio.Scanner + candidate int + port64 int64 + ) + + socket, err = os.Readlink(fmt.Sprintf("/proc/%d/fd/%d", pid, fd)) + if err != nil { + return + } + + match = socketRegex.FindStringSubmatch(socket) + if match == nil { + err = fmt.Errorf("Fd %d not a socket", fd) + return + } + + inode, err = strconv.Atoi(match[1]) + if err != nil { + return + } + + tcpFile, err = os.Open(fmt.Sprintf("/proc/%d/net/tcp", pid)) + if err != nil { + return + } + defer tcpFile.Close() + + scanner = bufio.NewScanner(tcpFile) + for scanner.Scan() { + match = tcpRegexp.FindStringSubmatch(scanner.Text()) + if match == nil { + continue + } + + candidate, err = strconv.Atoi(match[6]) + if err != nil { + return + } + if candidate != inode { + continue + } + + addr = make([]byte, 4) + if _, err = hex.Decode(addr, []byte(match[2])); err != nil { + return + } + addr[0], addr[1], addr[2], addr[3] = addr[3], addr[2], addr[1], addr[0] + + // use a 32 bit int for target, at the result is a uint16 + port64, err = strconv.ParseInt(match[3], 16, 32) + if err != nil { + return + } + port = uint16(port64) + + return + } + + if err = scanner.Err(); err != nil { + return + } + + err = fmt.Errorf("Fd %d not found for proc %d", fd, pid) + return +} + +// We want to get the listening address from /proc +func newListeningFd(pid, fd int) (*Fd, error) { + localAddr, localPort, err := getLocalAddr(pid, fd) + if err != nil { + return nil, err + } + + return &Fd{ + direction: listening, fd: fd, Start: time.Now().Unix(), + toAddr: localAddr, toPort: uint16(localPort), + }, nil +} + +// We intercepted a connect syscall +func newConnectionFd(pid, fd int, remoteAddr net.IP, remotePort uint16) (*Fd, error) { + localAddr, localPort, err := getLocalAddr(pid, fd) + if err != nil { + return nil, err + } + + return &Fd{ + direction: outgoing, fd: fd, Start: time.Now().Unix(), + FromAddr: localAddr, FromPort: uint16(localPort), + toAddr: remoteAddr, toPort: remotePort, + }, nil +} + +// We got a new connection on a listening socket +func (fd *Fd) newConnection(addr net.IP, port uint16, newFd int) (*Fd, error) { + if fd.direction != listening { + return nil, fmt.Errorf("New connection on non-listening fd!") + } + + return &Fd{ + direction: incoming, fd: newFd, Start: time.Now().Unix(), + toAddr: fd.toAddr, toPort: fd.toPort, + FromAddr: addr, FromPort: port, + }, nil +} + +func (fd *Fd) close() { + fd.stop = time.Now().Unix() +} diff --git a/experimental/tracer/ptrace/process.go b/experimental/tracer/ptrace/process.go new file mode 100644 index 000000000..6add841c7 --- /dev/null +++ b/experimental/tracer/ptrace/process.go @@ -0,0 +1,99 @@ +package ptrace + +import ( + "fmt" + "io/ioutil" + "log" + "strconv" + "sync" +) + +type process struct { + sync.Mutex + pid int + + detaching bool + detached chan struct{} + + tracer *PTracer + threads map[int]*thread + fds map[int]*Fd +} + +func newProcess(pid int, tracer *PTracer) *process { + return &process{ + pid: pid, + tracer: tracer, + threads: make(map[int]*thread), + fds: make(map[int]*Fd), + detached: make(chan struct{}), + } +} + +func (p *process) trace() { + go p.loop() +} + +// This doesn't actually guarantees we follow all the threads. Oops. +func (p *process) loop() { + var ( + attached int + ) + log.Printf("Tracing process %d", p.pid) + + for { + ps, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/task", p.pid)) + if err != nil { + log.Printf("ReadDir failed, pid=%d, err=%v", p.pid, err) + return + } + + attached = 0 + for _, file := range ps { + pid, err := strconv.Atoi(file.Name()) + if err != nil { + log.Printf("'%s' is not a pid: %v", file.Name(), err) + attached++ + continue + } + + p.Lock() + t, ok := p.threads[pid] + if !ok { + t = p.tracer.traceThread(pid, p) + p.threads[pid] = t + } + p.Unlock() + + if !t.attached { + continue + } + + attached++ + } + + // When we successfully attach to all threads + // we can be sure to catch new clones, so we + // can quit. + if attached == len(ps) { + break + } + } + + log.Printf("Successfully attached to %d threads", attached) +} + +func (p *process) newThread(thread *thread) { + p.Lock() + defer p.Unlock() + p.threads[thread.tid] = thread +} + +func (p *process) newFd(fd *Fd) error { + _, ok := p.fds[fd.fd] + if ok { + return fmt.Errorf("New fd %d, alread exists!", fd.fd) + } + p.fds[fd.fd] = fd + return nil +} diff --git a/experimental/tracer/ptrace/ptracer.go b/experimental/tracer/ptrace/ptracer.go new file mode 100644 index 000000000..431a0bf61 --- /dev/null +++ b/experimental/tracer/ptrace/ptracer.go @@ -0,0 +1,314 @@ +package ptrace + +import ( + "bufio" + "fmt" + "log" + "os" + "runtime" + "strconv" + "strings" + "syscall" +) + +const ( + ptraceOptions = syscall.PTRACE_O_TRACESYSGOOD | syscall.PTRACE_O_TRACECLONE + ptraceTracesysgoodBit = 0x80 +) + +// PTracer ptrace processed and threads +type PTracer struct { + + // All ptrace calls must come from the + // same thread. So we wait on a separate + // thread. + ops chan func() + stopped chan stopped + quit chan struct{} + childAttached chan struct{} //used to signal the wait loop + + threads map[int]*thread + processes map[int]*process +} + +type stopped struct { + pid int + status syscall.WaitStatus +} + +// NewPTracer creates a new ptracer. +func NewPTracer() PTracer { + t := PTracer{ + ops: make(chan func()), + stopped: make(chan stopped), + quit: make(chan struct{}), + childAttached: make(chan struct{}), + + threads: make(map[int]*thread), + processes: make(map[int]*process), + } + go t.waitLoop() + go t.loop() + return t +} + +// TraceProcess starts tracing the given pid +func (t *PTracer) TraceProcess(pid int) *process { + result := make(chan *process) + t.ops <- func() { + process := newProcess(pid, t) + t.processes[pid] = process + process.trace() + result <- process + } + return <-result +} + +// StopTracing stops tracing all threads for the given pid +func (t *PTracer) StopTracing(pid int) error { + log.Printf("Detaching from %d", pid) + + errors := make(chan error) + processes := make(chan *process) + + t.ops <- func() { + // send sigstop to all threads + process, ok := t.processes[pid] + if !ok { + errors <- fmt.Errorf("PID %d not found", pid) + return + } + + // This flag tells the thread to detach when it next stops + process.detaching = true + + // Now send sigstop to all threads. + for _, thread := range process.threads { + log.Printf("sending SIGSTOP to %d", thread.tid) + if err := syscall.Tgkill(pid, thread.tid, syscall.SIGSTOP); err != nil { + errors <- err + return + } + } + + processes <- process + } + + select { + case err := <-errors: + return err + case process := <-processes: + <-process.detached + return nil + } +} + +// AttachedPIDs list the currently attached processes. +func (t *PTracer) AttachedPIDs() []int { + result := make(chan []int) + t.ops <- func() { + var pids []int + for pid := range t.processes { + pids = append(pids, pid) + } + result <- pids + } + return <-result +} + +func (t *PTracer) traceThread(pid int, process *process) *thread { + result := make(chan *thread) + t.ops <- func() { + thread := newThread(pid, process, t) + + err := syscall.PtraceAttach(pid) + if err != nil { + log.Printf("Attach %d failed: %v", pid, err) + return + } + + var status syscall.WaitStatus + if _, err = syscall.Wait4(pid, &status, 0, nil); err != nil { + log.Printf("Wait %d failed: %v", pid, err) + return + } + + thread.attached = true + + err = syscall.PtraceSetOptions(pid, ptraceOptions) + if err != nil { + log.Printf("SetOptions failed, pid=%d, err=%v", pid, err) + return + } + + err = syscall.PtraceSyscall(pid, 0) + if err != nil { + log.Printf("PtraceSyscall failed, pid=%d, err=%v", pid, err) + return + } + + t.threads[pid] = thread + result <- thread + t.childAttached <- struct{}{} + } + return <-result +} + +func (t *PTracer) waitLoop() { + var ( + status syscall.WaitStatus + pid int + err error + ) + + for { + pid, err = syscall.Wait4(-1, &status, syscall.WALL, nil) + if err != nil && err.(syscall.Errno) == syscall.ECHILD { + log.Printf("No children to wait4") + <-t.childAttached + continue + } + + if err != nil { + log.Printf("Wait failed: %v %d", err, err.(syscall.Errno)) + return + } + + log.Printf("PID %d stopped", pid) + t.stopped <- stopped{pid, status} + } +} + +func (t *PTracer) loop() { + runtime.LockOSThread() + + for { + select { + case op := <-t.ops: + op() + case stopped := <-t.stopped: + t.handleStopped(stopped.pid, stopped.status) + case <-t.quit: + return + } + } +} + +func (t *PTracer) handleStopped(pid int, status syscall.WaitStatus) { + signal := syscall.Signal(0) + target, err := t.thread(pid) + if err != nil { + log.Printf("thread failed: %v", err) + return + } + + if status.Stopped() && status.StopSignal() == syscall.SIGTRAP|ptraceTracesysgoodBit { + // pid entered Syscall-enter-stop or syscall-exit-stop + target.syscallStopped() + } else if status.Stopped() && status.StopSignal() == syscall.SIGTRAP { + // pid entered PTRACE_EVENT stop + switch status.TrapCause() { + case syscall.PTRACE_EVENT_CLONE: + err := target.handleClone(pid) + if err != nil { + log.Printf("clone failed: %v", err) + return + } + default: + log.Printf("Unknown PTRACE_EVENT %d for pid %d", status.TrapCause(), pid) + } + } else if status.Exited() || status.Signaled() { + // "tracer can safely assume pid will exit" + t.threadExited(target) + return + } else if status.Stopped() { + // tracee recieved a non-trace related signal + signal = status.StopSignal() + + if signal == syscall.SIGSTOP && target.process.detaching { + t.detachThread(target) + return + } + } else { + // unknown stop - shouldn't happen! + log.Printf("Pid %d random stop with status %x", pid, status) + } + + // Restart stopped caller in syscall trap mode. + err = syscall.PtraceSyscall(pid, int(signal)) + if err != nil { + log.Printf("PtraceSyscall failed, pid=%d, err=%v", pid, err) + } +} + +func (t *PTracer) detachThread(thread *thread) { + syscall.PtraceDetach(thread.tid) + process := thread.process + delete(process.threads, thread.tid) + delete(t.threads, thread.tid) + if len(process.threads) == 0 { + delete(t.processes, process.pid) + close(process.detached) + log.Printf("Process %d detached", process.pid) + } +} + +func pidForTid(tid int) (pid int, err error) { + var ( + status *os.File + scanner *bufio.Scanner + splits []string + ) + + status, err = os.Open(fmt.Sprintf("/proc/%d/status", tid)) + if err != nil { + return + } + defer status.Close() + + scanner = bufio.NewScanner(status) + for scanner.Scan() { + splits = strings.Split(scanner.Text(), ":") + if splits[0] != "Tgid" { + continue + } + + pid, err = strconv.Atoi(strings.TrimSpace(splits[1])) + return + } + + if err = scanner.Err(); err != nil { + return + } + + err = fmt.Errorf("Pid not found for proc %d", tid) + return +} + +func (t *PTracer) thread(tid int) (*thread, error) { + thread, ok := t.threads[tid] + if !ok { + pid, err := pidForTid(tid) + if err != nil { + return nil, err + } + + proc, ok := t.processes[pid] + if !ok { + return nil, fmt.Errorf("Got new thread %d for unknown process", tid) + } + + thread = newThread(tid, proc, t) + t.threads[tid] = thread + log.Printf("New thread reported, tid=%d, pid=%d", tid, pid) + } + return thread, nil +} + +func (t *PTracer) threadExited(thread *thread) { + thread.handleExit() + delete(t.threads, thread.tid) + if thread.process != nil { + delete(thread.process.threads, thread.tid) + } +} diff --git a/experimental/tracer/ptrace/thread.go b/experimental/tracer/ptrace/thread.go new file mode 100644 index 000000000..f14dbe005 --- /dev/null +++ b/experimental/tracer/ptrace/thread.go @@ -0,0 +1,253 @@ +package ptrace + +import ( + "fmt" + "log" + "net" + "syscall" + "unsafe" +) + +// Syscall numbers +const ( + READ = 0 + WRITE = 1 + CLOSE = 3 + MMAP = 9 + MPROTECT = 10 + MADVISE = 28 + SOCKET = 41 + CONNECT = 42 + ACCEPT = 43 + SENDTO = 44 + RECVFROM = 45 + CLONE = 56 + GETID = 186 + SETROBUSTLIST = 273 + ACCEPT4 = 288 +) + +// States for a given thread +const ( + NORMAL = iota + INSYSCALL +) + +type thread struct { + tid int + attached bool + process *process // might be nil! + tracer *PTracer + + state int + callRegs syscall.PtraceRegs + resultRegs syscall.PtraceRegs + + currentConnection *Fd +} + +func newThread(pid int, process *process, tracer *PTracer) *thread { + t := &thread{ + tid: pid, + process: process, + tracer: tracer, + } + return t +} + +// trace thread calls this +func (t *thread) syscallStopped() { + var err error + + if t.state == NORMAL { + if err = syscall.PtraceGetRegs(t.tid, &t.callRegs); err != nil { + t.logf("GetRegs failed, pid=%d, err=%v", t.tid, err) + } + t.state = INSYSCALL + return + } + + t.state = NORMAL + + if err = syscall.PtraceGetRegs(t.tid, &t.resultRegs); err != nil { + t.logf("GetRegs failed, pid=%d, err=%v", t.tid, err) + return + } + + if t.process == nil { + t.logf("Got syscall, but don't know parent process!") + return + } + + switch t.callRegs.Orig_rax { + case ACCEPT, ACCEPT4: + t.handleAccept(&t.callRegs, &t.resultRegs) + + case CLOSE: + t.handleClose(&t.callRegs, &t.resultRegs) + + case CONNECT: + t.handleConnect(&t.callRegs, &t.resultRegs) + + case READ, WRITE, RECVFROM, SENDTO: + t.handleIO(&t.callRegs, &t.resultRegs) + + // we can ignore these syscalls + case SETROBUSTLIST, GETID, MMAP, MPROTECT, MADVISE, SOCKET, CLONE: + return + + default: + t.logf("syscall(%d)", t.callRegs.Orig_rax) + } +} + +func (t *thread) getSocketAddress(ptr uintptr) (addr net.IP, port uint16, err error) { + var ( + buf = make([]byte, syscall.SizeofSockaddrAny) + read int + ) + + if ptr == 0 { + err = fmt.Errorf("Null ptr") + return + } + + read, err = syscall.PtracePeekData(t.tid, ptr, buf) + if read != syscall.SizeofSockaddrAny || err != nil { + return + } + + var sockaddr4 = (*syscall.RawSockaddrInet4)(unsafe.Pointer(&buf[0])) + if sockaddr4.Family != syscall.AF_INET { + return + } + + addr = net.IP(sockaddr4.Addr[0:]) + port = sockaddr4.Port + return +} + +func (t *thread) handleAccept(call, result *syscall.PtraceRegs) { + var ( + err error + ok bool + listeningFdNum int + connectionFdNum int + addrPtr uintptr + addr net.IP + port uint16 + listeningFd *Fd + connection *Fd + ) + + listeningFdNum = int(result.Rdi) + connectionFdNum = int(result.Rax) + addrPtr = uintptr(result.Rsi) + addr, port, err = t.getSocketAddress(addrPtr) + if err != nil { + t.logf("failed to read sockaddr: %v", err) + return + } + + listeningFd, ok = t.process.fds[listeningFdNum] + if !ok { + listeningFd, err = newListeningFd(t.process.pid, listeningFdNum) + if err != nil { + t.logf("Failed to read listening port: %v", err) + return + } + t.process.fds[listeningFdNum] = listeningFd + } + + connection, err = listeningFd.newConnection(addr, port, connectionFdNum) + if err != nil { + t.logf("Failed to create connection fd: %v", err) + return + } + + t.process.newFd(connection) + + t.logf("Accepted connection from %s:%d -> %s:%d on fd %d, new fd %d", + addr, port, connection.toAddr, connection.toPort, listeningFdNum, connectionFdNum) +} + +func (t *thread) handleConnect(call, result *syscall.PtraceRegs) { + fd := int(result.Rdi) + ptr := result.Rsi + addr, port, err := t.getSocketAddress(uintptr(ptr)) + if err != nil { + t.logf("failed to read sockaddr: %v", err) + return + } + + connection, err := newConnectionFd(t.process.pid, fd, addr, port) + if err != nil { + t.logf("Failed to create connection fd: %v", err) + return + } + + t.process.newFd(connection) + if t.currentConnection != nil { + t.currentConnection.Children = append(t.currentConnection.Children, connection) + } + + t.logf("Made connection from %s:%d -> %s:%d on fd %d", + connection.toAddr, connection.toPort, connection.FromAddr, + connection.FromPort, fd) +} + +func (t *thread) handleClose(call, result *syscall.PtraceRegs) { + fdNum := int(call.Rdi) + fd, ok := t.process.fds[fdNum] + if !ok { + t.logf("Got close unknown fd %d", fdNum) + return + } + + fd.close() + delete(t.process.fds, fdNum) + + // clear current connection if we just closed it! + if t.currentConnection != nil && t.currentConnection.fd == fdNum { + t.currentConnection = nil + } + + // if this connection was incoming, add it to 'the registry' + //if fd.direction == incoming { + // t.tracer.store.RecordConnection(t.process.pid, fd) + //} +} + +func (t *thread) handleIO(call, result *syscall.PtraceRegs) { + fdNum := int(call.Rdi) + fd, ok := t.process.fds[fdNum] + if !ok { + t.logf("IO on unknown fd %d", fdNum) + return + } + + if fd.direction == incoming { + t.logf("IO on incoming connection %d; setting affinity", fdNum) + t.currentConnection = fd + } +} + +func (t *thread) handleClone(pid int) error { + // We can't use the pid in the process, as it may be in a namespace + newPid, err := syscall.PtraceGetEventMsg(pid) + if err != nil { + log.Printf("PtraceGetEventMsg failed: %v", err) + return err + } + + t.logf("New thread clone'd, pid=%d", newPid) + return nil +} + +func (t *thread) handleExit() { + t.logf("Exiting") +} + +func (t *thread) logf(fmt string, args ...interface{}) { + log.Printf("[thread %d] "+fmt, append([]interface{}{t.tid}, args...)...) +} diff --git a/experimental/tracer/tracer b/experimental/tracer/tracer new file mode 100755 index 000000000..4fba8b179 --- /dev/null +++ b/experimental/tracer/tracer @@ -0,0 +1,50 @@ +#!/bin/bash + +set -eu + +usage() { + echo "$0" +} + +PORT=6060 +CONTAINER_NAME=weavetracer + +[ $# -gt 0 ] || usage +COMMAND=$1 +shift 1 + +case "$COMMAND" in + + launch) + docker run --privileged --net=host --pid=host -d -v /var/run/docker.sock:/var/run/docker.sock \ + --name $CONTAINER_NAME weaveworks/tracer + for ip in $(hostname -I); do + echo http://$ip:6060 + done + ;; + + stop) + docker stop $CONTAINER_NAME || true + docker rm $CONTAINER_NAME >/dev/null || true + ;; + + attach) + PID=$1 + if [ -z "${PID##*[!0-9]*}" ]; then + PID=$(pgrep $PID) + fi + curl -X POST http://localhost:$PORT/pid/$PID + ;; + + detach) + PID=$1 + if [ -z "${PID##*[!0-9]*}" ]; then + PID=$(pgrep $PID) + fi + curl -X DELETE http://localhost:$PORT/pid/$PID + ;; + + traces) + curl http://localhost:$PORT/trace + ;; +esac diff --git a/experimental/tracer/ui/index.html b/experimental/tracer/ui/index.html new file mode 100644 index 000000000..141fc38da --- /dev/null +++ b/experimental/tracer/ui/index.html @@ -0,0 +1,98 @@ + + + + Weave Tracer + + + + + + + + + + + + + + + + + + + + + +
+
+

Weave Tracer

+
    +
+
+ +
+ +
+
+ + diff --git a/experimental/tracer/ui/sprintf.min.js b/experimental/tracer/ui/sprintf.min.js new file mode 100644 index 000000000..1cbc6b6eb --- /dev/null +++ b/experimental/tracer/ui/sprintf.min.js @@ -0,0 +1,4 @@ +/*! sprintf-js | Alexandru Marasteanu (http://alexei.ro/) | BSD-3-Clause */ + +!function(a){function b(){var a=arguments[0],c=b.cache;return c[a]&&c.hasOwnProperty(a)||(c[a]=b.parse(a)),b.format.call(null,c[a],arguments)}function c(a){return Object.prototype.toString.call(a).slice(8,-1).toLowerCase()}function d(a,b){return Array(b+1).join(a)}var e={not_string:/[^s]/,number:/[dief]/,json:/[j]/,not_json:/[^j]/,text:/^[^\x25]+/,modulo:/^\x25{2}/,placeholder:/^\x25(?:([1-9]\d*)\$|\(([^\)]+)\))?(\+)?(0|'[^$])?(-)?(\d+)?(?:\.(\d+))?([b-fijosuxX])/,key:/^([a-z_][a-z_\d]*)/i,key_access:/^\.([a-z_][a-z_\d]*)/i,index_access:/^\[(\d+)\]/,sign:/^[\+\-]/};b.format=function(a,f){var g,h,i,j,k,l,m,n=1,o=a.length,p="",q=[],r=!0,s="";for(h=0;o>h;h++)if(p=c(a[h]),"string"===p)q[q.length]=a[h];else if("array"===p){if(j=a[h],j[2])for(g=f[n],i=0;i=0),j[8]){case"b":g=g.toString(2);break;case"c":g=String.fromCharCode(g);break;case"d":case"i":g=parseInt(g,10);break;case"j":g=JSON.stringify(g,null,j[6]?parseInt(j[6]):0);break;case"e":g=j[7]?g.toExponential(j[7]):g.toExponential();break;case"f":g=j[7]?parseFloat(g).toFixed(j[7]):parseFloat(g);break;case"o":g=g.toString(8);break;case"s":g=(g=String(g))&&j[7]?g.substring(0,j[7]):g;break;case"u":g>>>=0;break;case"x":g=g.toString(16);break;case"X":g=g.toString(16).toUpperCase()}e.json.test(j[8])?q[q.length]=g:(!e.number.test(j[8])||r&&!j[3]?s="":(s=r?"+":"-",g=g.toString().replace(e.sign,"")),l=j[4]?"0"===j[4]?"0":j[4].charAt(1):" ",m=j[6]-(s+g).length,k=j[6]&&m>0?d(l,m):"",q[q.length]=j[5]?s+g+k:"0"===l?s+k+g:k+s+g)}return q.join("")},b.cache={},b.parse=function(a){for(var b=a,c=[],d=[],f=0;b;){if(null!==(c=e.text.exec(b)))d[d.length]=c[0];else if(null!==(c=e.modulo.exec(b)))d[d.length]="%";else{if(null===(c=e.placeholder.exec(b)))throw new SyntaxError("[sprintf] unexpected placeholder");if(c[2]){f|=1;var g=[],h=c[2],i=[];if(null===(i=e.key.exec(h)))throw new SyntaxError("[sprintf] failed to parse named argument key");for(g[g.length]=i[1];""!==(h=h.substring(i[0].length));)if(null!==(i=e.key_access.exec(h)))g[g.length]=i[1];else{if(null===(i=e.index_access.exec(h)))throw new SyntaxError("[sprintf] failed to parse named argument key");g[g.length]=i[1]}c[2]=g}else f|=2;if(3===f)throw new Error("[sprintf] mixing positional and named placeholders is not (yet) supported");d[d.length]=c}b=b.substring(c[0].length)}return d};var f=function(a,c,d){return d=(c||[]).slice(0),d.splice(0,0,a),b.apply(null,d)};"undefined"!=typeof exports?(exports.sprintf=b,exports.vsprintf=f):(a.sprintf=b,a.vsprintf=f,"function"==typeof define&&define.amd&&define(function(){return{sprintf:b,vsprintf:f}}))}("undefined"==typeof window?this:window); +//# sourceMappingURL=sprintf.min.js.map \ No newline at end of file diff --git a/probe/docker/container.go b/probe/docker/container.go index 5823e3739..8fadb4b3e 100644 --- a/probe/docker/container.go +++ b/probe/docker/container.go @@ -87,6 +87,8 @@ type Container interface { State() string HasTTY() bool + GetNode() report.Node + Container() *docker.Container StartGatheringStats() error StopGatheringStats() } @@ -146,6 +148,10 @@ func (c *container) State() string { return StateStopped } +func (c *container) Container() *docker.Container { + return c.container +} + func (c *container) StartGatheringStats() error { c.Lock() defer c.Unlock() From 31cadac623476047f0dc1f1aa2c0c09eb1a63dbf Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Wed, 16 Sep 2015 09:01:09 +0000 Subject: [PATCH 02/23] Make it stop --- experimental/tracer/main/main.go | 39 ++++++++++++++++++--------- experimental/tracer/ptrace/ptracer.go | 28 ++++++++++++++++--- experimental/tracer/tracer | 5 +--- experimental/tracer/ui/index.html | 35 +++++++++++++++++------- 4 files changed, 77 insertions(+), 30 deletions(-) diff --git a/experimental/tracer/main/main.go b/experimental/tracer/main/main.go index 2d6675245..6e3b07d34 100644 --- a/experimental/tracer/main/main.go +++ b/experimental/tracer/main/main.go @@ -24,6 +24,13 @@ type tracer struct { docker docker.Registry } +func (t *tracer) Stop() { + log.Printf("Shutting down...") + t.ptrace.Stop() + t.docker.Stop() + log.Printf("Done.") +} + func main() { dockerRegistry, err := docker.NewRegistry(pollInterval) if err != nil { @@ -35,20 +42,28 @@ func main() { store: newStore(), docker: dockerRegistry, } + defer tracer.Stop() + go tracer.http(6060) - handleSignals() + <-handleSignals() } -func handleSignals() { - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGQUIT) - buf := make([]byte, 1<<20) - for { - sig := <-sigs - switch sig { - case syscall.SIGQUIT: - stacklen := runtime.Stack(buf, true) - log.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf[:stacklen]) +func handleSignals() chan struct{} { + quit := make(chan struct{}, 10) + go func() { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGQUIT, syscall.SIGINT, syscall.SIGTERM) + buf := make([]byte, 1<<20) + for { + sig := <-sigs + switch sig { + case syscall.SIGINT, syscall.SIGTERM: + quit <- struct{}{} + case syscall.SIGQUIT: + stacklen := runtime.Stack(buf, true) + log.Printf("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end\n", buf[:stacklen]) + } } - } + } () + return quit } diff --git a/experimental/tracer/ptrace/ptracer.go b/experimental/tracer/ptrace/ptracer.go index 431a0bf61..2530bccb7 100644 --- a/experimental/tracer/ptrace/ptracer.go +++ b/experimental/tracer/ptrace/ptracer.go @@ -52,6 +52,21 @@ func NewPTracer() PTracer { return t } +func (t *PTracer) Stop() { + out := make(chan []int) + t.ops <- func() { + pids := []int{} + for pid := range t.processes { + pids = append(pids, pid) + } + out <- pids + } + for _, pid := range <-out { + t.StopTracing(pid) + } + t.quit <- struct{}{} +} + // TraceProcess starts tracing the given pid func (t *PTracer) TraceProcess(pid int) *process { result := make(chan *process) @@ -149,7 +164,10 @@ func (t *PTracer) traceThread(pid int, process *process) *thread { t.threads[pid] = thread result <- thread - t.childAttached <- struct{}{} + select { + case t.childAttached <- struct{}{}: + default: + } } return <-result } @@ -162,19 +180,20 @@ func (t *PTracer) waitLoop() { ) for { + log.Printf("Waiting...") pid, err = syscall.Wait4(-1, &status, syscall.WALL, nil) if err != nil && err.(syscall.Errno) == syscall.ECHILD { - log.Printf("No children to wait4") + log.Printf( "No children to wait4") <-t.childAttached continue } if err != nil { - log.Printf("Wait failed: %v %d", err, err.(syscall.Errno)) + log.Printf(" Wait failed: %v %d", err, err.(syscall.Errno)) return } - log.Printf("PID %d stopped", pid) + log.Printf(" PID %d stopped with signal %#x", pid, status) t.stopped <- stopped{pid, status} } } @@ -235,6 +254,7 @@ func (t *PTracer) handleStopped(pid int, status syscall.WaitStatus) { } // Restart stopped caller in syscall trap mode. + log.Printf("Restarting pid %d with signal %d", pid, int(signal)) err = syscall.PtraceSyscall(pid, int(signal)) if err != nil { log.Printf("PtraceSyscall failed, pid=%d, err=%v", pid, err) diff --git a/experimental/tracer/tracer b/experimental/tracer/tracer index 4fba8b179..3a35f0e5a 100755 --- a/experimental/tracer/tracer +++ b/experimental/tracer/tracer @@ -16,11 +16,8 @@ shift 1 case "$COMMAND" in launch) - docker run --privileged --net=host --pid=host -d -v /var/run/docker.sock:/var/run/docker.sock \ + docker run --privileged --net=host --pid=host -ti -v /var/run/docker.sock:/var/run/docker.sock \ --name $CONTAINER_NAME weaveworks/tracer - for ip in $(hostname -I); do - echo http://$ip:6060 - done ;; stop) diff --git a/experimental/tracer/ui/index.html b/experimental/tracer/ui/index.html index 141fc38da..c8a95dfb1 100644 --- a/experimental/tracer/ui/index.html +++ b/experimental/tracer/ui/index.html @@ -21,14 +21,29 @@ var containers = {}; var traces = {} - $.get("/containers").done(function (data) { - $.each(data, function(i, container) { - containers[container.Id] = container - $("ol.containers").append(sprintf("
  • %s
    • ", container.Id, container.Name)) - }); - }); + function updateContainers() { + $.get("/containers").done(function (data) { + $("ul.containers").html("") + data.sort(function (a, b) { + if (a.Name > b.Name) { + return 1; + } + if (a.Name < b.Name) { + return -1; + } + // a must be equal to b + return 0; + }); + $.each(data, function(i, container) { + containers[container.Id] = container + $("ul.containers").append(sprintf("
  • %s
    • ", container.Id, container.Name.substring(1))) + }); + window.setTimeout(updateContainers, 5 * 1000) + }); + } + updateContainers() - $("ol.containers").on("click", "li", function() { + $("ul.containers").on("click", "li", function() { var container = containers[$(this).attr("id")] var containerTraces = traces[$(this).attr("id")] || [] @@ -56,7 +71,7 @@ @@ -86,8 +101,8 @@

    Weave Tracer

    -
      -
    +
      +
    From a3a0907ffcd11f44aaa24154bff2181a09a16089 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Thu, 17 Sep 2015 04:05:52 +0000 Subject: [PATCH 03/23] It works again! --- experimental/example/qotd/qotd.c | 13 ++++---- experimental/tracer/ptrace/ptracer.go | 43 ++++++++++----------------- 2 files changed, 21 insertions(+), 35 deletions(-) diff --git a/experimental/example/qotd/qotd.c b/experimental/example/qotd/qotd.c index d61314966..082f99eaa 100644 --- a/experimental/example/qotd/qotd.c +++ b/experimental/example/qotd/qotd.c @@ -8,17 +8,13 @@ /* this function is run by the thread */ void *thread_func(void *sock) { struct sockaddr_in dest; - char buffer[1024]; + char in_buffer[1024]; + char out_buffer[1024]; int sockfd = (int)sock; int clientfd; printf("I'm thread %d\n", syscall(SYS_gettid)); - if (read(sockfd, buffer, sizeof(buffer)) < 0) { - perror("ERROR reading from socket"); - return; - } - clientfd = socket(AF_INET, SOCK_STREAM, 0); if (clientfd < 0) { perror("ERROR opening socket"); @@ -37,10 +33,11 @@ void *thread_func(void *sock) { return; } - int readbytes = recv(clientfd, buffer, sizeof(buffer), 0); + int readbytes = recv(clientfd, in_buffer, sizeof(in_buffer), 0); close(clientfd); - if (write(sockfd, buffer, read) < 0) { + int writtenbytes = snprintf(out_buffer, sizeof(out_buffer), "{\"qotd\": %s}\n", in_buffer); + if (write(sockfd, out_buffer, writtenbytes) < 0) { perror("ERROR writing to socket"); return; } diff --git a/experimental/tracer/ptrace/ptracer.go b/experimental/tracer/ptrace/ptracer.go index 2530bccb7..af7a89d85 100644 --- a/experimental/tracer/ptrace/ptracer.go +++ b/experimental/tracer/ptrace/ptracer.go @@ -142,28 +142,9 @@ func (t *PTracer) traceThread(pid int, process *process) *thread { return } - var status syscall.WaitStatus - if _, err = syscall.Wait4(pid, &status, 0, nil); err != nil { - log.Printf("Wait %d failed: %v", pid, err) - return - } - - thread.attached = true - - err = syscall.PtraceSetOptions(pid, ptraceOptions) - if err != nil { - log.Printf("SetOptions failed, pid=%d, err=%v", pid, err) - return - } - - err = syscall.PtraceSyscall(pid, 0) - if err != nil { - log.Printf("PtraceSyscall failed, pid=%d, err=%v", pid, err) - return - } - t.threads[pid] = thread result <- thread + select { case t.childAttached <- struct{}{}: default: @@ -180,20 +161,18 @@ func (t *PTracer) waitLoop() { ) for { - log.Printf("Waiting...") + //log.Printf("Waiting...") pid, err = syscall.Wait4(-1, &status, syscall.WALL, nil) if err != nil && err.(syscall.Errno) == syscall.ECHILD { - log.Printf( "No children to wait4") + //log.Printf( "No children to wait4") <-t.childAttached continue } - if err != nil { log.Printf(" Wait failed: %v %d", err, err.(syscall.Errno)) return } - - log.Printf(" PID %d stopped with signal %#x", pid, status) + //log.Printf(" PID %d stopped with signal %#x", pid, status) t.stopped <- stopped{pid, status} } } @@ -221,9 +200,19 @@ func (t *PTracer) handleStopped(pid int, status syscall.WaitStatus) { return } - if status.Stopped() && status.StopSignal() == syscall.SIGTRAP|ptraceTracesysgoodBit { + if !target.attached { + target.attached = true + + err = syscall.PtraceSetOptions(pid, ptraceOptions) + if err != nil { + log.Printf("SetOptions failed, pid=%d, err=%v", pid, err) + return + } + + } else if status.Stopped() && status.StopSignal() == syscall.SIGTRAP|ptraceTracesysgoodBit { // pid entered Syscall-enter-stop or syscall-exit-stop target.syscallStopped() + } else if status.Stopped() && status.StopSignal() == syscall.SIGTRAP { // pid entered PTRACE_EVENT stop switch status.TrapCause() { @@ -254,7 +243,7 @@ func (t *PTracer) handleStopped(pid int, status syscall.WaitStatus) { } // Restart stopped caller in syscall trap mode. - log.Printf("Restarting pid %d with signal %d", pid, int(signal)) + // log.Printf("Restarting pid %d with signal %d", pid, int(signal)) err = syscall.PtraceSyscall(pid, int(signal)) if err != nil { log.Printf("PtraceSyscall failed, pid=%d, err=%v", pid, err) From b6e43c8f3fc73a091f4a090c3ec40b717a099ce3 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Thu, 17 Sep 2015 04:20:31 +0000 Subject: [PATCH 04/23] Wire up recording of traces. --- experimental/tracer/main/main.go | 5 +++-- experimental/tracer/main/store.go | 5 +++++ experimental/tracer/ptrace/ptracer.go | 10 ++++++++-- experimental/tracer/ptrace/thread.go | 8 ++++---- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/experimental/tracer/main/main.go b/experimental/tracer/main/main.go index 6e3b07d34..a044bd4a0 100644 --- a/experimental/tracer/main/main.go +++ b/experimental/tracer/main/main.go @@ -37,9 +37,10 @@ func main() { log.Fatalf("Could start docker watcher: %v", err) } + store := newStore() tracer := tracer{ - ptrace: ptrace.NewPTracer(), - store: newStore(), + store: store, + ptrace: ptrace.NewPTracer(store), docker: dockerRegistry, } defer tracer.Stop() diff --git a/experimental/tracer/main/store.go b/experimental/tracer/main/store.go index beb2875b5..a021f362c 100644 --- a/experimental/tracer/main/store.go +++ b/experimental/tracer/main/store.go @@ -3,6 +3,7 @@ package main import ( "math/rand" "sync" + "log" "github.com/msackman/skiplist" @@ -65,12 +66,15 @@ func (s *store) RecordConnection(pid int, connection *ptrace.Fd) { newTrace := &trace{pid: pid, root: connection} newTraceKey := newKey(connection) + log.Printf("Recording trace: %+v", newTrace) + // First, see if this new conneciton is a child of an existing connection. // This indicates we have a parent connection to attach to. // If not, insert this connection. if parentNode := s.traces.Get(newTraceKey); parentNode != nil { parentNode.Remove() parentTrace := parentNode.Value.(*trace) + log.Printf(" Found parent trace: %+v", parentTrace) parentTrace.children = append(parentTrace.children, newTrace) } else { s.traces.Insert(newTraceKey, newTrace) @@ -84,6 +88,7 @@ func (s *store) RecordConnection(pid int, connection *ptrace.Fd) { if childNode := s.traces.Get(childTraceKey); childNode != nil { childNode.Remove() childTrace := childNode.Value.(*trace) + log.Printf(" Found child trace: %+v", childTrace) newTrace.children = append(newTrace.children, childTrace) } else { s.traces.Insert(childTraceKey, newTrace) diff --git a/experimental/tracer/ptrace/ptracer.go b/experimental/tracer/ptrace/ptracer.go index af7a89d85..b49d5f7ba 100644 --- a/experimental/tracer/ptrace/ptracer.go +++ b/experimental/tracer/ptrace/ptracer.go @@ -16,6 +16,10 @@ const ( ptraceTracesysgoodBit = 0x80 ) +type Store interface { + RecordConnection(int, *Fd) +} + // PTracer ptrace processed and threads type PTracer struct { @@ -29,6 +33,7 @@ type PTracer struct { threads map[int]*thread processes map[int]*process + store Store } type stopped struct { @@ -37,7 +42,7 @@ type stopped struct { } // NewPTracer creates a new ptracer. -func NewPTracer() PTracer { +func NewPTracer(store Store) PTracer { t := PTracer{ ops: make(chan func()), stopped: make(chan stopped), @@ -46,6 +51,7 @@ func NewPTracer() PTracer { threads: make(map[int]*thread), processes: make(map[int]*process), + store: store, } go t.waitLoop() go t.loop() @@ -135,6 +141,7 @@ func (t *PTracer) traceThread(pid int, process *process) *thread { result := make(chan *thread) t.ops <- func() { thread := newThread(pid, process, t) + t.threads[pid] = thread err := syscall.PtraceAttach(pid) if err != nil { @@ -142,7 +149,6 @@ func (t *PTracer) traceThread(pid int, process *process) *thread { return } - t.threads[pid] = thread result <- thread select { diff --git a/experimental/tracer/ptrace/thread.go b/experimental/tracer/ptrace/thread.go index f14dbe005..6e90d26a0 100644 --- a/experimental/tracer/ptrace/thread.go +++ b/experimental/tracer/ptrace/thread.go @@ -43,7 +43,7 @@ type thread struct { callRegs syscall.PtraceRegs resultRegs syscall.PtraceRegs - currentConnection *Fd + currentConnection *Fd // current incoming connection } func newThread(pid int, process *process, tracer *PTracer) *thread { @@ -213,9 +213,9 @@ func (t *thread) handleClose(call, result *syscall.PtraceRegs) { } // if this connection was incoming, add it to 'the registry' - //if fd.direction == incoming { - // t.tracer.store.RecordConnection(t.process.pid, fd) - //} + if fd.direction == incoming { + t.tracer.store.RecordConnection(t.process.pid, fd) + } } func (t *thread) handleIO(call, result *syscall.PtraceRegs) { From bdeed7219f2802c75bd505087e3ab7aa35dbd5d2 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Thu, 17 Sep 2015 05:40:51 +0000 Subject: [PATCH 05/23] Start showing traces in the UI --- experimental/tracer/main/http.go | 2 +- experimental/tracer/main/store.go | 17 ++- experimental/tracer/ptrace/fd.go | 25 ++-- experimental/tracer/ptrace/thread.go | 4 +- experimental/tracer/ui/index.html | 169 ++++++++++++++++----------- 5 files changed, 129 insertions(+), 88 deletions(-) diff --git a/experimental/tracer/main/http.go b/experimental/tracer/main/http.go index 414de7d4c..bec8c23bb 100644 --- a/experimental/tracer/main/http.go +++ b/experimental/tracer/main/http.go @@ -61,7 +61,7 @@ func (t *tracer) http(port int) { w.WriteHeader(204) }) - router.Methods("GET").Path("/trace").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + router.Methods("GET").Path("/traces").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { respondWith(w, http.StatusOK, t.store.Traces()) }) diff --git a/experimental/tracer/main/store.go b/experimental/tracer/main/store.go index a021f362c..2fe0a6b93 100644 --- a/experimental/tracer/main/store.go +++ b/experimental/tracer/main/store.go @@ -20,9 +20,9 @@ type key struct { } type trace struct { - pid int - root *ptrace.Fd - children []*trace + PID int + Root *ptrace.Fd + Children []*trace } type store struct { @@ -63,7 +63,7 @@ func (s *store) RecordConnection(pid int, connection *ptrace.Fd) { s.Lock() defer s.Unlock() - newTrace := &trace{pid: pid, root: connection} + newTrace := &trace{PID: pid, Root: connection} newTraceKey := newKey(connection) log.Printf("Recording trace: %+v", newTrace) @@ -75,7 +75,7 @@ func (s *store) RecordConnection(pid int, connection *ptrace.Fd) { parentNode.Remove() parentTrace := parentNode.Value.(*trace) log.Printf(" Found parent trace: %+v", parentTrace) - parentTrace.children = append(parentTrace.children, newTrace) + parentTrace.Children = append(parentTrace.Children, newTrace) } else { s.traces.Insert(newTraceKey, newTrace) } @@ -89,7 +89,7 @@ func (s *store) RecordConnection(pid int, connection *ptrace.Fd) { childNode.Remove() childTrace := childNode.Value.(*trace) log.Printf(" Found child trace: %+v", childTrace) - newTrace.children = append(newTrace.children, childTrace) + newTrace.Children = append(newTrace.Children, childTrace) } else { s.traces.Insert(childTraceKey, newTrace) } @@ -102,12 +102,9 @@ func (s *store) Traces() []*trace { var traces []*trace var cur = s.traces.First() - for { + for cur != nil { traces = append(traces, cur.Value.(*trace)) cur = cur.Next() - if cur == nil { - break - } } return traces } diff --git a/experimental/tracer/ptrace/fd.go b/experimental/tracer/ptrace/fd.go index 05138f4d0..b5f842fa5 100644 --- a/experimental/tracer/ptrace/fd.go +++ b/experimental/tracer/ptrace/fd.go @@ -35,14 +35,14 @@ type Fd struct { fd int Start int64 - stop int64 + Stop int64 sent int64 received int64 FromAddr net.IP FromPort uint16 - toAddr net.IP - toPort uint16 + ToAddr net.IP + ToPort uint16 // Fds are connections, and can have a causal-link to other Fds Children []*Fd @@ -120,6 +120,11 @@ func getLocalAddr(pid, fd int) (addr net.IP, port uint16, err error) { return } +// in milliseconds +func now() int64 { + return time.Now().UnixNano() / 1000000 +} + // We want to get the listening address from /proc func newListeningFd(pid, fd int) (*Fd, error) { localAddr, localPort, err := getLocalAddr(pid, fd) @@ -128,8 +133,8 @@ func newListeningFd(pid, fd int) (*Fd, error) { } return &Fd{ - direction: listening, fd: fd, Start: time.Now().Unix(), - toAddr: localAddr, toPort: uint16(localPort), + direction: listening, fd: fd, Start: now(), + ToAddr: localAddr, ToPort: uint16(localPort), }, nil } @@ -141,9 +146,9 @@ func newConnectionFd(pid, fd int, remoteAddr net.IP, remotePort uint16) (*Fd, er } return &Fd{ - direction: outgoing, fd: fd, Start: time.Now().Unix(), + direction: outgoing, fd: fd, Start: now(), FromAddr: localAddr, FromPort: uint16(localPort), - toAddr: remoteAddr, toPort: remotePort, + ToAddr: remoteAddr, ToPort: remotePort, }, nil } @@ -154,12 +159,12 @@ func (fd *Fd) newConnection(addr net.IP, port uint16, newFd int) (*Fd, error) { } return &Fd{ - direction: incoming, fd: newFd, Start: time.Now().Unix(), - toAddr: fd.toAddr, toPort: fd.toPort, + direction: incoming, fd: newFd, Start: now(), + ToAddr: fd.ToAddr, ToPort: fd.ToPort, FromAddr: addr, FromPort: port, }, nil } func (fd *Fd) close() { - fd.stop = time.Now().Unix() + fd.Stop = now() } diff --git a/experimental/tracer/ptrace/thread.go b/experimental/tracer/ptrace/thread.go index 6e90d26a0..8670e207c 100644 --- a/experimental/tracer/ptrace/thread.go +++ b/experimental/tracer/ptrace/thread.go @@ -168,7 +168,7 @@ func (t *thread) handleAccept(call, result *syscall.PtraceRegs) { t.process.newFd(connection) t.logf("Accepted connection from %s:%d -> %s:%d on fd %d, new fd %d", - addr, port, connection.toAddr, connection.toPort, listeningFdNum, connectionFdNum) + addr, port, connection.ToAddr, connection.ToPort, listeningFdNum, connectionFdNum) } func (t *thread) handleConnect(call, result *syscall.PtraceRegs) { @@ -192,7 +192,7 @@ func (t *thread) handleConnect(call, result *syscall.PtraceRegs) { } t.logf("Made connection from %s:%d -> %s:%d on fd %d", - connection.toAddr, connection.toPort, connection.FromAddr, + connection.ToAddr, connection.ToPort, connection.FromAddr, connection.FromPort, fd) } diff --git a/experimental/tracer/ui/index.html b/experimental/tracer/ui/index.html index c8a95dfb1..c6e71b631 100644 --- a/experimental/tracer/ui/index.html +++ b/experimental/tracer/ui/index.html @@ -12,102 +12,141 @@ + - + - - -
    -
    -

    Weave Tracer

    -
      -
    -
    - -
    - -
    -
    From bf85e621a22a965c5ebb591156d50e0786c635c5 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Thu, 17 Sep 2015 07:27:36 +0000 Subject: [PATCH 06/23] Make connection causality detection more reliable. --- experimental/tracer/ptrace/ptracer.go | 5 ++++ experimental/tracer/ptrace/thread.go | 41 ++++++++++++++++++--------- experimental/tracer/ui/index.html | 8 ++++-- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/experimental/tracer/ptrace/ptracer.go b/experimental/tracer/ptrace/ptracer.go index b49d5f7ba..306ce64a4 100644 --- a/experimental/tracer/ptrace/ptracer.go +++ b/experimental/tracer/ptrace/ptracer.go @@ -301,6 +301,11 @@ func pidForTid(tid int) (pid int, err error) { } func (t *PTracer) thread(tid int) (*thread, error) { + // unfortunately we can't propage fd affinitiy as we + // can reliably determin who clone'd new threads, as + // the pids returned by the ptrace calls are in the process' + // namespace, not ours. + thread, ok := t.threads[tid] if !ok { pid, err := pidForTid(tid) diff --git a/experimental/tracer/ptrace/thread.go b/experimental/tracer/ptrace/thread.go index 8670e207c..038f7050c 100644 --- a/experimental/tracer/ptrace/thread.go +++ b/experimental/tracer/ptrace/thread.go @@ -43,14 +43,17 @@ type thread struct { callRegs syscall.PtraceRegs resultRegs syscall.PtraceRegs - currentConnection *Fd // current incoming connection + currentIncoming map[int]*Fd + currentOutgoing map[int]*Fd } func newThread(pid int, process *process, tracer *PTracer) *thread { t := &thread{ - tid: pid, - process: process, - tracer: tracer, + tid: pid, + process: process, + tracer: tracer, + currentIncoming: map[int]*Fd{}, + currentOutgoing: map[int]*Fd{}, } return t } @@ -187,9 +190,6 @@ func (t *thread) handleConnect(call, result *syscall.PtraceRegs) { } t.process.newFd(connection) - if t.currentConnection != nil { - t.currentConnection.Children = append(t.currentConnection.Children, connection) - } t.logf("Made connection from %s:%d -> %s:%d on fd %d", connection.ToAddr, connection.ToPort, connection.FromAddr, @@ -205,16 +205,27 @@ func (t *thread) handleClose(call, result *syscall.PtraceRegs) { } fd.close() - delete(t.process.fds, fdNum) - - // clear current connection if we just closed it! - if t.currentConnection != nil && t.currentConnection.fd == fdNum { - t.currentConnection = nil - } // if this connection was incoming, add it to 'the registry' if fd.direction == incoming { + for _, outgoing := range t.currentOutgoing { + t.logf("Fd %d caused %d", fd.fd, outgoing.fd) + fd.Children = append(fd.Children, outgoing) + } + t.tracer.store.RecordConnection(t.process.pid, fd) + } else { + for _, incoming := range t.currentIncoming { + t.logf("Fd %d caused %d", incoming.fd, fd.fd) + incoming.Children = append(incoming.Children, fd) + } + } + + // now make sure we've remove it from everywhere + delete(t.process.fds, fdNum) + for _, thread := range t.process.threads { + delete(thread.currentIncoming, fdNum) + delete(t.currentOutgoing, fdNum) } } @@ -228,7 +239,9 @@ func (t *thread) handleIO(call, result *syscall.PtraceRegs) { if fd.direction == incoming { t.logf("IO on incoming connection %d; setting affinity", fdNum) - t.currentConnection = fd + t.currentIncoming[fdNum] = fd + } else { + t.currentOutgoing[fdNum] = fd } } diff --git a/experimental/tracer/ui/index.html b/experimental/tracer/ui/index.html index c6e71b631..4eeaa0e5d 100644 --- a/experimental/tracer/ui/index.html +++ b/experimental/tracer/ui/index.html @@ -32,12 +32,16 @@ Handlebars.registerHelper('duration', function(input) { var ms = input.Stop - input.Start if (ms < 60000) { - return sprintf("%0.2fs", ms / 1000) + return new Handlebars.SafeString(sprintf("%0.2fs", ms / 1000)); } var ds = moment.duration(ms).humanize(); return new Handlebars.SafeString(ds); }); + Handlebars.registerHelper('count', function(input) { + return new Handlebars.SafeString(input === null ? '0' : sprintf("%d", input.len)); + }); + function render() { var template = $('script#process-template').text(); template = Handlebars.compile(template); @@ -138,7 +142,7 @@ {{#traces}} {{ts Root.Start}}{{duration Root}} {{PID}}{{Root.FromAddr}}:{{Root.FromPort}} - {{Root.ToAddr}}:{{Root.ToPort}} + {{Root.ToAddr}}:{{Root.ToPort}}{{count Root.Children}} {{/traces}} From 38e7c7c560de401a90285b7ccb1efad75d9a59ac Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Thu, 17 Sep 2015 09:48:02 +0000 Subject: [PATCH 07/23] Show subtraces in the API and UI. --- experimental/tracer/main/store.go | 29 +++++++++++++--- experimental/tracer/ptrace/fd.go | 49 +++++++++++++++++++++------- experimental/tracer/ptrace/thread.go | 13 ++++---- experimental/tracer/ui/index.html | 38 +++++++++++++++++---- 4 files changed, 99 insertions(+), 30 deletions(-) diff --git a/experimental/tracer/main/store.go b/experimental/tracer/main/store.go index 2fe0a6b93..aecee9e02 100644 --- a/experimental/tracer/main/store.go +++ b/experimental/tracer/main/store.go @@ -21,8 +21,10 @@ type key struct { type trace struct { PID int - Root *ptrace.Fd + ServerDetails *ptrace.ConnectionDetails + ClientDetails *ptrace.ConnectionDetails Children []*trace + Level int } type store struct { @@ -63,10 +65,18 @@ func (s *store) RecordConnection(pid int, connection *ptrace.Fd) { s.Lock() defer s.Unlock() - newTrace := &trace{PID: pid, Root: connection} - newTraceKey := newKey(connection) + newTrace := &trace{ + PID: pid, + ServerDetails: &connection.ConnectionDetails, + } + for _, child := range connection.Children { + newTrace.Children = append(newTrace.Children, &trace{ + Level: 1, + ClientDetails: &child.ConnectionDetails, + }) + } - log.Printf("Recording trace: %+v", newTrace) + newTraceKey := newKey(connection) // First, see if this new conneciton is a child of an existing connection. // This indicates we have a parent connection to attach to. @@ -75,6 +85,7 @@ func (s *store) RecordConnection(pid int, connection *ptrace.Fd) { parentNode.Remove() parentTrace := parentNode.Value.(*trace) log.Printf(" Found parent trace: %+v", parentTrace) + newTrace.Level = parentTrace.Level + 1 parentTrace.Children = append(parentTrace.Children, newTrace) } else { s.traces.Insert(newTraceKey, newTrace) @@ -89,6 +100,7 @@ func (s *store) RecordConnection(pid int, connection *ptrace.Fd) { childNode.Remove() childTrace := childNode.Value.(*trace) log.Printf(" Found child trace: %+v", childTrace) + IncrementLevel(childTrace, newTrace.Level) newTrace.Children = append(newTrace.Children, childTrace) } else { s.traces.Insert(childTraceKey, newTrace) @@ -96,11 +108,18 @@ func (s *store) RecordConnection(pid int, connection *ptrace.Fd) { } } +func IncrementLevel(trace *trace, increment int) { + trace.Level += increment + for _, child := range trace.Children { + IncrementLevel(child, increment) + } +} + func (s *store) Traces() []*trace { s.RLock() defer s.RUnlock() - var traces []*trace + traces := []*trace{} var cur = s.traces.First() for cur != nil { traces = append(traces, cur.Value.(*trace)) diff --git a/experimental/tracer/ptrace/fd.go b/experimental/tracer/ptrace/fd.go index b5f842fa5..ffea4b11d 100644 --- a/experimental/tracer/ptrace/fd.go +++ b/experimental/tracer/ptrace/fd.go @@ -29,10 +29,8 @@ var ( tcpRegexp = regexp.MustCompile(tcpPattern) ) -// Fd represents a connect and subsequent connections caused by it. -type Fd struct { +type ConnectionDetails struct { direction int - fd int Start int64 Stop int64 @@ -43,6 +41,14 @@ type Fd struct { FromPort uint16 ToAddr net.IP ToPort uint16 +} + +// Fd represents a connect and subsequent connections caused by it. +type Fd struct { + fd int + closed bool + + ConnectionDetails // Fds are connections, and can have a causal-link to other Fds Children []*Fd @@ -133,8 +139,14 @@ func newListeningFd(pid, fd int) (*Fd, error) { } return &Fd{ - direction: listening, fd: fd, Start: now(), - ToAddr: localAddr, ToPort: uint16(localPort), + fd: fd, + + ConnectionDetails: ConnectionDetails{ + direction: listening, + Start: now(), + ToAddr: localAddr, + ToPort: uint16(localPort), + }, }, nil } @@ -146,9 +158,16 @@ func newConnectionFd(pid, fd int, remoteAddr net.IP, remotePort uint16) (*Fd, er } return &Fd{ - direction: outgoing, fd: fd, Start: now(), - FromAddr: localAddr, FromPort: uint16(localPort), - ToAddr: remoteAddr, ToPort: remotePort, + fd: fd, + + ConnectionDetails: ConnectionDetails{ + direction: outgoing, + Start: now(), + FromAddr: localAddr, + FromPort: uint16(localPort), + ToAddr: remoteAddr, + ToPort: remotePort, + }, }, nil } @@ -159,12 +178,20 @@ func (fd *Fd) newConnection(addr net.IP, port uint16, newFd int) (*Fd, error) { } return &Fd{ - direction: incoming, fd: newFd, Start: now(), - ToAddr: fd.ToAddr, ToPort: fd.ToPort, - FromAddr: addr, FromPort: port, + fd: newFd, + + ConnectionDetails: ConnectionDetails{ + direction: incoming, + Start: now(), + ToAddr: fd.ToAddr, + ToPort: fd.ToPort, + FromAddr: addr, + FromPort: port, + }, }, nil } func (fd *Fd) close() { + fd.closed = true fd.Stop = now() } diff --git a/experimental/tracer/ptrace/thread.go b/experimental/tracer/ptrace/thread.go index 038f7050c..09c6eb715 100644 --- a/experimental/tracer/ptrace/thread.go +++ b/experimental/tracer/ptrace/thread.go @@ -204,28 +204,26 @@ func (t *thread) handleClose(call, result *syscall.PtraceRegs) { return } + t.logf("Closing fd %d", fdNum) fd.close() // if this connection was incoming, add it to 'the registry' if fd.direction == incoming { + // collect all the outgoing connections this thread has made + // and treat them as caused by this incoming connections for _, outgoing := range t.currentOutgoing { - t.logf("Fd %d caused %d", fd.fd, outgoing.fd) + t.logf("Fd %d caused %d", fdNum, outgoing.fd) fd.Children = append(fd.Children, outgoing) } + t.currentOutgoing = map[int]*Fd{} t.tracer.store.RecordConnection(t.process.pid, fd) - } else { - for _, incoming := range t.currentIncoming { - t.logf("Fd %d caused %d", incoming.fd, fd.fd) - incoming.Children = append(incoming.Children, fd) - } } // now make sure we've remove it from everywhere delete(t.process.fds, fdNum) for _, thread := range t.process.threads { delete(thread.currentIncoming, fdNum) - delete(t.currentOutgoing, fdNum) } } @@ -241,6 +239,7 @@ func (t *thread) handleIO(call, result *syscall.PtraceRegs) { t.logf("IO on incoming connection %d; setting affinity", fdNum) t.currentIncoming[fdNum] = fd } else { + t.logf("IO on outgoing connection %d; setting affinity", fdNum) t.currentOutgoing[fdNum] = fd } } diff --git a/experimental/tracer/ui/index.html b/experimental/tracer/ui/index.html index 4eeaa0e5d..938f9a728 100644 --- a/experimental/tracer/ui/index.html +++ b/experimental/tracer/ui/index.html @@ -24,6 +24,10 @@ var containers = []; var traces = []; + Handlebars.registerHelper('spaces', function(input) { + return new Array(input + 1).join("> "); + }); + Handlebars.registerHelper('ts', function(input) { var ts = moment(input).format("LTS") return new Handlebars.SafeString(ts); @@ -39,9 +43,11 @@ }); Handlebars.registerHelper('count', function(input) { - return new Handlebars.SafeString(input === null ? '0' : sprintf("%d", input.len)); + return new Handlebars.SafeString(input === null ? '0' : sprintf("%d", input.length)); }); + Handlebars.registerPartial('traces', $("#traces").html()); + function render() { var template = $('script#process-template').text(); template = Handlebars.compile(template); @@ -115,6 +121,27 @@ width: 100%; } + + - @@ -24,6 +22,12 @@ var containers = []; var traces = []; + Handlebars.registerHelper('isSelected', function(input) { + if (currentContainer && input === currentContainer.Id) { + return 'class=selected'; + } + }); + Handlebars.registerHelper('spaces', function(input) { return new Array(input + 1).join("> "); }); @@ -75,6 +79,10 @@ $.each(data, function(i, container) { containersByID[container.Id] = container }); + // auto-select first container + if (containers.length) { + currentContainer = containersByID[containers[0].Id]; + } render(); window.setTimeout(updateContainers, 5 * 1000); }); @@ -113,13 +121,70 @@ }) diff --git a/experimental/tracer/ui/logo.svg b/experimental/tracer/ui/logo.svg new file mode 100644 index 000000000..406732034 --- /dev/null +++ b/experimental/tracer/ui/logo.svg @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/experimental/tracer/ui/package.json b/experimental/tracer/ui/package.json new file mode 100644 index 000000000..d2d91d3ed --- /dev/null +++ b/experimental/tracer/ui/package.json @@ -0,0 +1,15 @@ +{ + "name": "scope-tracer", + "version": "1.0.0", + "description": "", + "main": "server.js", + "scripts": { + "start": "node server.js", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "David Kaltschmidt ", + "license": "UNLICENSED", + "devDependencies": { + "express": "^4.13.3" + } +} diff --git a/experimental/tracer/ui/server.js b/experimental/tracer/ui/server.js new file mode 100644 index 000000000..e56d77342 --- /dev/null +++ b/experimental/tracer/ui/server.js @@ -0,0 +1,25 @@ +var express = require('express') + +var app = express(); + +app.get('/', function(req, res) { + res.sendFile(__dirname + '/index.html'); +}); + +app.get('/container', function(req, res) { + res.sendFile(__dirname + '/container.json'); +}); + +app.get('/traces', function(req, res) { + res.sendFile(__dirname + '/traces.json'); +}); + +app.use(express.static('./')); + +var port = process.env.PORT || 4050; +var server = app.listen(port, function () { + var host = server.address().address; + var port = server.address().port; + + console.log('Scope Tracer UI listening at http://%s:%s', host, port); +}); From 9f49f63fc67dea9b9bf22552f036d3ec2cf9f897 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Fri, 18 Sep 2015 03:50:06 +0000 Subject: [PATCH 10/23] Mime type --- experimental/tracer/main/http.go | 7 ++++--- experimental/tracer/ptrace/thread.go | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/experimental/tracer/main/http.go b/experimental/tracer/main/http.go index e1fcdb42d..53ca11d0c 100644 --- a/experimental/tracer/main/http.go +++ b/experimental/tracer/main/http.go @@ -4,11 +4,12 @@ import ( "encoding/json" "fmt" "log" + "mime" "net/http" "strconv" - "github.com/gorilla/mux" dockerClient "github.com/fsouza/go-dockerclient" + "github.com/gorilla/mux" "github.com/weaveworks/scope/probe/docker" "github.com/weaveworks/scope/probe/process" @@ -110,12 +111,12 @@ func (t *tracer) http(port int) { w.WriteHeader(204) }) - - router.Methods("GET").Path("/traces").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { respondWith(w, http.StatusOK, t.store.Traces()) }) + mime.AddExtensionType(".svg", "image/svg+xml") + router.Methods("GET").PathPrefix("/").Handler(http.FileServer(FS(false))) // everything else is static log.Printf("Launching HTTP API on port %d", port) diff --git a/experimental/tracer/ptrace/thread.go b/experimental/tracer/ptrace/thread.go index 61e3fdfbc..af663d2d4 100644 --- a/experimental/tracer/ptrace/thread.go +++ b/experimental/tracer/ptrace/thread.go @@ -16,6 +16,7 @@ const ( STAT = 4 MMAP = 9 MPROTECT = 10 + SELECT = 23 MADVISE = 28 SOCKET = 41 CONNECT = 42 @@ -97,7 +98,7 @@ func (t *thread) syscallStopped() { t.handleIO(&t.callRegs, &t.resultRegs) // we can ignore these syscalls - case SETROBUSTLIST, GETID, MMAP, MPROTECT, MADVISE, SOCKET, CLONE, STAT: + case SETROBUSTLIST, GETID, MMAP, MPROTECT, MADVISE, SOCKET, CLONE, STAT, SELECT: return default: From dfbd83b6584cf19b643da061f053c74692dc8dd1 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Fri, 18 Sep 2015 06:25:24 +0000 Subject: [PATCH 11/23] Expose keys, sort and dedupe traces, and hide show children in ui. --- experimental/tracer/main/store.go | 33 ++++++++++++++++++---- experimental/tracer/ui/index.html | 47 +++++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 7 deletions(-) diff --git a/experimental/tracer/main/store.go b/experimental/tracer/main/store.go index aecee9e02..dd1637037 100644 --- a/experimental/tracer/main/store.go +++ b/experimental/tracer/main/store.go @@ -4,13 +4,15 @@ import ( "math/rand" "sync" "log" + "fmt" + "sort" "github.com/msackman/skiplist" "github.com/weaveworks/scope/experimental/tracer/ptrace" ) -const epsilon = int64(5) +const epsilon = int64(5) * 1000 // milliseconds // Traces are indexed by from addr, from port, and start time. type key struct { @@ -19,14 +21,25 @@ type key struct { startTime int64 } +func (k key) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf("\"%x.%x.%x\"", k.fromAddr, k.fromPort, k.startTime)), nil +} + type trace struct { PID int + Key key ServerDetails *ptrace.ConnectionDetails ClientDetails *ptrace.ConnectionDetails Children []*trace Level int } +type byKey []*trace + +func (a byKey) Len() int { return len(a) } +func (a byKey) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byKey) Less(i, j int) bool { return a[i].Key.startTime < a[j].Key.startTime } + type store struct { sync.RWMutex traces *skiplist.SkipList @@ -67,6 +80,7 @@ func (s *store) RecordConnection(pid int, connection *ptrace.Fd) { newTrace := &trace{ PID: pid, + Key: newKey(connection), ServerDetails: &connection.ConnectionDetails, } for _, child := range connection.Children { @@ -76,7 +90,7 @@ func (s *store) RecordConnection(pid int, connection *ptrace.Fd) { }) } - newTraceKey := newKey(connection) + newTraceKey := newTrace.Key // First, see if this new conneciton is a child of an existing connection. // This indicates we have a parent connection to attach to. @@ -119,11 +133,20 @@ func (s *store) Traces() []*trace { s.RLock() defer s.RUnlock() - traces := []*trace{} + traces := map[key]*trace{} var cur = s.traces.First() for cur != nil { - traces = append(traces, cur.Value.(*trace)) + trace := cur.Value.(*trace) + traces[trace.Key] = trace cur = cur.Next() } - return traces + + result := []*trace{} + for _, trace := range traces { + result = append(result, trace) + } + + sort.Sort(byKey(result)) + + return result } diff --git a/experimental/tracer/ui/index.html b/experimental/tracer/ui/index.html index c9a1aed35..a8e29d187 100644 --- a/experimental/tracer/ui/index.html +++ b/experimental/tracer/ui/index.html @@ -18,6 +18,7 @@ + From baf7aa106c33144416ce4f83f522cc4d31c3bdf5 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Fri, 18 Sep 2015 07:38:12 +0000 Subject: [PATCH 12/23] Fix table column width; ntohl port numbers. --- experimental/example/run.sh | 8 ++------ experimental/tracer/ptrace/thread.go | 10 +++++++++- experimental/tracer/ui/index.html | 11 +++++++++-- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/experimental/example/run.sh b/experimental/example/run.sh index ceeac18ca..7a4dee446 100755 --- a/experimental/example/run.sh +++ b/experimental/example/run.sh @@ -19,10 +19,6 @@ start_container() { done } -start_container 1 elasticsearch elasticsearch -start_container 2 tomwilkie/searchapp searchapp -start_container 1 redis redis start_container 1 tomwilkie/qotd qotd -start_container 2 tomwilkie/app app -start_container 2 tomwilkie/frontend frontend --add-host=dns.weave.local:$(weave docker-bridge-ip) -start_container 1 tomwilkie/client client +start_container 1 tomwilkie/app app + diff --git a/experimental/tracer/ptrace/thread.go b/experimental/tracer/ptrace/thread.go index af663d2d4..ca67de89c 100644 --- a/experimental/tracer/ptrace/thread.go +++ b/experimental/tracer/ptrace/thread.go @@ -12,6 +12,7 @@ import ( const ( READ = 0 WRITE = 1 + OPEN = 2 CLOSE = 3 STAT = 4 MMAP = 9 @@ -101,11 +102,18 @@ func (t *thread) syscallStopped() { case SETROBUSTLIST, GETID, MMAP, MPROTECT, MADVISE, SOCKET, CLONE, STAT, SELECT: return + case OPEN: + return + default: t.logf("syscall(%d)", t.callRegs.Orig_rax) } } +func ntohl(b uint16) uint16 { + return (b << 8) | ((b & 0xff00) >> 8) +} + func (t *thread) getSocketAddress(ptr uintptr) (addr net.IP, port uint16, err error) { var ( buf = make([]byte, syscall.SizeofSockaddrAny) @@ -128,7 +136,7 @@ func (t *thread) getSocketAddress(ptr uintptr) (addr net.IP, port uint16, err er } addr = net.IP(sockaddr4.Addr[0:]) - port = sockaddr4.Port + port = ntohl(sockaddr4.Port) return } diff --git a/experimental/tracer/ui/index.html b/experimental/tracer/ui/index.html index a8e29d187..b479f377c 100644 --- a/experimental/tracer/ui/index.html +++ b/experimental/tracer/ui/index.html @@ -273,8 +273,15 @@ Stop
    - - +
    Start timeDurationPIDFromToSub-traces
    + + + + + + + + {{>traces traces}} From e58f0f756fc6aa89ae865dcfb33718391658ab08 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Fri, 18 Sep 2015 09:32:54 +0000 Subject: [PATCH 13/23] Make temporal join work - now can have traces with more than one child! --- experimental/tracer/main/store.go | 96 +++++++++++++++++----------- experimental/tracer/ptrace/thread.go | 16 +++-- experimental/tracer/ui/index.html | 20 ++++-- 3 files changed, 83 insertions(+), 49 deletions(-) diff --git a/experimental/tracer/main/store.go b/experimental/tracer/main/store.go index dd1637037..ad8c137db 100644 --- a/experimental/tracer/main/store.go +++ b/experimental/tracer/main/store.go @@ -1,11 +1,10 @@ package main import ( - "math/rand" - "sync" - "log" "fmt" + "math/rand" "sort" + "sync" "github.com/msackman/skiplist" @@ -26,12 +25,12 @@ func (k key) MarshalJSON() ([]byte, error) { } type trace struct { - PID int - Key key + PID int + Key key ServerDetails *ptrace.ConnectionDetails ClientDetails *ptrace.ConnectionDetails - Children []*trace - Level int + Children []*trace + Level int } type byKey []*trace @@ -57,7 +56,20 @@ func newKey(fd *ptrace.Fd) key { func (l key) LessThan(other skiplist.Comparable) bool { r := other.(key) - return l.fromAddr < r.fromAddr && l.fromPort < r.fromPort && l.startTime < r.startTime + + if l.fromAddr != r.fromAddr { + return l.fromAddr > r.fromAddr + } + + if l.fromPort != r.fromPort { + return l.fromPort < r.fromPort + } + + if l.Equal(other) { + return false + } + + return l.startTime < r.startTime } func (l key) Equal(other skiplist.Comparable) bool { @@ -74,50 +86,61 @@ func newStore() *store { return &store{traces: skiplist.New(rand.New(rand.NewSource(0)))} } +func (t *trace) addChild(child *trace) { + // find the child we're supposed to be replacing + for i, candidate := range t.Children { + if !candidate.Key.Equal(skiplist.Comparable(child.Key)) { + continue + } + + // Fix up some fields + child.ClientDetails = candidate.ClientDetails + child.PID = candidate.PID + IncrementLevel(child, t.Level+1) + + // Overwrite old record + t.Children[i] = child + return + } +} + func (s *store) RecordConnection(pid int, connection *ptrace.Fd) { s.Lock() defer s.Unlock() newTrace := &trace{ - PID: pid, - Key: newKey(connection), + PID: pid, + Key: newKey(connection), ServerDetails: &connection.ConnectionDetails, } for _, child := range connection.Children { newTrace.Children = append(newTrace.Children, &trace{ - Level: 1, + Level: 1, + Key: newKey(child), ClientDetails: &child.ConnectionDetails, }) } - newTraceKey := newTrace.Key - // First, see if this new conneciton is a child of an existing connection. // This indicates we have a parent connection to attach to. // If not, insert this connection. - if parentNode := s.traces.Get(newTraceKey); parentNode != nil { - parentNode.Remove() + if parentNode := s.traces.Get(newTrace.Key); parentNode != nil { parentTrace := parentNode.Value.(*trace) - log.Printf(" Found parent trace: %+v", parentTrace) - newTrace.Level = parentTrace.Level + 1 - parentTrace.Children = append(parentTrace.Children, newTrace) + parentTrace.addChild(newTrace) + parentNode.Remove() } else { - s.traces.Insert(newTraceKey, newTrace) + s.traces.Insert(newTrace.Key, newTrace) } // Next, see if we already know about the child connections // If not, insert each of our children. - for _, childConnection := range connection.Children { - childTraceKey := newKey(childConnection) - - if childNode := s.traces.Get(childTraceKey); childNode != nil { - childNode.Remove() + for _, child := range newTrace.Children { + if childNode := s.traces.Get(child.Key); childNode != nil { childTrace := childNode.Value.(*trace) - log.Printf(" Found child trace: %+v", childTrace) - IncrementLevel(childTrace, newTrace.Level) - newTrace.Children = append(newTrace.Children, childTrace) + newTrace.addChild(childTrace) + childNode.Remove() } else { - s.traces.Insert(childTraceKey, newTrace) + s.traces.Insert(child.Key, newTrace) } } } @@ -133,20 +156,17 @@ func (s *store) Traces() []*trace { s.RLock() defer s.RUnlock() - traces := map[key]*trace{} + traces := []*trace{} var cur = s.traces.First() for cur != nil { + key := cur.Key.(key) trace := cur.Value.(*trace) - traces[trace.Key] = trace + if trace.Key == key { + traces = append(traces, trace) + } cur = cur.Next() } - result := []*trace{} - for _, trace := range traces { - result = append(result, trace) - } - - sort.Sort(byKey(result)) - - return result + sort.Sort(byKey(traces)) + return traces } diff --git a/experimental/tracer/ptrace/thread.go b/experimental/tracer/ptrace/thread.go index ca67de89c..530c94928 100644 --- a/experimental/tracer/ptrace/thread.go +++ b/experimental/tracer/ptrace/thread.go @@ -17,6 +17,7 @@ const ( STAT = 4 MMAP = 9 MPROTECT = 10 + MUNMAP = 11 SELECT = 23 MADVISE = 28 SOCKET = 41 @@ -24,8 +25,11 @@ const ( ACCEPT = 43 SENDTO = 44 RECVFROM = 45 + SHUTDOWN = 48 CLONE = 56 + GETTIMEOFDAY = 96 GETID = 186 + FUTEX = 202 SETROBUSTLIST = 273 ACCEPT4 = 288 ) @@ -102,7 +106,7 @@ func (t *thread) syscallStopped() { case SETROBUSTLIST, GETID, MMAP, MPROTECT, MADVISE, SOCKET, CLONE, STAT, SELECT: return - case OPEN: + case OPEN, FUTEX, SHUTDOWN, GETTIMEOFDAY, MUNMAP: return default: @@ -214,7 +218,7 @@ func (t *thread) handleClose(call, result *syscall.PtraceRegs) { return } - t.logf("Closing fd %d", fdNum) + //t.logf("Closing fd %d", fdNum) fd.close() // if this connection was incoming, add it to 'the registry' @@ -222,7 +226,7 @@ func (t *thread) handleClose(call, result *syscall.PtraceRegs) { // collect all the outgoing connections this thread has made // and treat them as caused by this incoming connections for _, outgoing := range t.currentOutgoing { - t.logf("Fd %d caused %d", fdNum, outgoing.fd) + //t.logf("Fd %d caused %d", fdNum, outgoing.fd) fd.Children = append(fd.Children, outgoing) } t.currentOutgoing = map[int]*Fd{} @@ -241,15 +245,15 @@ func (t *thread) handleIO(call, result *syscall.PtraceRegs) { fdNum := int(call.Rdi) fd, ok := t.process.fds[fdNum] if !ok { - t.logf("IO on unknown fd %d", fdNum) + //t.logf("IO on unknown fd %d", fdNum) return } if fd.direction == incoming { - t.logf("IO on incoming connection %d; setting affinity", fdNum) + //t.logf("IO on incoming connection %d; setting affinity", fdNum) t.currentIncoming[fdNum] = fd } else { - t.logf("IO on outgoing connection %d; setting affinity", fdNum) + //t.logf("IO on outgoing connection %d; setting affinity", fdNum) t.currentOutgoing[fdNum] = fd } } diff --git a/experimental/tracer/ui/index.html b/experimental/tracer/ui/index.html index b479f377c..35e0b32da 100644 --- a/experimental/tracer/ui/index.html +++ b/experimental/tracer/ui/index.html @@ -53,8 +53,18 @@ return new Handlebars.SafeString(ds); }); + function numChildren(input) { + if (input.Children === null) { + return 0 + } + var count = input.Children.length + $.each(input.Children, function(i, child) { + count += numChildren(child) + }) + return count + } Handlebars.registerHelper('count', function(input) { - return new Handlebars.SafeString(input === null ? '0' : sprintf("%d", input.length)); + return sprintf("%d", numChildren(input)); }); Handlebars.registerPartial('traces', $("#traces").html()); @@ -214,13 +224,13 @@ {{#with ServerDetails}} - + {{/with}} {{else}} {{#with ClientDetails}} - + {{/with}} {{/if}} @@ -236,14 +246,14 @@ - + {{/with}} {{else}} {{#with ClientDetails}} - + {{/with}} {{/if}} {{>children Children}} From 3dbb94eed3d68adbdcedb1bae36488888b29ecba Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Fri, 18 Sep 2015 10:19:58 +0000 Subject: [PATCH 14/23] Send all the container pids to the client. --- experimental/tracer/README.md | 4 +++- experimental/tracer/main/http.go | 24 +++++++++++++++++++++--- experimental/tracer/main/store.go | 1 - experimental/tracer/ui/index.html | 12 ++++++------ 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/experimental/tracer/README.md b/experimental/tracer/README.md index 7a077cee8..23cc29f33 100644 --- a/experimental/tracer/README.md +++ b/experimental/tracer/README.md @@ -3,7 +3,9 @@ Run tracer: - ./tracer.sh start TODO: -- need to stich traces together +- need to stich traces together - deal with persistant connections - make it work for goroutines - test with jvm based app +- find way to get local ip address for an incoming connection +- make the container/process trace start/stop more reliable diff --git a/experimental/tracer/main/http.go b/experimental/tracer/main/http.go index 53ca11d0c..9ebdae3b4 100644 --- a/experimental/tracer/main/http.go +++ b/experimental/tracer/main/http.go @@ -7,8 +7,8 @@ import ( "mime" "net/http" "strconv" + "strings" - dockerClient "github.com/fsouza/go-dockerclient" "github.com/gorilla/mux" "github.com/weaveworks/scope/probe/docker" @@ -45,13 +45,31 @@ func (t *tracer) pidsForContainer(id string) ([]int, error) { return pidTree.GetChildren(container.PID()) } +type Container struct { + Id string + Name string + PIDs []int +} + func (t *tracer) http(port int) { router := mux.NewRouter() router.Methods("GET").Path("/container").HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - containers := []*dockerClient.Container{} + pidTree, err := process.NewTree(process.NewWalker("/proc")) + if err != nil { + respondWith(w, http.StatusBadRequest, err.Error()) + return + } + + containers := []Container{} t.docker.WalkContainers(func(container docker.Container) { - containers = append(containers, container.Container()) + children, _ := pidTree.GetChildren(container.PID()) + out := Container{ + Name: strings.TrimPrefix(container.Container().Name, "/"), + Id: container.ID(), + PIDs: children, + } + containers = append(containers, out) }) respondWith(w, http.StatusOK, containers) diff --git a/experimental/tracer/main/store.go b/experimental/tracer/main/store.go index ad8c137db..a47cfbd57 100644 --- a/experimental/tracer/main/store.go +++ b/experimental/tracer/main/store.go @@ -95,7 +95,6 @@ func (t *trace) addChild(child *trace) { // Fix up some fields child.ClientDetails = candidate.ClientDetails - child.PID = candidate.PID IncrementLevel(child, t.Level+1) // Overwrite old record diff --git a/experimental/tracer/ui/index.html b/experimental/tracer/ui/index.html index 35e0b32da..ca99672fd 100644 --- a/experimental/tracer/ui/index.html +++ b/experimental/tracer/ui/index.html @@ -220,14 +220,14 @@ + + + + + From 3d549546655db14105884a81c95a34b2dda779ea Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Fri, 18 Sep 2015 10:49:47 +0000 Subject: [PATCH 16/23] Protect against fd reuse. --- experimental/example/app/app.py | 1 + experimental/tracer/ptrace/thread.go | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/experimental/example/app/app.py b/experimental/example/app/app.py index f1fc4f00a..e8d3d6045 100644 --- a/experimental/example/app/app.py +++ b/experimental/example/app/app.py @@ -47,6 +47,7 @@ def ignore_error(f): @app.route('/') def hello(): qotd_msg = do_qotd() + qotd_msg += do_qotd() return qotd_msg if __name__ == "__main__": diff --git a/experimental/tracer/ptrace/thread.go b/experimental/tracer/ptrace/thread.go index 530c94928..f3cd468a6 100644 --- a/experimental/tracer/ptrace/thread.go +++ b/experimental/tracer/ptrace/thread.go @@ -52,6 +52,7 @@ type thread struct { currentIncoming map[int]*Fd currentOutgoing map[int]*Fd + closedOutgoing []*Fd } func newThread(pid int, process *process, tracer *PTracer) *thread { @@ -206,8 +207,8 @@ func (t *thread) handleConnect(call, result *syscall.PtraceRegs) { t.process.newFd(connection) t.logf("Made connection from %s:%d -> %s:%d on fd %d", - connection.ToAddr, connection.ToPort, connection.FromAddr, - connection.FromPort, fd) + connection.FromAddr, connection.FromPort, + connection.ToAddr, connection.ToPort, fd) } func (t *thread) handleClose(call, result *syscall.PtraceRegs) { @@ -231,6 +232,12 @@ func (t *thread) handleClose(call, result *syscall.PtraceRegs) { } t.currentOutgoing = map[int]*Fd{} + for _, outgoing := range t.closedOutgoing { + //t.logf("Fd %d caused %d", fdNum, outgoing.fd) + fd.Children = append(fd.Children, outgoing) + } + t.closedOutgoing = []*Fd{} + t.tracer.store.RecordConnection(t.process.pid, fd) } @@ -238,6 +245,11 @@ func (t *thread) handleClose(call, result *syscall.PtraceRegs) { delete(t.process.fds, fdNum) for _, thread := range t.process.threads { delete(thread.currentIncoming, fdNum) + + if _, ok := thread.currentOutgoing[fdNum]; ok { + thread.closedOutgoing = append(thread.closedOutgoing, fd) + } + delete(thread.currentOutgoing, fdNum) } } From b8d43a650df233b1ac591838746556882f404d1b Mon Sep 17 00:00:00 2001 From: David Kaltschmidt Date: Fri, 18 Sep 2015 12:50:45 +0200 Subject: [PATCH 17/23] allowing more boxes on same row in child traces --- experimental/tracer/ui/index.html | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/experimental/tracer/ui/index.html b/experimental/tracer/ui/index.html index d7c341835..ae53d4f27 100644 --- a/experimental/tracer/ui/index.html +++ b/experimental/tracer/ui/index.html @@ -73,7 +73,7 @@ var span = (this.Stop - this.Start) / parentSpan * 100; var offset = (this.Start - this.ParentStart) / parentSpan * 100; - return 'width:' + span + '%; margin-left:' + offset + '%;'; + return 'width:' + span + '%; left:' + offset + '%;'; }); Handlebars.registerHelper('childStyle', function() { @@ -266,6 +266,13 @@ overflow: hidden; text-overflow: ellipsis; } + .childBoxWrapper { + position: absolute; + } + .childRow { + position: relative; + height: 100px; + } - -
    Start timeDurationPIDFromToSub-traces
    {{spaces ../Level}}{{ts Start}}{{duration .}} {{../PID}}{{FromAddr}}:{{FromPort}}{{ToAddr}}:{{ToPort}}{{count ../Children}}{{ToAddr}}:{{ToPort}}{{count ../.}}{{spaces ../Level}}{{ts Start}}{{duration .}} {{../PID}}{{FromAddr}}:{{FromPort}}{{ToAddr}}:{{ToPort}}{{count ../Children}}{{ToAddr}}:{{ToPort}}{{count ../.}}
    {{spaces ../Level}}{{ts Start}}{{duration .}} {{../PID}}{{FromAddr}}:{{FromPort}}{{ToAddr}}:{{ToPort}}{{count ../Children}}
    {{ToAddr}}:{{ToPort}}{{count ../.}}
    {{spaces ../Level}}{{ts Start}}{{duration .}} {{../PID}}{{FromAddr}}:{{FromPort}}{{ToAddr}}:{{ToPort}}{{count ../Children}}
    {{ToAddr}}:{{ToPort}}{{count ../.}}