Show subtraces in the API and UI.

experimental/tracer/main/store.go

@@ -21,8 +21,10 @@ type key struct {
 
 type trace struct {
 	PID      int
-	Root     *ptrace.Fd
+	ServerDetails *ptrace.ConnectionDetails
+	ClientDetails *ptrace.ConnectionDetails
 	Children []*trace
+	Level    int
 }
 
 type store struct {
@@ -63,10 +65,18 @@ func (s *store) RecordConnection(pid int, connection *ptrace.Fd) {
 	s.Lock()
 	defer s.Unlock()
 
-	newTrace := &trace{PID: pid, Root: connection}
-	newTraceKey := newKey(connection)
+	newTrace := &trace{
+		PID: pid,
+		ServerDetails: &connection.ConnectionDetails,
+	}
+	for _, child := range connection.Children {
+		newTrace.Children = append(newTrace.Children, &trace{
+			Level: 1,
+			ClientDetails: &child.ConnectionDetails,
+		})
+	}
 
-	log.Printf("Recording trace: %+v", newTrace)
+	newTraceKey := newKey(connection)
 
 	// First, see if this new conneciton is a child of an existing connection.
 	// This indicates we have a parent connection to attach to.
@@ -75,6 +85,7 @@ func (s *store) RecordConnection(pid int, connection *ptrace.Fd) {
 		parentNode.Remove()
 		parentTrace := parentNode.Value.(*trace)
 		log.Printf(" Found parent trace: %+v", parentTrace)
+		newTrace.Level = parentTrace.Level + 1
 		parentTrace.Children = append(parentTrace.Children, newTrace)
 	} else {
 		s.traces.Insert(newTraceKey, newTrace)
@@ -89,6 +100,7 @@ func (s *store) RecordConnection(pid int, connection *ptrace.Fd) {
 			childNode.Remove()
 			childTrace := childNode.Value.(*trace)
 			log.Printf(" Found child trace: %+v", childTrace)
+			IncrementLevel(childTrace, newTrace.Level)
 			newTrace.Children = append(newTrace.Children, childTrace)
 		} else {
 			s.traces.Insert(childTraceKey, newTrace)
@@ -96,11 +108,18 @@ func (s *store) RecordConnection(pid int, connection *ptrace.Fd) {
 	}
 }
 
+func IncrementLevel(trace *trace, increment int) {
+	trace.Level += increment
+	for _, child := range trace.Children {
+		IncrementLevel(child, increment)
+	}
+}
+
 func (s *store) Traces() []*trace {
 	s.RLock()
 	defer s.RUnlock()
 
-	var traces []*trace
+	traces := []*trace{}
 	var cur = s.traces.First()
 	for cur != nil {
 		traces = append(traces, cur.Value.(*trace))

experimental/tracer/ptrace/fd.go

@@ -29,10 +29,8 @@ var (
 	tcpRegexp   = regexp.MustCompile(tcpPattern)
 )
 
-// Fd represents a connect and subsequent connections caused by it.
-type Fd struct {
+type ConnectionDetails struct {
 	direction int
-	fd        int
 
 	Start    int64
 	Stop     int64
@@ -43,6 +41,14 @@ type Fd struct {
 	FromPort uint16
 	ToAddr   net.IP
 	ToPort   uint16
+}
+
+// Fd represents a connect and subsequent connections caused by it.
+type Fd struct {
+	fd     int
+	closed bool
+
+	ConnectionDetails
 
 	// Fds are connections, and can have a causal-link to other Fds
 	Children []*Fd
@@ -133,8 +139,14 @@ func newListeningFd(pid, fd int) (*Fd, error) {
 	}
 
 	return &Fd{
-		direction: listening, fd: fd, Start: now(),
-		ToAddr: localAddr, ToPort: uint16(localPort),
+		fd: fd,
+
+		ConnectionDetails: ConnectionDetails{
+			direction: listening,
+			Start:     now(),
+			ToAddr:    localAddr,
+			ToPort:    uint16(localPort),
+		},
 	}, nil
 }
 
@@ -146,9 +158,16 @@ func newConnectionFd(pid, fd int, remoteAddr net.IP, remotePort uint16) (*Fd, er
 	}
 
 	return &Fd{
-		direction: outgoing, fd: fd, Start: now(),
-		FromAddr: localAddr, FromPort: uint16(localPort),
-		ToAddr: remoteAddr, ToPort: remotePort,
+		fd: fd,
+
+		ConnectionDetails: ConnectionDetails{
+			direction: outgoing,
+			Start:     now(),
+			FromAddr:  localAddr,
+			FromPort:  uint16(localPort),
+			ToAddr:    remoteAddr,
+			ToPort:    remotePort,
+		},
 	}, nil
 }
 
@@ -159,12 +178,20 @@ func (fd *Fd) newConnection(addr net.IP, port uint16, newFd int) (*Fd, error) {
 	}
 
 	return &Fd{
-		direction: incoming, fd: newFd, Start: now(),
-		ToAddr: fd.ToAddr, ToPort: fd.ToPort,
-		FromAddr: addr, FromPort: port,
+		fd: newFd,
+
+		ConnectionDetails: ConnectionDetails{
+			direction: incoming,
+			Start:     now(),
+			ToAddr:    fd.ToAddr,
+			ToPort:    fd.ToPort,
+			FromAddr:  addr,
+			FromPort:  port,
+		},
 	}, nil
 }
 
 func (fd *Fd) close() {
+	fd.closed = true
 	fd.Stop = now()
 }

experimental/tracer/ptrace/thread.go

@@ -204,28 +204,26 @@ func (t *thread) handleClose(call, result *syscall.PtraceRegs) {
 		return
 	}
 
+	t.logf("Closing fd %d", fdNum)
 	fd.close()
 
 	// if this connection was incoming, add it to 'the registry'
 	if fd.direction == incoming {
+		// collect all the outgoing connections this thread has made
+		// and treat them as caused by this incoming connections
 		for _, outgoing := range t.currentOutgoing {
-			t.logf("Fd %d caused %d", fd.fd, outgoing.fd)
+			t.logf("Fd %d caused %d", fdNum, outgoing.fd)
 			fd.Children = append(fd.Children, outgoing)
 		}
+		t.currentOutgoing = map[int]*Fd{}
 
 		t.tracer.store.RecordConnection(t.process.pid, fd)
-	} else {
-		for _, incoming := range t.currentIncoming {
-			t.logf("Fd %d caused %d", incoming.fd, fd.fd)
-			incoming.Children = append(incoming.Children, fd)
-		}
 	}
 
 	// now make sure we've remove it from everywhere
 	delete(t.process.fds, fdNum)
 	for _, thread := range t.process.threads {
 		delete(thread.currentIncoming, fdNum)
-		delete(t.currentOutgoing, fdNum)
 	}
 }
 
@@ -241,6 +239,7 @@ func (t *thread) handleIO(call, result *syscall.PtraceRegs) {
 		t.logf("IO on incoming connection %d; setting affinity", fdNum)
 		t.currentIncoming[fdNum] = fd
 	} else {
+		t.logf("IO on outgoing connection %d; setting affinity", fdNum)
 		t.currentOutgoing[fdNum] = fd
 	}
 }

experimental/tracer/ui/index.html

@@ -24,6 +24,10 @@
       var containers = [];
       var traces = [];
 
+      Handlebars.registerHelper('spaces', function(input) {
+        return new Array(input + 1).join("> ");
+      });
+
       Handlebars.registerHelper('ts', function(input) {
         var ts = moment(input).format("LTS")
         return new Handlebars.SafeString(ts);
@@ -39,9 +43,11 @@
       });
 
       Handlebars.registerHelper('count', function(input) {
-        return new Handlebars.SafeString(input === null ? '0' : sprintf("%d", input.len));
+        return new Handlebars.SafeString(input === null ? '0' : sprintf("%d", input.length));
       });
 
+      Handlebars.registerPartial('traces', $("#traces").html());
+
       function render() {
         var template = $('script#process-template').text();
         template = Handlebars.compile(template);
@@ -115,6 +121,27 @@
         width: 100%;
       }
     </style>
+    <script type="text/x-handlebars-template" id="traces">
+      {{#.}}
+      {{#if ServerDetails}}
+        {{#with ServerDetails}}
+        <tr>
+          <td>{{spaces ../Level}}{{ts Start}}</td><td>{{duration .}}</td>
+          <td>{{../PID}}</td><td>{{FromAddr}}:{{FromPort}}</td>
+          <td>{{ToAddr}}:{{ToPort}}</td><td>{{count ../Children}}</tr>
+        {{/with}}
+      {{else}}
+        {{#with ClientDetails}}
+        <tr>
+          <td>{{spaces ../Level}}{{ts Start}}</td><td>{{duration .}}</td>
+          <td>{{../PID}}</td><td>{{FromAddr}}:{{FromPort}}</td>
+          <td>{{ToAddr}}:{{ToPort}}</td><td>{{count ../Children}}</tr>
+        {{/with}}
+      {{/if}}
+      {{>traces Children}}
+      {{/.}}
+    </script>
+
     <script type="text/x-handlebars-template" id="process-template">
       <div class="container-fluid">
         <div class="col-md-4">
@@ -136,16 +163,13 @@
               <span class="glyphicon glyphicon-stop" aria-hidden="true"></span> Stop</button>
           </div>
 
-          <table>
+          <table padding="2">
             <thead><tr><th>Start time</th><th>Duration</th><th>PID<th>From</th><th>To</th><th>Sub-traces</th></tr></thead>
             <tbody>
-              {{#traces}}
-              <tr><td>{{ts Root.Start}}</td><td>{{duration Root}}</td>
-                <td>{{PID}}</td><td>{{Root.FromAddr}}:{{Root.FromPort}}</td>
-                <td>{{Root.ToAddr}}:{{Root.ToPort}}</td><td>{{count Root.Children}}</tr>
-              {{/traces}}
+          {{>traces traces}}
             </tbody>
           </table>
+
           {{/if}}
         </div>
       </div>