From b94751ac1015c8a6bad0b613105b60b22d17d48b Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Wed, 9 Dec 2015 10:59:27 +0000 Subject: [PATCH 1/2] Move procspy out of vendor into probe/endpoint. --- .../endpoint}/procspy/LICENSE | 0 .../procspy/benchmark_internal_test.go | 0 .../endpoint}/procspy/example_test.go | 0 .../endpoint}/procspy/fixture.go | 0 .../endpoint}/procspy/lsof.go | 0 .../endpoint/procspy/lsof_internal_test.go | 20 +++--- .../endpoint}/procspy/netstat.go | 0 .../endpoint/procspy/netstat_internal_test.go | 0 .../endpoint}/procspy/proc.go | 0 .../endpoint}/procspy/procnet.go | 0 .../endpoint/procspy/procnet_internal_test.go | 0 .../endpoint}/procspy/spy.go | 0 .../endpoint}/procspy/spy_darwin.go | 0 .../endpoint}/procspy/spy_linux.go | 0 probe/endpoint/reporter.go | 2 +- probe/endpoint/reporter_test.go | 2 +- vendor/github.com/weaveworks/procspy/Makefile | 20 ------ .../github.com/weaveworks/procspy/README.md | 64 ------------------- .../weaveworks/procspy/lsproc/lsproc.go | 18 ------ vendor/manifest | 8 +-- 20 files changed, 13 insertions(+), 121 deletions(-) rename {vendor/github.com/weaveworks => probe/endpoint}/procspy/LICENSE (100%) rename vendor/github.com/weaveworks/procspy/benchmark_test.go => probe/endpoint/procspy/benchmark_internal_test.go (100%) rename {vendor/github.com/weaveworks => probe/endpoint}/procspy/example_test.go (100%) rename {vendor/github.com/weaveworks => probe/endpoint}/procspy/fixture.go (100%) rename {vendor/github.com/weaveworks => probe/endpoint}/procspy/lsof.go (100%) rename vendor/github.com/weaveworks/procspy/lsof_test.go => probe/endpoint/procspy/lsof_internal_test.go (77%) rename {vendor/github.com/weaveworks => probe/endpoint}/procspy/netstat.go (100%) rename vendor/github.com/weaveworks/procspy/netstat_test.go => probe/endpoint/procspy/netstat_internal_test.go (100%) rename {vendor/github.com/weaveworks => probe/endpoint}/procspy/proc.go (100%) rename {vendor/github.com/weaveworks => probe/endpoint}/procspy/procnet.go (100%) rename vendor/github.com/weaveworks/procspy/procnet_test.go => probe/endpoint/procspy/procnet_internal_test.go (100%) rename {vendor/github.com/weaveworks => probe/endpoint}/procspy/spy.go (100%) rename {vendor/github.com/weaveworks => probe/endpoint}/procspy/spy_darwin.go (100%) rename {vendor/github.com/weaveworks => probe/endpoint}/procspy/spy_linux.go (100%) delete mode 100644 vendor/github.com/weaveworks/procspy/Makefile delete mode 100644 vendor/github.com/weaveworks/procspy/README.md delete mode 100644 vendor/github.com/weaveworks/procspy/lsproc/lsproc.go diff --git a/vendor/github.com/weaveworks/procspy/LICENSE b/probe/endpoint/procspy/LICENSE similarity index 100% rename from vendor/github.com/weaveworks/procspy/LICENSE rename to probe/endpoint/procspy/LICENSE diff --git a/vendor/github.com/weaveworks/procspy/benchmark_test.go b/probe/endpoint/procspy/benchmark_internal_test.go similarity index 100% rename from vendor/github.com/weaveworks/procspy/benchmark_test.go rename to probe/endpoint/procspy/benchmark_internal_test.go diff --git a/vendor/github.com/weaveworks/procspy/example_test.go b/probe/endpoint/procspy/example_test.go similarity index 100% rename from vendor/github.com/weaveworks/procspy/example_test.go rename to probe/endpoint/procspy/example_test.go diff --git a/vendor/github.com/weaveworks/procspy/fixture.go b/probe/endpoint/procspy/fixture.go similarity index 100% rename from vendor/github.com/weaveworks/procspy/fixture.go rename to probe/endpoint/procspy/fixture.go diff --git a/vendor/github.com/weaveworks/procspy/lsof.go b/probe/endpoint/procspy/lsof.go similarity index 100% rename from vendor/github.com/weaveworks/procspy/lsof.go rename to probe/endpoint/procspy/lsof.go diff --git a/vendor/github.com/weaveworks/procspy/lsof_test.go b/probe/endpoint/procspy/lsof_internal_test.go similarity index 77% rename from vendor/github.com/weaveworks/procspy/lsof_test.go rename to probe/endpoint/procspy/lsof_internal_test.go index 217046229..f716eedd5 100644 --- a/vendor/github.com/weaveworks/procspy/lsof_test.go +++ b/probe/endpoint/procspy/lsof_internal_test.go @@ -12,8 +12,8 @@ func TestLSOFParsing(t *testing.T) { "p25196\n" + "ccello-app\n" + "n127.0.0.1:48094->127.0.0.1:4039\n" + - "n*:4040\n": map[string]Proc{ - "127.0.0.1:48094": Proc{ + "n*:4040\n": { + "127.0.0.1:48094": { PID: 25196, Name: "cello-app", }, @@ -23,7 +23,7 @@ func TestLSOFParsing(t *testing.T) { "cdhclient\n" + "n*:68\n" + "n*:38282\n" + - "n*:40625\n": map[string]Proc{}, + "n*:40625\n": {}, // A bunch "p13100\n" + @@ -39,28 +39,28 @@ func TestLSOFParsing(t *testing.T) { "n192.168.2.111:56385->74.201.105.31:443\n" + "p21356\n" + "cssh\n" + - "n192.168.2.111:33963->192.168.2.71:22\n": map[string]Proc{ - "[::1]:6600": Proc{ + "n192.168.2.111:33963->192.168.2.71:22\n": { + "[::1]:6600": { PID: 13100, Name: "mpd", }, - "[2003:45:2b57:8900:1869:2947:f942:aba7]:55711": Proc{ + "[2003:45:2b57:8900:1869:2947:f942:aba7]:55711": { PID: 14612, Name: "chromium", }, - "192.168.2.111:37158": Proc{ + "192.168.2.111:37158": { PID: 14612, Name: "chromium", }, - "192.168.2.111:44013": Proc{ + "192.168.2.111:44013": { PID: 14612, Name: "chromium", }, - "192.168.2.111:56385": Proc{ + "192.168.2.111:56385": { PID: 14612, Name: "chromium", }, - "192.168.2.111:33963": Proc{ + "192.168.2.111:33963": { PID: 21356, Name: "ssh", }, diff --git a/vendor/github.com/weaveworks/procspy/netstat.go b/probe/endpoint/procspy/netstat.go similarity index 100% rename from vendor/github.com/weaveworks/procspy/netstat.go rename to probe/endpoint/procspy/netstat.go diff --git a/vendor/github.com/weaveworks/procspy/netstat_test.go b/probe/endpoint/procspy/netstat_internal_test.go similarity index 100% rename from vendor/github.com/weaveworks/procspy/netstat_test.go rename to probe/endpoint/procspy/netstat_internal_test.go diff --git a/vendor/github.com/weaveworks/procspy/proc.go b/probe/endpoint/procspy/proc.go similarity index 100% rename from vendor/github.com/weaveworks/procspy/proc.go rename to probe/endpoint/procspy/proc.go diff --git a/vendor/github.com/weaveworks/procspy/procnet.go b/probe/endpoint/procspy/procnet.go similarity index 100% rename from vendor/github.com/weaveworks/procspy/procnet.go rename to probe/endpoint/procspy/procnet.go diff --git a/vendor/github.com/weaveworks/procspy/procnet_test.go b/probe/endpoint/procspy/procnet_internal_test.go similarity index 100% rename from vendor/github.com/weaveworks/procspy/procnet_test.go rename to probe/endpoint/procspy/procnet_internal_test.go diff --git a/vendor/github.com/weaveworks/procspy/spy.go b/probe/endpoint/procspy/spy.go similarity index 100% rename from vendor/github.com/weaveworks/procspy/spy.go rename to probe/endpoint/procspy/spy.go diff --git a/vendor/github.com/weaveworks/procspy/spy_darwin.go b/probe/endpoint/procspy/spy_darwin.go similarity index 100% rename from vendor/github.com/weaveworks/procspy/spy_darwin.go rename to probe/endpoint/procspy/spy_darwin.go diff --git a/vendor/github.com/weaveworks/procspy/spy_linux.go b/probe/endpoint/procspy/spy_linux.go similarity index 100% rename from vendor/github.com/weaveworks/procspy/spy_linux.go rename to probe/endpoint/procspy/spy_linux.go diff --git a/probe/endpoint/reporter.go b/probe/endpoint/reporter.go index 9f8f66ff2..e0083a2d7 100644 --- a/probe/endpoint/reporter.go +++ b/probe/endpoint/reporter.go @@ -6,7 +6,7 @@ import ( "github.com/prometheus/client_golang/prometheus" - "github.com/weaveworks/procspy" + "github.com/weaveworks/scope/probe/endpoint/procspy" "github.com/weaveworks/scope/probe/process" "github.com/weaveworks/scope/report" ) diff --git a/probe/endpoint/reporter_test.go b/probe/endpoint/reporter_test.go index c7ec41995..9da3e4cac 100644 --- a/probe/endpoint/reporter_test.go +++ b/probe/endpoint/reporter_test.go @@ -5,9 +5,9 @@ import ( "strconv" "testing" - "github.com/weaveworks/procspy" "github.com/weaveworks/scope/probe/docker" "github.com/weaveworks/scope/probe/endpoint" + "github.com/weaveworks/scope/probe/endpoint/procspy" "github.com/weaveworks/scope/report" ) diff --git a/vendor/github.com/weaveworks/procspy/Makefile b/vendor/github.com/weaveworks/procspy/Makefile deleted file mode 100644 index 709dfae80..000000000 --- a/vendor/github.com/weaveworks/procspy/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -.PHONY: all build buildall test install bench -all: test build buildall install - -build: - go build - go vet - golint . - -buildall: - GOOS=darwin go build - GOOS=linux go build - -test: - go test - -install: - go install - -bench: - go test -bench . diff --git a/vendor/github.com/weaveworks/procspy/README.md b/vendor/github.com/weaveworks/procspy/README.md deleted file mode 100644 index 2daf68ffe..000000000 --- a/vendor/github.com/weaveworks/procspy/README.md +++ /dev/null @@ -1,64 +0,0 @@ -Go module to list all TCP connections, with an option to try to find the owning PID and processname. - -Works by reading /proc directly on Linux, and by executing `netstat` and `lsof -i` on Darwin. - -Works for IPv4 and IPv6 TCP connections. Only established connections are listed; ports where something is only listening or TIME_WAITs are skipped. - -If you want to find all processes you'll need to run this as root. - -Status: -------- - -Tested on Linux and Darwin (10.9). - -Install: --------- - -`go install` - -Usage: ------- - -Only list the connections: - -``` -cs, err := procspy.Connections(false) -for c := cs.Next(); c != nil; c = cs.Next() { - ... -} -``` - -List the connections and try to find the owning process: - -``` -cs, err := procspy.Connections(true) -for c := cs.Next(); c != nil; c = cs.Next() { - ... -} -``` - -(See ./example\_test.go) - -``` go - -package main - -import ( - "fmt" - - "github.com/weaveworks/procspy" -) - -func main() { - lookupProcesses := true - cs, err := procspy.Connections(lookupProcesses) - if err != nil { - panic(err) - } - - fmt.Printf("TCP Connections:\n") - for c := cs.Next(); c != nil; c = cs.Next() { - fmt.Printf(" - %v\n", c) - } -} -``` diff --git a/vendor/github.com/weaveworks/procspy/lsproc/lsproc.go b/vendor/github.com/weaveworks/procspy/lsproc/lsproc.go deleted file mode 100644 index d926b463c..000000000 --- a/vendor/github.com/weaveworks/procspy/lsproc/lsproc.go +++ /dev/null @@ -1,18 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/weaveworks/procspy" -) - -func main() { - cs, err := procspy.Connections(true) - if err != nil { - panic(err) - } - fmt.Printf("TCP Connections:\n") - for c := cs.Next(); c != nil; c = cs.Next() { - fmt.Printf(" - %+v\n", c) - } -} diff --git a/vendor/manifest b/vendor/manifest index f65a6c3dd..21351d08b 100644 --- a/vendor/manifest +++ b/vendor/manifest @@ -723,12 +723,6 @@ "branch": "master", "path": "/odp" }, - { - "importpath": "github.com/weaveworks/procspy", - "repository": "https://github.com/weaveworks/procspy", - "revision": "cb970aa190c374d1e47711dbffb3c2c6e9ef0dd1", - "branch": "master" - }, { "importpath": "github.com/weaveworks/weave/common", "repository": "https://github.com/weaveworks/weave", @@ -931,4 +925,4 @@ "branch": "master" } ] -} \ No newline at end of file +} From 5cadafcda48e5b796516fec805119eb233b292a0 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Wed, 9 Dec 2015 11:50:30 +0000 Subject: [PATCH 2/2] Extend testing of procspy. --- common/fs/fs.go | 11 + probe/endpoint/procspy/example_test.go | 20 -- probe/endpoint/procspy/proc.go | 48 ++--- probe/endpoint/procspy/proc_internal_test.go | 57 ++++++ test/fs/fs.go | 199 +++++++++++++++++++ 5 files changed, 291 insertions(+), 44 deletions(-) create mode 100644 common/fs/fs.go delete mode 100644 probe/endpoint/procspy/example_test.go create mode 100644 probe/endpoint/procspy/proc_internal_test.go create mode 100644 test/fs/fs.go diff --git a/common/fs/fs.go b/common/fs/fs.go new file mode 100644 index 000000000..50e41a3ab --- /dev/null +++ b/common/fs/fs.go @@ -0,0 +1,11 @@ +package fs + +import ( + "io" + "os" +) + +// Open is a mockable version of os.Open +var Open = func(path string) (io.ReadWriteCloser, error) { + return os.Open(path) +} diff --git a/probe/endpoint/procspy/example_test.go b/probe/endpoint/procspy/example_test.go deleted file mode 100644 index 26bbd9d02..000000000 --- a/probe/endpoint/procspy/example_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package procspy_test - -import ( - "fmt" - - "github.com/weaveworks/procspy" -) - -func Example() { - lookupProcesses := true - cs, err := procspy.Connections(lookupProcesses) - if err != nil { - panic(err) - } - - fmt.Printf("TCP Connections:\n") - for c := cs.Next(); c != nil; c = cs.Next() { - fmt.Printf(" - %v\n", c) - } -} diff --git a/probe/endpoint/procspy/proc.go b/probe/endpoint/procspy/proc.go index ac0e37957..8f00dc5c6 100644 --- a/probe/endpoint/procspy/proc.go +++ b/probe/endpoint/procspy/proc.go @@ -4,10 +4,13 @@ package procspy import ( "bytes" + "io/ioutil" "os" "path/filepath" "strconv" "syscall" + + "github.com/weaveworks/scope/common/fs" ) var ( @@ -19,17 +22,19 @@ func SetProcRoot(root string) { procRoot = root } +// made variables for mocking +var ( + readDir = ioutil.ReadDir + lstat = syscall.Lstat + stat = syscall.Stat + open = fs.Open +) + // walkProcPid walks over all numerical (PID) /proc entries, and sees if their // ./fd/* files are symlink to sockets. Returns a map from socket ID (inode) // to PID. Will return an error if /proc isn't there. func walkProcPid(buf *bytes.Buffer) (map[uint64]Proc, error) { - fh, err := os.Open(procRoot) - if err != nil { - return nil, err - } - - dirNames, err := fh.Readdirnames(-1) - fh.Close() + dirNames, err := readDir(procRoot) if err != nil { return nil, err } @@ -37,9 +42,10 @@ func walkProcPid(buf *bytes.Buffer) (map[uint64]Proc, error) { var ( res = map[uint64]Proc{} namespaces = map[uint64]struct{}{} - stat syscall.Stat_t + statT syscall.Stat_t ) - for _, dirName := range dirNames { + for _, entry := range dirNames { + dirName := entry.Name() pid, err := strconv.ParseUint(dirName, 10, 0) if err != nil { // Not a number, so not a PID subdir. @@ -47,41 +53,35 @@ func walkProcPid(buf *bytes.Buffer) (map[uint64]Proc, error) { } fdBase := filepath.Join(procRoot, dirName, "fd") - dfh, err := os.Open(fdBase) + fds, err := readDir(fdBase) if err != nil { // Process is be gone by now, or we don't have access. continue } - fdNames, err := dfh.Readdirnames(-1) - dfh.Close() - if err != nil { - continue - } - // Read network namespace, and if we haven't seen it before, // read /proc//net/tcp - err = syscall.Lstat(filepath.Join(procRoot, dirName, "/ns/net"), &stat) + err = lstat(filepath.Join(procRoot, dirName, "/ns/net"), &statT) if err != nil { continue } - if _, ok := namespaces[stat.Ino]; !ok { - namespaces[stat.Ino] = struct{}{} + if _, ok := namespaces[statT.Ino]; !ok { + namespaces[statT.Ino] = struct{}{} readFile(filepath.Join(procRoot, dirName, "/net/tcp"), buf) readFile(filepath.Join(procRoot, dirName, "/net/tcp6"), buf) } var name string - for _, fdName := range fdNames { + for _, fd := range fds { // Direct use of syscall.Stat() to save garbage. - err = syscall.Stat(filepath.Join(fdBase, fdName), &stat) + err = stat(filepath.Join(fdBase, fd.Name()), &statT) if err != nil { continue } // We want sockets only. - if stat.Mode&syscall.S_IFMT != syscall.S_IFSOCK { + if statT.Mode&syscall.S_IFMT != syscall.S_IFSOCK { continue } @@ -92,7 +92,7 @@ func walkProcPid(buf *bytes.Buffer) (map[uint64]Proc, error) { } } - res[stat.Ino] = Proc{ + res[statT.Ino] = Proc{ PID: uint(pid), Name: name, } @@ -104,7 +104,7 @@ func walkProcPid(buf *bytes.Buffer) (map[uint64]Proc, error) { // procName does a pid->name lookup. func procName(base string) string { - fh, err := os.Open(filepath.Join(base, "/comm")) + fh, err := open(filepath.Join(base, "/comm")) if err != nil { return "" } diff --git a/probe/endpoint/procspy/proc_internal_test.go b/probe/endpoint/procspy/proc_internal_test.go new file mode 100644 index 000000000..0dbf4882d --- /dev/null +++ b/probe/endpoint/procspy/proc_internal_test.go @@ -0,0 +1,57 @@ +package procspy + +import ( + "bytes" + "reflect" + "syscall" + "testing" + + "github.com/weaveworks/scope/test/fs" +) + +var mockFS = fs.Dir("", + fs.Dir("proc", + fs.Dir("1", + fs.Dir("fd", + fs.File{ + FName: "16", + FStat: syscall.Stat_t{ + Ino: 45, + Mode: syscall.S_IFSOCK, + }, + }, + ), + fs.File{ + FName: "comm", + FContents: "foo\n", + }, + fs.Dir("ns", + fs.File{ + FName: "net", + FStat: syscall.Stat_t{}, + }, + ), + ), + ), +) + +func TestWalkProcPid(t *testing.T) { + oldReadDir, oldLstat, oldStat, oldOpen := readDir, lstat, stat, open + defer func() { readDir, lstat, stat, open = oldReadDir, oldLstat, oldStat, oldOpen }() + readDir, lstat, stat, open = mockFS.ReadDir, mockFS.Lstat, mockFS.Stat, mockFS.Open + + buf := bytes.Buffer{} + have, err := walkProcPid(&buf) + if err != nil { + t.Fatal(err) + } + want := map[uint64]Proc{ + 45: { + PID: 1, + Name: "foo", + }, + } + if !reflect.DeepEqual(want, have) { + t.Fatalf("%+v", have) + } +} diff --git a/test/fs/fs.go b/test/fs/fs.go new file mode 100644 index 000000000..2a47a602e --- /dev/null +++ b/test/fs/fs.go @@ -0,0 +1,199 @@ +package fs + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "syscall" + "time" +) + +type mockInode struct{} + +type dir struct { + mockInode + name string + entries map[string]FS + stat syscall.Stat_t +} + +// File is a mock file +type File struct { + mockInode + FName string + FContents string + FStat syscall.Stat_t +} + +// FS is a mock filesystem +type FS interface { + os.FileInfo + ReadDir(string) ([]os.FileInfo, error) + ReadFile(string) ([]byte, error) + Lstat(string, *syscall.Stat_t) error + Stat(string, *syscall.Stat_t) error + Open(string) (io.ReadWriteCloser, error) +} + +// Dir creates a new directory with the given entries. +func Dir(name string, entries ...FS) FS { + result := dir{ + name: name, + entries: map[string]FS{}, + } + + for _, entry := range entries { + result.entries[entry.Name()] = entry + } + + return result +} + +func split(path string) (string, string) { + if !strings.HasPrefix(path, "/") { + panic(path) + } + + comps := strings.SplitN(path, "/", 3) + if len(comps) == 2 { + return comps[1], "/" + } + + return comps[1], "/" + comps[2] +} + +func (mockInode) Size() int64 { return 0 } +func (mockInode) Mode() os.FileMode { return 0 } +func (mockInode) ModTime() time.Time { return time.Now() } +func (mockInode) Sys() interface{} { return nil } + +func (p dir) Name() string { return p.name } +func (p dir) IsDir() bool { return true } + +func (p dir) ReadDir(path string) ([]os.FileInfo, error) { + if path == "/" { + result := []os.FileInfo{} + for _, v := range p.entries { + result = append(result, v) + } + return result, nil + } + + head, tail := split(path) + fs, ok := p.entries[head] + if !ok { + return nil, fmt.Errorf("Not found: %s", path) + } + + return fs.ReadDir(tail) +} + +func (p dir) ReadFile(path string) ([]byte, error) { + if path == "/" { + return nil, fmt.Errorf("I'm a directory!") + } + + head, tail := split(path) + fs, ok := p.entries[head] + if !ok { + return nil, fmt.Errorf("Not found: %s", path) + } + + return fs.ReadFile(tail) +} + +func (p dir) Lstat(path string, stat *syscall.Stat_t) error { + if path == "/" { + return nil + } + + head, tail := split(path) + fs, ok := p.entries[head] + if !ok { + return fmt.Errorf("Not found: %s", path) + } + + return fs.Lstat(tail, stat) +} + +func (p dir) Stat(path string, stat *syscall.Stat_t) error { + if path == "/" { + return nil + } + + head, tail := split(path) + fs, ok := p.entries[head] + if !ok { + return fmt.Errorf("Not found: %s", path) + } + + return fs.Stat(tail, stat) +} + +func (p dir) Open(path string) (io.ReadWriteCloser, error) { + if path == "/" { + return nil, fmt.Errorf("I'm a directory!") + } + + head, tail := split(path) + fs, ok := p.entries[head] + if !ok { + return nil, fmt.Errorf("Not found: %s", path) + } + + return fs.Open(tail) +} + +// Name implements os.FileInfo +func (p File) Name() string { return p.FName } + +// IsDir implements os.FileInfo +func (p File) IsDir() bool { return false } + +// ReadDir implements FS +func (p File) ReadDir(path string) ([]os.FileInfo, error) { + return nil, fmt.Errorf("I'm a file!") +} + +// ReadFile implements FS +func (p File) ReadFile(path string) ([]byte, error) { + if path != "/" { + return nil, fmt.Errorf("I'm a file!") + } + return []byte(p.FContents), nil +} + +// Lstat implements FS +func (p File) Lstat(path string, stat *syscall.Stat_t) error { + if path != "/" { + return fmt.Errorf("I'm a file!") + } + *stat = p.FStat + return nil +} + +// Stat implements FS +func (p File) Stat(path string, stat *syscall.Stat_t) error { + if path != "/" { + return fmt.Errorf("I'm a file!") + } + *stat = p.FStat + return nil +} + +// Open implements FS +func (p File) Open(path string) (io.ReadWriteCloser, error) { + if path != "/" { + return nil, fmt.Errorf("I'm a file!") + } + return struct { + io.ReadWriter + io.Closer + }{ + bytes.NewBuffer([]byte(p.FContents)), + ioutil.NopCloser(nil), + }, nil +}