Merge pull request #2456 from kinvolk/alban/open-files-count

proc walker: optimize open file counter

probe/process/walker_linux.go

@@ -102,7 +102,7 @@ func (w *walker) Walk(f func(Process, Process)) error {
 			continue
 		}
 
-		openFiles, err := fs.ReadDirNames(path.Join(w.procRoot, filename, "fd"))
+		openFilesCount, err := fs.ReadDirCount(path.Join(w.procRoot, filename, "fd"))
 		if err != nil {
 			continue
 		}
@@ -139,7 +139,7 @@ func (w *walker) Walk(f func(Process, Process)) error {
 			Jiffies:        jiffies,
 			RSSBytes:       rss,
 			RSSBytesLimit:  rssLimit,
-			OpenFilesCount: len(openFiles),
+			OpenFilesCount: openFilesCount,
 			OpenFilesLimit: openFilesLimit,
 		}, Process{})
 	}

vendor/github.com/weaveworks/common/fs/fs.go

@@ -11,6 +11,7 @@ import (
 type Interface interface {
 	ReadDir(string) ([]os.FileInfo, error)
 	ReadDirNames(string) ([]string, error)
+	ReadDirCount(string) (int, error)
 	ReadFile(string) ([]byte, error)
 	Lstat(string, *syscall.Stat_t) error
 	Stat(string, *syscall.Stat_t) error
@@ -63,6 +64,11 @@ func ReadDirNames(path string) ([]string, error) {
 	return fs.ReadDirNames(path)
 }
 
+// ReadDirCount is an optimized way to call len(ReadDirNames)
+func ReadDirCount(path string) (int, error) {
+	return fs.ReadDirCount(path)
+}
+
 // ReadFile see ioutil.ReadFile
 func ReadFile(path string) ([]byte, error) {
 	return fs.ReadFile(path)

vendor/github.com/weaveworks/common/fs/readdircount_linux_amd64.go

@@ -0,0 +1,62 @@
+// +build linux,amd64
+
+package fs
+
+import (
+	"fmt"
+	"os"
+	"unsafe"
+
+	"syscall"
+)
+
+func countDirEntries(buf []byte, n int) int {
+	count := 0
+	buf = buf[:n]
+	for len(buf) > 0 {
+		// see man page getdents(2) for struct linux_dirent64
+		reclenOffset := unsafe.Offsetof(syscall.Dirent{}.Reclen)
+		reclen := *(*uint16)(unsafe.Pointer(&buf[reclenOffset]))
+
+		inoOffset := unsafe.Offsetof(syscall.Dirent{}.Ino)
+		ino := *(*uint64)(unsafe.Pointer(&buf[inoOffset]))
+
+		if int(reclen) > len(buf) {
+			return count
+		}
+		buf = buf[reclen:]
+		if ino == 0 {
+			continue
+		}
+		count++
+	}
+	return count
+}
+
+// ReadDirCount is similar to ReadDirNames() and then counting with len() but
+// it is optimized to avoid parsing the entries
+func (realFS) ReadDirCount(dir string) (int, error) {
+	buf := make([]byte, 4096)
+	fh, err := os.Open(dir)
+	if err != nil {
+		return 0, err
+	}
+	defer fh.Close()
+
+	openFilesCount := 0
+	for {
+		n, err := syscall.ReadDirent(int(fh.Fd()), buf)
+		if err != nil {
+			return 0, fmt.Errorf("ReadDirent() failed: %v", err)
+		}
+		if n == 0 {
+			break
+		}
+
+		openFilesCount += countDirEntries(buf, n)
+	}
+
+	// "." and ".." don't count as files to be counted
+	nDotFiles := 2
+	return openFilesCount - nDotFiles, err
+}

vendor/github.com/weaveworks/common/fs/readdircount_unsupported.go

@@ -0,0 +1,9 @@
+// +build !linux !amd64
+
+package fs
+
+// ReadDirCount, unoptimized version
+func (realFS) ReadDirCount(path string) (int, error) {
+	names, err := ReadDirNames(path)
+	return len(names), err
+}

vendor/github.com/weaveworks/common/middleware/http_auth.go

@@ -9,7 +9,7 @@ import (
 // AuthenticateUser propagates the user ID from HTTP headers back to the request's context.
 var AuthenticateUser = Func(func(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		_, ctx, err := user.ExtractFromHTTPRequest(r)
+		_, ctx, err := user.ExtractOrgIDFromHTTPRequest(r)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusUnauthorized)
 			return

vendor/github.com/weaveworks/common/test/fs/fs.go

@@ -112,6 +112,11 @@ func (p dir) ReadDirNames(path string) ([]string, error) {
 	return fs.ReadDirNames(tail)
 }
 
+func (p dir) ReadDirCount(path string) (int, error) {
+	names, err := p.ReadDirNames(path)
+	return len(names), err
+}
+
 func (p dir) ReadFile(path string) ([]byte, error) {
 	if path == "/" {
 		return nil, fmt.Errorf("I'm a directory")
@@ -216,6 +221,11 @@ func (p File) ReadDirNames(path string) ([]string, error) {
 	return nil, fmt.Errorf("I'm a file")
 }
 
+// ReadDirCount implements FS
+func (p File) ReadDirCount(path string) (int, error) {
+	return 0, fmt.Errorf("I'm a file")
+}
+
 // ReadFile implements FS
 func (p File) ReadFile(path string) ([]byte, error) {
 	if path != "/" {

vendor/github.com/weaveworks/common/tools/runner/runner.go

@@ -148,9 +148,10 @@ func updateScheduler(test string, duration float64) {
 
 func getSchedule(tests []string) ([]string, error) {
 	var (
+		userName    = os.Getenv("CIRCLE_PROJECT_USERNAME")
 		project     = os.Getenv("CIRCLE_PROJECT_REPONAME")
 		buildNum    = os.Getenv("CIRCLE_BUILD_NUM")
-		testRun     = project + "-integration-" + buildNum
+		testRun     = userName + "-" + project + "-integration-" + buildNum
 		shardCount  = os.Getenv("CIRCLE_NODE_TOTAL")
 		shardID     = os.Getenv("CIRCLE_NODE_INDEX")
 		requestBody = &bytes.Buffer{}

vendor/github.com/weaveworks/common/user/grpc.go

@@ -10,20 +10,20 @@ import (
 func ExtractFromGRPCRequest(ctx context.Context) (string, context.Context, error) {
 	md, ok := metadata.FromContext(ctx)
 	if !ok {
-		return "", ctx, ErrNoUserID
+		return "", ctx, ErrNoOrgID
 	}
 
-	userIDs, ok := md[lowerOrgIDHeaderName]
-	if !ok || len(userIDs) != 1 {
-		return "", ctx, ErrNoUserID
+	orgIDs, ok := md[lowerOrgIDHeaderName]
+	if !ok || len(orgIDs) != 1 {
+		return "", ctx, ErrNoOrgID
 	}
 
-	return userIDs[0], Inject(ctx, userIDs[0]), nil
+	return orgIDs[0], InjectOrgID(ctx, orgIDs[0]), nil
 }
 
-// InjectIntoGRPCRequest injects the userID from the context into the request metadata.
+// InjectIntoGRPCRequest injects the orgID from the context into the request metadata.
 func InjectIntoGRPCRequest(ctx context.Context) (context.Context, error) {
-	userID, err := Extract(ctx)
+	orgID, err := ExtractOrgID(ctx)
 	if err != nil {
 		return ctx, err
 	}
@@ -33,17 +33,17 @@ func InjectIntoGRPCRequest(ctx context.Context) (context.Context, error) {
 		md = metadata.New(map[string]string{})
 	}
 	newCtx := ctx
-	if userIDs, ok := md[lowerOrgIDHeaderName]; ok {
-		if len(userIDs) == 1 {
-			if userIDs[0] != userID {
-				return ctx, ErrDifferentIDPresent
+	if orgIDs, ok := md[lowerOrgIDHeaderName]; ok {
+		if len(orgIDs) == 1 {
+			if orgIDs[0] != orgID {
+				return ctx, ErrDifferentOrgIDPresent
 			}
 		} else {
-			return ctx, ErrTooManyUserIDs
+			return ctx, ErrTooManyOrgIDs
 		}
 	} else {
 		md = md.Copy()
-		md[lowerOrgIDHeaderName] = []string{userID}
+		md[lowerOrgIDHeaderName] = []string{orgID}
 		newCtx = metadata.NewContext(ctx, md)
 	}
 

vendor/github.com/weaveworks/common/user/http.go

@@ -6,26 +6,59 @@ import (
 	"golang.org/x/net/context"
 )
 
-// ExtractFromHTTPRequest extracts the user ID from the request headers and returns
-// the user ID and a context with the user ID embbedded.
-func ExtractFromHTTPRequest(r *http.Request) (string, context.Context, error) {
-	userID := r.Header.Get(orgIDHeaderName)
-	if userID == "" {
-		return "", r.Context(), ErrNoUserID
+const (
+	// orgIDHeaderName is a legacy from scope as a service.
+	orgIDHeaderName  = "X-Scope-OrgID"
+	userIDHeaderName = "X-Scope-UserID"
+
+	// LowerOrgIDHeaderName as gRPC / HTTP2.0 headers are lowercased.
+	lowerOrgIDHeaderName = "x-scope-orgid"
+)
+
+// ExtractOrgIDFromHTTPRequest extracts the org ID from the request headers and returns
+// the org ID and a context with the org ID embedded.
+func ExtractOrgIDFromHTTPRequest(r *http.Request) (string, context.Context, error) {
+	orgID := r.Header.Get(orgIDHeaderName)
+	if orgID == "" {
+		return "", r.Context(), ErrNoOrgID
 	}
-	return userID, Inject(r.Context(), userID), nil
+	return orgID, InjectOrgID(r.Context(), orgID), nil
 }
 
-// InjectIntoHTTPRequest injects the userID from the context into the request headers.
-func InjectIntoHTTPRequest(ctx context.Context, r *http.Request) error {
-	userID, err := Extract(ctx)
+// InjectOrgIDIntoHTTPRequest injects the orgID from the context into the request headers.
+func InjectOrgIDIntoHTTPRequest(ctx context.Context, r *http.Request) error {
+	orgID, err := ExtractOrgID(ctx)
 	if err != nil {
 		return err
 	}
 	existingID := r.Header.Get(orgIDHeaderName)
-	if existingID != "" && existingID != userID {
-		return ErrDifferentIDPresent
+	if existingID != "" && existingID != orgID {
+		return ErrDifferentOrgIDPresent
 	}
-	r.Header.Set(orgIDHeaderName, userID)
+	r.Header.Set(orgIDHeaderName, orgID)
+	return nil
+}
+
+// ExtractUserIDFromHTTPRequest extracts the org ID from the request headers and returns
+// the org ID and a context with the org ID embedded.
+func ExtractUserIDFromHTTPRequest(r *http.Request) (string, context.Context, error) {
+	userID := r.Header.Get(userIDHeaderName)
+	if userID == "" {
+		return "", r.Context(), ErrNoUserID
+	}
+	return userID, InjectUserID(r.Context(), userID), nil
+}
+
+// InjectUserIDIntoHTTPRequest injects the userID from the context into the request headers.
+func InjectUserIDIntoHTTPRequest(ctx context.Context, r *http.Request) error {
+	userID, err := ExtractUserID(ctx)
+	if err != nil {
+		return err
+	}
+	existingID := r.Header.Get(userIDHeaderName)
+	if existingID != "" && existingID != userID {
+		return ErrDifferentUserIDPresent
+	}
+	r.Header.Set(userIDHeaderName, userID)
 	return nil
 }

vendor/github.com/weaveworks/common/user/id.go

@@ -9,25 +9,38 @@ import (
 type contextKey int
 
 const (
-	// UserIDContextKey is the key used in contexts to find the userid
-	userIDContextKey contextKey = 0
-
-	// orgIDHeaderName is a legacy from scope as a service.
-	orgIDHeaderName = "X-Scope-OrgID"
-
-	// LowerOrgIDHeaderName as gRPC / HTTP2.0 headers are lowercased.
-	lowerOrgIDHeaderName = "x-scope-orgid"
+	// Keys used in contexts to find the org or user ID
+	orgIDContextKey  contextKey = 0
+	userIDContextKey contextKey = 1
 )
 
 // Errors that we return
 const (
-	ErrNoUserID           = errors.Error("no user id")
-	ErrDifferentIDPresent = errors.Error("different user ID already present")
-	ErrTooManyUserIDs     = errors.Error("multiple user IDs present")
+	ErrNoOrgID               = errors.Error("no org id")
+	ErrDifferentOrgIDPresent = errors.Error("different org ID already present")
+	ErrTooManyOrgIDs         = errors.Error("multiple org IDs present")
+
+	ErrNoUserID               = errors.Error("no user id")
+	ErrDifferentUserIDPresent = errors.Error("different user ID already present")
+	ErrTooManyUserIDs         = errors.Error("multiple user IDs present")
 )
 
-// Extract gets the user ID from the context
-func Extract(ctx context.Context) (string, error) {
+// ExtractOrgID gets the org ID from the context.
+func ExtractOrgID(ctx context.Context) (string, error) {
+	orgID, ok := ctx.Value(orgIDContextKey).(string)
+	if !ok {
+		return "", ErrNoOrgID
+	}
+	return orgID, nil
+}
+
+// InjectOrgID returns a derived context containing the org ID.
+func InjectOrgID(ctx context.Context, userID string) context.Context {
+	return context.WithValue(ctx, interface{}(orgIDContextKey), userID)
+}
+
+// ExtractUserID gets the user ID from the context.
+func ExtractUserID(ctx context.Context) (string, error) {
 	userID, ok := ctx.Value(userIDContextKey).(string)
 	if !ok {
 		return "", ErrNoUserID
@@ -35,7 +48,7 @@ func Extract(ctx context.Context) (string, error) {
 	return userID, nil
 }
 
-// Inject returns a derived context containing the user ID.
-func Inject(ctx context.Context, userID string) context.Context {
+// InjectUserID returns a derived context containing the user ID.
+func InjectUserID(ctx context.Context, userID string) context.Context {
 	return context.WithValue(ctx, interface{}(userIDContextKey), userID)
 }

vendor/manifest

@@ -1422,7 +1422,7 @@
 			"importpath": "github.com/weaveworks/common",
 			"repository": "https://github.com/weaveworks/common",
 			"vcs": "git",
-			"revision": "f94043b3da140c7a735b1f2f286d72d19014b200",
+			"revision": "2faced4ddea5ec3b1ff8e88ba552714e3088cf41",
 			"branch": "master",
 			"notests": true
 		},
@@ -1756,4 +1756,4 @@
 			"branch": "master"
 		}
 	]
-}
+}