Merge pull request #124 from up9inc/mizu_telemetry

Adding telemetry reports

README.md

@@ -26,6 +26,49 @@ SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/
 ### Development (unstable) build
 Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page.
 
+## Prerequisites
+1. Set `KUBECONFIG` environment variable to your kubernetes configuration. If this is not set, mizu assumes that configuration is at `${HOME}/.kube/config`
+2. mizu needs following permissions on your kubernetes cluster to run
+```
+- apiGroups:
+  - ""
+  - apps
+  resources:
+  - pods
+  - services
+  verbs:
+  - list
+  - get
+  - create
+  - delete
+- apiGroups:
+  - ""
+  - apps
+  resources:
+  - daemonsets
+  verbs:
+  - list
+  - get
+  - create
+  - patch
+  - delete
+```
+3. Optionally, for resolving traffic ip to kubernetes service name, mizu needs below permissions
+```
+- apiGroups:
+  - ""
+  - apps
+  - "rbac.authorization.k8s.io"
+  resources:
+  - clusterroles
+  - clusterrolebindings
+  - serviceaccounts
+  verbs:
+  - get
+  - create
+  - delete
+```
+
 ## How to run
 
 1. Find pod you'd like to tap to in your Kubernetes cluster

api/pkg/resolver/loader.go

@@ -10,6 +10,7 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/util/homedir"
 	"path/filepath"
+	"os"
 )
 
 func NewFromInCluster(errOut chan error) (*Resolver, error) {
@@ -26,8 +27,13 @@ func NewFromInCluster(errOut chan error) (*Resolver, error) {
 
 func NewFromOutOfCluster(kubeConfigPath string, errOut chan error) (*Resolver, error) {
 	if kubeConfigPath == "" {
-		home := homedir.HomeDir()
-		kubeConfigPath = filepath.Join(home, ".kube", "config")
+		env := os.Getenv("KUBECONFIG") 
+		if env != "" {
+			kubeConfigPath = env
+		} else {
+			home := homedir.HomeDir()
+			kubeConfigPath = filepath.Join(home, ".kube", "config")
+		}
 	}
 
 	configPathList := filepath.SplitList(kubeConfigPath)

cli/cmd/fetch.go

@@ -18,6 +18,7 @@ var fetchCmd = &cobra.Command{
 	Use:   "fetch",
 	Short: "Download recorded traffic to files",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		go mizu.ReportRun("tap", mizuTapOptions)
 		if isCompatible, err := mizu.CheckVersionCompatibility(mizuFetchOptions.MizuPort); err != nil {
 			return err
 		} else if !isCompatible {

cli/cmd/tap.go

@@ -31,8 +31,8 @@ var mizuTapOptions = &MizuTapOptions{}
 var direction string
 var humanMaxEntriesDBSize string
 var regex *regexp.Regexp
-const maxEntriesDBSizeFlagName = "max-entries-db-size"
 
+const maxEntriesDBSizeFlagName = "max-entries-db-size"
 
 const analysisMessageToConfirm = `NOTE: running mizu with --analysis flag will upload recorded traffic
 to UP9 cloud for further analysis and enriched presentation options.
@@ -44,6 +44,7 @@ var tapCmd = &cobra.Command{
 	Long: `Record the ingoing traffic of a kubernetes pod.
 Supported protocols are HTTP and gRPC.`,
 	RunE: func(cmd *cobra.Command, args []string) error {
+		go mizu.ReportRun("tap", mizuTapOptions)
 		RunMizuTap(regex, mizuTapOptions)
 		return nil
 	},

cli/cmd/version.go

@@ -10,20 +10,20 @@ import (
 )
 
 type MizuVersionOptions struct {
-	DebugInfo                bool
+	DebugInfo bool
 }
 
-
 var mizuVersionOptions = &MizuVersionOptions{}
 
 var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print version info",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		go mizu.ReportRun("version", mizuVersionOptions)
 		if mizuVersionOptions.DebugInfo {
 			timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0)
 			fmt.Printf("Version: %s \nBranch: %s (%s) \n", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)
-			fmt.Printf("Build Time: %s (%s)\n", mizu.BuildTimestamp,  time.Unix(timeStampInt, 0))
+			fmt.Printf("Build Time: %s (%s)\n", mizu.BuildTimestamp, time.Unix(timeStampInt, 0))
 
 		} else {
 			fmt.Printf("Version: %s (%s)\n", mizu.SemVer, mizu.Branch)

cli/cmd/view.go

@@ -6,7 +6,7 @@ import (
 )
 
 type MizuViewOptions struct {
-	GuiPort                uint16
+	GuiPort uint16
 }
 
 var mizuViewOptions = &MizuViewOptions{}
@@ -15,6 +15,7 @@ var viewCmd = &cobra.Command{
 	Use:   "view",
 	Short: "Open GUI in browser",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		go mizu.ReportRun("view", mizuFetchOptions)
 		if isCompatible, err := mizu.CheckVersionCompatibility(mizuFetchOptions.MizuPort); err != nil {
 			return err
 		} else if !isCompatible {
@@ -23,7 +24,6 @@ var viewCmd = &cobra.Command{
 		runMizuView(mizuViewOptions)
 		return nil
 	},
-
 }
 
 func init() {

cli/mizu/telemetry.go

@@ -0,0 +1,30 @@
+package mizu
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/romana/rlog"
+	"net/http"
+)
+
+const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
+
+func ReportRun(cmd string, args interface{}) {
+	if Branch != "main" {
+		rlog.Debugf("reporting only on main branch")
+		return
+	}
+
+	argsBytes, _ := json.Marshal(args)
+	argsMap := map[string]string{"telemetry_type": "mizu_execution", "cmd": cmd, "args": string(argsBytes), "component": "mizu_cli"}
+	argsMap["message"] = fmt.Sprintf("mizu %v - %v", argsMap["cmd"], string(argsBytes))
+
+	jsonValue, _ := json.Marshal(argsMap)
+
+	if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
+		rlog.Debugf("error sending telemtry err: %v, response %v", err, resp)
+	} else {
+		rlog.Debugf("Successfully reported telemetry")
+	}
+}

tap/har_writer.go

@@ -1,9 +1,11 @@
 package tap
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
@@ -46,12 +48,27 @@ type HarFile struct {
 }
 
 func NewEntry(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time) (*har.Entry, error) {
-	harRequest, err := har.NewRequest(request, true)
+	harRequest, err := har.NewRequest(request, false)
 	if err != nil {
 		SilentError("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)", err, err, err)
 		return nil, errors.New("Failed converting request to HAR")
 	}
 
+	// For requests with multipart/form-data or application/x-www-form-urlencoded Content-Type,
+	// martian/har will parse the request body and place the parameters in harRequest.PostData.Params
+	// instead of harRequest.PostData.Text (as the HAR spec requires it).
+	// Mizu currently only looks at PostData.Text. Therefore, instead of letting martian/har set the content of
+	// PostData, always copy the request body to PostData.Text.
+	if (request.ContentLength > 0) {
+		reqBody, err := ioutil.ReadAll(request.Body)
+		if err != nil {
+			SilentError("read-request-body", "Failed converting request to HAR %s (%v,%+v)", err, err, err)
+			return nil, errors.New("Failed reading request body")
+		}
+		request.Body = ioutil.NopCloser(bytes.NewReader(reqBody))
+		harRequest.PostData.Text = string(reqBody)
+	}
+
 	harResponse, err := har.NewResponse(response, true)
 	if err != nil {
 		SilentError("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)", err, err, err)