Renaming ignored user agents var (#320)

* Supporting Mizu view from given url

agent/main.go

@@ -77,7 +77,7 @@ func main() {
 
 		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
 		tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, extensions, filteringOptions)
-		socketConnection, err := utils.ConnectToSocketServer(*apiServerAddress)
+		socketConnection, _, err := websocket.DefaultDialer.Dial(*apiServerAddress, nil)
 		if err != nil {
 			panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
 		}
@@ -230,7 +230,7 @@ func getTrafficFilteringOptions() *tapApi.TrafficFilteringOptions {
 	filteringOptionsJson := os.Getenv(shared.MizuFilteringOptionsEnvVar)
 	if filteringOptionsJson == "" {
 		return &tapApi.TrafficFilteringOptions{
-			HealthChecksUserAgentHeaders: []string{},
+			IgnoredUserAgents: []string{},
 		}
 	}
 	var filteringOptions tapApi.TrafficFilteringOptions
@@ -248,7 +248,7 @@ func filterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *ta
 			continue
 		}
 		// TODO: move this to tappers https://up9.atlassian.net/browse/TRA-3441
-		if isHealthCheckByUserAgent(message, filterOptions.HealthChecksUserAgentHeaders) {
+		if isIgnoredUserAgent(message, filterOptions.IgnoredUserAgents) {
 			continue
 		}
 
@@ -256,7 +256,7 @@ func filterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *ta
 	}
 }
 
-func isHealthCheckByUserAgent(item *tapApi.OutputChannelItem, userAgentsToIgnore []string) bool {
+func isIgnoredUserAgent(item *tapApi.OutputChannelItem, userAgentsToIgnore []string) bool {
 	if item.Protocol.Name != "http" {
 		return false
 	}

agent/pkg/utils/socket_client.go

@@ -1,38 +0,0 @@
-package utils
-
-import (
-	"github.com/gorilla/websocket"
-	"github.com/romana/rlog"
-	"time"
-)
-
-const (
-	DEFAULT_SOCKET_RETRIES          = 3
-	DEFAULT_SOCKET_RETRY_SLEEP_TIME = time.Second * 10
-)
-
-func ConnectToSocketServer(address string) (*websocket.Conn, error) {
-	var err error
-	var connection *websocket.Conn
-	try := 0
-
-	// Connection to server fails if client pod is up before server.
-	// Retries solve this issue.
-	for try < DEFAULT_SOCKET_RETRIES {
-		rlog.Infof("Trying to connect to websocket: %s, attempt: %v/%v", address, try, DEFAULT_SOCKET_RETRIES)
-		connection, _, err = websocket.DefaultDialer.Dial(address, nil)
-		if err != nil {
-			rlog.Warnf("Failed connecting to websocket: %s, attempt: %v/%v, err: %s, (%v,%+v)", address, try, DEFAULT_SOCKET_RETRIES, err, err, err)
-			try++
-		} else {
-			break
-		}
-		time.Sleep(DEFAULT_SOCKET_RETRY_SLEEP_TIME)
-	}
-
-	if err != nil {
-		return nil, err
-	}
-
-	return connection, nil
-}

cli/cmd/tapRunner.go

@@ -109,19 +109,17 @@ func RunMizuTap() {
 		return
 	}
 
-	nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
-
 	defer finishMizuExecution(kubernetesProvider)
-	if err := createMizuResources(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions, mizuValidationRules); err != nil {
+	if err := createMizuResources(ctx, kubernetesProvider, mizuApiFilteringOptions, mizuValidationRules); err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
 		return
 	}
 
-	go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel)
-	go goUtils.HandleExcWrapper(watchTapperPod, ctx, kubernetesProvider, cancel, nodeToTappedPodIPMap)
+	go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel, mizuApiFilteringOptions)
+	go goUtils.HandleExcWrapper(watchTapperPod, ctx, kubernetesProvider, cancel)
 	go goUtils.HandleExcWrapper(watchPodsForTapping, ctx, kubernetesProvider, targetNamespaces, cancel, mizuApiFilteringOptions)
 
-	//block until exit signal or error
+	// block until exit signal or error
 	waitForFinish(ctx, cancel)
 }
 
@@ -134,7 +132,7 @@ func readValidationRules(file string) (string, error) {
 	return string(newContent), nil
 }
 
-func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, mizuApiFilteringOptions *api.TrafficFilteringOptions, mizuValidationRules string) error {
+func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *api.TrafficFilteringOptions, mizuValidationRules string) error {
 	if !config.Config.IsNsRestrictedMode() {
 		if err := createMizuNamespace(ctx, kubernetesProvider); err != nil {
 			return err
@@ -145,10 +143,6 @@ func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Pro
 		return err
 	}
 
-	if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions); err != nil {
-		return err
-	}
-
 	if err := createMizuConfigmap(ctx, kubernetesProvider, mizuValidationRules); err != nil {
 		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err)))
 	}
@@ -222,13 +216,15 @@ func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) {
 	}
 
 	return &api.TrafficFilteringOptions{
-		PlainTextMaskingRegexes:      compiledRegexSlice,
-		HealthChecksUserAgentHeaders: config.Config.Tap.HealthChecksUserAgentHeaders,
-		DisableRedaction:             config.Config.Tap.DisableRedaction,
+		PlainTextMaskingRegexes: compiledRegexSlice,
+		IgnoredUserAgents:       config.Config.Tap.IgnoredUserAgents,
+		DisableRedaction:        config.Config.Tap.DisableRedaction,
 	}, nil
 }
 
-func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, mizuApiFilteringOptions *api.TrafficFilteringOptions) error {
+func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *api.TrafficFilteringOptions) error {
+	nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
+
 	if len(nodeToTappedPodIPMap) > 0 {
 		var serviceAccountName string
 		if state.mizuServiceAccountExists {
@@ -407,13 +403,8 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
 			logger.Log.Debugf("[Error] failed update tapped pods %v", err)
 		}
 
-		nodeToTappedPodIPMap := getNodeHostToTappedPodIpsMap(state.currentlyTappedPods)
-		if err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error building node to ips map: %v", errormessage.FormatError(err)))
-			cancel()
-		}
-		if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, mizuApiFilteringOptions); err != nil {
-			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating daemonset: %v", errormessage.FormatError(err)))
+		if err := updateMizuTappers(ctx, kubernetesProvider, mizuApiFilteringOptions); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating tappers: %v", errormessage.FormatError(err)))
 			cancel()
 		}
 	}
@@ -531,7 +522,7 @@ func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod {
 	return missingPods
 }
 
-func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
+func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, mizuApiFilteringOptions *api.TrafficFilteringOptions) {
 	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.ApiServerPodName))
 	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex)
 	isPodReady := false
@@ -561,6 +552,23 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 			}
 
 			logger.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase)
+
+			if modifiedPod.Status.Phase == core.PodPending {
+				if modifiedPod.Status.Conditions[0].Type == core.PodScheduled && modifiedPod.Status.Conditions[0].Status != core.ConditionTrue {
+					logger.Log.Debugf("Wasn't able to deploy the API server. Reason: \"%s\"", modifiedPod.Status.Conditions[0].Message)
+					logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Wasn't able to deploy the API server, for more info check logs at %s", logger.GetLogFilePath()))
+					cancel()
+					break
+				}
+
+				if len(modifiedPod.Status.ContainerStatuses) > 0 && modifiedPod.Status.ContainerStatuses[0].State.Waiting != nil && modifiedPod.Status.ContainerStatuses[0].State.Waiting.Reason == "ErrImagePull" {
+					logger.Log.Debugf("Wasn't able to deploy the API server. (ErrImagePull) Reason: \"%s\"", modifiedPod.Status.ContainerStatuses[0].State.Waiting.Message)
+					logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Wasn't able to deploy the API server: failed to pull the image, for more info check logs at %v", logger.GetLogFilePath()))
+					cancel()
+					break
+				}
+			}
+
 			if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
 				isPodReady = true
 				go startProxyReportErrorIfAny(kubernetesProvider, cancel)
@@ -571,6 +579,10 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 					cancel()
 					break
 				}
+				if err := updateMizuTappers(ctx, kubernetesProvider, mizuApiFilteringOptions); err != nil {
+					logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error updating tappers: %v", errormessage.FormatError(err)))
+					cancel()
+				}
 
 				logger.Log.Infof("Mizu is available at %s\n", url)
 				openBrowser(url)
@@ -579,13 +591,13 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 					logger.Log.Debugf("[Error] failed update tapped pods %v", err)
 				}
 			}
-		case _, ok := <-errorChan:
+		case err, ok := <-errorChan:
 			if !ok {
 				errorChan = nil
 				continue
 			}
 
-			logger.Log.Debugf("[ERROR] Agent creation, watching %v namespace", config.Config.MizuResourcesNamespace)
+			logger.Log.Debugf("[ERROR] Agent creation, watching %v namespace, error: %v", config.Config.MizuResourcesNamespace, err)
 			cancel()
 
 		case <-timeAfter:
@@ -600,14 +612,10 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 	}
 }
 
-func watchTapperPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, nodeToTappedPodIPMap map[string][]string) {
+func watchTapperPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", mizu.TapperDaemonSetName))
 	added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex)
 	var prevPodPhase core.PodPhase
-	var appendMetaname bool
-	if len(nodeToTappedPodIPMap) > 1 {
-		appendMetaname = true
-	}
 	for {
 		select {
 		case addedPod, ok := <-added:
@@ -616,22 +624,14 @@ func watchTapperPod(ctx context.Context, kubernetesProvider *kubernetes.Provider
 				continue
 			}
 
-			if appendMetaname {
-				logger.Log.Debugf("Tapper is created [%s]", addedPod.ObjectMeta.Name)
-			} else {
-				logger.Log.Debugf("Tapper is created")
-			}
+			logger.Log.Debugf("Tapper is created [%s]", addedPod.Name)
 		case removedPod, ok := <-removed:
 			if !ok {
 				removed = nil
 				continue
 			}
 
-			if appendMetaname {
-				logger.Log.Debugf("Tapper is removed [%s]", removedPod.ObjectMeta.Name)
-			} else {
-				logger.Log.Debugf("Tapper is removed")
-			}
+			logger.Log.Debugf("Tapper is removed [%s]", removedPod.Name)
 		case modifiedPod, ok := <-modified:
 			if !ok {
 				modified = nil
@@ -639,13 +639,14 @@ func watchTapperPod(ctx context.Context, kubernetesProvider *kubernetes.Provider
 			}
 
 			if modifiedPod.Status.Phase == core.PodPending && modifiedPod.Status.Conditions[0].Type == core.PodScheduled && modifiedPod.Status.Conditions[0].Status != core.ConditionTrue {
-				logger.Log.Infof(uiUtils.Red, "Cannot deploy the tapper. Reason: \"%s\"", modifiedPod.Status.Conditions[0].Message)
+				logger.Log.Infof(uiUtils.Red, "Wasn't able to deploy the tapper %s. Reason: \"%s\"", modifiedPod.Name, modifiedPod.Status.Conditions[0].Message)
 				cancel()
 				break
 			}
 
 			podStatus := modifiedPod.Status
 			if podStatus.Phase == core.PodPending && prevPodPhase == podStatus.Phase {
+				logger.Log.Debugf("Tapper %s is %s", modifiedPod.Name, strings.ToLower(string(podStatus.Phase)))
 				continue
 			}
 			prevPodPhase = podStatus.Phase
@@ -655,22 +656,19 @@ func watchTapperPod(ctx context.Context, kubernetesProvider *kubernetes.Provider
 				if state.Terminated != nil {
 					switch state.Terminated.Reason {
 					case "OOMKilled":
-						logger.Log.Infof(uiUtils.Red, "Tapper is terminated! OOMKilled. Increase pod resources.")
+						logger.Log.Infof(uiUtils.Red, "Tapper %s was terminated (reason: OOMKilled). You should consider increasing machine resources.", modifiedPod.Name)
 					}
-				} else {
-					logger.Log.Debugf("Tapper is %s", strings.ToLower(string(podStatus.Phase)))
 				}
-			} else {
-				logger.Log.Debugf("Tapper is %s", strings.ToLower(string(podStatus.Phase)))
 			}
 
-		case _, ok := <-errorChan:
+			logger.Log.Debugf("Tapper %s is %s", modifiedPod.Name, strings.ToLower(string(podStatus.Phase)))
+		case err, ok := <-errorChan:
 			if !ok {
 				errorChan = nil
 				continue
 			}
 
-			logger.Log.Errorf("[ERROR] Tapper creation, watching %v namespace", config.Config.MizuResourcesNamespace)
+			logger.Log.Debugf("[Error] Error in mizu tapper watch, err: %v", err)
 			cancel()
 
 		case <-ctx.Done():

cli/cmd/view.go

@@ -25,4 +25,7 @@ func init() {
 	defaults.Set(&defaultViewConfig)
 
 	viewCmd.Flags().Uint16P(configStructs.GuiPortViewName, "p", defaultViewConfig.GuiPort, "Provide a custom port for the web interface webserver")
+	viewCmd.Flags().StringP(configStructs.UrlViewName, "u", defaultViewConfig.Url, "Provide a custom host")
+
+	viewCmd.Flags().MarkHidden(configStructs.UrlViewName)
 }

cli/cmd/viewRunner.go

@@ -24,34 +24,39 @@ func runMizuView() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	exists, err := kubernetesProvider.DoesServicesExist(ctx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
-	if err != nil {
-		logger.Log.Errorf("Failed to found mizu service %v", err)
-		cancel()
-		return
-	}
-	if !exists {
-		logger.Log.Infof("%s service not found, you should run `mizu tap` command first", mizu.ApiServerPodName)
-		cancel()
-		return
-	}
+	url := config.Config.View.Url
 
-	url := GetApiServerUrl()
+	if url == "" {
+		exists, err := kubernetesProvider.DoesServicesExist(ctx, config.Config.MizuResourcesNamespace, mizu.ApiServerPodName)
+		if err != nil {
+			logger.Log.Errorf("Failed to found mizu service %v", err)
+			cancel()
+			return
+		}
+		if !exists {
+			logger.Log.Infof("%s service not found, you should run `mizu tap` command first", mizu.ApiServerPodName)
+			cancel()
+			return
+		}
 
-	response, err := http.Get(fmt.Sprintf("%s/", url))
-	if err == nil && response.StatusCode == 200 {
-		logger.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, config.Config.View.GuiPort)
-		return
-	}
-	logger.Log.Infof("Establishing connection to k8s cluster...")
-	go startProxyReportErrorIfAny(kubernetesProvider, cancel)
+		url = GetApiServerUrl()
 
-	if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl()); err != nil {
-		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", logger.GetLogFilePath()))
-		return
+		response, err := http.Get(fmt.Sprintf("%s/", url))
+		if err == nil && response.StatusCode == 200 {
+			logger.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, config.Config.View.GuiPort)
+			return
+		}
+		logger.Log.Infof("Establishing connection to k8s cluster...")
+		go startProxyReportErrorIfAny(kubernetesProvider, cancel)
+
+		if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl()); err != nil {
+			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", logger.GetLogFilePath()))
+			return
+		}
 	}
 
 	logger.Log.Infof("Mizu is available at %s\n", url)
+
 	openBrowser(url)
 	if isCompatible, err := version.CheckVersionCompatibility(); err != nil {
 		logger.Log.Errorf("Failed to check versions compatibility %v", err)

cli/config/configStructs/tapConfig.go

@@ -21,22 +21,22 @@ const (
 )
 
 type TapConfig struct {
-	AnalysisDestination          string    `yaml:"dest" default:"up9.app"`
-	SleepIntervalSec             int       `yaml:"upload-interval" default:"10"`
-	PodRegexStr                  string    `yaml:"regex" default:".*"`
-	GuiPort                      uint16    `yaml:"gui-port" default:"8899"`
-	Namespaces                   []string  `yaml:"namespaces"`
-	Analysis                     bool      `yaml:"analysis" default:"false"`
-	AllNamespaces                bool      `yaml:"all-namespaces" default:"false"`
-	PlainTextFilterRegexes       []string  `yaml:"regex-masking"`
-	HealthChecksUserAgentHeaders []string  `yaml:"ignored-user-agents"`
-	DisableRedaction             bool      `yaml:"no-redact" default:"false"`
-	HumanMaxEntriesDBSize        string    `yaml:"max-entries-db-size" default:"200MB"`
-	DryRun                       bool      `yaml:"dry-run" default:"false"`
-	EnforcePolicyFile            string    `yaml:"traffic-validation-file"`
-	EnforcePolicyFileDeprecated  string    `yaml:"test-rules,omitempty" readonly:""`
-	ApiServerResources           Resources `yaml:"api-server-resources"`
-	TapperResources              Resources `yaml:"tapper-resources"`
+	AnalysisDestination         string    `yaml:"dest" default:"up9.app"`
+	SleepIntervalSec            int       `yaml:"upload-interval" default:"10"`
+	PodRegexStr                 string    `yaml:"regex" default:".*"`
+	GuiPort                     uint16    `yaml:"gui-port" default:"8899"`
+	Namespaces                  []string  `yaml:"namespaces"`
+	Analysis                    bool      `yaml:"analysis" default:"false"`
+	AllNamespaces               bool      `yaml:"all-namespaces" default:"false"`
+	PlainTextFilterRegexes      []string  `yaml:"regex-masking"`
+	IgnoredUserAgents           []string  `yaml:"ignored-user-agents"`
+	DisableRedaction            bool      `yaml:"no-redact" default:"false"`
+	HumanMaxEntriesDBSize       string    `yaml:"max-entries-db-size" default:"200MB"`
+	DryRun                      bool      `yaml:"dry-run" default:"false"`
+	EnforcePolicyFile           string    `yaml:"traffic-validation-file"`
+	EnforcePolicyFileDeprecated string    `yaml:"test-rules,omitempty" readonly:""`
+	ApiServerResources          Resources `yaml:"api-server-resources"`
+	TapperResources             Resources `yaml:"tapper-resources"`
 }
 
 type Resources struct {

cli/config/configStructs/viewConfig.go

@@ -2,8 +2,10 @@ package configStructs
 
 const (
 	GuiPortViewName = "gui-port"
+	UrlViewName     = "url"
 )
 
 type ViewConfig struct {
 	GuiPort uint16 `yaml:"gui-port" default:"8899"`
+	Url     string `yaml:"url,omitempty" readonly:""`
 }

cli/kubernetes/watch.go

@@ -33,7 +33,10 @@ func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetName
 						return
 					}
 
-					pod := e.Object.(*corev1.Pod)
+					pod, ok := e.Object.(*corev1.Pod)
+					if !ok {
+						continue
+					}
 
 					if !podFilter.MatchString(pod.Name) {
 						continue

tap/api/options.go

@@ -1,7 +1,7 @@
 package api
 
 type TrafficFilteringOptions struct {
-	HealthChecksUserAgentHeaders []string
-	PlainTextMaskingRegexes      []*SerializableRegexp
-	DisableRedaction             bool
+	IgnoredUserAgents       []string
+	PlainTextMaskingRegexes []*SerializableRegexp
+	DisableRedaction        bool
 }