run acceptance tests for the latest code (and cancel all other jobs) (#238)

.github/workflows/acceptance_tests.yml

@@ -1,4 +1,5 @@
 name: acceptance tests
+
 on:
   pull_request:
     branches:
@@ -6,6 +7,11 @@ on:
   push:
     branches:
       - 'develop'
+
+concurrency:
+  group: mizu-acceptance-tests-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   run-acceptance-tests:
     name: Run acceptance tests

.github/workflows/publish.yml

@@ -1,9 +1,15 @@
 name: publish
+
 on:
   push:
     branches:
       - 'develop'
       - 'main'
+
+concurrency:
+  group: mizu-publish-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   docker:
     runs-on: ubuntu-latest
@@ -78,4 +84,3 @@ jobs:
           tag: ${{ steps.versioning.outputs.version }}
           prerelease: ${{ github.ref != 'refs/heads/main' }}
           bodyFile: 'cli/bin/README.md'
-

agent/pkg/sensitiveDataFiltering/messageSensitiveDataCleaner.go

@@ -158,9 +158,11 @@ func filterJsonBody(bytes []byte) ([]byte, error) {
 
 func filterJsonMap(jsonMap map[string] interface{}) {
 	for key, value := range jsonMap {
+		// Do not replace nil values with maskedFieldPlaceholderValue
 		if value == nil {
-			return
+			continue
 		}
+
 		nestedMap, isNested := value.(map[string] interface{})
 		if isNested {
 			filterJsonMap(nestedMap)

cli/cmd/logs.go

@@ -19,8 +19,9 @@ var logsCmd = &cobra.Command{
 	RunE: func(cmd *cobra.Command, args []string) error {
 		go telemetry.ReportRun("logs", config.Config.Logs)
 
-		kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath)
+		kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath())
 		if err != nil {
+			logger.Log.Error(err)
 			return nil
 		}
 		ctx, _ := context.WithCancel(context.Background())

cli/cmd/root.go

@@ -7,6 +7,8 @@ import (
 	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
+	"github.com/up9inc/mizu/cli/mizu/version"
+	"github.com/up9inc/mizu/cli/uiUtils"
 )
 
 var rootCmd = &cobra.Command{
@@ -15,14 +17,9 @@ var rootCmd = &cobra.Command{
 	Long: `A web traffic viewer for kubernetes
 Further info is available at https://github.com/up9inc/mizu`,
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-		if err := fsUtils.EnsureDir(mizu.GetMizuFolderPath()); err != nil {
-			logger.Log.Errorf("Failed to use mizu folder, %v", err)
-		}
-		logger.InitLogger()
 		if err := config.InitConfig(cmd); err != nil {
 			logger.Log.Fatal(err)
 		}
-
 		return nil
 	},
 }
@@ -31,8 +28,24 @@ func init() {
 	rootCmd.PersistentFlags().StringSlice(config.SetCommandName, []string{}, fmt.Sprintf("Override values using --%s", config.SetCommandName))
 }
 
+func printNewVersionIfNeeded(versionChan chan string) {
+	versionMsg := <-versionChan
+	if versionMsg != "" {
+		logger.Log.Infof(uiUtils.Yellow, versionMsg)
+	}
+}
+
 // Execute adds all child commands to the root command and sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the tapCmd.
 func Execute() {
+	if err := fsUtils.EnsureDir(mizu.GetMizuFolderPath()); err != nil {
+		logger.Log.Errorf("Failed to use mizu folder, %v", err)
+	}
+	logger.InitLogger()
+
+	versionChan := make(chan string)
+	defer printNewVersionIfNeeded(versionChan)
+	go version.CheckNewerVersion(versionChan)
+
 	cobra.CheckErr(rootCmd.Execute())
 }

cli/cmd/tap.go

@@ -33,10 +33,6 @@ Supported protocols are HTTP and gRPC.`,
 			return errors.New("unexpected number of arguments")
 		}
 
-		if err := config.Config.Validate(); err != nil {
-			return errormessage.FormatError(err)
-		}
-
 		if err := config.Config.Tap.Validate(); err != nil {
 			return errormessage.FormatError(err)
 		}

cli/cmd/tapRunner.go

@@ -10,7 +10,6 @@ import (
 	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu/fsUtils"
 	"github.com/up9inc/mizu/cli/mizu/goUtils"
-	"github.com/up9inc/mizu/cli/mizu/version"
 	"github.com/up9inc/mizu/cli/telemetry"
 	"net/http"
 	"net/url"
@@ -62,7 +61,7 @@ func RunMizuTap() {
 		}
 	}
 
-	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath)
+	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath())
 	if err != nil {
 		logger.Log.Error(err)
 		return
@@ -73,13 +72,21 @@ func RunMizuTap() {
 
 	targetNamespaces := getNamespaces(kubernetesProvider)
 
+	if config.Config.IsNsRestrictedMode() {
+		if len(targetNamespaces) != 1 || !mizu.Contains(targetNamespaces, config.Config.MizuResourcesNamespace) {
+			logger.Log.Errorf("Not supported mode. Mizu can't resolve IPs in other namespaces when running in namespace restricted mode.\n"+
+				"You can use the same namespace for --%s and --%s", configStructs.NamespacesTapName, config.MizuResourcesNamespaceConfigName)
+			return
+		}
+	}
+
 	var namespacesStr string
 	if !mizu.Contains(targetNamespaces, mizu.K8sAllNamespaces) {
 		namespacesStr = fmt.Sprintf("namespaces \"%s\"", strings.Join(targetNamespaces, "\", \""))
 	} else {
 		namespacesStr = "all namespaces"
 	}
-	version.CheckNewerVersion()
+
 	logger.Log.Infof("Tapping pods in %s", namespacesStr)
 
 	if err, _ := updateCurrentlyTappedPods(kubernetesProvider, ctx, targetNamespaces); err != nil {

cli/cmd/viewRunner.go

@@ -13,7 +13,7 @@ import (
 )
 
 func runMizuView() {
-	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath)
+	kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath())
 	if err != nil {
 		logger.Log.Error(err)
 		return

cli/config/config.go

@@ -3,7 +3,6 @@ package config
 import (
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/mizu"
 	"io/ioutil"
@@ -32,17 +31,6 @@ var (
 	cmdName string
 )
 
-func (config *ConfigStruct) Validate() error {
-	if config.IsNsRestrictedMode() {
-		if config.Tap.AllNamespaces || len(config.Tap.Namespaces) != 1 || config.Tap.Namespaces[0] != config.MizuResourcesNamespace {
-			return fmt.Errorf("Not supported mode. Mizu can't resolve IPs in other namespaces when running in namespace restricted mode.\n"+
-				"You can use the same namespace for --%s and --%s", configStructs.NamespacesTapName, MizuResourcesNamespaceConfigName)
-		}
-	}
-
-	return nil
-}
-
 func InitConfig(cmd *cobra.Command) error {
 	cmdName = cmd.Name()
 

cli/config/configStruct.go

@@ -5,6 +5,9 @@ import (
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/mizu"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/util/homedir"
+	"os"
+	"path/filepath"
 )
 
 const (
@@ -23,7 +26,7 @@ type ConfigStruct struct {
 	MizuResourcesNamespace string                      `yaml:"mizu-resources-namespace" default:"mizu"`
 	Telemetry              bool                        `yaml:"telemetry" default:"true"`
 	DumpLogs               bool                        `yaml:"dump-logs" default:"false"`
-	KubeConfigPath         string                      `yaml:"kube-config-path"`
+	KubeConfigPathStr      string                      `yaml:"kube-config-path"`
 }
 
 func (config *ConfigStruct) SetDefaults() {
@@ -37,3 +40,17 @@ func (config *ConfigStruct) ImagePullPolicy() v1.PullPolicy {
 func (config *ConfigStruct) IsNsRestrictedMode() bool {
 	return config.MizuResourcesNamespace != "mizu" // Notice "mizu" string must match the default MizuResourcesNamespace
 }
+
+func (config *ConfigStruct) KubeConfigPath() string {
+	if config.KubeConfigPathStr != "" {
+		return config.KubeConfigPathStr
+	}
+
+	envKubeConfigPath := os.Getenv("KUBECONFIG")
+	if envKubeConfigPath != "" {
+		return envKubeConfigPath
+	}
+
+	home := homedir.HomeDir()
+	return filepath.Join(home, ".kube", "config")
+}

cli/kubernetes/provider.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/logger"
-	"os"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -38,7 +37,6 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 	_ "k8s.io/client-go/tools/portforward"
 	watchtools "k8s.io/client-go/tools/watch"
-	"k8s.io/client-go/util/homedir"
 )
 
 type Provider struct {
@@ -57,13 +55,23 @@ func NewProvider(kubeConfigPath string) (*Provider, error) {
 	restClientConfig, err := kubernetesConfig.ClientConfig()
 	if err != nil {
 		if clientcmd.IsEmptyConfig(err) {
-			return nil, fmt.Errorf("Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'\n")
+			return nil, fmt.Errorf("couldn't find the kube config file, or file is empty (%s)\n" +
+				"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
 		}
 		if clientcmd.IsConfigurationInvalid(err) {
-			return nil, fmt.Errorf("Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'\n")
+			return nil, fmt.Errorf("invalid kube config file (%s)\n" +
+				"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
 		}
+
+		return nil, fmt.Errorf("error while using kube config (%s)\n" +
+			"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
+	}
+
+	clientSet, err := getClientSet(restClientConfig)
+	if err != nil {
+		return nil, fmt.Errorf("error while using kube config (%s)\n" +
+			"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
 	}
-	clientSet := getClientSet(restClientConfig)
 
 	return &Provider{
 		clientSet:        clientSet,
@@ -731,24 +739,16 @@ func (provider *Provider) GetPodLogs(namespace string, podName string, ctx conte
 	return str, nil
 }
 
-func getClientSet(config *restclient.Config) *kubernetes.Clientset {
+func getClientSet(config *restclient.Config) (*kubernetes.Clientset, error) {
 	clientSet, err := kubernetes.NewForConfig(config)
 	if err != nil {
-		panic(err.Error())
+		return nil, err
 	}
-	return clientSet
+
+	return clientSet, nil
 }
 
 func loadKubernetesConfiguration(kubeConfigPath string) clientcmd.ClientConfig {
-	if kubeConfigPath == "" {
-		kubeConfigPath = os.Getenv("KUBECONFIG")
-	}
-
-	if kubeConfigPath == "" {
-		home := homedir.HomeDir()
-		kubeConfigPath = filepath.Join(home, ".kube", "config")
-	}
-
 	logger.Log.Debugf("Using kube config %s", kubeConfigPath)
 	configPathList := filepath.SplitList(kubeConfigPath)
 	configLoadingRules := &clientcmd.ClientConfigLoadingRules{}

cli/mizu/version/versionCheck.go

@@ -52,7 +52,7 @@ func CheckVersionCompatibility(port uint16) (bool, error) {
 	return false, nil
 }
 
-func CheckNewerVersion() {
+func CheckNewerVersion(versionChan chan string) {
 	logger.Log.Debugf("Checking for newer version...")
 	start := time.Now()
 	client := github.NewClient(nil)
@@ -88,11 +88,13 @@ func CheckNewerVersion() {
 	}
 	gitHubVersion := string(data)
 	gitHubVersion = gitHubVersion[:len(gitHubVersion)-1]
-	logger.Log.Debugf("Finished version validation, took %v", time.Since(start))
 
 	gitHubVersionSemVer := semver.SemVersion(gitHubVersion)
 	currentSemVer := semver.SemVersion(mizu.SemVer)
+	logger.Log.Debugf("Finished version validation, github version %v, current version %v, took %v", gitHubVersion, currentSemVer, time.Since(start))
+
 	if gitHubVersionSemVer.GreaterThan(currentSemVer) {
-		logger.Log.Infof(uiUtils.Yellow, fmt.Sprintf("Update available! %v -> %v (%v)", mizu.SemVer, gitHubVersion, *latestRelease.HTMLURL))
+		versionChan <- fmt.Sprintf("Update available! %v -> %v (%v)", mizu.SemVer, gitHubVersion, *latestRelease.HTMLURL)
 	}
+	versionChan <- ""
 }

tap/http_reader.go

@@ -80,7 +80,7 @@ func (h *httpReader) Read(p []byte) (int, error) {
 			err := clientHello.Unmarshall(msg.bytes)
 			if err == nil {
 				statsTracker.incTlsConnectionsCount()
-				fmt.Printf("Detected TLS client hello with SNI %s\n", clientHello.SNI)
+				Debug("Detected TLS client hello with SNI %s\n", clientHello.SNI)
 				numericPort, _ := strconv.Atoi(h.tcpID.dstPort)
 				h.outboundLinkWriter.WriteOutboundLink(h.tcpID.srcIP, h.tcpID.dstIP, numericPort, clientHello.SNI, TLSProtocol)
 			}