unhide svcmap + CHANGELOG (#724)

* start dev commands

* env files

* no message

.github/workflows/pr_validation.yml

@@ -12,7 +12,7 @@ concurrency:
 
 jobs:
   build-cli:
-    name: Build CLI
+    name: Build CLI executable
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go 1.16
@@ -27,35 +27,11 @@ jobs:
         run: make cli
 
   build-agent:
-    name: Build Agent
+    name: Build Agent docker image
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.16
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.16'
-
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
 
-      - shell: bash
-        run: |
-          sudo apt-get install libpcap-dev
-
       - name: Build Agent
-        run: make agent
-
-  build-ui:
-    name: Build UI
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Node 16
-        uses: actions/setup-node@v2
-        with:
-          node-version: '16'
-
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
-      - name: Build UI
-        run: make ui
+        run: make agent-docker

Makefile

@@ -47,6 +47,10 @@ agent-debug: ## Build agent for debug.
 docker: ## Build and publish agent docker image.
 	$(MAKE) push-docker
 
+agent-docker: ## Build agent docker image.
+	@echo "Building agent docker image"
+	@docker build -t up9inc/mizu:devlatest .
+
 push: push-docker push-cli ## Build and publish agent docker image & CLI.
 
 push-docker: ## Build and publish agent docker image.

agent/main.go

@@ -62,7 +62,6 @@ const (
 	socketConnectionRetries    = 30
 	socketConnectionRetryDelay = time.Second * 2
 	socketHandshakeTimeout     = time.Second * 2
-	uiIndexPath                = "./site/index.html"
 )
 
 func main() {
@@ -169,7 +168,7 @@ func configureBasenineServer(host string, port string) {
 		wait.WithWait(200*time.Millisecond),
 		wait.WithBreak(50*time.Millisecond),
 		wait.WithDeadline(5*time.Second),
-		wait.WithDebug(true),
+		wait.WithDebug(config.Config.LogLevel == logging.DEBUG),
 	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
 		logger.Log.Panicf("Basenine is not available!")
 	}
@@ -248,16 +247,23 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 
 	app.Use(DisableRootStaticCache())
 
-	if err := setUIFlags(); err != nil {
-		logger.Log.Errorf("Error setting ui mode, err: %v", err)
+	var staticFolder string
+	if config.Config.StandaloneMode {
+		staticFolder = "./site-standalone"
+	} else {
+		staticFolder = "./site"
 	}
 
-	if config.Config.StandaloneMode {
-		app.Use(static.ServeRoot("/", "./site-standalone"))
-	} else {
-		app.Use(static.ServeRoot("/", "./site"))
+	indexStaticFile := staticFolder + "/index.html"
+	if err := setUIFlags(indexStaticFile); err != nil {
+		logger.Log.Errorf("Error setting ui flags, err: %v", err)
 	}
 
+	app.Use(static.ServeRoot("/", staticFolder))
+	app.NoRoute(func(c *gin.Context) {
+		c.File(indexStaticFile)
+	})
+
 	app.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
 
 	api.WebSocketRoutes(app, &eventHandlers, startTime)
@@ -278,7 +284,6 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
 	routes.EntriesRoutes(app)
 	routes.MetadataRoutes(app)
 	routes.StatusRoutes(app)
-	routes.NotFoundRoute(app)
 	utils.StartServer(app)
 }
 
@@ -293,7 +298,7 @@ func DisableRootStaticCache() gin.HandlerFunc {
 	}
 }
 
-func setUIFlags() error {
+func setUIFlags(uiIndexPath string) error {
 	read, err := ioutil.ReadFile(uiIndexPath)
 	if err != nil {
 		return err

agent/pkg/routes/not_found_route.go

@@ -1,18 +0,0 @@
-package routes
-
-import (
-	"github.com/gin-gonic/gin"
-	"net/http"
-)
-
-// NotFoundRoute defines the 404 Error route.
-func NotFoundRoute(app *gin.Engine) {
-	app.Use(
-		func(c *gin.Context) {
-			c.JSON(http.StatusNotFound, map[string]interface{}{
-				"error": true,
-				"msg":   "sorry, endpoint is not found",
-			})
-		},
-	)
-}

cli/apiserver/provider.go

@@ -23,8 +23,8 @@ type Provider struct {
 	client  *http.Client
 }
 
-const DefaultRetries = 20
-const DefaultTimeout = 5 * time.Second
+const DefaultRetries = 3
+const DefaultTimeout = 2 * time.Second
 
 func NewProvider(url string, retries int, timeout time.Duration) *Provider {
 	return &Provider{
@@ -36,16 +36,6 @@ func NewProvider(url string, retries int, timeout time.Duration) *Provider {
 	}
 }
 
-func NewProviderWithoutRetries(url string, timeout time.Duration) *Provider {
-	return &Provider{
-		url:     url,
-		retries: 1,
-		client: &http.Client{
-			Timeout: timeout,
-		},
-	}
-}
-
 func (provider *Provider) TestConnection() error {
 	retriesLeft := provider.retries
 	for retriesLeft > 0 {

cli/cmd/checkRunner.go

@@ -97,7 +97,7 @@ func checkServerConnection(kubernetesProvider *kubernetes.Provider) bool {
 
 	serverUrl := GetApiServerUrl()
 
-	apiServerProvider := apiserver.NewProviderWithoutRetries(serverUrl, apiserver.DefaultTimeout)
+	apiServerProvider := apiserver.NewProvider(serverUrl, 1, apiserver.DefaultTimeout)
 	if err := apiServerProvider.TestConnection(); err == nil {
 		logger.Log.Infof("%v found Mizu server tunnel available and connected successfully to API server", fmt.Sprintf(uiUtils.Green, "√"))
 		return true

cli/cmd/common.go

@@ -5,6 +5,10 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"path"
+	"regexp"
+	"time"
+
 	"github.com/up9inc/mizu/cli/apiserver"
 	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/errormessage"
@@ -13,9 +17,6 @@ import (
 	"github.com/up9inc/mizu/cli/resources"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
-	"path"
-	"regexp"
-	"time"
 
 	"github.com/up9inc/mizu/cli/config"
 	"github.com/up9inc/mizu/shared/kubernetes"
@@ -35,10 +36,10 @@ func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, ctx con
 		return
 	}
 
-	apiProvider = apiserver.NewProviderWithoutRetries(GetApiServerUrl(), time.Second) // short check for proxy
+	apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout)
 	if err := apiProvider.TestConnection(); err != nil {
 		logger.Log.Debugf("Couldn't connect using proxy, stopping proxy and trying to create port-forward")
-		if err := httpServer.Shutdown(context.Background()); err != nil {
+		if err := httpServer.Shutdown(ctx); err != nil {
 			logger.Log.Debugf("Error occurred while stopping proxy %v", errormessage.FormatError(err))
 		}
 
@@ -50,7 +51,7 @@ func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, ctx con
 			return
 		}
 
-		apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout) // long check for port-forward
+		apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout)
 		if err := apiProvider.TestConnection(); err != nil {
 			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
 			cancel()

cli/config/configStruct.go

@@ -27,7 +27,7 @@ type ConfigStruct struct {
 	Logs                   configStructs.LogsConfig    `yaml:"logs"`
 	Auth                   configStructs.AuthConfig    `yaml:"auth"`
 	Config                 configStructs.ConfigConfig  `yaml:"config,omitempty"`
-	AgentImage             string                      `yaml:"agent-image,omitempty"`
+	AgentImage             string                      `yaml:"agent-image,omitempty" readonly:""`
 	KratosImage            string                      `yaml:"kratos-image,omitempty" readonly:""`
 	KetoImage              string                      `yaml:"keto-image,omitempty" readonly:""`
 	ImagePullPolicyStr     string                      `yaml:"image-pull-policy" default:"Always"`
@@ -38,7 +38,7 @@ type ConfigStruct struct {
 	ConfigFilePath         string                      `yaml:"config-path,omitempty" readonly:""`
 	HeadlessMode           bool                        `yaml:"headless" default:"false"`
 	LogLevelStr            string                      `yaml:"log-level,omitempty" default:"INFO" readonly:""`
-	ServiceMap             bool                        `yaml:"service-map,omitempty" default:"false" readonly:""`
+	ServiceMap             bool                        `yaml:"service-map" default:"false"`
 	OAS                    bool                        `yaml:"oas,omitempty" default:"false" readonly:""`
 	Elastic                shared.ElasticConfig        `yaml:"elastic"`
 }

docs/CHANGELOG.md

@@ -2,6 +2,35 @@
 This document summarizes main and fixes changes published in stable (aka `main`) branch of this project.
 Ongoing work and development releases are under `develop` branch.
 
+## 0.24.0
+
+### main features
+* ARM64 support -- Mizu is now available for ARM 64bit architecture
+  * Now you can run Mizu with `minikube` on your Apple M1 laptop or any other ARM-based hosts
+* New command helps user verify Mizu deployment
+  * Run `mizu check` to verify Mizu was deployed successfully
+  * `mizu check` verifies version compatibility, resources and permissions required by Mizu
+* EXPERIMENTAL: Service Map - graph of all service interactions
+  * Arrow direction show client to server connection
+  * Graph edge width reflects volume of traffic captured between the services
+  * to enable this experimental feature use `--set service-map=true` flag
+
+### improvements
+* Mizu container images are now served from [Docker Hub](https://hub.docker.com/r/up9inc/mizu), as multi-architecture images (arm64, amd64)
+* in Mizu GUI the filter query can now be applied by pressing CONTROL/COMMAND + ENTER
+* try port-forwarding if http-proxy connection to Mizu API server is not available
+
+### notable bug fixes
+* Fixed HTTP/1.0 presentation which was shown as HTTP/1.1
+* Fixed handling of long-living TCP connections, improves capturing gRPC and HTTP/2 traffic, and helps in service-mesh setups (istio, linkerd)
+
+
+## 0.23.0
+### notable bug fixes
+* fixed errors in Redis protocol parser (better handling of Array and Bulk String message types)
+
+
+
 ## 0.22.0
 
 ### main features

tap/extensions/http/main.go

@@ -100,7 +100,6 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 		http2Assembler = createHTTP2Assembler(b)
 	}
 
-	dissected := false
 	switchingProtocolsHTTP2 := false
 	for {
 		if switchingProtocolsHTTP2 {
@@ -121,7 +120,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 			} else if err != nil {
 				continue
 			}
-			dissected = true
+			superIdentifier.Protocol = &http11protocol
 		} else if isClient {
 			var req *http.Request
 			switchingProtocolsHTTP2, req, err = handleHTTP1ClientStream(b, tcpID, counterPair, superTimer, emitter, options)
@@ -130,7 +129,7 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 			} else if err != nil {
 				continue
 			}
-			dissected = true
+			superIdentifier.Protocol = &http11protocol
 
 			// In case of an HTTP2 upgrade, duplicate the HTTP1 request into HTTP2 with stream ID 1
 			if switchingProtocolsHTTP2 {
@@ -161,14 +160,14 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
 			} else if err != nil {
 				continue
 			}
-			dissected = true
+			superIdentifier.Protocol = &http11protocol
 		}
 	}
 
-	if !dissected {
+	if superIdentifier.Protocol == nil {
 		return err
 	}
-	superIdentifier.Protocol = &http11protocol
+
 	return nil
 }
 

tap/extensions/kafka/response.go

@@ -251,10 +251,10 @@ func ReadResponse(r io.Reader, tcpID *api.TcpID, superTimer *api.SuperTimer, emi
 	}
 
 	connectionInfo := &api.ConnectionInfo{
-		ClientIP:   tcpID.SrcIP,
-		ClientPort: tcpID.SrcPort,
-		ServerIP:   tcpID.DstIP,
-		ServerPort: tcpID.DstPort,
+		ClientIP:   tcpID.DstIP,
+		ClientPort: tcpID.DstPort,
+		ServerIP:   tcpID.SrcIP,
+		ServerPort: tcpID.SrcPort,
 		IsOutgoing: true,
 	}
 

ui/.env.basic

@@ -1,2 +1 @@
-REACT_APP_OVERRIDE_WS_URL="ws://localhost:8899/ws"
-REACT_APP_OVERRIDE_API_URL="http://localhost:8899/"
+REACT_APP_OVERRIDE_IS_ENTERPRISE="false"

ui/.env.dev.basic

@@ -0,0 +1,3 @@
+REACT_APP_OVERRIDE_WS_URL="ws://localhost:8899/ws"
+REACT_APP_OVERRIDE_API_URL="http://localhost:8899/"
+REACT_APP_OVERRIDE_IS_ENTERPRISE="false"

ui/.env.dev.enterprise

@@ -0,0 +1,3 @@
+REACT_APP_OVERRIDE_WS_URL="ws://localhost:8899/ws"
+REACT_APP_OVERRIDE_API_URL="http://localhost:8899/"
+REACT_APP_OVERRIDE_IS_ENTERPRISE="true"

ui/.env.enterprise

@@ -1,3 +1 @@
-REACT_APP_OVERRIDE_WS_URL="ws://localhost:8899/ws"
-REACT_APP_OVERRIDE_API_URL="http://localhost:8899/"
 REACT_APP_OVERRIDE_IS_ENTERPRISE="true"

ui/package.json

@@ -49,7 +49,8 @@
   },
   "scripts": {
     "start": "craco start",
-    "start-ent": "./node_modules/.bin/env-cmd -f .env.enterprise craco start",
+    "start-dev": "./node_modules/.bin/env-cmd -f .env.dev.basic craco start",
+    "start-dev-ent": "./node_modules/.bin/env-cmd -f .env.dev.enterprise craco start",
     "build": "./node_modules/.bin/env-cmd -f .env.basic craco build",
     "build-ent": "BUILD_PATH='./build-ent' ./node_modules/.bin/env-cmd -f .env.enterprise craco build",
     "test": "craco test",

ui/src/components/EntryDetailed/EntrySections.tsx

@@ -137,12 +137,12 @@ export const EntryBodySection: React.FC<EntryBodySectionProps> = ({
         const chunk = body.slice(0, MAXIMUM_BYTES_TO_FORMAT);
         const bodyBuf = isBase64Encoding ? atob(chunk) : chunk;
 
-        if (!isPretty) return bodyBuf;
-
         try {
             if (jsonLikeFormats.some(format => contentType?.indexOf(format) > -1)) {
+                if (!isPretty) return bodyBuf;
                 return jsonBeautify(JSON.parse(bodyBuf), null, 2, 80);
             }  else if (xmlLikeFormats.some(format => contentType?.indexOf(format) > -1)) {
+                if (!isPretty) return bodyBuf;
                 return xmlBeautify(bodyBuf, {
                     indentation: '  ',
                     filter: (node) => node.type !== 'Comment',
@@ -152,7 +152,9 @@ export const EntryBodySection: React.FC<EntryBodySectionProps> = ({
             } else if (protobufFormats.some(format => contentType?.indexOf(format) > -1)) {
                 // Replace all non printable characters (ASCII)
                 const protobufDecoder = new ProtobufDecoder(bodyBuf, true);
-                return jsonBeautify(protobufDecoder.decode().toSimple(), null, 2, 80);
+                const protobufDecoded = protobufDecoder.decode().toSimple();
+                if (!isPretty) return JSON.stringify(protobufDecoded);
+                return jsonBeautify(protobufDecoded, null, 2, 80);
             }
         } catch (error) {
             console.error(error);

ui/src/components/OasModal/OasModal.tsx

@@ -21,7 +21,6 @@ const OasModal = ({ openModal, handleCloseModal }) => {
         const services = await api.getOasServices();
         setOasServices(services);
       } catch (e) {
-        toast.error("Error occurred while fetching services list");
         console.error(e);
       }
     })();

ui/src/helpers/api.js

@@ -1,14 +1,12 @@
 import * as axios from "axios";
 
-// When working locally cp `cp .env.example .env`
 export const MizuWebsocketURL = process.env.REACT_APP_OVERRIDE_WS_URL ? process.env.REACT_APP_OVERRIDE_WS_URL :
-                        window.location.protocol === 'https:' ? `wss://${window.location.host}/ws` : `ws://${window.location.host}/ws`;
+    window.location.protocol === 'https:' ? `wss://${window.location.host}/ws` : `ws://${window.location.host}/ws`;
 
 export const FormValidationErrorType = "formError";
 
 const CancelToken = axios.CancelToken;
 
-// When working locally cp `cp .env.example .env`
 const apiURL = process.env.REACT_APP_OVERRIDE_API_URL ? process.env.REACT_APP_OVERRIDE_API_URL : `${window.location.origin}/`;
 
 export default class Api {

ui/src/hooks/useKeyPress.ts

@@ -10,8 +10,11 @@ const useKeyPress = (eventConfigs, callback, node = null) => {
   // handle what happens on key press
   const handleKeyPress = useCallback(
     (event) => {
+      
       // check if one of the key is part of the ones we want
       if (eventConfigs.some((eventConfig) => Object.keys(eventConfig).every(nameKey => eventConfig[nameKey] === event[nameKey]))) {
+        event.stopPropagation()
+        event.preventDefault();
         callbackRef.current(event);
       }
     },