api server support sync workspace (#340)

agent/pkg/controllers/entries_controller.go

@@ -10,6 +10,7 @@ import (
 	"mizuserver/pkg/utils"
 	"mizuserver/pkg/validation"
 	"net/http"
+	"regexp"
 	"time"
 
 	"github.com/google/martian/har"
@@ -72,23 +73,46 @@ func SyncEntries(c *gin.Context) {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
+
 	if err := validation.Validate(syncParams); err != nil {
 		c.JSON(http.StatusBadRequest, err)
 		return
 	}
+
 	if up9.GetAnalyzeInfo().IsAnalyzing {
 		c.String(http.StatusBadRequest, "Cannot analyze, mizu is already analyzing")
 		return
 	}
 
-	rlog.Infof("Sync entries - creating token. env %s\n", syncParams.Env)
-	token, err := up9.CreateAnonymousToken(syncParams.Env)
-	if err != nil {
-		c.String(http.StatusServiceUnavailable, "Cannot analyze, mizu is already analyzing")
+	var (
+		token, model string
+		guestMode    bool
+	)
+	if syncParams.Token == "" {
+		rlog.Infof("Sync entries - creating token. env %s\n", syncParams.Env)
+		guestToken, err := up9.CreateAnonymousToken(syncParams.Env)
+		if err != nil {
+			c.String(http.StatusServiceUnavailable, "Failed creating anonymous token")
+			return
+		}
+
+		token = guestToken.Token
+		model = guestToken.Model
+		guestMode = true
+	} else {
+		token = fmt.Sprintf("bearer %s", syncParams.Token)
+		model = syncParams.Workspace
+		guestMode = false
+	}
+
+	modelRegex, _ := regexp.Compile("[A-Za-z0-9][-A-Za-z0-9_.]*[A-Za-z0-9]+$")
+	if len(model) > 63 || !modelRegex.MatchString(model) {
+		c.String(http.StatusBadRequest, "Invalid model name")
 		return
 	}
-	rlog.Infof("Sync entries - syncing. token: %s model: %s\n", token.Token, token.Model)
-	go up9.SyncEntriesImpl(token.Token, token.Model, syncParams.Env, syncParams.SleepIntervalSec)
+
+	rlog.Infof("Sync entries - syncing. token: %s, model: %s, guest mode: %v\n", token, model, guestMode)
+	go up9.SyncEntriesImpl(token, model, syncParams.Env, syncParams.UploadIntervalSec, guestMode)
 	c.String(http.StatusOK, "OK")
 }
 

agent/pkg/models/models.go

@@ -23,8 +23,10 @@ type EntriesFilter struct {
 }
 
 type SyncEntriesRequestQuery struct {
-	Env              string `form:"env"`
-	SleepIntervalSec int    `form:"interval"`
+	Token             string `form:"token"`
+	Env               string `form:"env"`
+	Workspace         string `form:"workspace"`
+	UploadIntervalSec int    `form:"interval"`
 }
 
 type HarFetchRequestQuery struct {

agent/pkg/up9/main.go

@@ -59,14 +59,16 @@ func GetRemoteUrl(analyzeDestination string, analyzeToken string) string {
 	return fmt.Sprintf("https://%s/share/%s", analyzeDestination, analyzeToken)
 }
 
-func CheckIfModelReady(analyzeDestination string, analyzeModel string, analyzeToken string) bool {
+func CheckIfModelReady(analyzeDestination string, analyzeModel string, analyzeToken string, guestMode bool) bool {
 	statusUrl, _ := url.Parse(fmt.Sprintf("https://trcc.%s/models/%s/status", analyzeDestination, analyzeModel))
+
+	authHeader := getAuthHeader(guestMode)
 	req := &http.Request{
 		Method: http.MethodGet,
 		URL:    statusUrl,
 		Header: map[string][]string{
 			"Content-Type": {"application/json"},
-			"Guest-Auth":   {analyzeToken},
+			authHeader:     {analyzeToken},
 		},
 	}
 	statusResp, err := http.DefaultClient.Do(req)
@@ -81,6 +83,14 @@ func CheckIfModelReady(analyzeDestination string, analyzeModel string, analyzeTo
 	return target.LastMajorGeneration > 0
 }
 
+func getAuthHeader(guestMode bool) string {
+	if guestMode {
+		return "Guest-Auth"
+	}
+
+	return "Authorization"
+}
+
 func GetTrafficDumpUrl(analyzeDestination string, analyzeModel string) *url.URL {
 	strUrl := fmt.Sprintf("https://traffic.%s/dumpTrafficBulk/%s", analyzeDestination, analyzeModel)
 	if strings.HasPrefix(analyzeDestination, "http") {
@@ -92,6 +102,7 @@ func GetTrafficDumpUrl(analyzeDestination string, analyzeModel string) *url.URL
 
 type AnalyzeInformation struct {
 	IsAnalyzing        bool
+	GuestMode          bool
 	SentCount          int
 	AnalyzedModel      string
 	AnalyzeToken       string
@@ -100,6 +111,7 @@ type AnalyzeInformation struct {
 
 func (info *AnalyzeInformation) Reset() {
 	info.IsAnalyzing = false
+	info.GuestMode = true
 	info.AnalyzedModel = ""
 	info.AnalyzeToken = ""
 	info.AnalyzeDestination = ""
@@ -112,19 +124,20 @@ func GetAnalyzeInfo() *shared.AnalyzeStatus {
 	return &shared.AnalyzeStatus{
 		IsAnalyzing:   analyzeInformation.IsAnalyzing,
 		RemoteUrl:     GetRemoteUrl(analyzeInformation.AnalyzeDestination, analyzeInformation.AnalyzeToken),
-		IsRemoteReady: CheckIfModelReady(analyzeInformation.AnalyzeDestination, analyzeInformation.AnalyzedModel, analyzeInformation.AnalyzeToken),
+		IsRemoteReady: CheckIfModelReady(analyzeInformation.AnalyzeDestination, analyzeInformation.AnalyzedModel, analyzeInformation.AnalyzeToken, analyzeInformation.GuestMode),
 		SentCount:     analyzeInformation.SentCount,
 	}
 }
 
-func SyncEntriesImpl(token string, model string, envPrefix string, sleepIntervalSec int) {
+func SyncEntriesImpl(token string, model string, envPrefix string, uploadIntervalSec int, guestMode bool) {
 	analyzeInformation.IsAnalyzing = true
+	analyzeInformation.GuestMode = guestMode
 	analyzeInformation.AnalyzedModel = model
 	analyzeInformation.AnalyzeToken = token
 	analyzeInformation.AnalyzeDestination = envPrefix
 	analyzeInformation.SentCount = 0
 
-	sleepTime := time.Second * time.Duration(sleepIntervalSec)
+	sleepTime := time.Second * time.Duration(uploadIntervalSec)
 
 	var timestampFrom int64 = 0
 
@@ -170,13 +183,14 @@ func SyncEntriesImpl(token string, model string, envPrefix string, sleepInterval
 			_ = w.Close()
 			reqBody := ioutil.NopCloser(bytes.NewReader(in.Bytes()))
 
+			authHeader := getAuthHeader(guestMode)
 			req := &http.Request{
 				Method: http.MethodPost,
 				URL:    GetTrafficDumpUrl(envPrefix, model),
 				Header: map[string][]string{
 					"Content-Encoding": {"deflate"},
 					"Content-Type":     {"application/octet-stream"},
-					"Guest-Auth":       {token},
+					authHeader:         {token},
 				},
 				Body: reqBody,
 			}

cli/auth/authProvider.go

@@ -4,12 +4,17 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/creasty/defaults"
+	"github.com/golang-jwt/jwt/v4"
 	"github.com/google/uuid"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/cli/config/configStructs"
 	"github.com/up9inc/mizu/cli/logger"
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"golang.org/x/oauth2"
 	"net"
 	"net/http"
+	"os"
 	"time"
 )
 
@@ -18,12 +23,60 @@ const loginTimeoutInMin = 2
 // Ports are configured in keycloak "cli" client as valid redirect URIs. A change here must be reflected there as well.
 var listenPorts = []int{3141, 4001, 5002, 6003, 7004, 8005, 9006, 10007}
 
-func LoginInteractively(envName string) (*oauth2.Token, error) {
+func IsTokenExpired(tokenString string) (bool, error) {
+	token, _, err := new(jwt.Parser).ParseUnverified(tokenString, jwt.MapClaims{})
+	if err != nil {
+		return true, fmt.Errorf("failed to parse token, err: %v", err)
+	}
+
+	claims, ok := token.Claims.(jwt.MapClaims)
+	if !ok {
+		return true, fmt.Errorf("can't convert token's claims to standard claims")
+	}
+
+	expiry := time.Unix(int64(claims["exp"].(float64)), 0)
+
+	return time.Now().After(expiry), nil
+}
+
+func Login() error {
+	token, err := loginInteractively()
+	if err != nil {
+		return fmt.Errorf("failed login interactively, err: %v", err)
+	}
+
+	authConfig := configStructs.AuthConfig{
+		EnvName: config.Config.Auth.EnvName,
+		Token:   token.AccessToken,
+	}
+
+	configFile := config.ConfigStruct{}
+	if err := defaults.Set(&configFile); err != nil {
+		return fmt.Errorf("failed inserting default values to config, err: %v", err)
+	}
+
+	if err := config.LoadConfigFile(config.Config.ConfigFilePath, &configFile); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed getting config file, err: %v", err)
+	}
+
+	configFile.Auth = authConfig
+
+	if err := config.WriteConfig(&configFile); err != nil {
+		return fmt.Errorf("failed writing config with auth, err: %v", err)
+	}
+
+	config.Config.Auth = authConfig
+
+	logger.Log.Infof("Login successfully, token stored in config path: %s", fmt.Sprintf(uiUtils.Purple, config.Config.ConfigFilePath))
+	return nil
+}
+
+func loginInteractively() (*oauth2.Token, error) {
 	tokenChannel := make(chan *oauth2.Token)
 	errorChannel := make(chan error)
 
 	server := http.Server{}
-	go startLoginServer(tokenChannel, errorChannel, envName, &server)
+	go startLoginServer(tokenChannel, errorChannel, &server)
 
 	defer func() {
 		if err := server.Shutdown(context.Background()); err != nil {
@@ -41,14 +94,14 @@ func LoginInteractively(envName string) (*oauth2.Token, error) {
 	}
 }
 
-func startLoginServer(tokenChannel chan *oauth2.Token, errorChannel chan error, envName string, server *http.Server) {
+func startLoginServer(tokenChannel chan *oauth2.Token, errorChannel chan error, server *http.Server) {
 	for _, port := range listenPorts {
-		var config = &oauth2.Config{
+		var authConfig = &oauth2.Config{
 			ClientID:    "cli",
 			RedirectURL: fmt.Sprintf("http://localhost:%v/callback", port),
 			Endpoint: oauth2.Endpoint{
-				AuthURL:  fmt.Sprintf("https://auth.%s/auth/realms/testr/protocol/openid-connect/auth", envName),
-				TokenURL: fmt.Sprintf("https://auth.%s/auth/realms/testr/protocol/openid-connect/token", envName),
+				AuthURL:  fmt.Sprintf("https://auth.%s/auth/realms/testr/protocol/openid-connect/auth", config.Config.Auth.EnvName),
+				TokenURL: fmt.Sprintf("https://auth.%s/auth/realms/testr/protocol/openid-connect/token", config.Config.Auth.EnvName),
 			},
 		}
 
@@ -56,7 +109,7 @@ func startLoginServer(tokenChannel chan *oauth2.Token, errorChannel chan error,
 
 		mux := http.NewServeMux()
 		server.Handler = mux
-		mux.Handle("/callback", loginCallbackHandler(tokenChannel, errorChannel, config, envName, state))
+		mux.Handle("/callback", loginCallbackHandler(tokenChannel, errorChannel, authConfig, state))
 
 		listener, listenErr := net.Listen("tcp", fmt.Sprintf("%s:%d", "127.0.0.1", port))
 		if listenErr != nil {
@@ -64,7 +117,7 @@ func startLoginServer(tokenChannel chan *oauth2.Token, errorChannel chan error,
 			continue
 		}
 
-		authorizationUrl := config.AuthCodeURL(state.String())
+		authorizationUrl := authConfig.AuthCodeURL(state.String())
 		uiUtils.OpenBrowser(authorizationUrl)
 
 		serveErr := server.Serve(listener)
@@ -83,7 +136,7 @@ func startLoginServer(tokenChannel chan *oauth2.Token, errorChannel chan error,
 	errorChannel <- fmt.Errorf("failed to start serving on all listen ports, ports: %v", listenPorts)
 }
 
-func loginCallbackHandler(tokenChannel chan *oauth2.Token, errorChannel chan error, config *oauth2.Config, envName string, state uuid.UUID) http.Handler {
+func loginCallbackHandler(tokenChannel chan *oauth2.Token, errorChannel chan error, authConfig *oauth2.Config, state uuid.UUID) http.Handler {
 	return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
 		if err := request.ParseForm(); err != nil {
 			errorMsg := fmt.Sprintf("failed to parse form, err: %v", err)
@@ -108,7 +161,7 @@ func loginCallbackHandler(tokenChannel chan *oauth2.Token, errorChannel chan err
 			return
 		}
 
-		token, err := config.Exchange(context.Background(), code)
+		token, err := authConfig.Exchange(context.Background(), code)
 		if err != nil {
 			errorMsg := fmt.Sprintf("failed to create token, err: %v", err)
 			http.Error(writer, errorMsg, http.StatusInternalServerError)
@@ -118,6 +171,6 @@ func loginCallbackHandler(tokenChannel chan *oauth2.Token, errorChannel chan err
 
 		tokenChannel <- token
 
-		http.Redirect(writer, request, fmt.Sprintf("https://%s/CliLogin", envName), http.StatusFound)
+		http.Redirect(writer, request, fmt.Sprintf("https://%s/CliLogin", config.Config.Auth.EnvName), http.StatusFound)
 	})
 }

cli/cmd/auth.go

@@ -1,15 +0,0 @@
-package cmd
-
-import (
-	"github.com/spf13/cobra"
-)
-
-var authCmd = &cobra.Command{
-	Use:   "auth",
-	Short: "Authenticate to up9 application",
-}
-
-func init() {
-	rootCmd.AddCommand(authCmd)
-}
-

cli/cmd/authLogin.go

@@ -1,44 +0,0 @@
-package cmd
-
-import (
-	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/auth"
-	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/logger"
-	"github.com/up9inc/mizu/cli/telemetry"
-)
-
-var authLoginCmd = &cobra.Command{
-	Use:   "login",
-	Short: "Login to up9 application",
-	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("authLogin", config.Config.Auth)
-
-		token, err := auth.LoginInteractively(config.Config.Auth.EnvName)
-		if err != nil {
-			logger.Log.Errorf("Failed login interactively, err: %v", err)
-			return nil
-		}
-
-		authConfig := configStructs.AuthConfig{
-			EnvName:      config.Config.Auth.EnvName,
-			Token:        token.AccessToken,
-		}
-
-		config.Config.Auth = authConfig
-
-		if err := config.WriteConfig(&config.Config); err != nil {
-			logger.Log.Errorf("Failed writing config with auth, err: %v", err)
-			return nil
-		}
-
-		logger.Log.Infof("Login successfully, token stored in config")
-
-		return nil
-	},
-}
-
-func init() {
-	authCmd.AddCommand(authLoginCmd)
-}

cli/cmd/authLogout.go

@@ -1,31 +0,0 @@
-package cmd
-
-import (
-	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/cli/logger"
-	"github.com/up9inc/mizu/cli/telemetry"
-)
-
-var authLogoutCmd = &cobra.Command{
-	Use:   "logout",
-	Short: "Logout from up9 application",
-	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("authLogout", config.Config.Auth)
-
-		config.Config.Auth.Token = ""
-
-		if err := config.WriteConfig(&config.Config); err != nil {
-			logger.Log.Errorf("Failed writing config with default auth, err: %v", err)
-			return nil
-		}
-
-		logger.Log.Infof("Logout successfully, token removed from config")
-
-		return nil
-	},
-}
-
-func init() {
-	authCmd.AddCommand(authLogoutCmd)
-}

cli/config/config.go

@@ -39,7 +39,7 @@ func InitConfig(cmd *cobra.Command) error {
 
 	configFilePathFlag := cmd.Flags().Lookup(ConfigFilePathCommandName)
 	configFilePath := configFilePathFlag.Value.String()
-	if err := mergeConfigFile(configFilePath); err != nil {
+	if err := LoadConfigFile(configFilePath, &Config); err != nil {
 		if configFilePathFlag.Changed || !os.IsNotExist(err) {
 			return fmt.Errorf("invalid config, %w\n"+
 				"you can regenerate the file by removing it (%v) and using `mizu config -r`", err, configFilePath)
@@ -80,7 +80,7 @@ func WriteConfig(config *ConfigStruct) error {
 	return nil
 }
 
-func mergeConfigFile(configFilePath string) error {
+func LoadConfigFile(configFilePath string, config *ConfigStruct) error {
 	reader, openErr := os.Open(configFilePath)
 	if openErr != nil {
 		return openErr
@@ -91,10 +91,11 @@ func mergeConfigFile(configFilePath string) error {
 		return readErr
 	}
 
-	if err := yaml.Unmarshal(buf, &Config); err != nil {
+	if err := yaml.Unmarshal(buf, config); err != nil {
 		return err
 	}
-	logger.Log.Debugf("Found config file, merged to default options")
+
+	logger.Log.Debugf("Found config file, config path: %s", configFilePath)
 
 	return nil
 }

cli/go.mod

@@ -5,6 +5,7 @@ go 1.16
 require (
 	github.com/creasty/defaults v1.5.1
 	github.com/denisbrodbeck/machineid v1.0.1
+	github.com/golang-jwt/jwt/v4 v4.1.0
 	github.com/google/go-github/v37 v37.0.0
 	github.com/google/uuid v1.1.2
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7

cli/go.sum

@@ -175,6 +175,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0=
+github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=

cli/mizu/version/versionCheck.go

@@ -75,6 +75,12 @@ func CheckNewerVersion(versionChan chan string) {
 
 	gitHubVersionSemVer := semver.SemVersion(gitHubVersion)
 	currentSemVer := semver.SemVersion(mizu.SemVer)
+	if !gitHubVersionSemVer.IsValid() || !currentSemVer.IsValid() {
+		logger.Log.Debugf("[ERROR] Semver version is not valid, github version %v, current version %v", gitHubVersion, currentSemVer)
+		versionChan <- ""
+		return
+	}
+
 	logger.Log.Debugf("Finished version validation, github version %v, current version %v, took %v", gitHubVersion, currentSemVer, time.Since(start))
 
 	if gitHubVersionSemVer.GreaterThan(currentSemVer) {

shared/semver/semver.go

@@ -6,9 +6,17 @@ import (
 
 type SemVersion string
 
+func (v SemVersion) IsValid() bool {
+	re := regexp.MustCompile(`\d+`)
+	breakdown := re.FindAllString(string(v), 3)
+
+	return len(breakdown) == 3
+}
+
 func (v SemVersion) Breakdown() (string, string, string) {
 	re := regexp.MustCompile(`\d+`)
 	breakdown := re.FindAllString(string(v), 3)
+
 	return breakdown[0], breakdown[1], breakdown[2]
 }
 

tap/extensions/redis/read.go

@@ -296,12 +296,28 @@ func (p *RedisProtocol) Read() (packet *RedisPacket, err error) {
 	switch x.(type) {
 	case []interface{}:
 		array := x.([]interface{})
-		packet.Command = RedisCommand(strings.ToUpper(string(array[0].([]uint8))))
-		if len(array) > 1 {
-			packet.Key = string(array[1].([]uint8))
-		}
-		if len(array) > 2 {
-			packet.Value = string(array[2].([]uint8))
+		switch array[0].(type) {
+		case []uint8:
+			packet.Command = RedisCommand(strings.ToUpper(string(array[0].([]uint8))))
+			if len(array) > 1 {
+				packet.Key = string(array[1].([]uint8))
+			}
+			if len(array) > 2 {
+				packet.Value = string(array[2].([]uint8))
+			}
+			if len(array) > 3 {
+				packet.Value = fmt.Sprintf("[%s", packet.Value)
+				for _, item := range array[3:] {
+					packet.Value = fmt.Sprintf("%s, %s", packet.Value, item.([]uint8))
+				}
+				packet.Value = strings.TrimSuffix(packet.Value, ", ")
+				packet.Value = fmt.Sprintf("%s]", packet.Value)
+			}
+		default:
+			msg := fmt.Sprintf("Unrecognized element in Redis array: %v\n", reflect.TypeOf(array[0]))
+			log.Printf(msg)
+			err = errors.New(msg)
+			return
 		}
 	case []uint8:
 		val := string(x.([]uint8))
@@ -316,6 +332,8 @@ func (p *RedisProtocol) Read() (packet *RedisPacket, err error) {
 		}
 	case string:
 		packet.Value = x.(string)
+	case int64:
+		packet.Value = fmt.Sprintf("%d", x.(int64))
 	default:
 		msg := fmt.Sprintf("Unrecognized Redis data type: %v\n", reflect.TypeOf(x))
 		log.Printf(msg)