organize routes (#348)

acceptanceTests/tap_test.go

@@ -66,7 +66,7 @@ func TestTap(t *testing.T) {
 			entriesCheckFunc := func() error {
 				timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-				entriesUrl := fmt.Sprintf("%v/api/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, entriesCount, timestamp)
+				entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, entriesCount, timestamp)
 				requestResult, requestErr := executeHttpGetRequest(entriesUrl)
 				if requestErr != nil {
 					return fmt.Errorf("failed to get entries, err: %v", requestErr)
@@ -79,7 +79,7 @@ func TestTap(t *testing.T) {
 
 				entry :=  entries[0].(map[string]interface{})
 
-				entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, entry["id"])
+				entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entry["id"])
 				requestResult, requestErr = executeHttpGetRequest(entryUrl)
 				if requestErr != nil {
 					return fmt.Errorf("failed to get entry, err: %v", requestErr)
@@ -188,7 +188,7 @@ func TestTapAllNamespaces(t *testing.T) {
 		return
 	}
 
-	podsUrl := fmt.Sprintf("%v/api/tapStatus", apiServerUrl)
+	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
 	requestResult, requestErr := executeHttpGetRequest(podsUrl)
 	if requestErr != nil {
 		t.Errorf("failed to get tap status, err: %v", requestErr)
@@ -269,7 +269,7 @@ func TestTapMultipleNamespaces(t *testing.T) {
 		return
 	}
 
-	podsUrl := fmt.Sprintf("%v/api/tapStatus", apiServerUrl)
+	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
 	requestResult, requestErr := executeHttpGetRequest(podsUrl)
 	if requestErr != nil {
 		t.Errorf("failed to get tap status, err: %v", requestErr)
@@ -352,7 +352,7 @@ func TestTapRegex(t *testing.T) {
 		return
 	}
 
-	podsUrl := fmt.Sprintf("%v/api/tapStatus", apiServerUrl)
+	podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl)
 	requestResult, requestErr := executeHttpGetRequest(podsUrl)
 	if requestErr != nil {
 		t.Errorf("failed to get tap status, err: %v", requestErr)
@@ -486,7 +486,7 @@ func TestTapRedact(t *testing.T) {
 	redactCheckFunc := func() error {
 		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-		entriesUrl := fmt.Sprintf("%v/api/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
+		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
 		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
 		if requestErr != nil {
 			return fmt.Errorf("failed to get entries, err: %v", requestErr)
@@ -499,7 +499,7 @@ func TestTapRedact(t *testing.T) {
 
 		firstEntry :=  entries[0].(map[string]interface{})
 
-		entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, firstEntry["id"])
+		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
 		requestResult, requestErr = executeHttpGetRequest(entryUrl)
 		if requestErr != nil {
 			return fmt.Errorf("failed to get entry, err: %v", requestErr)
@@ -601,7 +601,7 @@ func TestTapNoRedact(t *testing.T) {
 	redactCheckFunc := func() error {
 		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-		entriesUrl := fmt.Sprintf("%v/api/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
+		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
 		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
 		if requestErr != nil {
 			return fmt.Errorf("failed to get entries, err: %v", requestErr)
@@ -614,7 +614,7 @@ func TestTapNoRedact(t *testing.T) {
 
 		firstEntry :=  entries[0].(map[string]interface{})
 
-		entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, firstEntry["id"])
+		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
 		requestResult, requestErr = executeHttpGetRequest(entryUrl)
 		if requestErr != nil {
 			return fmt.Errorf("failed to get entry, err: %v", requestErr)
@@ -716,7 +716,7 @@ func TestTapRegexMasking(t *testing.T) {
 	redactCheckFunc := func() error {
 		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-		entriesUrl := fmt.Sprintf("%v/api/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
+		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
 		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
 		if requestErr != nil {
 			return fmt.Errorf("failed to get entries, err: %v", requestErr)
@@ -729,7 +729,7 @@ func TestTapRegexMasking(t *testing.T) {
 
 		firstEntry :=  entries[0].(map[string]interface{})
 
-		entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, firstEntry["id"])
+		entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
 		requestResult, requestErr = executeHttpGetRequest(entryUrl)
 		if requestErr != nil {
 			return fmt.Errorf("failed to get entry, err: %v", requestErr)
@@ -823,7 +823,7 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 	ignoredUserAgentsCheckFunc := func() error {
 		timestamp := time.Now().UnixNano() / int64(time.Millisecond)
 
-		entriesUrl := fmt.Sprintf("%v/api/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount * 2, timestamp)
+		entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount * 2, timestamp)
 		requestResult, requestErr := executeHttpGetRequest(entriesUrl)
 		if requestErr != nil {
 			return fmt.Errorf("failed to get entries, err: %v", requestErr)
@@ -835,7 +835,7 @@ func TestTapIgnoredUserAgents(t *testing.T) {
 		}
 
 		for _, entryInterface := range entries {
-			entryUrl := fmt.Sprintf("%v/api/entries/%v", apiServerUrl, entryInterface.(map[string]interface{})["id"])
+			entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entryInterface.(map[string]interface{})["id"])
 			requestResult, requestErr = executeHttpGetRequest(entryUrl)
 			if requestErr != nil {
 				return fmt.Errorf("failed to get entry, err: %v", requestErr)

agent/main.go

@@ -42,20 +42,20 @@ var extensionsMap map[string]*tapApi.Extension // global
 func main() {
 	flag.Parse()
 	loadExtensions()
-	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
-	tapOpts := &tap.TapOpts{HostMode: hostMode}
 
 	if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode {
 		panic("One of the flags --tap, --api or --standalone or --hars-read must be provided")
 	}
 
-	filteringOptions := getTrafficFilteringOptions()
-
 	if *standaloneMode {
 		api.StartResolving(*namespace)
 
 		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
 		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+
+		filteringOptions := getTrafficFilteringOptions()
+		hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
+		tapOpts := &tap.TapOpts{HostMode: hostMode}
 		tap.StartPassiveTapper(tapOpts, outputItemsChannel, extensions, filteringOptions)
 
 		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
@@ -75,6 +75,10 @@ func main() {
 		}
 
 		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+
+		filteringOptions := getTrafficFilteringOptions()
+		hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
+		tapOpts := &tap.TapOpts{HostMode: hostMode}
 		tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, extensions, filteringOptions)
 		socketConnection, _, err := websocket.DefaultDialer.Dial(*apiServerAddress, nil)
 		if err != nil {

agent/pkg/controllers/entries_controller.go

@@ -3,19 +3,13 @@ package controllers
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/gin-gonic/gin"
+	tapApi "github.com/up9inc/mizu/tap/api"
 	"mizuserver/pkg/database"
 	"mizuserver/pkg/models"
-	"mizuserver/pkg/providers"
-	"mizuserver/pkg/up9"
 	"mizuserver/pkg/utils"
 	"mizuserver/pkg/validation"
 	"net/http"
-	"time"
-
-	"github.com/google/martian/har"
-
-	"github.com/gin-gonic/gin"
-	tapApi "github.com/up9inc/mizu/tap/api"
 )
 
 var extensionsMap map[string]*tapApi.Extension // global
@@ -62,47 +56,6 @@ func GetEntries(c *gin.Context) {
 	c.JSON(http.StatusOK, baseEntries)
 }
 
-func GetFullEntries(c *gin.Context) {
-	entriesFilter := &models.HarFetchRequestQuery{}
-	if err := c.BindQuery(entriesFilter); err != nil {
-		c.JSON(http.StatusBadRequest, err)
-	}
-	err := validation.Validate(entriesFilter)
-	if err != nil {
-		c.JSON(http.StatusBadRequest, err)
-	}
-
-	var timestampFrom, timestampTo int64
-
-	if entriesFilter.From < 0 {
-		timestampFrom = 0
-	} else {
-		timestampFrom = entriesFilter.From
-	}
-	if entriesFilter.To <= 0 {
-		timestampTo = time.Now().UnixNano() / int64(time.Millisecond)
-	} else {
-		timestampTo = entriesFilter.To
-	}
-
-	entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo, nil)
-
-	result := make([]har.Entry, 0)
-	for _, data := range entriesArray {
-		var pair tapApi.RequestResponsePair
-		if err := json.Unmarshal([]byte(data.Entry), &pair); err != nil {
-			continue
-		}
-		harEntry, err := utils.NewEntry(&pair)
-		if err != nil {
-			continue
-		}
-		result = append(result, *harEntry)
-	}
-
-	c.JSON(http.StatusOK, result)
-}
-
 func GetEntry(c *gin.Context) {
 	var entryData tapApi.MizuEntry
 	database.GetEntriesTable().
@@ -133,30 +86,3 @@ func GetEntry(c *gin.Context) {
 		IsRulesEnabled: isRulesEnabled,
 	})
 }
-
-func DeleteAllEntries(c *gin.Context) {
-	database.GetEntriesTable().
-		Where("1 = 1").
-		Delete(&tapApi.MizuEntry{})
-
-	c.JSON(http.StatusOK, map[string]string{
-		"msg": "Success",
-	})
-
-}
-
-func GetGeneralStats(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.GetGeneralStats())
-}
-
-func GetTappingStatus(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.TapStatus)
-}
-
-func AnalyzeInformation(c *gin.Context) {
-	c.JSON(http.StatusOK, up9.GetAnalyzeInfo())
-}
-
-func GetRecentTLSLinks(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.GetAllRecentTLSAddresses())
-}

agent/pkg/controllers/resolving_controller.go

@@ -1,12 +0,0 @@
-package controllers
-
-import (
-	"github.com/gin-gonic/gin"
-	"mizuserver/pkg/holder"
-	"net/http"
-)
-
-func GetCurrentResolvingInformation(c *gin.Context) {
-	c.JSON(http.StatusOK, holder.GetResolver().GetMap())
-}
-

agent/pkg/controllers/status_controller.go

@@ -6,7 +6,9 @@ import (
 	"github.com/romana/rlog"
 	"github.com/up9inc/mizu/shared"
 	"mizuserver/pkg/api"
+	"mizuserver/pkg/holder"
 	"mizuserver/pkg/providers"
+	"mizuserver/pkg/up9"
 	"mizuserver/pkg/validation"
 	"net/http"
 )
@@ -44,3 +46,23 @@ func GetAuthStatus(c *gin.Context) {
 
 	c.JSON(http.StatusOK, authStatus)
 }
+
+func GetTappingStatus(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.TapStatus)
+}
+
+func AnalyzeInformation(c *gin.Context) {
+	c.JSON(http.StatusOK, up9.GetAnalyzeInfo())
+}
+
+func GetGeneralStats(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.GetGeneralStats())
+}
+
+func GetRecentTLSLinks(c *gin.Context) {
+	c.JSON(http.StatusOK, providers.GetAllRecentTLSAddresses())
+}
+
+func GetCurrentResolvingInformation(c *gin.Context) {
+	c.JSON(http.StatusOK, holder.GetResolver().GetMap())
+}

agent/pkg/models/models.go

@@ -22,11 +22,6 @@ type EntriesFilter struct {
 	Timestamp int64  `form:"timestamp" validate:"required,min=1"`
 }
 
-type HarFetchRequestQuery struct {
-	From int64 `form:"from"`
-	To   int64 `form:"to"`
-}
-
 type WebSocketEntryMessage struct {
 	*shared.WebSocketMessageMetadata
 	Data *tapApi.BaseEntryDetails `json:"data,omitempty"`

agent/pkg/routes/entries_routes.go

@@ -7,17 +7,8 @@ import (
 
 // EntriesRoutes defines the group of har entries routes.
 func EntriesRoutes(ginApp *gin.Engine) {
-	routeGroup := ginApp.Group("/api")
+	routeGroup := ginApp.Group("/entries")
 
-	routeGroup.GET("/entries", controllers.GetEntries)        // get entries (base/thin entries)
-	routeGroup.GET("/entries/:entryId", controllers.GetEntry) // get single (full) entry
-	routeGroup.GET("/exportEntries", controllers.GetFullEntries)
-	routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation)
-
-	routeGroup.GET("/resetDB", controllers.DeleteAllEntries)     // get single (full) entry
-	routeGroup.GET("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB
-
-	routeGroup.GET("/tapStatus", controllers.GetTappingStatus) // get tapping status
-	routeGroup.GET("/analyzeStatus", controllers.AnalyzeInformation)
-	routeGroup.GET("/recentTLSLinks", controllers.GetRecentTLSLinks)
+	routeGroup.GET("/", controllers.GetEntries)        // get entries (base/thin entries)
+	routeGroup.GET("/:entryId", controllers.GetEntry) // get single (full) entry
 }

agent/pkg/routes/status_routes.go

@@ -9,8 +9,16 @@ func StatusRoutes(ginApp *gin.Engine) {
 	routeGroup := ginApp.Group("/status")
 
 	routeGroup.POST("/tappedPods", controllers.PostTappedPods)
-
 	routeGroup.GET("/tappersCount", controllers.GetTappersCount)
+	routeGroup.GET("/tap", controllers.GetTappingStatus)
 
 	routeGroup.GET("/auth", controllers.GetAuthStatus)
+
+	routeGroup.GET("/analyze", controllers.AnalyzeInformation)
+
+	routeGroup.GET("/general", controllers.GetGeneralStats) // get general stats about entries in DB
+
+	routeGroup.GET("/recentTLSLinks", controllers.GetRecentTLSLinks)
+
+	routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation)
 }

cli/apiserver/provider.go

@@ -85,7 +85,7 @@ func (provider *apiServerProvider) GetGeneralStats() (map[string]interface{}, er
 	if !provider.isReady {
 		return nil, fmt.Errorf("trying to reach api server when not initialized yet")
 	}
-	generalStatsUrl := fmt.Sprintf("%s/api/generalStats", provider.url)
+	generalStatsUrl := fmt.Sprintf("%s/status/general", provider.url)
 
 	response, requestErr := http.Get(generalStatsUrl)
 	if requestErr != nil {

cli/cmd/tapRunner.go

@@ -103,7 +103,7 @@ func RunMizuTap() {
 	}
 
 	defer finishMizuExecution(kubernetesProvider)
-	if err := createMizuResources(ctx, kubernetesProvider, mizuApiFilteringOptions, mizuValidationRules); err != nil {
+	if err := createMizuResources(ctx, kubernetesProvider, mizuValidationRules); err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
 		return
 	}
@@ -125,14 +125,14 @@ func readValidationRules(file string) (string, error) {
 	return string(newContent), nil
 }
 
-func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *api.TrafficFilteringOptions, mizuValidationRules string) error {
+func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuValidationRules string) error {
 	if !config.Config.IsNsRestrictedMode() {
 		if err := createMizuNamespace(ctx, kubernetesProvider); err != nil {
 			return err
 		}
 	}
 
-	if err := createMizuApiServer(ctx, kubernetesProvider, mizuApiFilteringOptions); err != nil {
+	if err := createMizuApiServer(ctx, kubernetesProvider); err != nil {
 		return err
 	}
 
@@ -153,7 +153,7 @@ func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Pro
 	return err
 }
 
-func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuApiFilteringOptions *api.TrafficFilteringOptions) error {
+func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider) error {
 	var err error
 
 	state.mizuServiceAccountExists, err = createRBACIfNecessary(ctx, kubernetesProvider)
@@ -174,7 +174,6 @@ func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Pro
 		PodImage:                config.Config.AgentImage,
 		ServiceAccountName:      serviceAccountName,
 		IsNamespaceRestricted:   config.Config.IsNsRestrictedMode(),
-		MizuApiFilteringOptions: mizuApiFilteringOptions,
 		SyncEntriesConfig:       getSyncEntriesConfig(),
 		MaxEntriesDBSizeBytes:   config.Config.Tap.MaxEntriesDBSizeBytes(),
 		Resources:               config.Config.Tap.ApiServerResources,

cli/kubernetes/provider.go

@@ -151,7 +151,6 @@ type ApiServerOptions struct {
 	PodImage                string
 	ServiceAccountName      string
 	IsNamespaceRestricted   bool
-	MizuApiFilteringOptions *api.TrafficFilteringOptions
 	SyncEntriesConfig       *shared.SyncEntriesConfig
 	MaxEntriesDBSizeBytes   int64
 	Resources               configStructs.Resources
@@ -159,15 +158,10 @@ type ApiServerOptions struct {
 }
 
 func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiServerOptions) (*core.Pod, error) {
-	marshaledFilteringOptions, err := json.Marshal(opts.MizuApiFilteringOptions)
-	if err != nil {
-		return nil, err
-	}
-
 	var marshaledSyncEntriesConfig []byte
 	if opts.SyncEntriesConfig != nil {
-		marshaledSyncEntriesConfig, err = json.Marshal(opts.SyncEntriesConfig)
-		if err != nil {
+		var err error
+		if marshaledSyncEntriesConfig, err = json.Marshal(opts.SyncEntriesConfig); err != nil {
 			return nil, err
 		}
 	}
@@ -219,14 +213,6 @@ func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, opts *ApiS
 					},
 					Command: command,
 					Env: []core.EnvVar{
-						{
-							Name:  shared.HostModeEnvVar,
-							Value: "1",
-						},
-						{
-							Name:  shared.MizuFilteringOptionsEnvVar,
-							Value: string(marshaledFilteringOptions),
-						},
 						{
 							Name:  shared.SyncEntriesConfigEnvVar,
 							Value: string(marshaledSyncEntriesConfig),

ui/src/helpers/api.js

@@ -22,27 +22,27 @@ export default class Api {
     }
 
     tapStatus = async () => {
-        const response = await this.client.get("/api/tapStatus");
+        const response = await this.client.get("/status/tap");
         return response.data;
     }
 
     analyzeStatus = async () => {
-        const response = await this.client.get("/api/analyzeStatus");
+        const response = await this.client.get("/status/analyze");
         return response.data;
     }
 
     getEntry = async (entryId) => {
-        const response = await this.client.get(`/api/entries/${entryId}`);
+        const response = await this.client.get(`/entries/${entryId}`);
         return response.data;
     }
 
     fetchEntries = async (operator, timestamp) => {
-        const response = await this.client.get(`/api/entries?limit=50&operator=${operator}&timestamp=${timestamp}`);
+        const response = await this.client.get(`/entries?limit=50&operator=${operator}&timestamp=${timestamp}`);
         return response.data;
     }
 
     getRecentTLSLinks = async () => {
-        const response = await this.client.get("/api/recentTLSLinks");
+        const response = await this.client.get("/status/recentTLSLinks");
         return response.data;
     }