fix elastic time picker ui css (#1204)

Co-authored-by: Leon <>

acceptanceTests/cypress.config.js

@@ -11,7 +11,6 @@ module.exports = defineConfig({
     testUrl: 'http://localhost:8899/',
     redactHeaderContent: 'User-Header[REDACTED]',
     redactBodyContent: '{ "User": "[REDACTED]" }',
-    regexMaskingBodyContent: '[REDACTED]',
     greenFilterColor: 'rgb(210, 250, 210)',
     redFilterColor: 'rgb(250, 214, 220)',
     bodyJsonClass: '.hljs',

acceptanceTests/cypress/e2e/tests/RegexMasking.js

@@ -1,7 +0,0 @@
-import {isValueExistsInElement} from "../testHelpers/TrafficHelper";
-
-it('Loading Mizu', function () {
-    cy.visit(Cypress.env('testUrl'));
-});
-
-isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), Cypress.env('bodyJsonClass'));

acceptanceTests/tap_test.go

@@ -2,10 +2,8 @@ package acceptanceTests
 
 import (
 	"archive/zip"
-	"bytes"
 	"fmt"
 	"io/ioutil"
-	"net/http"
 	"os/exec"
 	"path"
 	"strings"
@@ -343,7 +341,7 @@ func TestTapRedact(t *testing.T) {
 
 	tapNamespace := GetDefaultTapNamespace()
 	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
-	tapCmdArgs = append(tapCmdArgs, "--redact")
+	tapCmdArgs = append(tapCmdArgs, "--redact", "--set", "tap.redact-patterns.request-headers=User-Header", "--set", "tap.redact-patterns.request-body=User")
 
 	tapCmd := exec.Command(cliPath, tapCmdArgs...)
 	t.Logf("running command: %v", tapCmd.String())
@@ -429,60 +427,6 @@ func TestTapNoRedact(t *testing.T) {
 	RunCypressTests(t, "npx cypress run --spec  \"cypress/e2e/tests/NoRedact.js\"")
 }
 
-func TestTapRegexMasking(t *testing.T) {
-	if testing.Short() {
-		t.Skip("ignored acceptance test")
-	}
-
-	cliPath, cliPathErr := GetCliPath()
-	if cliPathErr != nil {
-		t.Errorf("failed to get cli path, err: %v", cliPathErr)
-		return
-	}
-
-	tapCmdArgs := GetDefaultTapCommandArgs()
-
-	tapNamespace := GetDefaultTapNamespace()
-	tapCmdArgs = append(tapCmdArgs, tapNamespace...)
-
-	tapCmdArgs = append(tapCmdArgs, "--redact")
-
-	tapCmdArgs = append(tapCmdArgs, "-r", "Mizu")
-
-	tapCmd := exec.Command(cliPath, tapCmdArgs...)
-	t.Logf("running command: %v", tapCmd.String())
-
-	t.Cleanup(func() {
-		if err := CleanupCommand(tapCmd); err != nil {
-			t.Logf("failed to cleanup tap command, err: %v", err)
-		}
-	})
-
-	if err := tapCmd.Start(); err != nil {
-		t.Errorf("failed to start tap command, err: %v", err)
-		return
-	}
-
-	apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
-
-	if err := WaitTapPodsReady(apiServerUrl); err != nil {
-		t.Errorf("failed to start tap pods on time, err: %v", err)
-		return
-	}
-
-	proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
-	for i := 0; i < DefaultEntriesCount; i++ {
-		response, requestErr := http.Post(fmt.Sprintf("%v/post", proxyUrl), "text/plain", bytes.NewBufferString("Mizu"))
-		if _, requestErr = ExecuteHttpRequest(response, requestErr); requestErr != nil {
-			t.Errorf("failed to send proxy request, err: %v", requestErr)
-			return
-		}
-	}
-
-	RunCypressTests(t, "npx cypress run --spec \"cypress/e2e/tests/RegexMasking.js\"")
-
-}
-
 func TestTapIgnoredUserAgents(t *testing.T) {
 	if testing.Short() {
 		t.Skip("ignored acceptance test")

agent/pkg/controllers/status_controller.go

@@ -1,7 +1,10 @@
 package controllers
 
 import (
+	"fmt"
 	"net/http"
+	"strconv"
+	"time"
 
 	core "k8s.io/api/core/v1"
 
@@ -80,7 +83,24 @@ func GetGeneralStats(c *gin.Context) {
 }
 
 func GetTrafficStats(c *gin.Context) {
-	c.JSON(http.StatusOK, providers.GetTrafficStats())
+	startTime, endTime, err := getStartEndTime(c)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+	c.JSON(http.StatusOK, providers.GetTrafficStats(startTime, endTime))
+}
+
+func getStartEndTime(c *gin.Context) (time.Time, time.Time, error) {
+	startTimeValue, err := strconv.Atoi(c.Query("startTimeMs"))
+	if err != nil {
+		return time.UnixMilli(0), time.UnixMilli(0), fmt.Errorf("invalid start time: %v", err)
+	}
+	endTimeValue, err := strconv.Atoi(c.Query("endTimeMs"))
+	if err != nil {
+		return time.UnixMilli(0), time.UnixMilli(0), fmt.Errorf("invalid end time: %v", err)
+	}
+	return time.UnixMilli(int64(startTimeValue)), time.UnixMilli(int64(endTimeValue)), nil
 }
 
 func GetCurrentResolvingInformation(c *gin.Context) {

agent/pkg/providers/stats_provider.go

@@ -5,7 +5,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"reflect"
-	"strings"
 	"sync"
 	"time"
 
@@ -82,13 +81,12 @@ func GetGeneralStats() *GeneralStats {
 
 func InitProtocolToColor(protocolMap map[string]*api.Protocol) {
 	for item, value := range protocolMap {
-		splitted := strings.SplitN(item, "/", 3)
-		protocolToColor[splitted[len(splitted)-1]] = value.BackgroundColor
+		protocolToColor[api.GetProtocolSummary(item).Abbreviation] = value.BackgroundColor
 	}
 }
 
-func GetTrafficStats() *TrafficStatsResponse {
-	bucketsStatsCopy := getBucketStatsCopy()
+func GetTrafficStats(startTime time.Time, endTime time.Time) *TrafficStatsResponse {
+	bucketsStatsCopy := getFilteredBucketStatsCopy(startTime, endTime)
 
 	return &TrafficStatsResponse{
 		Protocols:     getAvailableProtocols(bucketsStatsCopy),
@@ -264,7 +262,7 @@ func convertAccumulativeStatsDictToArray(methodsPerProtocolAggregated map[string
 	return protocolsData
 }
 
-func getBucketStatsCopy() BucketStats {
+func getFilteredBucketStatsCopy(startTime time.Time, endTime time.Time) BucketStats {
 	bucketStatsCopy := BucketStats{}
 	bucketStatsLocker.Lock()
 	if err := copier.Copy(&bucketStatsCopy, bucketsStats); err != nil {
@@ -272,7 +270,18 @@ func getBucketStatsCopy() BucketStats {
 		return nil
 	}
 	bucketStatsLocker.Unlock()
-	return bucketStatsCopy
+
+	filteredBucketStatsCopy := BucketStats{}
+	interval := InternalBucketThreshold
+
+	for _, bucket := range bucketStatsCopy {
+		if (bucket.BucketTime.After(startTime.Add(-1*interval/2).Round(interval)) && bucket.BucketTime.Before(endTime.Add(-1*interval/2).Round(interval))) ||
+			bucket.BucketTime.Equal(startTime.Add(-1*interval/2).Round(interval)) ||
+			bucket.BucketTime.Equal(endTime.Add(-1*interval/2).Round(interval)) {
+			filteredBucketStatsCopy = append(filteredBucketStatsCopy, bucket)
+		}
+	}
+	return filteredBucketStatsCopy
 }
 
 func getAggregatedResultTiming(stats BucketStats, interval time.Duration) map[time.Time]map[string]map[string]*AccumulativeStatsCounter {

cli/cmd/tap.go

@@ -48,7 +48,6 @@ func init() {
 	tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver")
 	tapCmd.Flags().StringSliceP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector")
 	tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces")
-	tapCmd.Flags().StringSliceP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
 	tapCmd.Flags().Bool(configStructs.EnableRedactionTapName, defaultTapConfig.EnableRedaction, "Enables redaction of potentially sensitive request/response headers and body values")
 	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
 	tapCmd.Flags().String(configStructs.InsertionFilterName, defaultTapConfig.InsertionFilter, "Set the insertion filter. Accepts string or a file path.")

cli/cmd/tapRunner.go

@@ -151,17 +151,18 @@ func printTappedPodsPreview(ctx context.Context, kubernetesProvider *kubernetes.
 	}
 }
 
-func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, mizuApiFilteringOptions api.TrafficFilteringOptions, startTime time.Time) error {
+func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, startTime time.Time) error {
 	tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
-		TargetNamespaces:         targetNamespaces,
-		PodFilterRegex:           *config.Config.Tap.PodRegex(),
-		MizuResourcesNamespace:   config.Config.MizuResourcesNamespace,
-		AgentImage:               config.Config.AgentImage,
-		TapperResources:          config.Config.Tap.TapperResources,
-		ImagePullPolicy:          config.Config.ImagePullPolicy(),
-		LogLevel:                 config.Config.LogLevel(),
-		IgnoredUserAgents:        config.Config.Tap.IgnoredUserAgents,
-		MizuApiFilteringOptions:  mizuApiFilteringOptions,
+		TargetNamespaces:       targetNamespaces,
+		PodFilterRegex:         *config.Config.Tap.PodRegex(),
+		MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
+		AgentImage:             config.Config.AgentImage,
+		TapperResources:        config.Config.Tap.TapperResources,
+		ImagePullPolicy:        config.Config.ImagePullPolicy(),
+		LogLevel:               config.Config.LogLevel(),
+		MizuApiFilteringOptions: api.TrafficFilteringOptions{
+			IgnoredUserAgents: config.Config.Tap.IgnoredUserAgents,
+		},
 		MizuServiceAccountExists: state.mizuServiceAccountExists,
 		ServiceMesh:              config.Config.Tap.ServiceMesh,
 		Tls:                      config.Config.Tap.Tls,
@@ -229,27 +230,6 @@ func getErrorDisplayTextForK8sTapManagerError(err kubernetes.K8sTapManagerError)
 	}
 }
 
-func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) {
-	var compiledRegexSlice []*api.SerializableRegexp
-
-	if config.Config.Tap.PlainTextFilterRegexes != nil && len(config.Config.Tap.PlainTextFilterRegexes) > 0 {
-		compiledRegexSlice = make([]*api.SerializableRegexp, 0)
-		for _, regexStr := range config.Config.Tap.PlainTextFilterRegexes {
-			compiledRegex, err := api.CompileRegexToSerializableRegexp(regexStr)
-			if err != nil {
-				return nil, err
-			}
-			compiledRegexSlice = append(compiledRegexSlice, compiledRegex)
-		}
-	}
-
-	return &api.TrafficFilteringOptions{
-		PlainTextMaskingRegexes: compiledRegexSlice,
-		IgnoredUserAgents:       config.Config.Tap.IgnoredUserAgents,
-		EnableRedaction:        config.Config.Tap.EnableRedaction,
-	}, nil
-}
-
 func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", kubernetes.ApiServerPodName))
 	podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
@@ -367,8 +347,7 @@ func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Pr
 func postApiServerStarted(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
 	startProxyReportErrorIfAny(kubernetesProvider, ctx, cancel, config.Config.Tap.GuiPort)
 
-	options, _ := getMizuApiFilteringOptions()
-	if err := startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, *options, state.startTime); err != nil {
+	if err := startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, state.startTime); err != nil {
 		logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error starting mizu tapper syncer: %v", errormessage.FormatError(err)))
 		cancel()
 	}

cli/config/configStructs/tapConfig.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"regexp"
+	"strings"
 
 	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
@@ -15,38 +16,43 @@ import (
 )
 
 const (
-	GuiPortTapName                = "gui-port"
-	NamespacesTapName             = "namespaces"
-	AllNamespacesTapName          = "all-namespaces"
-	PlainTextFilterRegexesTapName = "regex-masking"
-	EnableRedactionTapName        = "redact"
-	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
-	InsertionFilterName           = "insertion-filter"
-	DryRunTapName                 = "dry-run"
-	ServiceMeshName               = "service-mesh"
-	TlsName                       = "tls"
-	ProfilerName                  = "profiler"
-	MaxLiveStreamsName            = "max-live-streams"
+	GuiPortTapName               = "gui-port"
+	NamespacesTapName            = "namespaces"
+	AllNamespacesTapName         = "all-namespaces"
+	EnableRedactionTapName       = "redact"
+	HumanMaxEntriesDBSizeTapName = "max-entries-db-size"
+	InsertionFilterName          = "insertion-filter"
+	DryRunTapName                = "dry-run"
+	ServiceMeshName              = "service-mesh"
+	TlsName                      = "tls"
+	ProfilerName                 = "profiler"
+	MaxLiveStreamsName           = "max-live-streams"
 )
 
 type TapConfig struct {
-	PodRegexStr            string           `yaml:"regex" default:".*"`
-	GuiPort                uint16           `yaml:"gui-port" default:"8899"`
-	ProxyHost              string           `yaml:"proxy-host" default:"127.0.0.1"`
-	Namespaces             []string         `yaml:"namespaces"`
-	AllNamespaces          bool             `yaml:"all-namespaces" default:"false"`
-	PlainTextFilterRegexes []string         `yaml:"regex-masking"`
-	IgnoredUserAgents      []string         `yaml:"ignored-user-agents"`
-	EnableRedaction        bool             `yaml:"redact" default:"false"`
-	HumanMaxEntriesDBSize  string           `yaml:"max-entries-db-size" default:"200MB"`
-	InsertionFilter        string           `yaml:"insertion-filter" default:""`
-	DryRun                 bool             `yaml:"dry-run" default:"false"`
-	ApiServerResources     shared.Resources `yaml:"api-server-resources"`
-	TapperResources        shared.Resources `yaml:"tapper-resources"`
-	ServiceMesh            bool             `yaml:"service-mesh" default:"false"`
-	Tls                    bool             `yaml:"tls" default:"false"`
-	Profiler               bool             `yaml:"profiler" default:"false"`
-	MaxLiveStreams         int              `yaml:"max-live-streams" default:"500"`
+	PodRegexStr       string   `yaml:"regex" default:".*"`
+	GuiPort           uint16   `yaml:"gui-port" default:"8899"`
+	ProxyHost         string   `yaml:"proxy-host" default:"127.0.0.1"`
+	Namespaces        []string `yaml:"namespaces"`
+	AllNamespaces     bool     `yaml:"all-namespaces" default:"false"`
+	IgnoredUserAgents []string `yaml:"ignored-user-agents"`
+	EnableRedaction   bool     `yaml:"redact" default:"false"`
+	RedactPatterns    struct {
+		RequestHeaders     []string `yaml:"request-headers"`
+		ResponseHeaders    []string `yaml:"response-headers"`
+		RequestBody        []string `yaml:"request-body"`
+		ResponseBody       []string `yaml:"response-body"`
+		RequestQueryParams []string `yaml:"request-query-params"`
+	} `yaml:"redact-patterns"`
+	HumanMaxEntriesDBSize string           `yaml:"max-entries-db-size" default:"200MB"`
+	InsertionFilter       string           `yaml:"insertion-filter" default:""`
+	DryRun                bool             `yaml:"dry-run" default:"false"`
+	ApiServerResources    shared.Resources `yaml:"api-server-resources"`
+	TapperResources       shared.Resources `yaml:"tapper-resources"`
+	ServiceMesh           bool             `yaml:"service-mesh" default:"false"`
+	Tls                   bool             `yaml:"tls" default:"false"`
+	Profiler              bool             `yaml:"profiler" default:"false"`
+	MaxLiveStreams        int              `yaml:"max-live-streams" default:"500"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {
@@ -71,9 +77,48 @@ func (config *TapConfig) GetInsertionFilter() string {
 			}
 		}
 	}
+
+	redactFilter := getRedactFilter(config)
+	if insertionFilter != "" && redactFilter != "" {
+		return fmt.Sprintf("(%s) and (%s)", insertionFilter, redactFilter)
+	} else if insertionFilter == "" && redactFilter != "" {
+		return redactFilter
+	}
+
 	return insertionFilter
 }
 
+func getRedactFilter(config *TapConfig) string {
+	if !config.EnableRedaction {
+		return ""
+	}
+
+	var redactValues []string
+	for _, requestHeader := range config.RedactPatterns.RequestHeaders {
+		redactValues = append(redactValues, fmt.Sprintf("request.headers['%s']", requestHeader))
+	}
+	for _, responseHeader := range config.RedactPatterns.ResponseHeaders {
+		redactValues = append(redactValues, fmt.Sprintf("response.headers['%s']", responseHeader))
+	}
+
+	for _, requestBody := range config.RedactPatterns.RequestBody {
+		redactValues = append(redactValues, fmt.Sprintf("request.postData.text.json()...%s", requestBody))
+	}
+	for _, responseBody := range config.RedactPatterns.ResponseBody {
+		redactValues = append(redactValues, fmt.Sprintf("response.content.text.json()...%s", responseBody))
+	}
+
+	for _, requestQueryParams := range config.RedactPatterns.RequestQueryParams {
+		redactValues = append(redactValues, fmt.Sprintf("request.queryString['%s']", requestQueryParams))
+	}
+
+	if len(redactValues) == 0 {
+		return ""
+	}
+
+	return fmt.Sprintf("redact(\"%s\")", strings.Join(redactValues, "\",\""))
+}
+
 func (config *TapConfig) Validate() error {
 	_, compileErr := regexp.Compile(config.PodRegexStr)
 	if compileErr != nil {

performance_analysis/run_tapper_benchmark.sh

@@ -57,7 +57,7 @@ log "Writing output to $MIZU_BENCHMARK_OUTPUT_DIR"
 cd $MIZU_HOME || exit 1
 
 export HOST_MODE=0
-export SENSITIVE_DATA_FILTERING_OPTIONS='{"EnableRedaction": false}'
+export SENSITIVE_DATA_FILTERING_OPTIONS='{}'
 export MIZU_DEBUG_DISABLE_PCAP=false
 export MIZU_DEBUG_DISABLE_TCP_REASSEMBLY=false
 export MIZU_DEBUG_DISABLE_TCP_STREAM=false

shared/kubernetes/mizuTapperSyncer.go

@@ -43,7 +43,6 @@ type TapperSyncerConfig struct {
 	TapperResources          shared.Resources
 	ImagePullPolicy          core.PullPolicy
 	LogLevel                 logging.Level
-	IgnoredUserAgents        []string
 	MizuApiFilteringOptions  api.TrafficFilteringOptions
 	MizuServiceAccountExists bool
 	ServiceMesh              bool

tap/api/api.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"fmt"
 	"net"
+	"strings"
 	"sync"
 	"time"
 
@@ -25,6 +26,15 @@ func (protocol *ProtocolSummary) ToString() string {
 	return fmt.Sprintf("%s?%s?%s", protocol.Name, protocol.Version, protocol.Abbreviation)
 }
 
+func GetProtocolSummary(inputString string) *ProtocolSummary {
+	splitted := strings.SplitN(inputString, "?", 3)
+	return &ProtocolSummary{
+		Name:         splitted[0],
+		Version:      splitted[1],
+		Abbreviation: splitted[2],
+	}
+}
+
 type Protocol struct {
 	ProtocolSummary
 	LongName        string   `json:"longName"`

tap/api/options.go

@@ -1,7 +1,5 @@
 package api
 
 type TrafficFilteringOptions struct {
-	IgnoredUserAgents       []string
-	PlainTextMaskingRegexes []*SerializableRegexp
-	EnableRedaction         bool
+	IgnoredUserAgents []string
 }

tap/extensions/http/handlers.go

@@ -18,10 +18,6 @@ func filterAndEmit(item *api.OutputChannelItem, emitter api.Emitter, options *ap
 		return
 	}
 
-	if options.EnableRedaction {
-		FilterSensitiveData(item, options)
-	}
-
 	replaceForwardedFor(item)
 
 	emitter.Emit(item)

tap/extensions/http/sensitive_data_cleaner.go

@@ -1,30 +1,14 @@
 package http
 
 import (
-	"bytes"
-	"encoding/json"
-	"encoding/xml"
-	"errors"
-	"fmt"
-	"io/ioutil"
 	"net/http"
-	"net/url"
 	"strings"
 
-	"github.com/beevik/etree"
 	"github.com/up9inc/mizu/tap/api"
 )
 
-const maskedFieldPlaceholderValue = "[REDACTED]"
 const userAgent = "user-agent"
 
-//these values MUST be all lower case and contain no `-` or `_` characters
-var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password",
-	"username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt",
-	"bearer", "clientid", "clientsecret", "redirecturi", "phonenumber",
-	"zip", "zipcode", "address", "country", "firstname", "lastname",
-	"middlename", "fname", "lname", "birthdate"}
-
 func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) bool {
 	if item.Protocol.Name != "http" {
 		return false
@@ -48,192 +32,3 @@ func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteri
 
 	return false
 }
-
-func FilterSensitiveData(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) {
-	request := item.Pair.Request.Payload.(HTTPPayload).Data.(*http.Request)
-	response := item.Pair.Response.Payload.(HTTPPayload).Data.(*http.Response)
-
-	filterHeaders(&request.Header)
-	filterHeaders(&response.Header)
-	filterUrl(request.URL)
-	filterRequestBody(request, options)
-	filterResponseBody(response, options)
-}
-
-func filterRequestBody(request *http.Request, options *api.TrafficFilteringOptions) {
-	contenType := getContentTypeHeaderValue(request.Header)
-	body, err := ioutil.ReadAll(request.Body)
-	if err != nil {
-		return
-	}
-	filteredBody, err := filterHttpBody(body, contenType, options)
-	if err == nil {
-		request.Body = ioutil.NopCloser(bytes.NewBuffer(filteredBody))
-	} else {
-		request.Body = ioutil.NopCloser(bytes.NewBuffer(body))
-	}
-}
-
-func filterResponseBody(response *http.Response, options *api.TrafficFilteringOptions) {
-	contentType := getContentTypeHeaderValue(response.Header)
-	body, err := ioutil.ReadAll(response.Body)
-	if err != nil {
-		return
-	}
-	filteredBody, err := filterHttpBody(body, contentType, options)
-	if err == nil {
-		response.Body = ioutil.NopCloser(bytes.NewBuffer(filteredBody))
-	} else {
-		response.Body = ioutil.NopCloser(bytes.NewBuffer(body))
-	}
-}
-
-func filterHeaders(headers *http.Header) {
-	for key := range *headers {
-		if strings.ToLower(key) == userAgent {
-			continue
-		}
-
-		if strings.ToLower(key) == "cookie" {
-			headers.Del(key)
-		} else if isFieldNameSensitive(key) {
-			headers.Set(key, maskedFieldPlaceholderValue)
-		}
-	}
-}
-
-func getContentTypeHeaderValue(headers http.Header) string {
-	for key := range headers {
-		if strings.ToLower(key) == "content-type" {
-			return headers.Get(key)
-		}
-	}
-	return ""
-}
-
-func isFieldNameSensitive(fieldName string) bool {
-	if fieldName == ":authority" {
-		return false
-	}
-
-	name := strings.ToLower(fieldName)
-	name = strings.ReplaceAll(name, "_", "")
-	name = strings.ReplaceAll(name, "-", "")
-	name = strings.ReplaceAll(name, " ", "")
-
-	for _, sensitiveField := range personallyIdentifiableDataFields {
-		if strings.Contains(name, sensitiveField) {
-			return true
-		}
-	}
-
-	return false
-}
-
-func filterHttpBody(bytes []byte, contentType string, options *api.TrafficFilteringOptions) ([]byte, error) {
-	mimeType := strings.Split(contentType, ";")[0]
-	switch strings.ToLower(mimeType) {
-	case "application/json":
-		return filterJsonBody(bytes)
-	case "text/html":
-		fallthrough
-	case "application/xhtml+xml":
-		fallthrough
-	case "text/xml":
-		fallthrough
-	case "application/xml":
-		return filterXmlEtree(bytes)
-	case "text/plain":
-		if options != nil && options.PlainTextMaskingRegexes != nil {
-			return filterPlainText(bytes, options), nil
-		}
-	}
-	return bytes, nil
-}
-
-func filterPlainText(bytes []byte, options *api.TrafficFilteringOptions) []byte {
-	for _, regex := range options.PlainTextMaskingRegexes {
-		bytes = regex.ReplaceAll(bytes, []byte(maskedFieldPlaceholderValue))
-	}
-	return bytes
-}
-
-func filterXmlEtree(bytes []byte) ([]byte, error) {
-	if !IsValidXML(bytes) {
-		return nil, errors.New("Invalid XML")
-	}
-	xmlDoc := etree.NewDocument()
-	err := xmlDoc.ReadFromBytes(bytes)
-	if err != nil {
-		return nil, err
-	} else {
-		filterXmlElement(xmlDoc.Root())
-	}
-	return xmlDoc.WriteToBytes()
-}
-
-func IsValidXML(data []byte) bool {
-	return xml.Unmarshal(data, new(interface{})) == nil
-}
-
-func filterXmlElement(element *etree.Element) {
-	for i, attribute := range element.Attr {
-		if isFieldNameSensitive(attribute.Key) {
-			element.Attr[i].Value = maskedFieldPlaceholderValue
-		}
-	}
-	if element.ChildElements() == nil || len(element.ChildElements()) == 0 {
-		if isFieldNameSensitive(element.Tag) {
-			element.SetText(maskedFieldPlaceholderValue)
-		}
-	} else {
-		for _, element := range element.ChildElements() {
-			filterXmlElement(element)
-		}
-	}
-}
-
-func filterJsonBody(bytes []byte) ([]byte, error) {
-	var bodyJsonMap map[string]interface{}
-	err := json.Unmarshal(bytes, &bodyJsonMap)
-	if err != nil {
-		return nil, err
-	}
-	filterJsonMap(bodyJsonMap)
-	return json.Marshal(bodyJsonMap)
-}
-
-func filterJsonMap(jsonMap map[string]interface{}) {
-	for key, value := range jsonMap {
-		// Do not replace nil values with maskedFieldPlaceholderValue
-		if value == nil {
-			continue
-		}
-
-		nestedMap, isNested := value.(map[string]interface{})
-		if isNested {
-			filterJsonMap(nestedMap)
-		} else {
-			if isFieldNameSensitive(key) {
-				jsonMap[key] = maskedFieldPlaceholderValue
-			}
-		}
-	}
-}
-
-func filterUrl(url *url.URL) {
-	if len(url.RawQuery) > 0 {
-		newQueryArgs := make([]string, 0)
-		for urlQueryParamName, urlQueryParamValues := range url.Query() {
-			newValues := urlQueryParamValues
-			if isFieldNameSensitive(urlQueryParamName) {
-				newValues = []string{maskedFieldPlaceholderValue}
-			}
-			for _, paramValue := range newValues {
-				newQueryArgs = append(newQueryArgs, fmt.Sprintf("%s=%s", urlQueryParamName, paramValue))
-			}
-		}
-
-		url.RawQuery = strings.Join(newQueryArgs, "&")
-	}
-}

ui-common/package.json

@@ -34,6 +34,7 @@
   },
   "dependencies": {
     "@craco/craco": "^6.4.3",
+    "@elastic/eui": "^60.2.0",
     "@emotion/react": "^11.9.0",
     "@emotion/styled": "^11.8.1",
     "@mui/icons-material": "^5.8.2",
@@ -85,7 +86,7 @@
     "rollup-plugin-postcss": "^4.0.2",
     "rollup-plugin-sass": "^1.2.12",
     "rollup-plugin-scss": "^3.0.0",
-    "typescript": "^4.7.2"
+    "typescript": "^4.5.3"
   },
   "eslintConfig": {
     "extends": [
@@ -94,6 +95,7 @@
     ]
   },
   "files": [
+    "src/*.scss",
     "dist"
   ]
 }

ui-common/src/components/EntryDetailed/EntryViewer/AutoRepresentation.tsx

@@ -32,7 +32,7 @@ export const AutoRepresentation: React.FC<any> = ({ representation, color, opene
                 badge: null
             }]
 
-        if (response) {
+        if (response && response.length > 0) {
             arr.push({
                 tab: 'Response',
                 badge: null
@@ -71,7 +71,7 @@ export const AutoRepresentation: React.FC<any> = ({ representation, color, opene
             {getOpenedTabIndex() === TabsEnum.Request && <React.Fragment>
                 <SectionsRepresentation data={request} color={color} requestRepresentation={request} />
             </React.Fragment>}
-            {response && getOpenedTabIndex() === TabsEnum.Response && <React.Fragment>
+            {response && response.length > 0 && getOpenedTabIndex() === TabsEnum.Response && <React.Fragment>
                 <SectionsRepresentation data={response} color={color} />
             </React.Fragment>}
         </div>}

ui-common/src/components/EntryDetailed/EntryViewer/SectionsRepresentation.tsx

@@ -28,10 +28,6 @@ const SectionsRepresentation: React.FC<any> = ({ data, color }) => {
         }
     }
 
-    if (sections.length === 0) {
-        sections.push(<div>This request or response has no data.</div>);
-    }
-
     return <React.Fragment>{sections}</React.Fragment>;
 }
 

ui-common/src/components/modals/ServiceMapModal/ServiceMapModal.module.sass

@@ -97,4 +97,7 @@ $modalMargin-from-edge : 35px
   overflow: hidden
 
 .servicesFilterList
-  height: calc(100% - 30px - 52px)
+  height: calc(100% - 30px)
+
+.protocolsFilterList
+  height: 100%

ui-common/src/components/modals/ServiceMapModal/ServiceMapModal.tsx

@@ -228,7 +228,7 @@ export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onClos
                                 <div className={styles.filterWrapper}>
                                     <div className={styles.card}>
                                         <SelectList items={getProtocolsForFilter} checkBoxWidth="5%" tableName={"PROTOCOLS"} multiSelect={true}
-                                            checkedValues={checkedProtocols} setCheckedValues={onProtocolsChange} tableClassName={styles.filters}
+                                            checkedValues={checkedProtocols} setCheckedValues={onProtocolsChange} tableClassName={styles.filters + ` ${styles.protocolsFilterList}`}
                                             inputSearchClass={styles.servicesFilterSearch} isFilterable={false} />
                                     </div>
                                     <div className={styles.servicesFilterWrapper + ` ${styles.card}`}>

ui-common/src/components/modals/TrafficStatsModal/TimelineBarChart/TimeRangePicker/TimeTangePicker.tsx

@@ -0,0 +1,96 @@
+import React, { useState, Fragment } from 'react';
+import { EuiProvider } from '@elastic/eui';
+
+import {
+    EuiSuperDatePicker,
+    EuiSpacer,
+} from '@elastic/eui';
+import dateMath from '@elastic/datemath';
+import '@elastic/eui/dist/eui_theme_light.css';
+
+interface TimeRangePickerProps {
+    refreshStats: (startTime, endTime) => void;
+}
+
+export const TimeRangePicker: React.FC<TimeRangePickerProps> = ({ refreshStats }) => {
+    const [recentlyUsedRanges, setRecentlyUsedRanges] = useState([]);
+    const [isLoading, setIsLoading] = useState(false);
+    const [start, setStart] = useState('now-30m');
+    const [end, setEnd] = useState('now');
+    const [isPaused, setIsPaused] = useState(true);
+    const [refreshInterval, setRefreshInterval] = useState();
+
+    const dateConvertor = (inputStart, inputEnd) => {
+        const startMoment = dateMath.parse(inputStart);
+        if (!startMoment || !startMoment.isValid()) {
+            console.error("Unable to parse start string");
+        }
+        const endMoment = dateMath.parse(inputEnd, { roundUp: true });
+        if (!endMoment || !endMoment.isValid()) {
+            console.error("Unable to parse end string");
+        }
+        return { startMoment: startMoment.format("x"), endMoment: endMoment.format("x") }
+    }
+
+    const onTimeChange = ({ start, end }) => {
+        const recentlyUsedRange = recentlyUsedRanges.filter(recentlyUsedRange => {
+            const isDuplicate =
+                recentlyUsedRange.start === start && recentlyUsedRange.end === end;
+            return !isDuplicate;
+        });
+        recentlyUsedRange.unshift({ start, end });
+        setStart(start);
+        setEnd(end);
+        setRecentlyUsedRanges(
+            recentlyUsedRange.length > 10
+                ? recentlyUsedRange.slice(0, 9)
+                : recentlyUsedRange
+        );
+        const { startMoment, endMoment } = dateConvertor(start, end)
+        refreshStats(startMoment, endMoment)
+        setIsLoading(true);
+        startLoading();
+    };
+
+    const onRefresh = ({ start, end, refreshInterval }) => {
+        return new Promise(resolve => {
+            setTimeout(resolve, 100);
+        }).then(() => {
+            const { startMoment, endMoment } = dateConvertor(start, end)
+            refreshStats(startMoment, endMoment)
+        });
+    };
+
+    const startLoading = () => {
+        setTimeout(stopLoading, 1000);
+    };
+    const stopLoading = () => {
+        setIsLoading(false);
+    };
+
+    const onRefreshChange = ({ isPaused, refreshInterval }) => {
+        setIsPaused(isPaused);
+        setRefreshInterval(refreshInterval);
+    };
+
+    return (
+        <EuiProvider>
+            <Fragment>
+                <EuiSpacer />
+                <EuiSuperDatePicker
+                    width='auto'
+                    isLoading={isLoading}
+                    start={start}
+                    end={end}
+                    onTimeChange={onTimeChange}
+                    onRefresh={onRefresh}
+                    isPaused={isPaused}
+                    refreshInterval={refreshInterval}
+                    onRefreshChange={onRefreshChange}
+                    recentlyUsedRanges={recentlyUsedRanges}
+                />
+                <EuiSpacer />
+            </Fragment>
+        </EuiProvider>
+    );
+};

ui-common/src/components/modals/TrafficStatsModal/TrafficStatsModal.module.sass

@@ -13,12 +13,12 @@
   top: 20px
 
 .mainContainer
-  padding: 30px
   text-align: center
 
 .selectContainer
   display: flex
   justify-content: space-evenly
+  align-items: center
   margin-bottom: 4%
 
 .select

ui-common/src/components/modals/TrafficStatsModal/TrafficStatsModal.tsx

@@ -1,13 +1,12 @@
 import React, { useCallback, useEffect, useState } from "react";
-import { Backdrop, Box, Button, debounce, Fade, Modal } from "@mui/material";
+import { Backdrop, Box, debounce, Fade, Modal } from "@mui/material";
 import styles from "./TrafficStatsModal.module.sass";
 import closeIcon from "assets/close.svg";
 import { TrafficPieChart } from "./TrafficPieChart/TrafficPieChart";
 import { TimelineBarChart } from "./TimelineBarChart/TimelineBarChart";
-import refreshIcon from "assets/refresh.svg";
-import { useCommonStyles } from "../../../helpers/commonStyle";
 import { LoadingWrapper } from "../../UI/withLoading/withLoading";
 import { ALL_PROTOCOLS, StatsMode } from "./consts";
+import { TimeRangePicker } from "./TimelineBarChart/TimeRangePicker/TimeTangePicker";
 
 const modalStyle = {
   position: 'absolute',
@@ -15,7 +14,7 @@ const modalStyle = {
   left: '50%',
   transform: 'translate(-50%, 0%)',
   width: '60vw',
-  height: '82vh',
+  height: '90vh',
   bgcolor: 'background.paper',
   borderRadius: '5px',
   boxShadow: 24,
@@ -26,11 +25,10 @@ const modalStyle = {
 interface TrafficStatsModalProps {
   isOpen: boolean;
   onClose: () => void;
-  getTrafficStatsDataApi: () => Promise<any>
+  getTrafficStatsDataApi: (start?, end?) => Promise<any>
 }
 
 export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, onClose, getTrafficStatsDataApi }) => {
-
   const modes = Object.keys(StatsMode).filter(x => !(parseInt(x) >= 0));
   const [statsMode, setStatsMode] = useState(modes[0]);
   const [selectedProtocol, setSelectedProtocol] = useState(ALL_PROTOCOLS);
@@ -38,14 +36,13 @@ export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, on
   const [timelineStatsData, setTimelineStatsData] = useState(null);
   const [protocols, setProtocols] = useState([])
   const [isLoading, setIsLoading] = useState(false);
-  const commonClasses = useCommonStyles();
 
-  const getTrafficStats = useCallback(async () => {
+  const getTrafficStats = useCallback(async (startTime, endTime) => {
     if (isOpen && getTrafficStatsDataApi) {
       (async () => {
         try {
           setIsLoading(true);
-          const statsData = await getTrafficStatsDataApi();
+          const statsData = await getTrafficStatsDataApi(startTime, endTime);
           setPieStatsData(statsData.pie);
           setTimelineStatsData(statsData.timeline);
           setProtocols(statsData.protocols)
@@ -59,11 +56,13 @@ export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, on
   }, [isOpen, getTrafficStatsDataApi, setPieStatsData, setTimelineStatsData])
 
   useEffect(() => {
-    getTrafficStats();
+    const now = new Date().getTime();
+    const halfAnHourAgo = now - (30 * 60 * 1000);
+    getTrafficStats(halfAnHourAgo, now);
   }, [getTrafficStats])
 
-  const refreshStats = debounce(() => {
-    getTrafficStats();
+  const refreshStats = debounce((newStartTime, newEndTime) => {
+    getTrafficStats(newStartTime, newEndTime);
   }, 500);
 
   return (
@@ -82,18 +81,12 @@ export const TrafficStatsModal: React.FC<TrafficStatsModalProps> = ({ isOpen, on
           </div>
           <div className={styles.headlineContainer}>
             <div className={styles.title}>Traffic Statistics</div>
-            <Button style={{ marginLeft: "2%", textTransform: 'unset' }}
-              startIcon={<img src={refreshIcon} className="custom" alt="refresh"></img>}
-              size="medium"
-              variant="contained"
-              className={commonClasses.outlinedButton + " " + commonClasses.imagedButton}
-              onClick={refreshStats}
-            >
-              Refresh
-            </Button>
           </div>
           <div className={styles.mainContainer}>
             <div className={styles.selectContainer}>
+              <div>
+                <TimeRangePicker refreshStats={refreshStats} />
+              </div>
               <div>
                 <span style={{ marginRight: 15 }}>Breakdown By</span>
                 <select className={styles.select} value={statsMode} onChange={(e) => setStatsMode(e.target.value)}>

ui-common/src/variables.module.scss

@@ -25,4 +25,5 @@ $light-gray: #8F9BB2;
   failureColor: $failure-color;
   blueGray: $blue-gray;
   lightGray: $light-gray;
+  contentSectionColor: $content-section-color;
 }

ui/package.json

@@ -4,6 +4,8 @@
   "private": true,
   "dependencies": {
     "@craco/craco": "^6.4.3",
+    "@elastic/datemath": "^5.0.3",
+    "@elastic/eui": "^60.2.0",
     "@emotion/react": "^11.9.0",
     "@emotion/styled": "^11.8.1",
     "@mui/material": "^5.8.2",
@@ -23,7 +25,6 @@
     "mobx": "^6.6.0",
     "moment": "^2.29.3",
     "node-fetch": "^3.2.4",
-    "sass": "^1.52.3",
     "numeral": "^2.0.6",
     "react": "^17.0.2",
     "react-copy-to-clipboard": "^5.1.0",
@@ -35,8 +36,9 @@
     "react-syntax-highlighter": "^15.5.0",
     "react-toastify": "^8.2.0",
     "redoc": "^2.0.0-rc.71",
+    "sass": "^1.52.3",
     "styled-components": "^5.3.5",
-    "typescript": "^4.7.2",
+    "typescript": "^4.5.3",
     "web-vitals": "^2.1.4",
     "xml-formatter": "^2.6.1"
   },

ui/src/App.tsx

@@ -8,7 +8,7 @@ import oasModalOpenAtom from './recoil/oasModalOpen/atom';
 import trafficStatsModalOpenAtom from "./recoil/trafficStatsModalOpen";
 import { OasModal } from '@up9/mizu-common';
 import Api from './helpers/api';
-import {ThemeProvider, StyledEngineProvider, createTheme} from '@mui/material';
+import { ThemeProvider, StyledEngineProvider, createTheme } from '@mui/material';
 import { TrafficStatsModal } from '@up9/mizu-common';
 
 const api = Api.getInstance()
@@ -36,7 +36,7 @@ const App = () => {
                         openModal={oasModalOpen}
                         handleCloseModal={() => setOasModalOpen(false)}
                     />}
-                    <TrafficStatsModal isOpen={trafficStatsModalOpen} onClose={() => setTrafficStatsModalOpen(false)} getTrafficStatsDataApi={api.getTrafficStats}/>
+                    <TrafficStatsModal isOpen={trafficStatsModalOpen} onClose={() => setTrafficStatsModalOpen(false)} getTrafficStatsDataApi={api.getTrafficStats} />
                 </div>
             </ThemeProvider>
         </StyledEngineProvider>

ui/src/helpers/api.js

@@ -116,8 +116,8 @@ export default class Api {
         });
     }
 
-    getTrafficStats = async () => {
-        const response = await client.get("/status/trafficStats");
+    getTrafficStats = async (startTimeMs, endTimeMs) => {
+        const response = await client.get("/status/trafficStats", {params: {startTimeMs, endTimeMs}});
         return response.data;
     }
 }