Refactor to acceptance tests setup (#875)

* Add the ability to set the insertion filter into CLI

* Resolve the conflict

* Upgrade Basenine to `v0.5.0` and check the insertion filter against syntax errors inside CLI

* Upgrade Basenine to `v0.5.1`

* Update the sums

.github/workflows/acceptance_tests.yml

@@ -22,7 +22,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Setup acceptance test
-        run: source ./acceptanceTests/setup.sh
+        run: ./acceptanceTests/setup.sh
 
       - name: Create k8s users and change context
         env:

.gitignore

@@ -30,7 +30,7 @@ build
 pprof/*
 
 # Database Files
-*.bin
+*.db
 *.gob
 
 # Nohup Files - https://man7.org/linux/man-pages/man1/nohup.1p.html

Dockerfile

@@ -78,8 +78,8 @@ RUN go build -ldflags="-extldflags=-static -s -w \
     -X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum
-ADD https://github.com/up9inc/basenine/releases/download/v0.4.17/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
-ADD https://github.com/up9inc/basenine/releases/download/v0.4.17/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
+ADD https://github.com/up9inc/basenine/releases/download/v0.5.4/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
+ADD https://github.com/up9inc/basenine/releases/download/v0.5.4/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
 RUN shasum -a 256 -c basenine_linux_${GOARCH}.sha256
 RUN chmod +x ./basenine_linux_${GOARCH}
 RUN mv ./basenine_linux_${GOARCH} ./basenine

Makefile

@@ -31,9 +31,6 @@ cli: ## Build CLI.
 cli-debug: ## Build CLI.
 	@echo "building cli"; cd cli && $(MAKE) build-debug
 
-build-cli-ci: ## Build CLI for CI.
-	@echo "building cli for ci"; cd cli && $(MAKE) build GIT_BRANCH=ci SUFFIX=ci
-
 agent: ## Build agent.
 	@(echo "building mizu agent .." )
 	@(cd agent; go build -o build/mizuagent main.go)
@@ -57,10 +54,6 @@ push-docker: ## Build and publish agent docker image.
 	@echo "publishing Docker image .. "
 	devops/build-push-featurebranch.sh
 
-build-docker-ci: ## Build agent docker image for CI.
-	@echo "building docker image for ci"
-	devops/build-agent-ci.sh
-
 push-cli: ## Build and publish CLI.
 	@echo "publishing CLI .. "
 	@cd cli; $(MAKE) build-all

acceptanceTests/setup.sh

@@ -61,10 +61,10 @@ echo "Setting minikube docker env"
 eval $(minikube docker-env)
 
 echo "Build agent image"
-make build-docker-ci
+docker build -t mizu/ci:0.0 .
 
 echo "Build cli"
-make build-cli-ci
+cd cli && make build GIT_BRANCH=ci SUFFIX=ci
 
 echo "Starting tunnel"
 minikube tunnel &

acceptanceTests/testsUtils.go

@@ -214,7 +214,7 @@ func DeleteKubeFile(kubeContext string, namespace string, filename string) error
 func getDefaultCommandArgs() []string {
 	setFlag := "--set"
 	telemetry := "telemetry=false"
-	agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0"
+	agentImage := "agent-image=mizu/ci:0.0"
 	imagePullPolicy := "image-pull-policy=IfNotPresent"
 	headless := "headless=true"
 
@@ -265,11 +265,11 @@ func RunCypressTests(t *testing.T, cypressRunCmd string) {
 	t.Logf("%s", out)
 }
 
-func retriesExecute(retriesCount int, executeFunc func() error) error {
+func RetriesExecute(retriesCount int, executeFunc func() error) error {
 	var lastError interface{}
 
 	for i := 0; i < retriesCount; i++ {
-		if err := tryExecuteFunc(executeFunc); err != nil {
+		if err := TryExecuteFunc(executeFunc); err != nil {
 			lastError = err
 
 			time.Sleep(1 * time.Second)
@@ -282,7 +282,7 @@ func retriesExecute(retriesCount int, executeFunc func() error) error {
 	return fmt.Errorf("reached max retries count, retries count: %v, last err: %v", retriesCount, lastError)
 }
 
-func tryExecuteFunc(executeFunc func() error) (err interface{}) {
+func TryExecuteFunc(executeFunc func() error) (err interface{}) {
 	defer func() {
 		if panicErr := recover(); panicErr != nil {
 			err = panicErr
@@ -308,10 +308,10 @@ func WaitTapPodsReady(apiServerUrl string) error {
 		return nil
 	}
 
-	return retriesExecute(LongRetriesCount, tapPodsReadyFunc)
+	return RetriesExecute(LongRetriesCount, tapPodsReadyFunc)
 }
 
-func jsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
+func JsonBytesToInterface(jsonBytes []byte) (interface{}, error) {
 	var result interface{}
 	if parseErr := json.Unmarshal(jsonBytes, &result); parseErr != nil {
 		return nil, parseErr
@@ -334,7 +334,7 @@ func ExecuteHttpRequest(response *http.Response, requestErr error) (interface{},
 		return nil, readErr
 	}
 
-	return jsonBytesToInterface(data)
+	return JsonBytesToInterface(data)
 }
 
 func ExecuteHttpGetRequestWithHeaders(url string, headers map[string]string) (interface{}, error) {

agent/go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/ory/kratos-client-go v0.8.2-alpha.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/stretchr/testify v1.7.0
-	github.com/up9inc/basenine/client/go v0.0.0-20220220204122-0ef8cb24fab1
+	github.com/up9inc/basenine/client/go v0.0.0-20220302182733-74dc40dc2ef0
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0

agent/go.sum

@@ -855,6 +855,12 @@ github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ
 github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
 github.com/up9inc/basenine/client/go v0.0.0-20220220204122-0ef8cb24fab1 h1:0XN8s3HtwUBr9hbWRAFulFMsu1f2cabfJbwpz/sOoLA=
 github.com/up9inc/basenine/client/go v0.0.0-20220220204122-0ef8cb24fab1/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20220301135911-d2111357b14e h1:nv/A/AeF8PcU91aHAj6o2cU8fl/46v0ZLj7wgIKjv+o=
+github.com/up9inc/basenine/client/go v0.0.0-20220301135911-d2111357b14e/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20220302073458-c32e0adf1500 h1:T1QHxt65NMete/GobVSvcHnwZAQibvahhrMTCgtnSS4=
+github.com/up9inc/basenine/client/go v0.0.0-20220302073458-c32e0adf1500/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20220302182733-74dc40dc2ef0 h1:mSqZuJJV4UZyaAoC8x7/AO7DLidlXepFyU18Vm3rFiA=
+github.com/up9inc/basenine/client/go v0.0.0-20220302182733-74dc40dc2ef0/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=

agent/main.go

@@ -86,7 +86,7 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) *gin.Engin
 	app := gin.Default()
 
 	app.GET("/echo", func(c *gin.Context) {
-		c.String(http.StatusOK, "Here is Mizu agent")
+		c.JSON(http.StatusOK, "Here is Mizu agent")
 	})
 
 	eventHandlers := api.RoutesEventHandlers{
@@ -140,7 +140,7 @@ func runInApiServerMode(namespace string) *gin.Engine {
 	if err := config.LoadConfig(); err != nil {
 		logger.Log.Fatalf("Error loading config file %v", err)
 	}
-	app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort, config.Config.MaxDBSizeBytes, config.Config.LogLevel)
+	app.ConfigureBasenineServer(shared.BasenineHost, shared.BaseninePort, config.Config.MaxDBSizeBytes, config.Config.LogLevel, config.Config.InsertionFilter)
 	startTime = time.Now().UnixNano() / int64(time.Millisecond)
 	api.StartResolving(namespace)
 

agent/pkg/app/main.go

@@ -62,7 +62,7 @@ func LoadExtensions() {
 	controllers.InitExtensionsMap(ExtensionsMap)
 }
 
-func ConfigureBasenineServer(host string, port string, dbSize int64, logLevel logging.Level) {
+func ConfigureBasenineServer(host string, port string, dbSize int64, logLevel logging.Level, insertionFilter string) {
 	if !wait.New(
 		wait.WithProto("tcp"),
 		wait.WithWait(200*time.Millisecond),
@@ -86,6 +86,11 @@ func ConfigureBasenineServer(host string, port string, dbSize int64, logLevel lo
 			}
 		}
 	}
+
+	// Set the insertion filter that comes from the config
+	if err := basenine.InsertionFilter(host, port, insertionFilter); err != nil {
+		logger.Log.Errorf("Error while setting the insertion filter: %v", err)
+	}
 }
 
 func GetEntryInputChannel() chan *tapApi.OutputChannelItem {

cli/cmd/tap.go

@@ -3,9 +3,10 @@ package cmd
 import (
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/cli/up9"
 	"os"
 
+	"github.com/up9inc/mizu/cli/up9"
+
 	"github.com/creasty/defaults"
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/auth"
@@ -115,6 +116,7 @@ func init() {
 	tapCmd.Flags().StringSliceP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
 	tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values")
 	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
+	tapCmd.Flags().String(configStructs.InsertionFilterName, defaultTapConfig.InsertionFilter, "Set the insertion filter. Accepts string or a file path.")
 	tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them")
 	tapCmd.Flags().StringP(configStructs.WorkspaceTapName, "w", defaultTapConfig.Workspace, "Uploads traffic to your UP9 workspace for further analysis (requires auth)")
 	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file path with policy rules")

cli/cmd/tapRunner.go

@@ -154,6 +154,7 @@ func finishTapExecution(kubernetesProvider *kubernetes.Provider) {
 func getTapMizuAgentConfig() *shared.MizuAgentConfig {
 	mizuAgentConfig := shared.MizuAgentConfig{
 		MaxDBSizeBytes:         config.Config.Tap.MaxEntriesDBSizeBytes(),
+		InsertionFilter:        config.Config.Tap.GetInsertionFilter(),
 		AgentImage:             config.Config.AgentImage,
 		PullPolicy:             config.Config.ImagePullPolicyStr,
 		LogLevel:               config.Config.LogLevel(),

cli/config/configStructs/tapConfig.go

@@ -3,10 +3,16 @@ package configStructs
 import (
 	"errors"
 	"fmt"
+	"io/fs"
+	"io/ioutil"
+	"os"
 	"regexp"
 
+	"github.com/up9inc/mizu/cli/uiUtils"
 	"github.com/up9inc/mizu/shared"
 
+	basenine "github.com/up9inc/basenine/server/lib"
+	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/shared/units"
 )
 
@@ -18,6 +24,7 @@ const (
 	PlainTextFilterRegexesTapName = "regex-masking"
 	DisableRedactionTapName       = "no-redact"
 	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
+	InsertionFilterName           = "insertion-filter"
 	DryRunTapName                 = "dry-run"
 	WorkspaceTapName              = "workspace"
 	EnforcePolicyFile             = "traffic-validation-file"
@@ -27,26 +34,27 @@ const (
 )
 
 type TapConfig struct {
-	UploadIntervalSec       int              `yaml:"upload-interval" default:"10"`
-	PodRegexStr             string           `yaml:"regex" default:".*"`
-	GuiPort                 uint16           `yaml:"gui-port" default:"8899"`
-	ProxyHost               string           `yaml:"proxy-host" default:"127.0.0.1"`
-	Namespaces              []string         `yaml:"namespaces"`
-	Analysis                bool             `yaml:"analysis" default:"false"`
-	AllNamespaces           bool             `yaml:"all-namespaces" default:"false"`
-	PlainTextFilterRegexes  []string         `yaml:"regex-masking"`
-	IgnoredUserAgents       []string         `yaml:"ignored-user-agents"`
-	DisableRedaction        bool             `yaml:"no-redact" default:"false"`
-	HumanMaxEntriesDBSize   string           `yaml:"max-entries-db-size" default:"200MB"`
-	DryRun                  bool             `yaml:"dry-run" default:"false"`
-	Workspace               string           `yaml:"workspace"`
-	EnforcePolicyFile       string           `yaml:"traffic-validation-file"`
-	ContractFile            string           `yaml:"contract"`
-	AskUploadConfirmation   bool             `yaml:"ask-upload-confirmation" default:"true"`
-	ApiServerResources      shared.Resources `yaml:"api-server-resources"`
-	TapperResources         shared.Resources `yaml:"tapper-resources"`
-	ServiceMesh             bool             `yaml:"service-mesh" default:"false"`
-	Tls                     bool             `yaml:"tls" default:"false"`
+	UploadIntervalSec      int              `yaml:"upload-interval" default:"10"`
+	PodRegexStr            string           `yaml:"regex" default:".*"`
+	GuiPort                uint16           `yaml:"gui-port" default:"8899"`
+	ProxyHost              string           `yaml:"proxy-host" default:"127.0.0.1"`
+	Namespaces             []string         `yaml:"namespaces"`
+	Analysis               bool             `yaml:"analysis" default:"false"`
+	AllNamespaces          bool             `yaml:"all-namespaces" default:"false"`
+	PlainTextFilterRegexes []string         `yaml:"regex-masking"`
+	IgnoredUserAgents      []string         `yaml:"ignored-user-agents"`
+	DisableRedaction       bool             `yaml:"no-redact" default:"false"`
+	HumanMaxEntriesDBSize  string           `yaml:"max-entries-db-size" default:"200MB"`
+	InsertionFilter        string           `yaml:"insertion-filter" default:""`
+	DryRun                 bool             `yaml:"dry-run" default:"false"`
+	Workspace              string           `yaml:"workspace"`
+	EnforcePolicyFile      string           `yaml:"traffic-validation-file"`
+	ContractFile           string           `yaml:"contract"`
+	AskUploadConfirmation  bool             `yaml:"ask-upload-confirmation" default:"true"`
+	ApiServerResources     shared.Resources `yaml:"api-server-resources"`
+	TapperResources        shared.Resources `yaml:"tapper-resources"`
+	ServiceMesh            bool             `yaml:"service-mesh" default:"false"`
+	Tls                    bool             `yaml:"tls" default:"false"`
 }
 
 func (config *TapConfig) PodRegex() *regexp.Regexp {
@@ -59,6 +67,25 @@ func (config *TapConfig) MaxEntriesDBSizeBytes() int64 {
 	return maxEntriesDBSizeBytes
 }
 
+func (config *TapConfig) GetInsertionFilter() string {
+	insertionFilter := config.InsertionFilter
+	if fs.ValidPath(insertionFilter) {
+		if _, err := os.Stat(insertionFilter); err == nil {
+			b, err := ioutil.ReadFile(insertionFilter)
+			if err != nil {
+				logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Couldn't read the file on path: %s, err: %v", insertionFilter, err))
+			} else {
+				insertionFilter = string(b)
+			}
+		}
+	}
+	_, err := basenine.Parse(insertionFilter)
+	if err != nil {
+		logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Insertion filter syntax error: %v", err))
+	}
+	return insertionFilter
+}
+
 func (config *TapConfig) Validate() error {
 	_, compileErr := regexp.Compile(config.PodRegexStr)
 	if compileErr != nil {

cli/go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/spf13/cobra v1.3.0
 	github.com/spf13/pflag v1.0.5
+	github.com/up9inc/basenine/server/lib v0.0.0-20220302182733-74dc40dc2ef0
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
@@ -32,8 +33,10 @@ require (
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/alecthomas/participle/v2 v2.0.0-alpha7 // indirect
 	github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dlclark/regexp2 v1.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
@@ -68,6 +71,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/ohler55/ojg v1.12.13 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

cli/go.sum

@@ -83,6 +83,10 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/alecthomas/participle/v2 v2.0.0-alpha7 h1:cK4vjj0VSgb3lN1nuKA5F7dw+1s1pWBe5bx7nNCnN+c=
+github.com/alecthomas/participle/v2 v2.0.0-alpha7/go.mod h1:NumScqsC42o9x+dGj8/YqsIfhrIQjFEOFovxotbBirA=
+github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E=
+github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -149,6 +153,8 @@ github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMS
 github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
+github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
@@ -479,6 +485,8 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/ohler55/ojg v1.12.13 h1:FvfVpYzLgMraLcg3rrXiRXaihOP6fnzQNEU9YyZ/AmM=
+github.com/ohler55/ojg v1.12.13/go.mod h1:LBbIVRAgoFbYBXQhRhuEpaJIqq+goSO63/FQ+nyJU88=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -590,6 +598,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/up9inc/basenine/server/lib v0.0.0-20220302182733-74dc40dc2ef0 h1:9PQamOq285DyVsRlS4KB/x2+xkr5QlpiT9Y/BPutS4A=
+github.com/up9inc/basenine/server/lib v0.0.0-20220302182733-74dc40dc2ef0/go.mod h1:R9bG4y/iq89jNC0xZ25uKDqenyKFTR3X9acGDOkKWSE=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk=

deploy/kubernetes/helm-chart/.helmignore

@@ -1,23 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*.orig
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj
-.vscode/

deploy/kubernetes/helm-chart/Chart.yaml

@@ -1,7 +0,0 @@
-apiVersion: v2
-name: mizuhelm
-description: Mizu helm chart for Kubernetes
-type: application
-version: 0.1.1
-kubeVersion: ">= 1.16.0-0"
-appVersion: "0.21.29"

deploy/kubernetes/helm-chart/templates/PersistentVolumeClaim.yaml

@@ -1,13 +0,0 @@
-kind: PersistentVolumeClaim
-apiVersion: v1
-metadata:
-  name: {{ .Values.volumeClaim.name }}
-  namespace: {{ .Release.Namespace }}
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    limits:
-      storage: 700M
-    requests:
-      storage: 700M

deploy/kubernetes/helm-chart/templates/clusterRole.yaml

@@ -1,30 +0,0 @@
-{{- if .Values.rbac.create -}}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: {{ .Values.rbac.name }}
-  namespace: {{ .Release.Namespace }}
-  labels:
-    mizu-cli-version: {{ .Chart.AppVersion }}
-    heritage: {{ .Release.Service }}
-    release: {{ .Release.Name }}
-rules:
-  - apiGroups: [ "", "extensions", "apps" ]
-    resources: [ "endpoints", "pods", "services", "namespaces" ]
-    verbs: [ "get", "list", "watch" ]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: {{ .Values.rbac.roleBindingName }}
-  labels:
-    mizu-cli-version: {{ .Chart.AppVersion }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: {{ .Values.rbac.name }}
-subjects:
-  - kind: ServiceAccount
-    name: {{ .Values.serviceAccountName }}
-    namespace: {{ .Release.Namespace }}
-  {{- end -}}

deploy/kubernetes/helm-chart/templates/configmap.yaml

@@ -1,8 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ .Values.configMap.name }}
-  namespace: {{ .Release.Namespace }}
-data:
-  mizu-config.json: >-
-    {"maxDBSizeBytes":200000000,"agentImage":"{{ .Values.container.tapper.image.repository }}:{{ .Values.container.tapper.image.tag }}","pullPolicy":"Always","logLevel":4,"tapperResources":{"CpuLimit":"750m","MemoryLimit":"1Gi","CpuRequests":"50m","MemoryRequests":"50Mi"},"mizuResourceNamespace":"{{ .Release.Namespace }}","agentDatabasePath":"/app/data/","standaloneMode":true}

deploy/kubernetes/helm-chart/templates/deployment.yaml

@@ -1,128 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ .Values.pod.name }}
-  namespace: {{ .Release.Namespace }}
-  labels:
-    app: {{ .Values.pod.name }}
-spec:
-  replicas: {{ .Values.deployment.replicaCount }}
-  selector:
-    matchLabels:
-      app: {{ .Values.pod.name }}
-  template:
-    metadata:
-      name: {{ .Values.pod.name }}
-      creationTimestamp: null
-      labels:
-        app: {{ .Values.pod.name }}
-    spec:
-      volumes:
-        - name: {{ .Values.configMap.name }}
-          configMap:
-            name: {{ .Values.configMap.name }}
-            defaultMode: 420
-        - name: {{ .Values.volumeClaim.name }}
-          persistentVolumeClaim:
-            claimName: {{ .Values.volumeClaim.name }}
-      containers:
-        - name: {{ .Values.pod.name }}
-          image: "{{ .Values.container.mizuAgent.image.repository }}:{{ .Values.container.mizuAgent.image.tag | default .Chart.AppVersion }}"
-          command:
-            - ./mizuagent
-            - '--api-server'
-          env:
-            - name: SYNC_ENTRIES_CONFIG
-            - name: LOG_LEVEL
-              value: INFO
-          resources:
-            limits:
-              cpu: 750m
-              memory: 1Gi
-            requests:
-              cpu: 50m
-              memory: 50Mi
-          volumeMounts:
-            - name: {{ .Values.configMap.name }}
-              mountPath: /app/config/
-            - name: {{ .Values.volumeClaim.name }}
-              mountPath: /app/data/
-          livenessProbe:
-            httpGet:
-              path: /echo
-              port: {{ .Values.pod.port }}
-              scheme: HTTP
-            initialDelaySeconds: 1
-            timeoutSeconds: 1
-            periodSeconds: 10
-            successThreshold: 1
-            failureThreshold: 3
-          terminationMessagePath: /dev/termination-log
-          terminationMessagePolicy: File
-          imagePullPolicy: Always
-        - name: {{ .Values.container.basenine.name }}
-          image: "{{ .Values.container.basenine.image.repository }}:{{ .Values.container.basenine.image.tag | default .Chart.AppVersion }}"
-          command:
-            - /basenine
-          args:
-            - '-addr'
-            - 0.0.0.0
-            - '-port'
-            - '9099'
-            - '-persistent'
-          workingDir: /app/data/
-          resources:
-            limits:
-              cpu: 750m
-              memory: 1Gi
-            requests:
-              cpu: 50m
-              memory: 50Mi
-          volumeMounts:
-            - name: {{ .Values.configMap.name }}
-              mountPath: /app/config/
-            - name: {{ .Values.volumeClaim.name }}
-              mountPath: /app/data/
-          readinessProbe:
-            tcpSocket:
-              port: 9099
-            timeoutSeconds: 1
-            periodSeconds: 1
-            successThreshold: 1
-            failureThreshold: 3
-          terminationMessagePath: /dev/termination-log
-          terminationMessagePolicy: File
-          imagePullPolicy: Always
-        - name: kratos
-          image: "{{ .Values.container.kratos.image.repository }}:{{ .Values.container.kratos.image.tag | default .Chart.AppVersion }}"
-          resources:
-            limits:
-              cpu: 750m
-              memory: 1Gi
-            requests:
-              cpu: 50m
-              memory: 50Mi
-          volumeMounts:
-            - name: {{ .Values.configMap.name }}
-              mountPath: /app/config/
-            - name: {{ .Values.volumeClaim.name }}
-              mountPath: /app/data/
-          readinessProbe:
-            httpGet:
-              path: /health/ready
-              port: 4433
-              scheme: HTTP
-            timeoutSeconds: 1
-            periodSeconds: 1
-            successThreshold: 1
-            failureThreshold: 3
-          terminationMessagePath: /dev/termination-log
-          terminationMessagePolicy: File
-          imagePullPolicy: Always
-      restartPolicy: Always
-      terminationGracePeriodSeconds: 0
-      dnsPolicy: ClusterFirstWithHostNet
-      serviceAccountName: {{ .Values.serviceAccountName }}
-      serviceAccount: {{ .Values.serviceAccountName }}
-      securityContext: { }
-      schedulerName: default-scheduler

deploy/kubernetes/helm-chart/templates/role.yaml

@@ -1,29 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: {{ .Values.roleName }}
-  namespace: {{ .Release.Namespace }}
-  labels:
-    mizu-cli-version: {{ .Chart.AppVersion }}
-rules:
-  - apiGroups: [ "apps" ]
-    resources: [ "daemonsets" ]
-    verbs: [ "patch", "get", "list", "create", "delete" ]
-  - apiGroups: [ "events.k8s.i" ]
-    resources: [ "events" ]
-    verbs: [ "list", "watch" ]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: {{ .Values.roleBindingName }}
-  namespace: {{ .Release.Namespace }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: {{ .Values.roleName }}
-subjects:
-  - kind: ServiceAccount
-    name: {{ .Values.serviceAccountName }}
-    namespace: {{ .Release.Namespace }}
----

deploy/kubernetes/helm-chart/templates/service.yaml

@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ .Values.service.name }}
-  namespace: {{ .Release.Namespace }}
-spec:
-  type: {{ .Values.service.type }}
-  ports:
-    - name: api
-      port: {{ .Values.service.port }}
-      targetPort: {{ .Values.pod.port }}
-      protocol: TCP
-  selector:
-    app: {{ .Values.pod.name }}

deploy/kubernetes/helm-chart/templates/serviceAccount.yaml

@@ -1,7 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ .Values.serviceAccountName }}
-  namespace: {{ .Release.Namespace }}
-  labels:
-    mizu-cli-version: {{ .Chart.AppVersion }}

deploy/kubernetes/helm-chart/values.yaml

@@ -1,51 +0,0 @@
-# Default values for mizu.
-rbac:
-  create: true
-  name: "mizu-cluster-role"
-  roleBindingName: "mizu-role-binding"
-
-serviceAccountName: "mizu-service-account"
-
-roleName: "mizu-role-daemon"
-roleBindingName: "mizu-role-binding-daemon"
-
-service:
-  name: "mizu-api-server"
-  type: ClusterIP
-  port: 80
-
-pod:
-  name: "mizu-api-server"
-  port: 8899
-
-container:
-  mizuAgent:
-    image:
-      repository: "gcr.io/up9-docker-hub/mizu/main"
-      tag: "0.22.0"
-  tapper:
-    image:
-      repository: "gcr.io/up9-docker-hub/mizu/main"
-      tag: "0.22.0"
-  basenine:
-    name: "basenine"
-    port: 9099
-    image:
-      repository: "ghcr.io/up9inc/basenine"
-      tag: "v0.3.0"
-  kratos:
-    name: "kratos"
-    port: 4433
-    image:
-      repository: "gcr.io/up9-docker-hub/mizu-kratos/stable"
-      tag: "0.0.0"
-
-deployment:
-  replicaCount: 1
-
-configMap:
-  name: "mizu-config"
-
-volumeClaim:
-  create: true
-  name: "mizu-volume-claim"

devops/build-agent-ci.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-set -e
-
-GCP_PROJECT=up9-docker-hub
-REPOSITORY=gcr.io/$GCP_PROJECT
-SERVER_NAME=mizu
-GIT_BRANCH=ci
-
-DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH
-VER=${VER=0.0}
-
-DOCKER_TAGGED_BUILD="$DOCKER_REPO:$VER"
-
-echo "building $DOCKER_TAGGED_BUILD"
-docker build -t ${DOCKER_TAGGED_BUILD} --build-arg VER=${VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .

shared/models.go

@@ -35,6 +35,7 @@ type Resources struct {
 
 type MizuAgentConfig struct {
 	MaxDBSizeBytes         int64         `json:"maxDBSizeBytes"`
+	InsertionFilter        string        `json:"insertionFilter"`
 	AgentImage             string        `json:"agentImage"`
 	PullPolicy             string        `json:"pullPolicy"`
 	LogLevel               logging.Level `json:"logLevel"`