TRA-4263 extensibility refactor (#770)

* Refactor routers

* refactor extension loading

* refactor server creation code

* Update main.go

.github/workflows/acceptance_tests.yml

@@ -8,19 +8,15 @@ on:
     branches:
       - 'develop'
 
-concurrency:
-  group: mizu-acceptance-tests-${{ github.ref }}
-  cancel-in-progress: true
-
 jobs:
   run-acceptance-tests:
     name: Run acceptance tests
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.16
+      - name: Set up Go 1.17
         uses: actions/setup-go@v2
         with:
-          go-version: '^1.16'
+          go-version: '^1.17'
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2

.github/workflows/build.yml

@@ -15,10 +15,10 @@ jobs:
     name: CLI executable build
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.16
+      - name: Set up Go 1.17
         uses: actions/setup-go@v2
         with:
-          go-version: '1.16'
+          go-version: '1.17'
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2

.github/workflows/release.yml

@@ -229,10 +229,10 @@ jobs:
     runs-on: ubuntu-latest
     needs: [docker-manifest, gcp-registry]
     steps:
-      - name: Set up Go 1.16
+      - name: Set up Go 1.17
         uses: actions/setup-go@v2
         with:
-          go-version: '1.16'
+          go-version: '1.17'
 
       - name: Check out the repo
         uses: actions/checkout@v2

.github/workflows/static_code_analysis.yml

@@ -17,7 +17,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-            go-version: '^1.16'
+            go-version: '^1.17'
 
       - name: Install dependencies
         run: |

.github/workflows/test.yml

@@ -19,10 +19,10 @@ jobs:
     name: Unit Tests
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.16
+      - name: Set up Go 1.17
         uses: actions/setup-go@v2
         with:
-          go-version: '^1.16'
+          go-version: '^1.17'
 
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2

Dockerfile

@@ -14,7 +14,7 @@ RUN npm run build
 RUN npm run build-ent
 
 ### Base builder image for native builds architecture
-FROM golang:1.16-alpine AS builder-native-base
+FROM golang:1.17-alpine AS builder-native-base
 ENV CGO_ENABLED=1 GOOS=linux
 RUN apk add libpcap-dev g++ perl-utils
 

acceptanceTests/go.mod

@@ -1,6 +1,6 @@
 module github.com/up9inc/mizu/tests
 
-go 1.16
+go 1.17
 
 require (
 	github.com/go-redis/redis/v8 v8.11.4
@@ -10,6 +10,40 @@ require (
 	k8s.io/client-go v0.21.2
 )
 
+require (
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.1.0 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/googleapis/gnostic v0.4.1 // indirect
+	github.com/imdario/mergo v0.3.5 // indirect
+	github.com/json-iterator/go v1.1.10 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
+	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 // indirect
+	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
+	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/api v0.21.2 // indirect
+	k8s.io/klog/v2 v2.8.0 // indirect
+	k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.1.0 // indirect
+	sigs.k8s.io/yaml v1.2.0 // indirect
+)
+
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
 
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api

agent/go.mod

@@ -1,6 +1,6 @@
 module github.com/up9inc/mizu/agent
 
-go 1.16
+go 1.17
 
 require (
 	github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
@@ -37,6 +37,93 @@ require (
 	k8s.io/client-go v0.21.2
 )
 
+require (
+	cloud.google.com/go v0.65.0 // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.12 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.5 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/logger v0.2.0 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
+	github.com/beevik/etree v1.1.0 // indirect
+	github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 // indirect
+	github.com/chanced/dynamic v0.0.0-20210502140838-c010b5fc3e44 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fatih/camelcase v1.0.0 // indirect
+	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/go-openapi/analysis v0.20.0 // indirect
+	github.com/go-openapi/errors v0.20.1 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.5 // indirect
+	github.com/go-openapi/loads v0.20.2 // indirect
+	github.com/go-openapi/runtime v0.20.0 // indirect
+	github.com/go-openapi/spec v0.20.3 // indirect
+	github.com/go-openapi/strfmt v0.20.3 // indirect
+	github.com/go-openapi/swag v0.19.15 // indirect
+	github.com/go-openapi/validate v0.20.3 // indirect
+	github.com/go-stack/stack v1.8.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.1.0 // indirect
+	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/gopacket v1.1.19 // indirect
+	github.com/google/martian v2.1.0+incompatible // indirect
+	github.com/googleapis/gnostic v0.4.1 // indirect
+	github.com/hashicorp/golang-lru v0.5.1 // indirect
+	github.com/imdario/mergo v0.3.5 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.10 // indirect
+	github.com/klauspost/compress v1.14.1 // indirect
+	github.com/leodido/go-urn v1.2.0 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/ohler55/ojg v1.12.12 // indirect
+	github.com/oklog/ulid v1.3.1 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pierrec/lz4 v2.6.0+incompatible // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 // indirect
+	github.com/segmentio/kafka-go v0.4.27 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/tidwall/gjson v1.12.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tidwall/sjson v1.2.3 // indirect
+	github.com/ugorji/go/codec v1.1.7 // indirect
+	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
+	go.mongodb.org/mongo-driver v1.5.1 // indirect
+	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect
+	golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 // indirect
+	golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558 // indirect
+	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 // indirect
+	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
+	golang.org/x/text v0.3.5 // indirect
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
+	google.golang.org/appengine v1.6.6 // indirect
+	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	k8s.io/klog/v2 v2.8.0 // indirect
+	k8s.io/kubectl v0.21.2 // indirect
+	k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.1.0 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)
+
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
 
 replace github.com/up9inc/mizu/tap v0.0.0 => ../tap

agent/main.go

@@ -5,48 +5,27 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"os/signal"
-	"sort"
-	"strconv"
-	"strings"
 	"syscall"
 	"time"
 
-	"github.com/up9inc/mizu/agent/pkg/middlewares"
 	"github.com/up9inc/mizu/agent/pkg/models"
-	"github.com/up9inc/mizu/agent/pkg/oas"
-	"github.com/up9inc/mizu/agent/pkg/routes"
-	"github.com/up9inc/mizu/agent/pkg/servicemap"
-	"github.com/up9inc/mizu/agent/pkg/up9"
 	"github.com/up9inc/mizu/agent/pkg/utils"
 
-	"github.com/up9inc/mizu/agent/pkg/elastic"
-
-	"github.com/up9inc/mizu/agent/pkg/controllers"
-
 	"github.com/up9inc/mizu/agent/pkg/api"
+	"github.com/up9inc/mizu/agent/pkg/app"
 	"github.com/up9inc/mizu/agent/pkg/config"
 
 	v1 "k8s.io/api/core/v1"
 
-	"github.com/antelman107/net-wait-go/wait"
-	"github.com/gin-contrib/static"
-	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
 	"github.com/op/go-logging"
-	basenine "github.com/up9inc/basenine/client/go"
 	"github.com/up9inc/mizu/shared"
 	"github.com/up9inc/mizu/shared/logger"
 	"github.com/up9inc/mizu/tap"
 	tapApi "github.com/up9inc/mizu/tap/api"
-
-	amqpExt "github.com/up9inc/mizu/tap/extensions/amqp"
-	httpExt "github.com/up9inc/mizu/tap/extensions/http"
-	kafkaExt "github.com/up9inc/mizu/tap/extensions/kafka"
-	redisExt "github.com/up9inc/mizu/tap/extensions/redis"
 )
 
 var tapperMode = flag.Bool("tap", false, "Run in tapper mode without API")
@@ -57,11 +36,6 @@ var namespace = flag.String("namespace", "", "Resolve IPs if they belong to reso
 var harsReaderMode = flag.Bool("hars-read", false, "Run in hars-read mode")
 var harsDir = flag.String("hars-dir", "", "Directory to read hars from")
 
-var extensions []*tapApi.Extension             // global
-var extensionsMap map[string]*tapApi.Extension // global
-
-var startTime int64
-
 const (
 	socketConnectionRetries    = 30
 	socketConnectionRetryDelay = time.Second * 2
@@ -75,78 +49,20 @@ func main() {
 	if err := config.LoadConfig(); err != nil {
 		logger.Log.Fatalf("Error loading config file %v", err)
 	}
-	loadExtensions()
+	app.LoadExtensions()
 
 	if !*tapperMode && !*apiServerMode && !*standaloneMode && !*harsReaderMode {
 		panic("One of the flags --tap, --api or --standalone or --hars-read must be provided")
 	}
 
 	if *standaloneMode {
-		api.StartResolving(*namespace)
-
-		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
-		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-
-		filteringOptions := getTrafficFilteringOptions()
-		hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
-		tapOpts := &tap.TapOpts{HostMode: hostMode}
-		tap.StartPassiveTapper(tapOpts, outputItemsChannel, extensions, filteringOptions)
-
-		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
-		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
-
-		hostApi(nil)
+		runInStandaloneMode()
 	} else if *tapperMode {
-		logger.Log.Infof("Starting tapper, websocket address: %s", *apiServerAddress)
-		if *apiServerAddress == "" {
-			panic("API server address must be provided with --api-server-address when using --tap")
-		}
-
-		hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
-		tapOpts := &tap.TapOpts{HostMode: hostMode}
-		tapTargets := getTapTargets()
-		if tapTargets != nil {
-			tapOpts.FilterAuthorities = tapTargets
-			logger.Log.Infof("Filtering for the following authorities: %v", tapOpts.FilterAuthorities)
-		}
-
-		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-
-		filteringOptions := getTrafficFilteringOptions()
-		tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, extensions, filteringOptions)
-		socketConnection, err := dialSocketWithRetry(*apiServerAddress, socketConnectionRetries, socketConnectionRetryDelay)
-		if err != nil {
-			panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
-		}
-		logger.Log.Infof("Connected successfully to websocket %s", *apiServerAddress)
-
-		go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
+		runInTapperMode()
 	} else if *apiServerMode {
-		configureBasenineServer(shared.BasenineHost, shared.BaseninePort)
-		startTime = time.Now().UnixNano() / int64(time.Millisecond)
-		api.StartResolving(*namespace)
-
-		outputItemsChannel := make(chan *tapApi.OutputChannelItem)
-		filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
-		enableExpFeatureIfNeeded()
-		go filterItems(outputItemsChannel, filteredOutputItemsChannel)
-		go api.StartReadingEntries(filteredOutputItemsChannel, nil, extensionsMap)
-
-		syncEntriesConfig := getSyncEntriesConfig()
-		if syncEntriesConfig != nil {
-			if err := up9.SyncEntries(syncEntriesConfig); err != nil {
-				logger.Log.Error("Error syncing entries, err: %v", err)
-			}
-		}
-
-		hostApi(outputItemsChannel)
+		utils.StartServer(app.RunInApiServerMode(*namespace))
 	} else if *harsReaderMode {
-		outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000)
-		filteredHarChannel := make(chan *tapApi.OutputChannelItem)
-
-		go filterItems(outputItemsChannel, filteredHarChannel)
-		go api.StartReadingEntries(filteredHarChannel, harsDir, extensionsMap)
-		hostApi(nil)
+		runInHarReaderMode()
 	}
 
 	signalChan := make(chan os.Signal, 1)
@@ -156,167 +72,59 @@ func main() {
 	logger.Log.Info("Exiting")
 }
 
-func enableExpFeatureIfNeeded() {
-	if config.Config.OAS {
-		oas.GetOasGeneratorInstance().Start()
-	}
-	if config.Config.ServiceMap {
-		servicemap.GetInstance().SetConfig(config.Config)
-	}
-	elastic.GetInstance().Configure(config.Config.Elastic)
-}
-
-func configureBasenineServer(host string, port string) {
-	if !wait.New(
-		wait.WithProto("tcp"),
-		wait.WithWait(200*time.Millisecond),
-		wait.WithBreak(50*time.Millisecond),
-		wait.WithDeadline(5*time.Second),
-		wait.WithDebug(config.Config.LogLevel == logging.DEBUG),
-	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
-		logger.Log.Panicf("Basenine is not available!")
+func runInTapperMode() {
+	logger.Log.Infof("Starting tapper, websocket address: %s", *apiServerAddress)
+	if *apiServerAddress == "" {
+		panic("API server address must be provided with --api-server-address when using --tap")
 	}
 
-	// Limit the database size to default 200MB
-	err := basenine.Limit(host, port, config.Config.MaxDBSizeBytes)
+	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
+	tapOpts := &tap.TapOpts{HostMode: hostMode}
+	tapTargets := getTapTargets()
+	if tapTargets != nil {
+		tapOpts.FilterAuthorities = tapTargets
+		logger.Log.Infof("Filtering for the following authorities: %v", tapOpts.FilterAuthorities)
+	}
+
+	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+
+	filteringOptions := getTrafficFilteringOptions()
+	tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, app.Extensions, filteringOptions)
+	socketConnection, err := dialSocketWithRetry(*apiServerAddress, socketConnectionRetries, socketConnectionRetryDelay)
 	if err != nil {
-		logger.Log.Panicf("Error while limiting database size: %v", err)
+		panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
 	}
+	logger.Log.Infof("Connected successfully to websocket %s", *apiServerAddress)
 
-	// Define the macros
-	for _, extension := range extensions {
-		macros := extension.Dissector.Macros()
-		for macro, expanded := range macros {
-			err = basenine.Macro(host, port, macro, expanded)
-			if err != nil {
-				logger.Log.Panicf("Error while adding a macro: %v", err)
-			}
-		}
-	}
+	go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
 }
 
-func loadExtensions() {
-	extensions = make([]*tapApi.Extension, 4)
-	extensionsMap = make(map[string]*tapApi.Extension)
+func runInStandaloneMode() {
+	api.StartResolving(*namespace)
 
-	extensionAmqp := &tapApi.Extension{}
-	dissectorAmqp := amqpExt.NewDissector()
-	dissectorAmqp.Register(extensionAmqp)
-	extensionAmqp.Dissector = dissectorAmqp
-	extensions[0] = extensionAmqp
-	extensionsMap[extensionAmqp.Protocol.Name] = extensionAmqp
+	outputItemsChannel := make(chan *tapApi.OutputChannelItem)
+	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
 
-	extensionHttp := &tapApi.Extension{}
-	dissectorHttp := httpExt.NewDissector()
-	dissectorHttp.Register(extensionHttp)
-	extensionHttp.Dissector = dissectorHttp
-	extensions[1] = extensionHttp
-	extensionsMap[extensionHttp.Protocol.Name] = extensionHttp
+	filteringOptions := getTrafficFilteringOptions()
+	hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
+	tapOpts := &tap.TapOpts{HostMode: hostMode}
+	tap.StartPassiveTapper(tapOpts, outputItemsChannel, app.Extensions, filteringOptions)
 
-	extensionKafka := &tapApi.Extension{}
-	dissectorKafka := kafkaExt.NewDissector()
-	dissectorKafka.Register(extensionKafka)
-	extensionKafka.Dissector = dissectorKafka
-	extensions[2] = extensionKafka
-	extensionsMap[extensionKafka.Protocol.Name] = extensionKafka
+	go app.FilterItems(outputItemsChannel, filteredOutputItemsChannel)
+	go api.StartReadingEntries(filteredOutputItemsChannel, nil, app.ExtensionsMap)
 
-	extensionRedis := &tapApi.Extension{}
-	dissectorRedis := redisExt.NewDissector()
-	dissectorRedis.Register(extensionRedis)
-	extensionRedis.Dissector = dissectorRedis
-	extensions[3] = extensionRedis
-	extensionsMap[extensionRedis.Protocol.Name] = extensionRedis
-
-	sort.Slice(extensions, func(i, j int) bool {
-		return extensions[i].Protocol.Priority < extensions[j].Protocol.Priority
-	})
-
-	for _, extension := range extensions {
-		logger.Log.Infof("Extension Properties: %+v", extension)
-	}
-
-	controllers.InitExtensionsMap(extensionsMap)
+	ginApp := app.HostApi(nil)
+	utils.StartServer(ginApp)
 }
 
-func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
-	app := gin.Default()
+func runInHarReaderMode() {
+	outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000)
+	filteredHarChannel := make(chan *tapApi.OutputChannelItem)
 
-	app.GET("/echo", func(c *gin.Context) {
-		c.String(http.StatusOK, "Here is Mizu agent")
-	})
-
-	eventHandlers := api.RoutesEventHandlers{
-		SocketOutChannel: socketHarOutputChannel,
-	}
-
-	app.Use(DisableRootStaticCache())
-
-	var staticFolder string
-	if config.Config.StandaloneMode {
-		staticFolder = "./site-standalone"
-	} else {
-		staticFolder = "./site"
-	}
-
-	indexStaticFile := staticFolder + "/index.html"
-	if err := setUIFlags(indexStaticFile); err != nil {
-		logger.Log.Errorf("Error setting ui flags, err: %v", err)
-	}
-
-	app.Use(static.ServeRoot("/", staticFolder))
-	app.NoRoute(func(c *gin.Context) {
-		c.File(indexStaticFile)
-	})
-
-	app.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
-
-	api.WebSocketRoutes(app, &eventHandlers, startTime)
-
-	if config.Config.StandaloneMode {
-		routes.ConfigRoutes(app)
-		routes.UserRoutes(app)
-		routes.InstallRoutes(app)
-	}
-	if config.Config.OAS {
-		routes.OASRoutes(app)
-	}
-	if config.Config.ServiceMap {
-		routes.ServiceMapRoutes(app)
-	}
-
-	routes.QueryRoutes(app)
-	routes.EntriesRoutes(app)
-	routes.MetadataRoutes(app)
-	routes.StatusRoutes(app)
-	utils.StartServer(app)
-}
-
-func DisableRootStaticCache() gin.HandlerFunc {
-	return func(c *gin.Context) {
-		if c.Request.RequestURI == "/" {
-			// Disable cache only for the main static route
-			c.Writer.Header().Set("Cache-Control", "no-store")
-		}
-
-		c.Next()
-	}
-}
-
-func setUIFlags(uiIndexPath string) error {
-	read, err := ioutil.ReadFile(uiIndexPath)
-	if err != nil {
-		return err
-	}
-
-	replacedContent := strings.Replace(string(read), "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS), 1)
-	replacedContent = strings.Replace(replacedContent, "__IS_SERVICE_MAP_ENABLED__", strconv.FormatBool(config.Config.ServiceMap), 1)
-
-	err = ioutil.WriteFile(uiIndexPath, []byte(replacedContent), 0)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	go app.FilterItems(outputItemsChannel, filteredHarChannel)
+	go api.StartReadingEntries(filteredHarChannel, harsDir, app.ExtensionsMap)
+	ginApp := app.HostApi(nil)
+	utils.StartServer(ginApp)
 }
 
 func parseEnvVar(env string) map[string][]v1.Pod {
@@ -357,16 +165,6 @@ func getTrafficFilteringOptions() *tapApi.TrafficFilteringOptions {
 	return &filteringOptions
 }
 
-func filterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem) {
-	for message := range inChannel {
-		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
-			continue
-		}
-
-		outChannel <- message
-	}
-}
-
 func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tapApi.OutputChannelItem) {
 	if connection == nil {
 		panic("Websocket connection is nil")
@@ -402,21 +200,6 @@ func pipeTapChannelToSocket(connection *websocket.Conn, messageDataChannel <-cha
 	}
 }
 
-func getSyncEntriesConfig() *shared.SyncEntriesConfig {
-	syncEntriesConfigJson := os.Getenv(shared.SyncEntriesConfigEnvVar)
-	if syncEntriesConfigJson == "" {
-		return nil
-	}
-
-	var syncEntriesConfig = &shared.SyncEntriesConfig{}
-	err := json.Unmarshal([]byte(syncEntriesConfigJson), syncEntriesConfig)
-	if err != nil {
-		panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.SyncEntriesConfig struct, err: %v", shared.SyncEntriesConfigEnvVar, syncEntriesConfigJson, err))
-	}
-
-	return syncEntriesConfig
-}
-
 func determineLogLevel() (logLevel logging.Level) {
 	logLevel, err := logging.LogLevel(os.Getenv(shared.LogLevelEnvVar))
 	if err != nil {

agent/pkg/api/socket_routes.go

@@ -30,14 +30,18 @@ type SocketConnection struct {
 	isTapper      bool
 }
 
-var websocketUpgrader = websocket.Upgrader{
-	ReadBufferSize:  1024,
-	WriteBufferSize: 1024,
-}
+var (
+	websocketUpgrader = websocket.Upgrader{
+		ReadBufferSize:  1024,
+		WriteBufferSize: 1024,
+	}
 
-var websocketIdsLock = sync.Mutex{}
-var connectedWebsockets map[int]*SocketConnection
-var connectedWebsocketIdCounter = 0
+	websocketIdsLock            = sync.Mutex{}
+	connectedWebsockets         map[int]*SocketConnection
+	connectedWebsocketIdCounter = 0
+	SocketGetBrowserHandler     gin.HandlerFunc
+	SocketGetTapperHandler      gin.HandlerFunc
+)
 
 func init() {
 	websocketUpgrader.CheckOrigin = func(r *http.Request) bool { return true } // like cors for web socket
@@ -45,12 +49,20 @@ func init() {
 }
 
 func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) {
-	app.GET("/ws", func(c *gin.Context) {
+	SocketGetBrowserHandler = func(c *gin.Context) {
 		websocketHandler(c.Writer, c.Request, eventHandlers, false, startTime)
+	}
+
+	SocketGetTapperHandler = func(c *gin.Context) {
+		websocketHandler(c.Writer, c.Request, eventHandlers, true, startTime)
+	}
+
+	app.GET("/ws", func(c *gin.Context) {
+		SocketGetBrowserHandler(c)
 	})
 
 	app.GET("/wsTapper", func(c *gin.Context) { // TODO: add m2m authentication to this route
-		websocketHandler(c.Writer, c.Request, eventHandlers, true, startTime)
+		SocketGetTapperHandler(c)
 	})
 }
 

agent/pkg/app/extensions.go

@@ -0,0 +1,62 @@
+package app
+
+import (
+	"sort"
+
+	"github.com/up9inc/mizu/agent/pkg/controllers"
+	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
+
+	amqpExt "github.com/up9inc/mizu/tap/extensions/amqp"
+	httpExt "github.com/up9inc/mizu/tap/extensions/http"
+	kafkaExt "github.com/up9inc/mizu/tap/extensions/kafka"
+	redisExt "github.com/up9inc/mizu/tap/extensions/redis"
+)
+
+var (
+	Extensions    []*tapApi.Extension          // global
+	ExtensionsMap map[string]*tapApi.Extension // global
+)
+
+func LoadExtensions() {
+	Extensions = make([]*tapApi.Extension, 4)
+	ExtensionsMap = make(map[string]*tapApi.Extension)
+
+	extensionAmqp := &tapApi.Extension{}
+	dissectorAmqp := amqpExt.NewDissector()
+	dissectorAmqp.Register(extensionAmqp)
+	extensionAmqp.Dissector = dissectorAmqp
+	Extensions[0] = extensionAmqp
+	ExtensionsMap[extensionAmqp.Protocol.Name] = extensionAmqp
+
+	extensionHttp := &tapApi.Extension{}
+	dissectorHttp := httpExt.NewDissector()
+	dissectorHttp.Register(extensionHttp)
+	extensionHttp.Dissector = dissectorHttp
+	Extensions[1] = extensionHttp
+	ExtensionsMap[extensionHttp.Protocol.Name] = extensionHttp
+
+	extensionKafka := &tapApi.Extension{}
+	dissectorKafka := kafkaExt.NewDissector()
+	dissectorKafka.Register(extensionKafka)
+	extensionKafka.Dissector = dissectorKafka
+	Extensions[2] = extensionKafka
+	ExtensionsMap[extensionKafka.Protocol.Name] = extensionKafka
+
+	extensionRedis := &tapApi.Extension{}
+	dissectorRedis := redisExt.NewDissector()
+	dissectorRedis.Register(extensionRedis)
+	extensionRedis.Dissector = dissectorRedis
+	Extensions[3] = extensionRedis
+	ExtensionsMap[extensionRedis.Protocol.Name] = extensionRedis
+
+	sort.Slice(Extensions, func(i, j int) bool {
+		return Extensions[i].Protocol.Priority < Extensions[j].Protocol.Priority
+	})
+
+	for _, extension := range Extensions {
+		logger.Log.Infof("Extension Properties: %+v", extension)
+	}
+
+	controllers.InitExtensionsMap(ExtensionsMap)
+}

agent/pkg/app/server.go

@@ -0,0 +1,210 @@
+package app
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/antelman107/net-wait-go/wait"
+	"github.com/gin-contrib/static"
+	"github.com/gin-gonic/gin"
+	"github.com/op/go-logging"
+	basenine "github.com/up9inc/basenine/client/go"
+	"github.com/up9inc/mizu/agent/pkg/api"
+	"github.com/up9inc/mizu/agent/pkg/config"
+	"github.com/up9inc/mizu/agent/pkg/elastic"
+	"github.com/up9inc/mizu/agent/pkg/middlewares"
+	"github.com/up9inc/mizu/agent/pkg/oas"
+	"github.com/up9inc/mizu/agent/pkg/routes"
+	"github.com/up9inc/mizu/agent/pkg/servicemap"
+	"github.com/up9inc/mizu/agent/pkg/up9"
+	"github.com/up9inc/mizu/shared"
+	"github.com/up9inc/mizu/shared/logger"
+	tapApi "github.com/up9inc/mizu/tap/api"
+)
+
+var (
+	ConfigRoutes     *gin.RouterGroup
+	UserRoutes       *gin.RouterGroup
+	InstallRoutes    *gin.RouterGroup
+	OASRoutes        *gin.RouterGroup
+	ServiceMapRoutes *gin.RouterGroup
+	QueryRoutes      *gin.RouterGroup
+	EntriesRoutes    *gin.RouterGroup
+	MetadataRoutes   *gin.RouterGroup
+	StatusRoutes     *gin.RouterGroup
+
+	startTime int64
+)
+
+func HostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) *gin.Engine {
+	app := gin.Default()
+
+	app.GET("/echo", func(c *gin.Context) {
+		c.String(http.StatusOK, "Here is Mizu agent")
+	})
+
+	eventHandlers := api.RoutesEventHandlers{
+		SocketOutChannel: socketHarOutputChannel,
+	}
+
+	app.Use(disableRootStaticCache())
+
+	var staticFolder string
+	if config.Config.StandaloneMode {
+		staticFolder = "./site-standalone"
+	} else {
+		staticFolder = "./site"
+	}
+
+	indexStaticFile := staticFolder + "/index.html"
+	if err := setUIFlags(indexStaticFile); err != nil {
+		logger.Log.Errorf("Error setting ui flags, err: %v", err)
+	}
+
+	app.Use(static.ServeRoot("/", staticFolder))
+	app.NoRoute(func(c *gin.Context) {
+		c.File(indexStaticFile)
+	})
+
+	app.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
+
+	api.WebSocketRoutes(app, &eventHandlers, startTime)
+
+	if config.Config.StandaloneMode {
+		ConfigRoutes = routes.ConfigRoutes(app)
+		UserRoutes = routes.UserRoutes(app)
+		InstallRoutes = routes.InstallRoutes(app)
+	}
+	if config.Config.OAS {
+		OASRoutes = routes.OASRoutes(app)
+	}
+	if config.Config.ServiceMap {
+		ServiceMapRoutes = routes.ServiceMapRoutes(app)
+	}
+
+	QueryRoutes = routes.QueryRoutes(app)
+	EntriesRoutes = routes.EntriesRoutes(app)
+	MetadataRoutes = routes.MetadataRoutes(app)
+	StatusRoutes = routes.StatusRoutes(app)
+
+	return app
+}
+
+func RunInApiServerMode(namespace string) *gin.Engine {
+	configureBasenineServer(shared.BasenineHost, shared.BaseninePort)
+	startTime = time.Now().UnixNano() / int64(time.Millisecond)
+	api.StartResolving(namespace)
+
+	outputItemsChannel := make(chan *tapApi.OutputChannelItem)
+	filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
+	enableExpFeatureIfNeeded()
+	go FilterItems(outputItemsChannel, filteredOutputItemsChannel)
+	go api.StartReadingEntries(filteredOutputItemsChannel, nil, ExtensionsMap)
+
+	syncEntriesConfig := getSyncEntriesConfig()
+	if syncEntriesConfig != nil {
+		if err := up9.SyncEntries(syncEntriesConfig); err != nil {
+			logger.Log.Error("Error syncing entries, err: %v", err)
+		}
+	}
+
+	return HostApi(outputItemsChannel)
+}
+
+func configureBasenineServer(host string, port string) {
+	if !wait.New(
+		wait.WithProto("tcp"),
+		wait.WithWait(200*time.Millisecond),
+		wait.WithBreak(50*time.Millisecond),
+		wait.WithDeadline(5*time.Second),
+		wait.WithDebug(config.Config.LogLevel == logging.DEBUG),
+	).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
+		logger.Log.Panicf("Basenine is not available!")
+	}
+
+	// Limit the database size to default 200MB
+	err := basenine.Limit(host, port, config.Config.MaxDBSizeBytes)
+	if err != nil {
+		logger.Log.Panicf("Error while limiting database size: %v", err)
+	}
+
+	// Define the macros
+	for _, extension := range Extensions {
+		macros := extension.Dissector.Macros()
+		for macro, expanded := range macros {
+			err = basenine.Macro(host, port, macro, expanded)
+			if err != nil {
+				logger.Log.Panicf("Error while adding a macro: %v", err)
+			}
+		}
+	}
+}
+
+func getSyncEntriesConfig() *shared.SyncEntriesConfig {
+	syncEntriesConfigJson := os.Getenv(shared.SyncEntriesConfigEnvVar)
+	if syncEntriesConfigJson == "" {
+		return nil
+	}
+
+	var syncEntriesConfig = &shared.SyncEntriesConfig{}
+	err := json.Unmarshal([]byte(syncEntriesConfigJson), syncEntriesConfig)
+	if err != nil {
+		panic(fmt.Sprintf("env var %s's value of %s is invalid! json must match the shared.SyncEntriesConfig struct, err: %v", shared.SyncEntriesConfigEnvVar, syncEntriesConfigJson, err))
+	}
+
+	return syncEntriesConfig
+}
+
+func FilterItems(inChannel <-chan *tapApi.OutputChannelItem, outChannel chan *tapApi.OutputChannelItem) {
+	for message := range inChannel {
+		if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
+			continue
+		}
+
+		outChannel <- message
+	}
+}
+
+func enableExpFeatureIfNeeded() {
+	if config.Config.OAS {
+		oas.GetOasGeneratorInstance().Start()
+	}
+	if config.Config.ServiceMap {
+		servicemap.GetInstance().SetConfig(config.Config)
+	}
+	elastic.GetInstance().Configure(config.Config.Elastic)
+}
+
+func disableRootStaticCache() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		if c.Request.RequestURI == "/" {
+			// Disable cache only for the main static route
+			c.Writer.Header().Set("Cache-Control", "no-store")
+		}
+
+		c.Next()
+	}
+}
+
+func setUIFlags(uiIndexPath string) error {
+	read, err := ioutil.ReadFile(uiIndexPath)
+	if err != nil {
+		return err
+	}
+
+	replacedContent := strings.Replace(string(read), "__IS_OAS_ENABLED__", strconv.FormatBool(config.Config.OAS), 1)
+	replacedContent = strings.Replace(replacedContent, "__IS_SERVICE_MAP_ENABLED__", strconv.FormatBool(config.Config.ServiceMap), 1)
+
+	err = ioutil.WriteFile(uiIndexPath, []byte(replacedContent), 0)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

agent/pkg/routes/config_routes.go

@@ -7,10 +7,17 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-func ConfigRoutes(ginApp *gin.Engine) {
+var (
+	ConfigPostTapConfigHandler = controllers.PostTapConfig
+	ConfigGetTapConfigHandler  = controllers.GetTapConfig
+)
+
+func ConfigRoutes(ginApp *gin.Engine) *gin.RouterGroup {
 	routeGroup := ginApp.Group("/config")
 	routeGroup.Use(middlewares.RequiresAuth())
 
-	routeGroup.POST("/tap", middlewares.RequiresAdmin(), controllers.PostTapConfig)
-	routeGroup.GET("/tap", controllers.GetTapConfig)
+	routeGroup.POST("/tap", middlewares.RequiresAdmin(), func(c *gin.Context) { ConfigPostTapConfigHandler(c) })
+	routeGroup.GET("/tap", func(c *gin.Context) { ConfigGetTapConfigHandler(c) })
+
+	return routeGroup
 }

agent/pkg/routes/entries_routes.go

@@ -7,11 +7,18 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
+var (
+	EntriesGetHandler       = controllers.GetEntries
+	EntriesGetSingleHandler = controllers.GetEntry
+)
+
 // EntriesRoutes defines the group of har entries routes.
-func EntriesRoutes(ginApp *gin.Engine) {
+func EntriesRoutes(ginApp *gin.Engine) *gin.RouterGroup {
 	routeGroup := ginApp.Group("/entries")
 	routeGroup.Use(middlewares.RequiresAuth())
 
-	routeGroup.GET("/", controllers.GetEntries)  // get entries (base/thin entries) and metadata
-	routeGroup.GET("/:id", controllers.GetEntry) // get single (full) entry
+	routeGroup.GET("/", func(c *gin.Context) { EntriesGetHandler(c) })          // get entries (base/thin entries) and metadata
+	routeGroup.GET("/:id", func(c *gin.Context) { EntriesGetSingleHandler(c) }) // get single (full) entry
+
+	return routeGroup
 }

agent/pkg/routes/install_routes.go

@@ -6,9 +6,16 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-func InstallRoutes(ginApp *gin.Engine) {
+var (
+	InstallGetIsNeededHandler = controllers.IsSetupNecessary
+	InstallPostAdminHandler   = controllers.SetupAdminUser
+)
+
+func InstallRoutes(ginApp *gin.Engine) *gin.RouterGroup {
 	routeGroup := ginApp.Group("/install")
 
-	routeGroup.GET("/isNeeded", controllers.IsSetupNecessary)
-	routeGroup.POST("/admin", controllers.SetupAdminUser)
+	routeGroup.GET("/isNeeded", func(c *gin.Context) { InstallGetIsNeededHandler(c) })
+	routeGroup.POST("/admin", func(c *gin.Context) { InstallPostAdminHandler(c) })
+
+	return routeGroup
 }

agent/pkg/routes/metadata_routes.go

@@ -6,9 +6,15 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
+var (
+	MetadataGetVersionHandler = controllers.GetVersion
+)
+
 // MetadataRoutes defines the group of metadata routes.
-func MetadataRoutes(app *gin.Engine) {
+func MetadataRoutes(app *gin.Engine) *gin.RouterGroup {
 	routeGroup := app.Group("/metadata")
 
-	routeGroup.GET("/version", controllers.GetVersion)
+	routeGroup.GET("/version", func(c *gin.Context) { MetadataGetVersionHandler(c) })
+
+	return routeGroup
 }

agent/pkg/routes/oas_routes.go

@@ -7,12 +7,20 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
+var (
+	OASGetServersHandler    = controllers.GetOASServers
+	OASGetAllSpecsHandler   = controllers.GetOASAllSpecs
+	OASGetSingleSpecHandler = controllers.GetOASSpec
+)
+
 // OASRoutes methods to access OAS spec
-func OASRoutes(ginApp *gin.Engine) {
+func OASRoutes(ginApp *gin.Engine) *gin.RouterGroup {
 	routeGroup := ginApp.Group("/oas")
 	routeGroup.Use(middlewares.RequiresAuth())
 
-	routeGroup.GET("/", controllers.GetOASServers)     // list of servers in OAS map
-	routeGroup.GET("/all", controllers.GetOASAllSpecs) // list of servers in OAS map
-	routeGroup.GET("/:id", controllers.GetOASSpec)     // get OAS spec for given server
+	routeGroup.GET("/", func(c *gin.Context) { OASGetServersHandler(c) })       // list of servers in OAS map
+	routeGroup.GET("/all", func(c *gin.Context) { OASGetAllSpecsHandler(c) })   // list of servers in OAS map
+	routeGroup.GET("/:id", func(c *gin.Context) { OASGetSingleSpecHandler(c) }) // get OAS spec for given server
+
+	return routeGroup
 }

agent/pkg/routes/query_routes.go

@@ -7,9 +7,15 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-func QueryRoutes(ginApp *gin.Engine) {
+var (
+	QueryPostValidateHandler = controllers.PostValidate
+)
+
+func QueryRoutes(ginApp *gin.Engine) *gin.RouterGroup {
 	routeGroup := ginApp.Group("/query")
 	routeGroup.Use(middlewares.RequiresAuth())
 
-	routeGroup.POST("/validate", controllers.PostValidate)
+	routeGroup.POST("/validate", func(c *gin.Context) { QueryPostValidateHandler(c) })
+
+	return routeGroup
 }

agent/pkg/routes/service_map_routes.go

@@ -7,13 +7,25 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-func ServiceMapRoutes(ginApp *gin.Engine) {
+var (
+	ServiceMapGetStatus gin.HandlerFunc
+	ServiceMapGet       gin.HandlerFunc
+	ServiceMapReset     gin.HandlerFunc
+)
+
+func ServiceMapRoutes(ginApp *gin.Engine) *gin.RouterGroup {
 	routeGroup := ginApp.Group("/servicemap")
 	routeGroup.Use(middlewares.RequiresAuth())
 
 	controller := controllers.NewServiceMapController()
 
-	routeGroup.GET("/status", controller.Status)
-	routeGroup.GET("/get", controller.Get)
-	routeGroup.GET("/reset", controller.Reset)
+	ServiceMapGetStatus = controller.Status
+	ServiceMapGet = controller.Get
+	ServiceMapReset = controller.Reset
+
+	routeGroup.GET("/status", func(c *gin.Context) { ServiceMapGetStatus(c) })
+	routeGroup.GET("/get", func(c *gin.Context) { ServiceMapGet(c) })
+	routeGroup.GET("/reset", func(c *gin.Context) { ServiceMapReset(c) })
+
+	return routeGroup
 }

agent/pkg/routes/status_routes.go

@@ -7,24 +7,39 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-func StatusRoutes(ginApp *gin.Engine) {
+var (
+	StatusGetHealthCheck                 = controllers.HealthCheck
+	StatusPostTappedPods                 = controllers.PostTappedPods
+	StatusPostTapperStatus               = controllers.PostTapperStatus
+	StatusGetConnectedTappersCount       = controllers.GetConnectedTappersCount
+	StatusGetTappingStatus               = controllers.GetTappingStatus
+	StatusGetAuthStatus                  = controllers.GetAuthStatus
+	StatusGetAnalyzeInformation          = controllers.AnalyzeInformation
+	StatusGetGeneralStats                = controllers.GetGeneralStats
+	StatusGetRecentTLSLinks              = controllers.GetRecentTLSLinks
+	StatusGetCurrentResolvingInformation = controllers.GetCurrentResolvingInformation
+)
+
+func StatusRoutes(ginApp *gin.Engine) *gin.RouterGroup {
 	routeGroup := ginApp.Group("/status")
 	routeGroup.Use(middlewares.RequiresAuth())
 
-	routeGroup.GET("/health", controllers.HealthCheck)
+	routeGroup.GET("/health", func(c *gin.Context) { StatusGetHealthCheck(c) })
 
-	routeGroup.POST("/tappedPods", controllers.PostTappedPods)
-	routeGroup.POST("/tapperStatus", controllers.PostTapperStatus)
-	routeGroup.GET("/connectedTappersCount", controllers.GetConnectedTappersCount)
-	routeGroup.GET("/tap", controllers.GetTappingStatus)
+	routeGroup.POST("/tappedPods", func(c *gin.Context) { StatusPostTappedPods(c) })
+	routeGroup.POST("/tapperStatus", func(c *gin.Context) { StatusPostTapperStatus(c) })
+	routeGroup.GET("/connectedTappersCount", func(c *gin.Context) { StatusGetConnectedTappersCount(c) })
+	routeGroup.GET("/tap", func(c *gin.Context) { StatusGetTappingStatus(c) })
 
-	routeGroup.GET("/auth", controllers.GetAuthStatus)
+	routeGroup.GET("/auth", func(c *gin.Context) { StatusGetAuthStatus(c) })
 
-	routeGroup.GET("/analyze", controllers.AnalyzeInformation)
+	routeGroup.GET("/analyze", func(c *gin.Context) { StatusGetAnalyzeInformation(c) })
 
-	routeGroup.GET("/general", controllers.GetGeneralStats) // get general stats about entries in DB
+	routeGroup.GET("/general", func(c *gin.Context) { StatusGetGeneralStats(c) }) // get general stats about entries in DB
 
-	routeGroup.GET("/recentTLSLinks", controllers.GetRecentTLSLinks)
+	routeGroup.GET("/recentTLSLinks", func(c *gin.Context) { StatusGetRecentTLSLinks(c) })
 
-	routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation)
+	routeGroup.GET("/resolving", func(c *gin.Context) { StatusGetCurrentResolvingInformation(c) })
+
+	return routeGroup
 }

agent/pkg/routes/user_routes.go

@@ -6,10 +6,18 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-func UserRoutes(ginApp *gin.Engine) {
+var (
+	UserPostLogin    = controllers.Login
+	UserPostLogout   = controllers.Logout
+	UserPostRegister = controllers.Register
+)
+
+func UserRoutes(ginApp *gin.Engine) *gin.RouterGroup {
 	routeGroup := ginApp.Group("/user")
 
-	routeGroup.POST("/login", controllers.Login)
-	routeGroup.POST("/logout", controllers.Logout)
-	routeGroup.POST("/register", controllers.Register)
+	routeGroup.POST("/login", func(c *gin.Context) { UserPostLogin(c) })
+	routeGroup.POST("/logout", func(c *gin.Context) { UserPostLogout(c) })
+	routeGroup.POST("/register", func(c *gin.Context) { UserPostRegister(c) })
+
+	return routeGroup
 }

cli/cmd/tapRunner.go

@@ -339,7 +339,7 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
 					continue
 				}
 
-				logger.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase)
+				logger.Log.Debugf("Watching API Server pod loop, modified: %v, containers statuses: %v", modifiedPod.Status.Phase, modifiedPod.Status.ContainerStatuses)
 
 				if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
 					isPodReady = true
@@ -406,7 +406,7 @@ func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Pr
 			case "FailedScheduling", "Failed":
 				logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Mizu API Server status: %s - %s", event.Reason, event.Note))
 				cancel()
-				
+
 			}
 		case err, ok := <-errorChan:
 			if !ok {

cli/go.mod

@@ -1,6 +1,6 @@
 module github.com/up9inc/mizu/cli
 
-go 1.16
+go 1.17
 
 require (
 	github.com/creasty/defaults v1.5.1
@@ -20,6 +20,54 @@ require (
 	k8s.io/client-go v0.22.3
 )
 
+require (
+	cloud.google.com/go v0.54.0 // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.18 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/logger v0.2.1 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/swag v0.19.5 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt/v4 v4.1.0 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/go-querystring v1.0.0 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/martian v2.1.0+incompatible // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/imdario/mergo v0.3.5 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.11 // indirect
+	github.com/mailru/easyjson v0.7.0 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect
+	golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 // indirect
+	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect
+	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
+	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/klog/v2 v2.9.0 // indirect
+	k8s.io/kubectl v0.21.2 // indirect
+	k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
+	sigs.k8s.io/yaml v1.2.0 // indirect
+)
+
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
 
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api

devops/linux-arm64-musl-go-libpcap/Dockerfile

@@ -1,8 +1,8 @@
 FROM dockcross/linux-arm64-musl:latest AS builder-from-amd64-to-arm64v8
 
 # Install Go
-RUN curl https://go.dev/dl/go1.16.13.linux-amd64.tar.gz -Lo ./go.linux-amd64.tar.gz
-RUN curl https://go.dev/dl/go1.16.13.linux-amd64.tar.gz.asc -Lo ./go.linux-amd64.tar.gz.asc
+RUN curl https://go.dev/dl/go1.17.6.linux-amd64.tar.gz -Lo ./go.linux-amd64.tar.gz
+RUN curl https://go.dev/dl/go1.17.6.linux-amd64.tar.gz.asc -Lo ./go.linux-amd64.tar.gz.asc
 RUN curl https://dl.google.com/dl/linux/linux_signing_key.pub  -Lo linux_signing_key.pub
 RUN gpg --import linux_signing_key.pub && gpg --verify ./go.linux-amd64.tar.gz.asc ./go.linux-amd64.tar.gz
 RUN rm -rf /usr/local/go && tar -C /usr/local -xzf go.linux-amd64.tar.gz

shared/go.mod

@@ -1,6 +1,6 @@
 module github.com/up9inc/mizu/shared
 
-go 1.16
+go 1.17
 
 require (
 	github.com/docker/go-units v0.4.0
@@ -14,4 +14,46 @@ require (
 	k8s.io/kubectl v0.21.2
 )
 
+require (
+	cloud.google.com/go v0.54.0 // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.12 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.5 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/logger v0.2.0 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
+	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.4.3 // indirect
+	github.com/google/go-cmp v0.5.4 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/martian v2.1.0+incompatible // indirect
+	github.com/googleapis/gnostic v0.4.1 // indirect
+	github.com/hashicorp/golang-lru v0.5.1 // indirect
+	github.com/imdario/mergo v0.3.5 // indirect
+	github.com/json-iterator/go v1.1.10 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect
+	golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 // indirect
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
+	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 // indirect
+	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
+	golang.org/x/text v0.3.4 // indirect
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/protobuf v1.25.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/klog/v2 v2.8.0 // indirect
+	k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.1.0 // indirect
+	sigs.k8s.io/yaml v1.2.0 // indirect
+)
+
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../tap/api

shared/kubernetes/eventWatchHelper.go

@@ -29,12 +29,10 @@ func (wh *EventWatchHelper) Filter(wEvent *WatchEvent) (bool, error) {
 	if err != nil {
 		return false, nil
 	}
-
 	if !wh.NameRegexFilter.MatchString(event.Name) {
 		return false, nil
 	}
-
-	if strings.EqualFold(event.Regarding.Kind, wh.Kind) {
+	if !strings.EqualFold(event.Regarding.Kind, wh.Kind) {
 		return false, nil
 	}
 

tap/api/go.mod

@@ -1,5 +1,5 @@
 module github.com/up9inc/mizu/tap/api
 
-go 1.16
+go 1.17
 
 require github.com/google/martian v2.1.0+incompatible

tap/extensions/amqp/go.mod

@@ -1,7 +1,9 @@
 module github.com/up9inc/mizu/tap/extensions/amqp
 
-go 1.16
+go 1.17
 
 require github.com/up9inc/mizu/tap/api v0.0.0
 
+require github.com/google/martian v2.1.0+incompatible // indirect
+
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api

tap/extensions/http/go.mod

@@ -1,11 +1,15 @@
 module github.com/up9inc/mizu/tap/extensions/http
 
-go 1.16
+go 1.17
 
 require (
 	github.com/beevik/etree v1.1.0
 	github.com/up9inc/mizu/tap/api v0.0.0
 	golang.org/x/net v0.0.0-20210224082022-3d97a244fca7
+)
+
+require (
+	github.com/google/martian v2.1.0+incompatible // indirect
 	golang.org/x/text v0.3.5 // indirect
 )
 

tap/extensions/kafka/go.mod

@@ -1,13 +1,19 @@
 module github.com/up9inc/mizu/tap/extensions/kafka
 
-go 1.16
+go 1.17
 
 require (
 	github.com/fatih/camelcase v1.0.0
-	github.com/klauspost/compress v1.14.1 // indirect
 	github.com/ohler55/ojg v1.12.12
 	github.com/segmentio/kafka-go v0.4.27
 	github.com/up9inc/mizu/tap/api v0.0.0
 )
 
+require (
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/google/martian v2.1.0+incompatible // indirect
+	github.com/klauspost/compress v1.14.1 // indirect
+	github.com/pierrec/lz4 v2.6.0+incompatible // indirect
+)
+
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api

tap/extensions/redis/go.mod

@@ -1,7 +1,9 @@
 module github.com/up9inc/mizu/tap/extensions/redis
 
-go 1.16
+go 1.17
 
 require github.com/up9inc/mizu/tap/api v0.0.0
 
+require github.com/google/martian v2.1.0+incompatible // indirect
+
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api

tap/go.mod

@@ -1,6 +1,6 @@
 module github.com/up9inc/mizu/tap
 
-go 1.16
+go 1.17
 
 require (
 	github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4
@@ -11,6 +11,26 @@ require (
 	k8s.io/api v0.21.2
 )
 
+require (
+	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/go-cmp v0.5.4 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/martian v2.1.0+incompatible // indirect
+	github.com/json-iterator/go v1.1.10 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 // indirect
+	golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 // indirect
+	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 // indirect
+	golang.org/x/text v0.3.4 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/apimachinery v0.21.2 // indirect
+	k8s.io/klog/v2 v2.8.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.1.0 // indirect
+)
+
 replace github.com/up9inc/mizu/tap/api v0.0.0 => ./api
 
 replace github.com/up9inc/mizu/shared v0.0.0 => ../shared