From b01fe9c9182b0c4a169739880c94275c9276af34 Mon Sep 17 00:00:00 2001 From: Roee Gadot Date: Wed, 28 Apr 2021 17:54:32 +0300 Subject: [PATCH 01/10] no message --- api/.dockerignore => .dockerignore | 1 + Dockerfile | 44 ++++++++++++++++++++++++++++++ api/Dockerfile | 29 -------------------- api/pkg/inserter/main.go | 1 + tap/src/tcp_stream_factory.go | 2 ++ 5 files changed, 48 insertions(+), 29 deletions(-) rename api/.dockerignore => .dockerignore (89%) create mode 100644 Dockerfile delete mode 100644 api/Dockerfile diff --git a/api/.dockerignore b/.dockerignore similarity index 89% rename from api/.dockerignore rename to .dockerignore index 5f5b607e8..464de808c 100644 --- a/api/.dockerignore +++ b/.dockerignore @@ -14,3 +14,4 @@ LICENSE .git/ .github/ build/ +**/node_modules/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..19fdf3004 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,44 @@ +FROM node:14-slim AS site-build + +WORKDIR /ui-build + +COPY ui ./ +RUN npm i +RUN npm run build + + +FROM golang:1.16-alpine AS builder +# Set necessary environment variables needed for our image. +ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64 + +RUN apk add libpcap-dev gcc g++ make + +# Move to tapper working directory (/tap-build). +WORKDIR /tap-build +COPY tap/go.mod tap/go.sum ./ +RUN go mod download +# Copy and build tapper code +COPY tap/src ./ +RUN go build -ldflags="-s -w" -o passivetapper . + +# Move to api working directory (/api-build). +WORKDIR /api-build +COPY api/go.mod api/go.sum ./ +RUN go mod download +# Copy and build api code +COPY api . +RUN go build -ldflags="-s -w" -o apiserver . + + +FROM alpine:3.13.5 +RUN apk add parallel libpcap-dev +RUN apk add tcpdump +# Copy binary and config files from /build to root folder of scratch container. +COPY --from=builder ["/api-build/apiserver", "/"] +COPY --from=builder ["/tap-build/passivetapper", "/"] +COPY --from=site-build ["/ui-build/build", "/site"] + +# parallel will exit if one of the executables exits, ensuring this container does not run without one of the processes + +ENV HOST_MODE="1" +CMD parallel --halt now,done=1 ::: './apiserver' './passivetapper -i any -hardump -hardir /tmp/up9hars -harentriesperfile 10' \ No newline at end of file diff --git a/api/Dockerfile b/api/Dockerfile deleted file mode 100644 index 5fac361f5..000000000 --- a/api/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -FROM golang:1.16 AS builder - -# Move to working directory (/site). -WORKDIR /build - -# Copy and download dependency using go mod. -COPY go.mod go.sum ./ -RUN go mod download - -# Copy the code into the container. -COPY . . - -# Set necessary environmet variables needed for our image and site the API server. -ENV GOOS=linux GOARCH=amd64 -RUN go build -ldflags="-s -w" -o apiserver . - -FROM golang:1.16 - -# Copy binary and config files from /site to root folder of scratch container. -COPY --from=builder ["/build/apiserver", "/"] - -# Export necessary port. -EXPOSE 8899 - -COPY site /go/site -COPY entries.db /go/entries.db - -# Command to run when starting the container. -ENTRYPOINT ["/apiserver"] diff --git a/api/pkg/inserter/main.go b/api/pkg/inserter/main.go index 16969afbc..b9d61e1dd 100644 --- a/api/pkg/inserter/main.go +++ b/api/pkg/inserter/main.go @@ -41,6 +41,7 @@ func StartReadingFiles(workingDir string) { utils.CheckErr(decErr) for _, entry := range inputHar.Log.Entries { + time.Sleep(time.Millisecond * 250) SaveHarToDb(*entry, "") } rmErr := os.Remove(inputFilePath) diff --git a/tap/src/tcp_stream_factory.go b/tap/src/tcp_stream_factory.go index bf69f2622..ae37896ef 100644 --- a/tap/src/tcp_stream_factory.go +++ b/tap/src/tcp_stream_factory.go @@ -84,6 +84,8 @@ func (factory *tcpStreamFactory) WaitGoRoutines() { } func (factory *tcpStreamFactory) shouldTap(dstIP string, dstPort int) bool { + return true + if hostMode { return inArrayString(hostAppAddresses, fmt.Sprintf("%s:%d", dstIP, dstPort)) } else { From 8078eb3728f88369a80b4d8dfd29b59904c0ab1a Mon Sep 17 00:00:00 2001 From: Roee Gadot Date: Thu, 29 Apr 2021 08:13:10 +0300 Subject: [PATCH 02/10] start the pod with host network and privileged --- cli/kubernetes/provider.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index 7fa0bcf68..c08a15599 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -9,10 +9,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" _ "k8s.io/client-go/plugin/pkg/client/auth/azure" - _ "k8s.io/client-go/plugin/pkg/client/auth/openstack" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" + _ "k8s.io/client-go/plugin/pkg/client/auth/openstack" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" _ "k8s.io/client-go/tools/portforward" @@ -71,17 +71,22 @@ func (provider *Provider) GetPods(ctx context.Context) { } func (provider *Provider) CreatePod(ctx context.Context, podName string, podImage string) (*core.Pod, error) { + privileged := true pod := &core.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, Namespace: provider.Namespace, }, Spec: core.PodSpec{ + HostNetwork: true, // very important to make passive tapper see traffic Containers: []core.Container{ { Name: podName, Image: podImage, ImagePullPolicy: core.PullAlways, + SecurityContext: &core.SecurityContext{ + Privileged: &privileged, // must be privileged to get node level traffic + }, }, }, TerminationGracePeriodSeconds: new(int64), From 81c6b66999e83491c73896bdb4dca779acf0b2bd Mon Sep 17 00:00:00 2001 From: Roee Gadot Date: Thu, 29 Apr 2021 08:21:30 +0300 Subject: [PATCH 03/10] fix multi runner passive tapper command --- api/main.go | 6 ++++-- api/scripts/multi-runner.sh | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/api/main.go b/api/main.go index e1890d668..d81effeb2 100644 --- a/api/main.go +++ b/api/main.go @@ -1,6 +1,7 @@ package main import ( + "flag" "github.com/gofiber/fiber/v2" "mizuserver/pkg/inserter" "mizuserver/pkg/middleware" @@ -14,10 +15,11 @@ func main() { app := fiber.New() - go inserter.StartReadingFiles("/tmp/up9hars") // process to read files and insert to DB + var harDir = flag.String("hardir", "/tmp/mizuhars", "Directory in which to store output har files") + go inserter.StartReadingFiles(*harDir) // process to read files and insert to DB + middleware.FiberMiddleware(app) // Register Fiber's middleware for app. - app.Static("/", "./site") //Simple route to know server is running diff --git a/api/scripts/multi-runner.sh b/api/scripts/multi-runner.sh index fc2523514..b33967ee9 100755 --- a/api/scripts/multi-runner.sh +++ b/api/scripts/multi-runner.sh @@ -2,6 +2,6 @@ # this script runs both executables and exits everything if one fails ./apiserver & -./passivetapper -i eth0 & +./passivetapper -i any -hardump -hardir /tmp/mizuhars -harentriesperfile 50 & wait -n pkill -P $$ From 1dd36dc2081f4434617d52f1f781b047d25c263d Mon Sep 17 00:00:00 2001 From: Roee Gadot Date: Thu, 29 Apr 2021 08:22:42 +0300 Subject: [PATCH 04/10] add HOST_MODE env var --- cli/kubernetes/provider.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cli/kubernetes/provider.go b/cli/kubernetes/provider.go index c08a15599..15a5ed728 100644 --- a/cli/kubernetes/provider.go +++ b/cli/kubernetes/provider.go @@ -87,6 +87,12 @@ func (provider *Provider) CreatePod(ctx context.Context, podName string, podImag SecurityContext: &core.SecurityContext{ Privileged: &privileged, // must be privileged to get node level traffic }, + Env: []core.EnvVar{ + { + Name: "HOST_MODE", + Value: "1", + }, + }, }, }, TerminationGracePeriodSeconds: new(int64), From 0cfc32cb35b2d43feadd56f2c0556800841f9545 Mon Sep 17 00:00:00 2001 From: Roee Gadot Date: Thu, 29 Apr 2021 08:23:20 +0300 Subject: [PATCH 05/10] do not return true in the should tap function --- tap/src/tcp_stream_factory.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/tap/src/tcp_stream_factory.go b/tap/src/tcp_stream_factory.go index ae37896ef..bf69f2622 100644 --- a/tap/src/tcp_stream_factory.go +++ b/tap/src/tcp_stream_factory.go @@ -84,8 +84,6 @@ func (factory *tcpStreamFactory) WaitGoRoutines() { } func (factory *tcpStreamFactory) shouldTap(dstIP string, dstPort int) bool { - return true - if hostMode { return inArrayString(hostAppAddresses, fmt.Sprintf("%s:%d", dstIP, dstPort)) } else { From 494b924dcbecd9c5d0d90a478e45cee00b514920 Mon Sep 17 00:00:00 2001 From: Roee Gadot Date: Thu, 29 Apr 2021 08:24:07 +0300 Subject: [PATCH 06/10] remove line in the end --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4bb6a4572..84d902b73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,5 +56,4 @@ COPY --from=site-build ["/ui-build/build", "site"] COPY api/scripts/multi-runner.sh ./ # this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process -CMD "./multi-runner.sh" - +CMD "./multi-runner.sh" \ No newline at end of file From 930f5178a7faa7b63f07cfca62130adf03ea9579 Mon Sep 17 00:00:00 2001 From: Roee Gadot Date: Thu, 29 Apr 2021 08:27:05 +0300 Subject: [PATCH 07/10] default value in api is input fix description and pass the parameter in the multi runner script --- api/main.go | 2 +- api/scripts/multi-runner.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/main.go b/api/main.go index d81effeb2..fc7357b25 100644 --- a/api/main.go +++ b/api/main.go @@ -15,7 +15,7 @@ func main() { app := fiber.New() - var harDir = flag.String("hardir", "/tmp/mizuhars", "Directory in which to store output har files") + var harDir = flag.String("hardir", "input", "Directory in which we read har files from") go inserter.StartReadingFiles(*harDir) // process to read files and insert to DB diff --git a/api/scripts/multi-runner.sh b/api/scripts/multi-runner.sh index b33967ee9..4e53c475f 100755 --- a/api/scripts/multi-runner.sh +++ b/api/scripts/multi-runner.sh @@ -1,7 +1,7 @@ #!/bin/bash # this script runs both executables and exits everything if one fails -./apiserver & +./apiserver -hardir /tmp/mizuhars & ./passivetapper -i any -hardump -hardir /tmp/mizuhars -harentriesperfile 50 & wait -n pkill -P $$ From 43d82cdf7be0ca371a48e4957cfa3724c35525f3 Mon Sep 17 00:00:00 2001 From: Roee Gadot Date: Thu, 29 Apr 2021 08:44:24 +0300 Subject: [PATCH 08/10] missing flag.parse --- api/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/main.go b/api/main.go index fc7357b25..9342d14d5 100644 --- a/api/main.go +++ b/api/main.go @@ -16,6 +16,8 @@ func main() { app := fiber.New() var harDir = flag.String("hardir", "input", "Directory in which we read har files from") + flag.Parse() + go inserter.StartReadingFiles(*harDir) // process to read files and insert to DB From 318247ea73bc7189034bf266ff166cd516b0bbb8 Mon Sep 17 00:00:00 2001 From: Roee Gadot Date: Thu, 29 Apr 2021 10:16:57 +0300 Subject: [PATCH 09/10] no message --- cli/README.md | 2 +- cli/cmd/root.go | 2 +- tap/src/tcp_stream_factory.go | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cli/README.md b/cli/README.md index 01dd24ba7..f88b6f0ab 100644 --- a/cli/README.md +++ b/cli/README.md @@ -7,7 +7,7 @@ | flag | default | purpose | |----------------------|------------------|--------------------------------------------------------------------------------------------------------------| | `--no-dashboard` | `false` | Don't host the dashboard (not applicable at the moment) | -| `--dashboard-port` | `3000` | local port that dashboard will be forwarded to | +| `--dashboard-port` | `8899` | local port that dashboard will be forwarded to | | `--namespace` | | use namespace different than the one found in kubeconfig | | `--kubeconfig` | | Path to custom kubeconfig file | diff --git a/cli/cmd/root.go b/cli/cmd/root.go index ae192e4d6..fdcd29b44 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -32,7 +32,7 @@ func init() { rootCmd.Flags().BoolVarP(&config.Configuration.DisplayVersion, "version", "v", false, "Print the version and exit") rootCmd.Flags().BoolVarP(&config.Configuration.Quiet, "quiet", "q", false, "No stdout output") rootCmd.Flags().BoolVarP(&config.Configuration.NoDashboard, "no-dashboard", "", false, "Dont host a dashboard") - rootCmd.Flags().Uint16VarP(&config.Configuration.DashboardPort, "dashboard-port", "p", 3000, "Provide a custom port for the dashboard webserver") + rootCmd.Flags().Uint16VarP(&config.Configuration.DashboardPort, "dashboard-port", "p", 8899, "Provide a custom port for the dashboard webserver") rootCmd.Flags().StringVarP(&config.Configuration.Namespace, "namespace", "n", "", "Namespace selector") rootCmd.Flags().BoolVarP(&config.Configuration.AllNamespaces, "all-namespaces", "A", false, "Select all namespaces") rootCmd.Flags().StringVarP(&config.Configuration.KubeConfigPath, "kubeconfig", "k", "", "Path to kubeconfig file") diff --git a/tap/src/tcp_stream_factory.go b/tap/src/tcp_stream_factory.go index bf69f2622..fc510a8cc 100644 --- a/tap/src/tcp_stream_factory.go +++ b/tap/src/tcp_stream_factory.go @@ -84,6 +84,8 @@ func (factory *tcpStreamFactory) WaitGoRoutines() { } func (factory *tcpStreamFactory) shouldTap(dstIP string, dstPort int) bool { + return true // TODO: this is only for checking it now + if hostMode { return inArrayString(hostAppAddresses, fmt.Sprintf("%s:%d", dstIP, dstPort)) } else { From 8791f9991f585eb90c21151752d2a861586a2475 Mon Sep 17 00:00:00 2001 From: Roee Gadot Date: Thu, 29 Apr 2021 10:20:49 +0300 Subject: [PATCH 10/10] fix image --- cli/cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/cmd/root.go b/cli/cmd/root.go index fdcd29b44..cb99a9c01 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -36,7 +36,7 @@ func init() { rootCmd.Flags().StringVarP(&config.Configuration.Namespace, "namespace", "n", "", "Namespace selector") rootCmd.Flags().BoolVarP(&config.Configuration.AllNamespaces, "all-namespaces", "A", false, "Select all namespaces") rootCmd.Flags().StringVarP(&config.Configuration.KubeConfigPath, "kubeconfig", "k", "", "Path to kubeconfig file") - rootCmd.Flags().StringVarP(&config.Configuration.MizuImage, "mizu-image", "", "gcr.io/up9-docker-hub/mizu/develop/v1", "Custom image for mizu collector") + rootCmd.Flags().StringVarP(&config.Configuration.MizuImage, "mizu-image", "", "gcr.io/up9-docker-hub/mizu/develop/v6", "Custom image for mizu collector") rootCmd.Flags().Uint16VarP(&config.Configuration.MizuPodPort, "mizu-port", "", 8899, "Port which mizu cli will attempt to forward from the mizu collector pod") }