Compare commits

..

34 Commits

Author SHA1 Message Date
Alex Haiut
fd5b72a66e #minor 2021-06-20 14:49:54 +03:00
Alex Haiut
826f2cfb04 fixed release readme 2021-06-20 14:33:37 +03:00
Alex Haiut
e6f140adb2 use separate checksum files 2021-06-20 14:24:03 +03:00
Alex Haiut
1af2cd728d using markdown for release text 2021-06-20 14:08:03 +03:00
Alex Haiut
d5f6093084 fixed build error - created bin directory upfront 2021-06-20 13:56:07 +03:00
Alex Haiut
32ad2f17c3 added checksum calc to CLI makefile 2021-06-20 13:52:10 +03:00
Roee Gadot
553433622e no message 2021-06-20 10:42:27 +03:00
Roee Gadot
b6917ed0b7 no message 2021-06-20 08:53:12 +03:00
Roee Gadot
d22e5a9620 #minor 2021-06-20 08:24:04 +03:00
Roee Gadot
e880a9224f no message 2021-06-20 08:17:48 +03:00
Roee Gadot
67c3e5aab5 no message 2021-06-19 10:06:20 +03:00
Roee Gadot
ed791c988e no message 2021-06-19 10:00:22 +03:00
Roee Gadot
3567f5c00b no message 2021-06-19 09:40:19 +03:00
Roee Gadot
605c67c22f no message 2021-06-17 16:57:30 +03:00
Roee Gadot
9b44838fed no message 2021-06-17 15:59:43 +03:00
Roee Gadot
9f6abf7a32 no message 2021-06-17 11:23:05 +03:00
gadotroee
609f85e6ae Update tag-temp.yaml 2021-06-15 09:33:47 +03:00
gadotroee
d2188bfc22 Update tag-temp.yaml 2021-06-15 09:12:58 +03:00
gadotroee
d5942c3c49 Create tag-temp.yaml 2021-06-15 08:59:28 +03:00
Roee Gadot
f03c2adce4 remove main.yml and fix branches 2021-06-13 08:38:20 +03:00
Roee Gadot
93ebec09b9 no message 2021-06-10 21:41:20 +03:00
Roee Gadot
06c7b9c748 no message 2021-06-10 21:36:44 +03:00
Roee Gadot
db66119182 missing ) 2021-06-10 21:33:22 +03:00
Roee Gadot
2ae6153631 no message 2021-06-10 21:32:17 +03:00
Roee Gadot
2dfb0b2549 no message 2021-06-10 21:23:40 +03:00
Roee Gadot
a2b774ae1f no message 2021-06-10 21:21:11 +03:00
Roee Gadot
862f4252c3 yaml error 2021-06-10 21:17:07 +03:00
Roee Gadot
d5f057156e no message 2021-06-10 21:16:06 +03:00
Roee Gadot
96ce694945 trying new approach 2021-06-10 21:13:15 +03:00
gadotroee
2d7ff95426 Update main.yml 2021-06-08 15:17:15 +03:00
gadotroee
af9daf91cf Update main.yml 2021-06-08 15:08:55 +03:00
gadotroee
ea82d1779b Update main.yml 2021-06-08 15:05:55 +03:00
gadotroee
51ddbf3815 Update main.yml 2021-06-08 15:01:23 +03:00
gadotroee
8bbdcb6877 Create main.yml 2021-06-08 14:54:13 +03:00
89 changed files with 1133 additions and 2554 deletions

View File

@@ -1,9 +1,10 @@
name: publish
name: public-cli
on:
push:
branches:
- 'develop'
- 'main'
- develop
- main
- my-temp-release-check
jobs:
docker:
runs-on: ubuntu-latest
@@ -27,43 +28,12 @@ jobs:
with:
releaseType: ${{ steps.condval.outputs.value }}
github_token: ${{ secrets.GITHUB_TOKEN }}
- name: Get version parameters
- name: Get base image name
shell: bash
run: |
echo "##[set-output name=build_timestamp;]$(echo $(date +%s))"
echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
id: version_parameters
- name: Get base image name
shell: bash
run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
id: base_image_step
- name: Docker meta
id: meta
uses: crazy-max/ghaction-docker-meta@v2
with:
images: ${{ steps.base_image_step.outputs.image }}
tags: |
type=sha
type=raw,${{ github.sha }}
type=raw,${{ steps.versioning.outputs.version }}
- name: Login to DockerHub
uses: docker/login-action@v1
with:
registry: gcr.io
username: _json_key
password: ${{ secrets.GCR_JSON_KEY }}
- name: Build and push
uses: docker/build-push-action@v2
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
SEM_VER=${{ steps.versioning.outputs.version }}
BUILD_TIMESTAMP=${{ steps.version_parameters.outputs.build_timestamp }}
GIT_BRANCH=${{ steps.version_parameters.outputs.branch }}
COMMIT_HASH=${{ github.sha }}
- name: Build and Push CLI
run: make push-cli SEM_VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
- name: publish

39
.github/workflows/publish-docker.yml vendored Normal file
View File

@@ -0,0 +1,39 @@
name: publish-docker
on:
push:
branches:
- 'develop'
- 'main'
jobs:
docker:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Get base image name
shell: bash
run: echo "##[set-output name=image;]$(echo gcr.io/up9-docker-hub/mizu/${GITHUB_REF#refs/heads/})"
id: base_image_step
- name: Docker meta
id: meta
uses: crazy-max/ghaction-docker-meta@v2
with:
images: ${{ steps.base_image_step.outputs.image }}
tags: |
type=sha
type=raw,${{ github.sha }}
type=raw,latest
- name: Login to DockerHub
uses: docker/login-action@v1
with:
registry: gcr.io
username: _json_key
password: ${{ secrets.GCR_JSON_KEY }}
- name: Build and push
uses: docker/build-push-action@v2
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

View File

@@ -18,25 +18,14 @@ WORKDIR /app/api-build
COPY api/go.mod api/go.sum ./
COPY shared/go.mod shared/go.mod ../shared/
COPY tap/go.mod tap/go.mod ../tap/
RUN go mod download
# cheap trick to make the build faster (As long as go.mod wasn't changes)
RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
ARG COMMIT_HASH
ARG GIT_BRANCH
ARG BUILD_TIMESTAMP
ARG SEM_VER
# Copy and build api code
COPY shared ../shared
COPY tap ../tap
COPY api .
RUN go build -ldflags="-s -w \
-X 'mizuserver/pkg/version.GitCommitHash=${COMMIT_HASH}' \
-X 'mizuserver/pkg/version.Branch=${GIT_BRANCH}' \
-X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
-X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
RUN go build -ldflags="-s -w" -o mizuagent .
FROM alpine:3.13.5

View File

@@ -1,5 +1,5 @@
# 水 mizu
A simple-yet-powerful API traffic viewer for Kubernetes to help you troubleshoot and debug your microservices. Think TCPDump and Chrome Dev Tools combined.
standalone web app traffic viewer for Kubernetes
## Download
@@ -7,107 +7,18 @@ Download `mizu` for your platform and operating system
### Latest stable release
* for MacOS - Intel
```
curl -Lo mizu \
https://github.com/up9inc/mizu/releases/latest/download/mizu_darwin_amd64 \
&& chmod 755 mizu
```
* for Linux - Intel 64bit
```
curl -Lo mizu \
https://github.com/up9inc/mizu/releases/latest/download/mizu_linux_amd64 \
&& chmod 755 mizu
```
SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/releases) page.
* for MacOS - `curl -o mizu https://github.com/up9inc/mizu/releases/download/latest/mizu_darwin_amd64 && chmod 755 mizu`
* for Linux - `curl -o mizu https://github.com/up9inc/mizu/releases/download/latest/mizu_linux_amd64 && chmod 755 mizu`
### Development (unstable) build
Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page.
## Prerequisites
1. Set `KUBECONFIG` environment variable to your kubernetes configuration. If this is not set, mizu assumes that configuration is at `${HOME}/.kube/config`
2. mizu needs following permissions on your kubernetes cluster to run
```
- apiGroups:
- ""
- apps
resources:
- pods
- services
verbs:
- list
- get
- create
- delete
- apiGroups:
- ""
- apps
resources:
- daemonsets
verbs:
- list
- get
- create
- patch
- delete
```
3. Optionally, for resolving traffic ip to kubernetes service name, mizu needs below permissions
```
- apiGroups:
- ""
- apps
- "rbac.authorization.k8s.io"
resources:
- clusterroles
- clusterrolebindings
- serviceaccounts
verbs:
- get
- create
- delete
```
## How to run
1. Find pod you'd like to tap to in your Kubernetes cluster
2. Run `mizu tap PODNAME` or `mizu tap REGEX`
2. Run `mizu PODNAME` or `mizu REGEX`
3. Open browser on `http://localhost:8899` as instructed ..
4. Watch the WebAPI traffic flowing ..
5. Type ^C to stop
## Examples
Run `mizu help` for usage options
To tap specific pod -
```
$ kubectl get pods
NAME READY STATUS RESTARTS AGE
front-end-649fc5fd6-kqbtn 2/2 Running 0 7m
..
$ mizu tap front-end-649fc5fd6-kqbtn
+front-end-649fc5fd6-kqbtn
Web interface is now available at http://localhost:8899
^C
```
To tap multiple pods using regex -
```
$ kubectl get pods
NAME READY STATUS RESTARTS AGE
carts-66c77f5fbb-fq65r 2/2 Running 0 20m
catalogue-5f4cb7cf5-7zrmn 2/2 Running 0 20m
front-end-649fc5fd6-kqbtn 2/2 Running 0 20m
..
$ mizu tap "^ca.*"
+carts-66c77f5fbb-fq65r
+catalogue-5f4cb7cf5-7zrmn
Web interface is now available at http://localhost:8899
^C
```
TBD

View File

@@ -11,21 +11,20 @@ require (
github.com/go-playground/universal-translator v0.17.0
github.com/go-playground/validator/v10 v10.5.0
github.com/gofiber/fiber/v2 v2.8.0
github.com/google/gopacket v1.1.19
github.com/google/martian v2.1.0+incompatible
github.com/gorilla/websocket v1.4.2
github.com/leodido/go-urn v1.2.1 // indirect
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/up9inc/mizu/shared v0.0.0
github.com/up9inc/mizu/tap v0.0.0
go.mongodb.org/mongo-driver v1.5.1
golang.org/x/net v0.0.0-20210421230115-4e50805a0758
gorm.io/driver/sqlite v1.1.4
gorm.io/gorm v1.21.8
k8s.io/api v0.21.0
k8s.io/apimachinery v0.21.0
k8s.io/client-go v0.21.0
github.com/fsnotify/fsnotify v1.4.9
)
replace github.com/up9inc/mizu/shared v0.0.0 => ../shared
replace github.com/up9inc/mizu/tap v0.0.0 => ../tap

View File

@@ -61,8 +61,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg=
github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -75,8 +73,6 @@ github.com/fasthttp/websocket v1.4.3-beta.1/go.mod h1:JGrgLaT02bL9NuJkZbHN8mVV2t
github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -255,6 +251,7 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1pzKW+1SsBnJDOHppJN9stOEwS+CRWyUtyYGU=
github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
@@ -267,8 +264,6 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI=
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0=
github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY=
github.com/savsgio/gotils v0.0.0-20200616100644-13ff1fd2c28c h1:KKqhycXW1WVNkX7r4ekTV2gFkbhdyihlWD8c0/FiWmk=
github.com/savsgio/gotils v0.0.0-20200616100644-13ff1fd2c28c/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8=
@@ -415,7 +410,6 @@ golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

View File

@@ -5,20 +5,17 @@ import (
"flag"
"fmt"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/cors"
"github.com/gorilla/websocket"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"
"mizuserver/pkg/api"
"mizuserver/pkg/middleware"
"mizuserver/pkg/models"
"mizuserver/pkg/routes"
"mizuserver/pkg/sensitiveDataFiltering"
"mizuserver/pkg/tap"
"mizuserver/pkg/utils"
"os"
"os/signal"
"strings"
)
var shouldTap = flag.Bool("tap", false, "Run in tapper mode without API")
@@ -26,23 +23,19 @@ var aggregator = flag.Bool("aggregator", false, "Run in aggregator mode with API
var standalone = flag.Bool("standalone", false, "Run in standalone tapper and API mode")
var aggregatorAddress = flag.String("aggregator-address", "", "Address of mizu collector for tapping")
func main() {
flag.Parse()
hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
tapOpts := &tap.TapOpts{HostMode: hostMode}
if !*shouldTap && !*aggregator && !*standalone {
if !*shouldTap && !*aggregator && !*standalone{
panic("One of the flags --tap, --api or --standalone must be provided")
}
if *standalone {
harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts)
harOutputChannel := tap.StartPassiveTapper()
filteredHarChannel := make(chan *tap.OutputChannelItem)
go filterHarItems(harOutputChannel, filteredHarChannel, getTrafficFilteringOptions())
go filterHarHeaders(harOutputChannel, filteredHarChannel, getTrafficFilteringOptions())
go api.StartReadingEntries(filteredHarChannel, nil)
go api.StartReadingOutbound(outboundLinkOutputChannel)
hostApi(nil)
} else if *shouldTap {
if *aggregatorAddress == "" {
@@ -51,26 +44,21 @@ func main() {
tapTargets := getTapTargets()
if tapTargets != nil {
tap.SetFilterAuthorities(tapTargets)
rlog.Infof("Filtering for the following authorities: %v", tap.GetFilterIPs())
tap.HostAppAddresses = tapTargets
fmt.Println("Filtering for the following addresses:", tap.HostAppAddresses)
}
harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts)
harOutputChannel := tap.StartPassiveTapper()
socketConnection, err := shared.ConnectToSocketServer(*aggregatorAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false)
if err != nil {
panic(fmt.Sprintf("Error connecting to socket server at %s %v", *aggregatorAddress, err))
}
go pipeChannelToSocket(socketConnection, harOutputChannel)
go api.StartReadingOutbound(outboundLinkOutputChannel)
} else if *aggregator {
socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000)
filteredHarChannel := make(chan *tap.OutputChannelItem)
go filterHarItems(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions())
go api.StartReadingEntries(filteredHarChannel, nil)
go filterHarHeaders(socketHarOutChannel, filteredHarChannel, getTrafficFilteringOptions())
hostApi(socketHarOutChannel)
}
@@ -78,17 +66,13 @@ func main() {
signal.Notify(signalChan, os.Interrupt)
<-signalChan
rlog.Info("Exiting")
fmt.Println("Exiting")
}
func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) {
app := fiber.New()
app.Use(cors.New(cors.Config{
AllowOrigins: "*",
AllowMethods: "*",
AllowHeaders: "*",
}))
middleware.FiberMiddleware(app) // Register Fiber's middleware for app.
app.Static("/", "./site")
@@ -101,12 +85,12 @@ func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) {
}
routes.WebSocketRoutes(app, &eventHandlers)
routes.EntriesRoutes(app)
routes.MetadataRoutes(app)
routes.NotFoundRoute(app)
utils.StartServer(app)
}
func getTapTargets() []string {
nodeName := os.Getenv(shared.NodeNameEnvVar)
var tappedAddressesPerNodeDict map[string][]string
@@ -131,38 +115,13 @@ func getTrafficFilteringOptions() *shared.TrafficFilteringOptions {
return &filteringOptions
}
var userAgentsToFilter = []string{"kube-probe", "prometheus"}
func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) {
func filterHarHeaders(inChannel <- chan *tap.OutputChannelItem, outChannel chan *tap.OutputChannelItem, filterOptions *shared.TrafficFilteringOptions) {
for message := range inChannel {
if message.ConnectionInfo.IsOutgoing && api.CheckIsServiceIP(message.ConnectionInfo.ServerIP) {
continue
}
// TODO: move this to tappers https://up9.atlassian.net/browse/TRA-3441
if filterOptions.HideHealthChecks && isHealthCheckByUserAgent(message) {
continue
}
sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions)
outChannel <- message
}
}
func isHealthCheckByUserAgent(message *tap.OutputChannelItem) bool {
for _, header := range message.HarEntry.Request.Headers {
if strings.ToLower(header.Name) == "user-agent" {
for _, userAgent := range userAgentsToFilter {
if strings.Contains(strings.ToLower(header.Value), userAgent) {
return true
}
}
return false
}
}
return false
}
func pipeChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *tap.OutputChannelItem) {
if connection == nil {
panic("Websocket connection is nil")
@@ -175,13 +134,13 @@ func pipeChannelToSocket(connection *websocket.Conn, messageDataChannel <-chan *
for messageData := range messageDataChannel {
marshaledData, err := models.CreateWebsocketTappedEntryMessage(messageData)
if err != nil {
rlog.Infof("error converting message to json %s, (%v,%+v)\n", err, err, err)
fmt.Printf("error converting message to json %s, (%v,%+v)\n", err, err, err)
continue
}
err = connection.WriteMessage(websocket.TextMessage, marshaledData)
if err != nil {
rlog.Infof("error sending message through socket server %s, (%v,%+v)\n", err, err, err)
fmt.Printf("error sending message through socket server %s, (%v,%+v)\n", err, err, err)
continue
}
}

View File

@@ -6,21 +6,17 @@ import (
"encoding/json"
"fmt"
"github.com/google/martian/har"
"github.com/romana/rlog"
"github.com/up9inc/mizu/tap"
"go.mongodb.org/mongo-driver/bson/primitive"
"mizuserver/pkg/holder"
"mizuserver/pkg/database"
"mizuserver/pkg/models"
"mizuserver/pkg/resolver"
"mizuserver/pkg/tap"
"mizuserver/pkg/utils"
"net/url"
"os"
"path"
"sort"
"strings"
"time"
"mizuserver/pkg/database"
"mizuserver/pkg/models"
"mizuserver/pkg/resolver"
"mizuserver/pkg/utils"
)
var k8sResolver *resolver.Resolver
@@ -29,7 +25,7 @@ func init() {
errOut := make(chan error, 100)
res, err := resolver.NewFromInCluster(errOut)
if err != nil {
rlog.Infof("error creating k8s resolver %s", err)
fmt.Printf("error creating k8s resolver %s", err)
return
}
ctx := context.Background()
@@ -38,13 +34,12 @@ func init() {
for {
select {
case err := <-errOut:
rlog.Infof("name resolving error %s", err)
fmt.Printf("name resolving error %s", err)
}
}
}()
k8sResolver = res
holder.SetResolver(res)
}
func StartReadingEntries(harChannel <-chan *tap.OutputChannelItem, workingDir *string) {
@@ -62,21 +57,14 @@ func startReadingFiles(workingDir string) {
for true {
dir, _ := os.Open(workingDir)
dirFiles, _ := dir.Readdir(-1)
sort.Sort(utils.ByModTime(dirFiles))
var harFiles []os.FileInfo
for _, fileInfo := range dirFiles {
if strings.HasSuffix(fileInfo.Name(), ".har") {
harFiles = append(harFiles, fileInfo)
}
}
sort.Sort(utils.ByModTime(harFiles))
if len(harFiles) == 0 {
rlog.Infof("Waiting for new files\n")
if len(dirFiles) == 0 {
fmt.Printf("Waiting for new files\n")
time.Sleep(3 * time.Second)
continue
}
fileInfo := harFiles[0]
fileInfo := dirFiles[0]
inputFilePath := path.Join(workingDir, fileInfo.Name())
file, err := os.Open(inputFilePath)
utils.CheckErr(err)
@@ -87,14 +75,7 @@ func startReadingFiles(workingDir string) {
for _, entry := range inputHar.Log.Entries {
time.Sleep(time.Millisecond * 250)
connectionInfo := &tap.ConnectionInfo{
ClientIP: fileInfo.Name(),
ClientPort: "",
ServerIP: "",
ServerPort: "",
IsOutgoing: false,
}
saveHarToDb(entry, connectionInfo)
saveHarToDb(entry, fileInfo.Name())
}
rmErr := os.Remove(inputFilePath)
utils.CheckErr(rmErr)
@@ -107,45 +88,22 @@ func startReadingChannel(outputItems <-chan *tap.OutputChannelItem) {
}
for item := range outputItems {
saveHarToDb(item.HarEntry, item.ConnectionInfo)
saveHarToDb(item.HarEntry, item.RequestSenderIp)
}
}
func StartReadingOutbound(outboundLinkChannel <-chan *tap.OutboundLink) {
// tcpStreamFactory will block on write to channel. Empty channel to unblock.
// TODO: Make write to channel optional.
for range outboundLinkChannel {
}
}
func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
func saveHarToDb(entry *har.Entry, sender string) {
entryBytes, _ := json.Marshal(entry)
serviceName, urlPath := getServiceNameFromUrl(entry.Request.URL)
serviceName, urlPath, serviceHostName := getServiceNameFromUrl(entry.Request.URL)
entryId := primitive.NewObjectID().Hex()
var (
resolvedSource string
resolvedDestination string
)
if k8sResolver != nil {
unresolvedSource := connectionInfo.ClientIP
resolvedSource = k8sResolver.Resolve(unresolvedSource)
if resolvedSource == "" {
rlog.Debugf("Cannot find resolved name to source: %s\n", unresolvedSource)
if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" {
return
}
}
unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort)
resolvedDestination = k8sResolver.Resolve(unresolvedDestination)
if resolvedDestination == "" {
rlog.Debugf("Cannot find resolved name to dest: %s\n", unresolvedDestination)
if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" {
return
}
}
resolvedSource = k8sResolver.Resolve(sender)
resolvedDestination = k8sResolver.Resolve(serviceHostName)
}
mizuEntry := models.MizuEntry{
EntryId: entryId,
Entry: string(entryBytes), // simple way to store it and not convert to bytes
@@ -154,48 +112,20 @@ func saveHarToDb(entry *har.Entry, connectionInfo *tap.ConnectionInfo) {
Path: urlPath,
Method: entry.Request.Method,
Status: entry.Response.Status,
RequestSenderIp: connectionInfo.ClientIP,
RequestSenderIp: sender,
Timestamp: entry.StartedDateTime.UnixNano() / int64(time.Millisecond),
ResolvedSource: resolvedSource,
ResolvedDestination: resolvedDestination,
IsOutgoing: connectionInfo.IsOutgoing,
}
mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry)
database.CreateEntry(&mizuEntry)
database.GetEntriesTable().Create(&mizuEntry)
baseEntry := models.BaseEntryDetails{}
if err := models.GetEntry(&mizuEntry, &baseEntry); err != nil {
return
}
baseEntry := utils.GetResolvedBaseEntry(mizuEntry)
baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(&baseEntry)
broadcastToBrowserClients(baseEntryBytes)
}
func getServiceNameFromUrl(inputUrl string) (string, string) {
func getServiceNameFromUrl(inputUrl string) (string, string, string) {
parsed, err := url.Parse(inputUrl)
utils.CheckErr(err)
return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host), parsed.Path
}
func CheckIsServiceIP(address string) bool {
return k8sResolver.CheckIsServiceIP(address)
}
// gives a rough estimate of the size this will take up in the db, good enough for maintaining db size limit accurately
func getEstimatedEntrySizeBytes(mizuEntry models.MizuEntry) int {
sizeBytes := len(mizuEntry.Entry)
sizeBytes += len(mizuEntry.EntryId)
sizeBytes += len(mizuEntry.Service)
sizeBytes += len(mizuEntry.Url)
sizeBytes += len(mizuEntry.Method)
sizeBytes += len(mizuEntry.RequestSenderIp)
sizeBytes += len(mizuEntry.ResolvedDestination)
sizeBytes += len(mizuEntry.ResolvedSource)
sizeBytes += 8 // Status bytes (sqlite integer is always 8 bytes)
sizeBytes += 8 // Timestamp bytes
sizeBytes += 8 // SizeBytes bytes
sizeBytes += 1 // IsOutgoing bytes
return sizeBytes
return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host), parsed.Path, parsed.Host
}

View File

@@ -2,14 +2,13 @@ package api
import (
"encoding/json"
"fmt"
"github.com/antoniodipinto/ikisocket"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"
"mizuserver/pkg/controllers"
"mizuserver/pkg/models"
"mizuserver/pkg/routes"
"mizuserver/pkg/up9"
"mizuserver/pkg/tap"
)
var browserClientSocketUUIDs = make([]string, 0)
@@ -19,24 +18,21 @@ type RoutesEventHandlers struct {
SocketHarOutChannel chan<- *tap.OutputChannelItem
}
func init() {
go up9.UpdateAnalyzeStatus(broadcastToBrowserClients)
}
func (h *RoutesEventHandlers) WebSocketConnect(ep *ikisocket.EventPayload) {
if ep.Kws.GetAttribute("is_tapper") == true {
rlog.Infof("Websocket Connection event - Tapper connected: %s", ep.SocketUUID)
fmt.Println(fmt.Sprintf("Websocket Connection event - Tapper connected: %s", ep.SocketUUID))
} else {
rlog.Infof("Websocket Connection event - Browser socket connected: %s", ep.SocketUUID)
fmt.Println(fmt.Sprintf("Websocket Connection event - Browser socket connected: %s", ep.SocketUUID))
browserClientSocketUUIDs = append(browserClientSocketUUIDs, ep.SocketUUID)
}
}
func (h *RoutesEventHandlers) WebSocketDisconnect(ep *ikisocket.EventPayload) {
if ep.Kws.GetAttribute("is_tapper") == true {
rlog.Infof("Disconnection event - Tapper connected: %s", ep.SocketUUID)
fmt.Println(fmt.Sprintf("Disconnection event - Tapper connected: %s", ep.SocketUUID))
} else {
rlog.Infof("Disconnection event - Browser socket connected: %s", ep.SocketUUID)
fmt.Println(fmt.Sprintf("Disconnection event - Browser socket connected: %s", ep.SocketUUID))
removeSocketUUIDFromBrowserSlice(ep.SocketUUID)
}
}
@@ -47,29 +43,29 @@ func broadcastToBrowserClients(message []byte) {
func (h *RoutesEventHandlers) WebSocketClose(ep *ikisocket.EventPayload) {
if ep.Kws.GetAttribute("is_tapper") == true {
rlog.Infof("Websocket Close event - Tapper connected: %s", ep.SocketUUID)
fmt.Println(fmt.Sprintf("Websocket Close event - Tapper connected: %s", ep.SocketUUID))
} else {
rlog.Infof("Websocket Close event - Browser socket connected: %s", ep.SocketUUID)
fmt.Println(fmt.Sprintf("Websocket Close event - Browser socket connected: %s", ep.SocketUUID))
removeSocketUUIDFromBrowserSlice(ep.SocketUUID)
}
}
func (h *RoutesEventHandlers) WebSocketError(ep *ikisocket.EventPayload) {
rlog.Infof("Socket error - Socket uuid : %s %v", ep.SocketUUID, ep.Error)
fmt.Println(fmt.Sprintf("Socket error - Socket uuid : %s %v", ep.SocketUUID, ep.Error))
}
func (h *RoutesEventHandlers) WebSocketMessage(ep *ikisocket.EventPayload) {
var socketMessageBase shared.WebSocketMessageMetadata
err := json.Unmarshal(ep.Data, &socketMessageBase)
if err != nil {
rlog.Infof("Could not unmarshal websocket message %v\n", err)
fmt.Printf("Could not unmarshal websocket message %v\n", err)
} else {
switch socketMessageBase.MessageType {
case shared.WebSocketMessageTypeTappedEntry:
var tappedEntryMessage models.WebSocketTappedEntryMessage
err := json.Unmarshal(ep.Data, &tappedEntryMessage)
if err != nil {
rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
fmt.Printf("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
} else {
h.SocketHarOutChannel <- tappedEntryMessage.Data
}
@@ -77,17 +73,18 @@ func (h *RoutesEventHandlers) WebSocketMessage(ep *ikisocket.EventPayload) {
var statusMessage shared.WebSocketStatusMessage
err := json.Unmarshal(ep.Data, &statusMessage)
if err != nil {
rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
fmt.Printf("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
} else {
controllers.TapStatus = statusMessage.TappingStatus
broadcastToBrowserClients(ep.Data)
}
default:
rlog.Infof("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
fmt.Printf("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
}
}
}
func removeSocketUUIDFromBrowserSlice(uuidToRemove string) {
newUUIDSlice := make([]string, 0, len(browserClientSocketUUIDs))
for _, uuid := range browserClientSocketUUIDs {

View File

@@ -5,14 +5,28 @@ import (
"fmt"
"github.com/gofiber/fiber/v2"
"github.com/google/martian/har"
"github.com/romana/rlog"
"mizuserver/pkg/database"
"mizuserver/pkg/models"
"mizuserver/pkg/up9"
"mizuserver/pkg/utils"
"mizuserver/pkg/validation"
"strings"
"time"
)
const (
OrderDesc = "desc"
OrderAsc = "asc"
LT = "lt"
GT = "gt"
)
var (
operatorToSymbolMapping = map[string]string{
LT: "<",
GT: ">",
}
operatorToOrderMapping = map[string]string{
LT: OrderDesc,
GT: OrderAsc,
}
)
func GetEntries(c *fiber.Ctx) error {
@@ -26,8 +40,8 @@ func GetEntries(c *fiber.Ctx) error {
return c.Status(fiber.StatusBadRequest).JSON(err)
}
order := database.OperatorToOrderMapping[entriesFilter.Operator]
operatorSymbol := database.OperatorToSymbolMapping[entriesFilter.Operator]
order := operatorToOrderMapping[entriesFilter.Operator]
operatorSymbol := operatorToSymbolMapping[entriesFilter.Operator]
var entries []models.MizuEntry
database.GetEntriesTable().
Order(fmt.Sprintf("timestamp %s", order)).
@@ -36,18 +50,15 @@ func GetEntries(c *fiber.Ctx) error {
Limit(entriesFilter.Limit).
Find(&entries)
if len(entries) > 0 && order == database.OrderDesc {
if len(entries) > 0 && order == OrderDesc {
// the entries always order from oldest to newest so we should revers
utils.ReverseSlice(entries)
}
baseEntries := make([]models.BaseEntryDetails, 0)
for _, data := range entries {
harEntry := models.BaseEntryDetails{}
if err := models.GetEntry(&data, &harEntry); err != nil {
continue
}
baseEntries = append(baseEntries, harEntry)
// Convert to base entries
baseEntries := make([]models.BaseEntryDetails, 0, entriesFilter.Limit)
for _, entry := range entries {
baseEntries = append(baseEntries, utils.GetResolvedBaseEntry(entry))
}
return c.Status(fiber.StatusOK).JSON(baseEntries)
@@ -55,7 +66,7 @@ func GetEntries(c *fiber.Ctx) error {
func GetHARs(c *fiber.Ctx) error {
entriesFilter := &models.HarFetchRequestBody{}
order := database.OrderDesc
order := OrderDesc
if err := c.QueryParser(entriesFilter); err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
}
@@ -64,23 +75,11 @@ func GetHARs(c *fiber.Ctx) error {
return c.Status(fiber.StatusBadRequest).JSON(err)
}
var timestampFrom, timestampTo int64
if entriesFilter.From < 0 {
timestampFrom = 0
} else {
timestampFrom = entriesFilter.From
}
if entriesFilter.To <= 0 {
timestampTo = time.Now().UnixNano() / int64(time.Millisecond)
} else {
timestampTo = entriesFilter.To
}
var entries []models.MizuEntry
database.GetEntriesTable().
Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)).
Order(fmt.Sprintf("timestamp %s", order)).
// Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)).
Limit(1000).
Find(&entries)
if len(entries) > 0 {
@@ -93,20 +92,9 @@ func GetHARs(c *fiber.Ctx) error {
for _, entryData := range entries {
var harEntry har.Entry
_ = json.Unmarshal([]byte(entryData.Entry), &harEntry)
if entryData.ResolvedDestination != "" {
harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entryData.ResolvedDestination)
}
var fileName string
sourceOfEntry := entryData.ResolvedSource
if sourceOfEntry != "" {
// naively assumes the proper service source is http
sourceOfEntry = fmt.Sprintf("http://%s", sourceOfEntry)
//replace / from the file name cause they end up creating a corrupted folder
fileName = fmt.Sprintf("%s.har", strings.ReplaceAll(sourceOfEntry, "/", "_"))
} else {
fileName = "unknown_source.har"
}
fileName := fmt.Sprintf("%s.har", sourceOfEntry)
if harOfSource, ok := harsObject[fileName]; ok {
harOfSource.Log.Entries = append(harOfSource.Log.Entries, &harEntry)
} else {
@@ -120,14 +108,11 @@ func GetHARs(c *fiber.Ctx) error {
Name: "mizu",
Version: "0.0.2",
},
Source: sourceOfEntry,
},
Entries: entriesHar,
},
}
// leave undefined when no source is present, otherwise modeler assumes source is empty string ""
if sourceOfEntry != "" {
harsObject[fileName].Log.Creator.Source = &sourceOfEntry
}
}
}
@@ -140,78 +125,21 @@ func GetHARs(c *fiber.Ctx) error {
return c.Status(fiber.StatusOK).SendStream(buffer)
}
func UploadEntries(c *fiber.Ctx) error {
rlog.Infof("Upload entries - started\n")
uploadRequestBody := &models.UploadEntriesRequestBody{}
if err := c.QueryParser(uploadRequestBody); err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
}
if err := validation.Validate(uploadRequestBody); err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
}
if up9.GetAnalyzeInfo().IsAnalyzing {
return c.Status(fiber.StatusBadRequest).SendString("Cannot analyze, mizu is already analyzing")
}
rlog.Infof("Upload entries - creating token. dest %s\n", uploadRequestBody.Dest)
token, err := up9.CreateAnonymousToken(uploadRequestBody.Dest)
if err != nil {
return c.Status(fiber.StatusServiceUnavailable).SendString("Can't get token")
}
rlog.Infof("Upload entries - uploading. token: %s model: %s\n", token.Token, token.Model)
go up9.UploadEntriesImpl(token.Token, token.Model, uploadRequestBody.Dest, uploadRequestBody.SleepIntervalSec)
return c.Status(fiber.StatusOK).SendString("OK")
}
func GetFullEntries(c *fiber.Ctx) error {
entriesFilter := &models.HarFetchRequestBody{}
if err := c.QueryParser(entriesFilter); err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
}
err := validation.Validate(entriesFilter)
if err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
}
var timestampFrom, timestampTo int64
if entriesFilter.From < 0 {
timestampFrom = 0
} else {
timestampFrom = entriesFilter.From
}
if entriesFilter.To <= 0 {
timestampTo = time.Now().UnixNano() / int64(time.Millisecond)
} else {
timestampTo = entriesFilter.To
}
entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo)
result := make([]models.FullEntryDetails, 0)
for _, data := range entriesArray {
harEntry := models.FullEntryDetails{}
if err := models.GetEntry(&data, &harEntry); err != nil {
continue
}
result = append(result, harEntry)
}
return c.Status(fiber.StatusOK).JSON(result)
}
func GetEntry(c *fiber.Ctx) error {
var entryData models.MizuEntry
var entryData models.EntryData
database.GetEntriesTable().
Select("entry", "resolvedDestination").
Where(map[string]string{"entryId": c.Params("entryId")}).
First(&entryData)
fullEntry := models.FullEntryDetails{}
if err := models.GetEntry(&entryData, &fullEntry); err != nil {
return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
"error": true,
"msg": "Can't get entry details",
})
var fullEntry har.Entry
unmarshallErr := json.Unmarshal([]byte(entryData.Entry), &fullEntry)
utils.CheckErr(unmarshallErr)
if entryData.ResolvedDestination != "" {
fullEntry.Request.URL = utils.SetHostname(fullEntry.Request.URL, entryData.ResolvedDestination)
}
return c.Status(fiber.StatusOK).JSON(fullEntry)
}

View File

@@ -1,12 +0,0 @@
package controllers
import (
"github.com/gofiber/fiber/v2"
"github.com/up9inc/mizu/shared"
"mizuserver/pkg/version"
)
func GetVersion(c *fiber.Ctx) error {
resp := shared.VersionResponse{SemVer: version.SemVer}
return c.Status(fiber.StatusOK).JSON(resp)
}

View File

@@ -1,11 +0,0 @@
package controllers
import (
"github.com/gofiber/fiber/v2"
"mizuserver/pkg/holder"
)
func GetCurrentResolvingInformation(c *fiber.Ctx) error {
return c.Status(fiber.StatusOK).JSON(holder.GetResolver().GetMap())
}

View File

@@ -3,7 +3,6 @@ package controllers
import (
"github.com/gofiber/fiber/v2"
"github.com/up9inc/mizu/shared"
"mizuserver/pkg/up9"
)
var TapStatus shared.TapStatus
@@ -11,7 +10,3 @@ var TapStatus shared.TapStatus
func GetTappingStatus(c *fiber.Ctx) error {
return c.Status(fiber.StatusOK).JSON(TapStatus)
}
func AnalyzeInformation(c *fiber.Ctx) error {
return c.Status(fiber.StatusOK).JSON(up9.GetAnalyzeInfo())
}

View File

@@ -1,73 +1,25 @@
package database
import (
"fmt"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"mizuserver/pkg/models"
"mizuserver/pkg/utils"
"time"
)
const (
DBPath = "./entries.db"
OrderDesc = "desc"
OrderAsc = "asc"
LT = "lt"
GT = "gt"
)
var (
DB *gorm.DB
IsDBLocked = false
OperatorToSymbolMapping = map[string]string{
LT: "<",
GT: ">",
}
OperatorToOrderMapping = map[string]string{
LT: OrderDesc,
GT: OrderAsc,
}
)
func init() {
DB = initDataBase(DBPath)
go StartEnforcingDatabaseSize()
}
)
func GetEntriesTable() *gorm.DB {
return DB.Table("mizu_entries")
}
func CreateEntry(entry *models.MizuEntry) {
if IsDBLocked {
return
}
GetEntriesTable().Create(entry)
}
func initDataBase(databasePath string) *gorm.DB {
temp, _ := gorm.Open(sqlite.Open(databasePath), &gorm.Config{
Logger: &utils.TruncatingLogger{LogLevel: logger.Warn, SlowThreshold: 500 * time.Millisecond},
})
temp, _ := gorm.Open(sqlite.Open(databasePath), &gorm.Config{})
_ = temp.AutoMigrate(&models.MizuEntry{}) // this will ensure table is created
return temp
}
func GetEntriesFromDb(timestampFrom int64, timestampTo int64) []models.MizuEntry {
order := OrderDesc
var entries []models.MizuEntry
GetEntriesTable().
Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)).
Order(fmt.Sprintf("timestamp %s", order)).
Find(&entries)
if len(entries) > 0 {
// the entries always order from oldest to newest so we should revers
utils.ReverseSlice(entries)
}
return entries
}

View File

@@ -1,121 +0,0 @@
package database
import (
"fmt"
"github.com/fsnotify/fsnotify"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/debounce"
"github.com/up9inc/mizu/shared/units"
"log"
"mizuserver/pkg/models"
"os"
"strconv"
"time"
)
const percentageOfMaxSizeBytesToPrune = 15
const defaultMaxDatabaseSizeBytes int64 = 200 * 1000 * 1000
func StartEnforcingDatabaseSize() {
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Fatalf("Error creating filesystem watcher for db size enforcement: %v\n", err)
return
}
maxEntriesDBByteSize, err := getMaxEntriesDBByteSize()
if err != nil {
log.Fatalf("Error parsing max db size: %v\n", err)
return
}
checkFileSizeDebouncer := debounce.NewDebouncer(5*time.Second, func() {
checkFileSize(maxEntriesDBByteSize)
})
go func() {
for {
select {
case event, ok := <-watcher.Events:
if !ok {
return // closed channel
}
if event.Op == fsnotify.Write {
checkFileSizeDebouncer.SetOn()
}
case err, ok := <-watcher.Errors:
if !ok {
return // closed channel
}
fmt.Printf("filesystem watcher encountered error:%v\n", err)
}
}
}()
err = watcher.Add(DBPath)
if err != nil {
log.Fatalf("Error adding %s to filesystem watcher for db size enforcement: %v\n", DBPath, err)
}
}
func getMaxEntriesDBByteSize() (int64, error) {
maxEntriesDBByteSize := defaultMaxDatabaseSizeBytes
var err error
maxEntriesDBSizeByteSEnvVarValue := os.Getenv(shared.MaxEntriesDBSizeByteSEnvVar)
if maxEntriesDBSizeByteSEnvVarValue != "" {
maxEntriesDBByteSize, err = strconv.ParseInt(maxEntriesDBSizeByteSEnvVarValue, 10, 64)
}
return maxEntriesDBByteSize, err
}
func checkFileSize(maxSizeBytes int64) {
fileStat, err := os.Stat(DBPath)
if err != nil {
fmt.Printf("Error checking %s file size: %v\n", DBPath, err)
} else {
if fileStat.Size() > maxSizeBytes {
pruneOldEntries(fileStat.Size())
}
}
}
func pruneOldEntries(currentFileSize int64) {
// sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here
IsDBLocked = true
defer func() {IsDBLocked = false}()
amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune)
rows, err := GetEntriesTable().Limit(10000).Order("id").Rows()
if err != nil {
fmt.Printf("Error getting 10000 first db rows: %v\n", err)
return
}
entryIdsToRemove := make([]uint, 0)
bytesToBeRemoved := int64(0)
for rows.Next() {
if bytesToBeRemoved >= amountOfBytesToTrim {
break
}
var entry models.MizuEntry
err = DB.ScanRows(rows, &entry)
if err != nil {
fmt.Printf("Error scanning db row: %v\n", err)
continue
}
entryIdsToRemove = append(entryIdsToRemove, entry.ID)
bytesToBeRemoved += int64(entry.EstimatedSizeBytes)
}
if len(entryIdsToRemove) > 0 {
GetEntriesTable().Where(entryIdsToRemove).Delete(models.MizuEntry{})
// VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this
DB.Exec("VACUUM")
fmt.Printf("Removed %d rows and cleared %s\n", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved))
} else {
fmt.Println("Found no rows to remove when pruning")
}
}

View File

@@ -1,14 +0,0 @@
package holder
import "mizuserver/pkg/resolver"
var k8sResolver *resolver.Resolver
func SetResolver(param *resolver.Resolver) {
k8sResolver = param
}
func GetResolver() *resolver.Resolver {
return k8sResolver
}

View File

@@ -4,36 +4,25 @@ import (
"encoding/json"
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"
"mizuserver/pkg/utils"
"mizuserver/pkg/tap"
"time"
)
type DataUnmarshaler interface {
UnmarshalData(*MizuEntry) error
}
func GetEntry(r *MizuEntry, v DataUnmarshaler) error {
return v.UnmarshalData(r)
}
type MizuEntry struct {
ID uint `gorm:"primarykey"`
CreatedAt time.Time
UpdatedAt time.Time
Entry string `json:"entry,omitempty" gorm:"column:entry"`
EntryId string `json:"entryId" gorm:"column:entryId"`
Url string `json:"url" gorm:"column:url"`
Method string `json:"method" gorm:"column:method"`
Status int `json:"status" gorm:"column:status"`
RequestSenderIp string `json:"requestSenderIp" gorm:"column:requestSenderIp"`
Service string `json:"service" gorm:"column:service"`
Timestamp int64 `json:"timestamp" gorm:"column:timestamp"`
Path string `json:"path" gorm:"column:path"`
Entry string `json:"entry,omitempty" gorm:"column:entry"`
EntryId string `json:"entryId" gorm:"column:entryId"`
Url string `json:"url" gorm:"column:url"`
Method string `json:"method" gorm:"column:method"`
Status int `json:"status" gorm:"column:status"`
RequestSenderIp string `json:"requestSenderIp" gorm:"column:requestSenderIp"`
Service string `json:"service" gorm:"column:service"`
Timestamp int64 `json:"timestamp" gorm:"column:timestamp"`
Path string `json:"path" gorm:"column:path"`
ResolvedSource string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"`
ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
IsOutgoing bool `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"`
EstimatedSizeBytes int `json:"-" gorm:"column:estimatedSizeBytes"`
}
type BaseEntryDetails struct {
@@ -45,64 +34,10 @@ type BaseEntryDetails struct {
StatusCode int `json:"statusCode,omitempty"`
Method string `json:"method,omitempty"`
Timestamp int64 `json:"timestamp,omitempty"`
IsOutgoing bool `json:"isOutgoing,omitempty"`
}
type FullEntryDetails struct {
har.Entry
}
type FullEntryDetailsExtra struct {
har.Entry
}
func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error {
entryUrl := entry.Url
service := entry.Service
if entry.ResolvedDestination != "" {
entryUrl = utils.SetHostname(entryUrl, entry.ResolvedDestination)
service = utils.SetHostname(service, entry.ResolvedDestination)
}
bed.Id = entry.EntryId
bed.Url = entryUrl
bed.Service = service
bed.Path = entry.Path
bed.StatusCode = entry.Status
bed.Method = entry.Method
bed.Timestamp = entry.Timestamp
bed.RequestSenderIp = entry.RequestSenderIp
bed.IsOutgoing = entry.IsOutgoing
return nil
}
func (fed *FullEntryDetails) UnmarshalData(entry *MizuEntry) error {
if err := json.Unmarshal([]byte(entry.Entry), &fed.Entry); err != nil {
return err
}
if entry.ResolvedDestination != "" {
fed.Entry.Request.URL = utils.SetHostname(fed.Entry.Request.URL, entry.ResolvedDestination)
}
return nil
}
func (fedex *FullEntryDetailsExtra) UnmarshalData(entry *MizuEntry) error {
if err := json.Unmarshal([]byte(entry.Entry), &fedex.Entry); err != nil {
return err
}
if entry.ResolvedSource != "" {
fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-source", Value: entry.ResolvedSource})
}
if entry.ResolvedDestination != "" {
fedex.Entry.Request.Headers = append(fedex.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entry.ResolvedDestination})
fedex.Entry.Request.URL = utils.SetHostname(fedex.Entry.Request.URL, entry.ResolvedDestination)
}
return nil
}
type EntryData struct {
Entry string `json:"entry,omitempty"`
Entry string `json:"entry,omitempty"`
ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
}
@@ -112,14 +47,8 @@ type EntriesFilter struct {
Timestamp int64 `query:"timestamp" validate:"required,min=1"`
}
type UploadEntriesRequestBody struct {
Dest string `query:"dest"`
SleepIntervalSec int `query:"interval"`
}
type HarFetchRequestBody struct {
From int64 `query:"from"`
To int64 `query:"to"`
Limit int `query:"limit"`
}
type WebSocketEntryMessage struct {
@@ -127,6 +56,7 @@ type WebSocketEntryMessage struct {
Data *BaseEntryDetails `json:"data,omitempty"`
}
type WebSocketTappedEntryMessage struct {
*shared.WebSocketMessageMetadata
Data *tap.OutputChannelItem
@@ -152,6 +82,7 @@ func CreateWebsocketTappedEntryMessage(base *tap.OutputChannelItem) ([]byte, err
return json.Marshal(message)
}
// ExtendedHAR is the top level object of a HAR log.
type ExtendedHAR struct {
Log *ExtendedLog `json:"log"`
@@ -169,5 +100,5 @@ type ExtendedLog struct {
type ExtendedCreator struct {
*har.Creator
Source *string `json:"_source"`
}
Source string `json:"_source"`
}

View File

@@ -10,7 +10,6 @@ import (
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
"path/filepath"
"os"
)
func NewFromInCluster(errOut chan error) (*Resolver, error) {
@@ -22,18 +21,13 @@ func NewFromInCluster(errOut chan error) (*Resolver, error) {
if err != nil {
return nil, err
}
return &Resolver{clientConfig: config, clientSet: clientset, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}, nil
return &Resolver{clientConfig: config, clientSet: clientset, nameMap: make(map[string]string), errOut: errOut}, nil
}
func NewFromOutOfCluster(kubeConfigPath string, errOut chan error) (*Resolver, error) {
if kubeConfigPath == "" {
env := os.Getenv("KUBECONFIG")
if env != "" {
kubeConfigPath = env
} else {
home := homedir.HomeDir()
kubeConfigPath = filepath.Join(home, ".kube", "config")
}
home := homedir.HomeDir()
kubeConfigPath = filepath.Join(home, ".kube", "config")
}
configPathList := filepath.SplitList(kubeConfigPath)
@@ -59,9 +53,9 @@ func NewFromOutOfCluster(kubeConfigPath string, errOut chan error) (*Resolver, e
return nil, err
}
return &Resolver{clientConfig: clientConfig, clientSet: clientset, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}, nil
return &Resolver{clientConfig: clientConfig, clientSet: clientset, nameMap: make(map[string]string), errOut: errOut}, nil
}
func NewFromExisting(clientConfig *restclient.Config, clientSet *kubernetes.Clientset, errOut chan error) *Resolver {
return &Resolver{clientConfig: clientConfig, clientSet: clientSet, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}
return &Resolver{clientConfig: clientConfig, clientSet: clientSet, nameMap: make(map[string]string), errOut: errOut}
}

View File

@@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"github.com/romana/rlog"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
corev1 "k8s.io/api/core/v1"
@@ -21,7 +20,6 @@ type Resolver struct {
clientConfig *restclient.Config
clientSet *kubernetes.Clientset
nameMap map[string]string
serviceMap map[string]string
isStarted bool
errOut chan error
}
@@ -43,15 +41,6 @@ func (resolver *Resolver) Resolve(name string) string {
return resolvedName
}
func (resolver *Resolver) GetMap() map[string]string {
return resolver.nameMap
}
func (resolver *Resolver) CheckIsServiceIP(address string) bool {
_, isFound := resolver.serviceMap[address]
return isFound
}
func (resolver *Resolver) watchPods(ctx context.Context) error {
// empty namespace makes the client watch all namespaces
watcher, err := resolver.clientSet.CoreV1().Pods("").Watch(ctx, metav1.ListOptions{Watch: true})
@@ -135,7 +124,6 @@ func (resolver *Resolver) watchServices(ctx context.Context) error {
serviceHostname := fmt.Sprintf("%s.%s", service.Name, service.Namespace)
if service.Spec.ClusterIP != "" && service.Spec.ClusterIP != kubClientNullString {
resolver.saveResolvedName(service.Spec.ClusterIP, serviceHostname, event.Type)
resolver.saveServiceIP(service.Spec.ClusterIP, serviceHostname, event.Type)
}
if service.Status.LoadBalancer.Ingress != nil {
for _, ingress := range service.Status.LoadBalancer.Ingress {
@@ -152,18 +140,10 @@ func (resolver *Resolver) watchServices(ctx context.Context) error {
func (resolver *Resolver) saveResolvedName(key string, resolved string, eventType watch.EventType) {
if eventType == watch.Deleted {
delete(resolver.nameMap, key)
rlog.Infof("setting %s=nil\n", key)
// fmt.Printf("setting %s=nil\n", key)
} else {
resolver.nameMap[key] = resolved
rlog.Infof("setting %s=%s\n", key, resolved)
}
}
func (resolver *Resolver) saveServiceIP(key string, resolved string, eventType watch.EventType) {
if eventType == watch.Deleted {
delete(resolver.serviceMap, key)
} else {
resolver.serviceMap[key] = resolved
// fmt.Printf("setting %s=%s\n", key, resolved)
}
}
@@ -176,7 +156,7 @@ func (resolver *Resolver) infiniteErrorHandleRetryFunc(ctx context.Context, fun
var statusError *k8serrors.StatusError
if errors.As(err, &statusError) {
if statusError.ErrStatus.Reason == metav1.StatusReasonForbidden {
rlog.Infof("Resolver loop encountered permission error, aborting event listening - %v\n", err)
fmt.Printf("Resolver loop encountered permission error, aborting event listening - %v\n", err)
return
}
}

View File

@@ -1,13 +0,0 @@
package routes
import (
"github.com/gofiber/fiber/v2"
"mizuserver/pkg/controllers"
)
// MetadataRoutes defines the group of metadata routes.
func MetadataRoutes(fiberApp *fiber.App) {
routeGroup := fiberApp.Group("/metadata")
routeGroup.Get("/version", controllers.GetVersion)
}

View File

@@ -2,7 +2,7 @@ package routes
import "github.com/gofiber/fiber/v2"
// NotFoundRoute defines the 404 Error route.
// NotFoundRoute func for describe 404 Error route.
func NotFoundRoute(fiberApp *fiber.App) {
fiberApp.Use(
func(c *fiber.Ctx) error {

View File

@@ -5,21 +5,16 @@ import (
"mizuserver/pkg/controllers"
)
// EntriesRoutes defines the group of har entries routes.
// EntriesRoutes func for describe group of public routes.
func EntriesRoutes(fiberApp *fiber.App) {
routeGroup := fiberApp.Group("/api")
routeGroup.Get("/entries", controllers.GetEntries) // get entries (base/thin entries)
routeGroup.Get("/entries/:entryId", controllers.GetEntry) // get single (full) entry
routeGroup.Get("/exportEntries", controllers.GetFullEntries)
routeGroup.Get("/uploadEntries", controllers.UploadEntries)
routeGroup.Get("/resolving", controllers.GetCurrentResolvingInformation)
routeGroup.Get("/har", controllers.GetHARs)
routeGroup.Get("/resetDB", controllers.DeleteAllEntries) // get single (full) entry
routeGroup.Get("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB
routeGroup.Get("/tapStatus", controllers.GetTappingStatus) // get tapping status
routeGroup.Get("/analyzeStatus", controllers.AnalyzeInformation)
}

View File

@@ -5,7 +5,7 @@ import (
"encoding/xml"
"errors"
"fmt"
"github.com/up9inc/mizu/tap"
"mizuserver/pkg/tap"
"net/url"
"strings"
@@ -15,8 +15,8 @@ import (
)
func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, options *shared.TrafficFilteringOptions) {
harOutputItem.HarEntry.Request.Headers = filterHarHeaders(harOutputItem.HarEntry.Request.Headers)
harOutputItem.HarEntry.Response.Headers = filterHarHeaders(harOutputItem.HarEntry.Response.Headers)
filterHarHeaders(harOutputItem.HarEntry.Request.Headers)
filterHarHeaders(harOutputItem.HarEntry.Response.Headers)
harOutputItem.HarEntry.Request.Cookies = make([]har.Cookie, 0, 0)
harOutputItem.HarEntry.Response.Cookies = make([]har.Cookie, 0, 0)
@@ -44,19 +44,12 @@ func FilterSensitiveInfoFromHarRequest(harOutputItem *tap.OutputChannelItem, opt
}
}
func filterHarHeaders(headers []har.Header) []har.Header {
newHeaders := make([]har.Header, 0)
func filterHarHeaders(headers []har.Header) {
for i, header := range headers {
if strings.ToLower(header.Name) == "cookie" {
continue
} else if isFieldNameSensitive(header.Name) {
newHeaders = append(newHeaders, har.Header{Name: header.Name, Value: maskedFieldPlaceholderValue})
if isFieldNameSensitive(header.Name) {
headers[i].Value = maskedFieldPlaceholderValue
} else {
newHeaders = append(newHeaders, header)
}
}
return newHeaders
}
func getContentTypeHeaderValue(headers []har.Header) string {

View File

@@ -84,14 +84,14 @@ type GrpcAssembler struct {
framer *http2.Framer
}
func (ga *GrpcAssembler) readMessage() (uint32, interface{}, error) {
func (ga *GrpcAssembler) readMessage() (uint32, interface{}, string, error) {
// Exactly one Framer is used for each half connection.
// (Instead of creating a new Framer for each ReadFrame operation)
// This is needed in order to decompress the headers,
// because the compression context is updated with each requests/response.
frame, err := ga.framer.ReadFrame()
if err != nil {
return 0, nil, err
return 0, nil, "", err
}
streamID := frame.Header().StreamID
@@ -99,7 +99,7 @@ func (ga *GrpcAssembler) readMessage() (uint32, interface{}, error) {
ga.fragmentsByStream.appendFrame(streamID, frame)
if !(ga.isStreamEnd(frame)) {
return 0, nil, nil
return 0, nil, "", nil
}
headers, data := ga.fragmentsByStream.pop(streamID)
@@ -137,10 +137,10 @@ func (ga *GrpcAssembler) readMessage() (uint32, interface{}, error) {
ContentLength: int64(len(dataString)),
}
} else {
return 0, nil, errors.New("Failed to assemble stream: neither a request nor a message")
return 0, nil, "", errors.New("Failed to assemble stream: neither a request nor a message")
}
return streamID, messageHTTP1, nil
return streamID, messageHTTP1, dataString, nil
}
func (ga *GrpcAssembler) isStreamEnd(frame http2.Frame) bool {

View File

@@ -4,7 +4,6 @@ import (
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"os"
"path/filepath"
@@ -16,8 +15,7 @@ import (
)
const readPermission = 0644
const harFilenameSuffix = ".har"
const tempFilenameSuffix = ".har.tmp"
const tempFilenamePrefix = "har_writer"
type PairChanItem struct {
Request *http.Request
@@ -25,13 +23,12 @@ type PairChanItem struct {
Response *http.Response
ResponseTime time.Time
RequestSenderIp string
ConnectionInfo *ConnectionInfo
}
func openNewHarFile(filename string) *HarFile {
file, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, readPermission)
if err != nil {
log.Panicf("Failed to open output file: %s (%v,%+v)", err, err, err)
panic(fmt.Sprintf("Failed to open output file: %s (%v,%+v)", err, err, err))
}
harFile := HarFile{file: file, entryCount: 0}
@@ -48,13 +45,13 @@ type HarFile struct {
func NewEntry(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time) (*har.Entry, error) {
harRequest, err := har.NewRequest(request, true)
if err != nil {
SilentError("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)", err, err, err)
SilentError("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)\n", err, err, err)
return nil, errors.New("Failed converting request to HAR")
}
harResponse, err := har.NewResponse(response, true)
if err != nil {
SilentError("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)", err, err, err)
SilentError("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)\n", err, err, err)
return nil, errors.New("Failed converting response to HAR")
}
@@ -65,7 +62,7 @@ func NewEntry(request *http.Request, requestTime time.Time, response *http.Respo
status, err := strconv.Atoi(response.Header.Get(":status"))
if err != nil {
SilentError("convert-response-status-for-har", "Failed converting status to int %s (%v,%+v)", err, err, err)
SilentError("convert-response-status-for-har", "Failed converting status to int %s (%v,%+v)\n", err, err, err)
return nil, errors.New("Failed converting response status to int for HAR")
}
harResponse.Status = status
@@ -105,7 +102,7 @@ func NewEntry(request *http.Request, requestTime time.Time, response *http.Respo
func (f *HarFile) WriteEntry(harEntry *har.Entry) {
harEntryJson, err := json.Marshal(harEntry)
if err != nil {
SilentError("har-entry-marshal", "Failed converting har entry object to JSON%s (%v,%+v)", err, err, err)
SilentError("har-entry-marshal", "Failed converting har entry object to JSON%s (%v,%+v)\n", err, err, err)
return
}
@@ -119,7 +116,7 @@ func (f *HarFile) WriteEntry(harEntry *har.Entry) {
harEntryString := append([]byte(separator), harEntryJson...)
if _, err := f.file.Write(harEntryString); err != nil {
log.Panicf("Failed to write to output file: %s (%v,%+v)", err, err, err)
panic(fmt.Sprintf("Failed to write to output file: %s (%v,%+v)", err, err, err))
}
f.entryCount++
@@ -134,21 +131,21 @@ func (f *HarFile) Close() {
err := f.file.Close()
if err != nil {
log.Panicf("Failed to close output file: %s (%v,%+v)", err, err, err)
panic(fmt.Sprintf("Failed to close output file: %s (%v,%+v)", err, err, err))
}
}
func (f*HarFile) writeHeader() {
header := []byte(`{"log": {"version": "1.2", "creator": {"name": "Mizu", "version": "0.0.1"}, "entries": [`)
if _, err := f.file.Write(header); err != nil {
log.Panicf("Failed to write header to output file: %s (%v,%+v)", err, err, err)
panic(fmt.Sprintf("Failed to write header to output file: %s (%v,%+v)", err, err, err))
}
}
func (f*HarFile) writeTrailer() {
trailer := []byte("]}}")
if _, err := f.file.Write(trailer); err != nil {
log.Panicf("Failed to write trailer to output file: %s (%v,%+v)", err, err, err)
panic(fmt.Sprintf("Failed to write trailer to output file: %s (%v,%+v)", err, err, err))
}
}
@@ -164,8 +161,8 @@ func NewHarWriter(outputDir string, maxEntries int) *HarWriter {
}
type OutputChannelItem struct {
HarEntry *har.Entry
ConnectionInfo *ConnectionInfo
HarEntry *har.Entry
RequestSenderIp string
}
type HarWriter struct {
@@ -177,20 +174,20 @@ type HarWriter struct {
done chan bool
}
func (hw *HarWriter) WritePair(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time, connectionInfo *ConnectionInfo) {
func (hw *HarWriter) WritePair(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time, requestSenderIp string) {
hw.PairChan <- &PairChanItem{
Request: request,
RequestTime: requestTime,
Response: response,
ResponseTime: responseTime,
ConnectionInfo: connectionInfo,
Request: request,
RequestTime: requestTime,
Response: response,
ResponseTime: responseTime,
RequestSenderIp: requestSenderIp,
}
}
func (hw *HarWriter) Start() {
if hw.OutputDirPath != "" {
if err := os.MkdirAll(hw.OutputDirPath, os.ModePerm); err != nil {
log.Panicf("Failed to create output directory: %s (%v,%+v)", err, err, err)
panic(fmt.Sprintf("Failed to create output directory: %s (%v,%+v)", err, err, err))
}
}
@@ -213,8 +210,8 @@ func (hw *HarWriter) Start() {
}
} else {
hw.OutChan <- &OutputChannelItem{
HarEntry: harEntry,
ConnectionInfo: pair.ConnectionInfo,
HarEntry: harEntry,
RequestSenderIp: pair.RequestSenderIp,
}
}
}
@@ -229,11 +226,10 @@ func (hw *HarWriter) Start() {
func (hw *HarWriter) Stop() {
close(hw.PairChan)
<-hw.done
close(hw.OutChan)
}
func (hw *HarWriter) openNewFile() {
filename := buildFilename(hw.OutputDirPath, time.Now(), tempFilenameSuffix)
filename := filepath.Join(os.TempDir(), fmt.Sprintf("%s_%d", tempFilenamePrefix, time.Now().UnixNano()))
hw.currentFile = openNewHarFile(filename)
}
@@ -242,15 +238,15 @@ func (hw *HarWriter) closeFile() {
tmpFilename := hw.currentFile.file.Name()
hw.currentFile = nil
filename := buildFilename(hw.OutputDirPath, time.Now(), harFilenameSuffix)
filename := buildFilename(hw.OutputDirPath, time.Now())
err := os.Rename(tmpFilename, filename)
if err != nil {
SilentError("Rename-file", "cannot rename file: %s (%v,%+v)", err, err, err)
SilentError("Rename-file", "cannot rename file: %s (%v,%+v)\n", err, err, err)
}
}
func buildFilename(dir string, t time.Time, suffix string) string {
func buildFilename(dir string, t time.Time) string {
// (epoch time in nanoseconds)__(YYYY_Month_DD__hh-mm-ss).har
filename := fmt.Sprintf("%d__%s%s", t.UnixNano(), t.Format("2006_Jan_02__15-04-05"), suffix)
filename := fmt.Sprintf("%d__%s.har", t.UnixNano(), t.Format("2006_Jan_02__15-04-05"))
return filepath.Join(dir, filename)
}

209
api/pkg/tap/http_matcher.go Normal file
View File

@@ -0,0 +1,209 @@
package tap
import (
"fmt"
"net/http"
"strconv"
"strings"
"time"
"github.com/orcaman/concurrent-map"
)
type requestResponsePair struct {
Request httpMessage `json:"request"`
Response httpMessage `json:"response"`
}
type envoyMessageWrapper struct {
HttpBufferedTrace requestResponsePair `json:"http_buffered_trace"`
}
type headerKeyVal struct {
Key string `json:"key"`
Value string `json:"value"`
}
type messageBody struct {
Truncated bool `json:"truncated"`
AsBytes string `json:"as_bytes"`
}
type httpMessage struct {
IsRequest bool
Headers []headerKeyVal `json:"headers"`
HTTPVersion string `json:"httpVersion"`
Body messageBody `json:"body"`
captureTime time.Time
orig interface {}
requestSenderIp string
}
// Key is {client_addr}:{client_port}->{dest_addr}:{dest_port}
type requestResponseMatcher struct {
openMessagesMap cmap.ConcurrentMap
}
func createResponseRequestMatcher() requestResponseMatcher {
newMatcher := &requestResponseMatcher{openMessagesMap: cmap.New()}
return *newMatcher
}
func (matcher *requestResponseMatcher) registerRequest(ident string, request *http.Request, captureTime time.Time, body string, isHTTP2 bool) *envoyMessageWrapper {
split := splitIdent(ident)
key := genKey(split)
messageExtraHeaders := []headerKeyVal{
{Key: "x-up9-source", Value: split[0]},
{Key: "x-up9-destination", Value: split[1] + ":" + split[3]},
}
requestHTTPMessage := requestToMessage(request, captureTime, body, &messageExtraHeaders, isHTTP2, split[0])
if response, found := matcher.openMessagesMap.Pop(key); found {
// Type assertion always succeeds because all of the map's values are of httpMessage type
responseHTTPMessage := response.(*httpMessage)
if responseHTTPMessage.IsRequest {
SilentError("Request-Duplicate", "Got duplicate request with same identifier\n")
return nil
}
Debug("Matched open Response for %s\n", key)
return matcher.preparePair(&requestHTTPMessage, responseHTTPMessage)
}
matcher.openMessagesMap.Set(key, &requestHTTPMessage)
Debug("Registered open Request for %s\n", key)
return nil
}
func (matcher *requestResponseMatcher) registerResponse(ident string, response *http.Response, captureTime time.Time, body string, isHTTP2 bool) *envoyMessageWrapper {
split := splitIdent(ident)
key := genKey(split)
responseHTTPMessage := responseToMessage(response, captureTime, body, isHTTP2)
if request, found := matcher.openMessagesMap.Pop(key); found {
// Type assertion always succeeds because all of the map's values are of httpMessage type
requestHTTPMessage := request.(*httpMessage)
if !requestHTTPMessage.IsRequest {
SilentError("Response-Duplicate", "Got duplicate response with same identifier\n")
return nil
}
Debug("Matched open Request for %s\n", key)
return matcher.preparePair(requestHTTPMessage, &responseHTTPMessage)
}
matcher.openMessagesMap.Set(key, &responseHTTPMessage)
Debug("Registered open Response for %s\n", key)
return nil
}
func (matcher *requestResponseMatcher) preparePair(requestHTTPMessage *httpMessage, responseHTTPMessage *httpMessage) *envoyMessageWrapper {
matcher.addDuration(requestHTTPMessage, responseHTTPMessage)
return &envoyMessageWrapper{
HttpBufferedTrace: requestResponsePair{
Request: *requestHTTPMessage,
Response: *responseHTTPMessage,
},
}
}
func requestToMessage(request *http.Request, captureTime time.Time, body string, messageExtraHeaders *[]headerKeyVal, isHTTP2 bool, requestSenderIp string) httpMessage {
messageHeaders := make([]headerKeyVal, 0)
for key, value := range request.Header {
messageHeaders = append(messageHeaders, headerKeyVal{Key: key, Value: value[0]})
}
if !isHTTP2 {
messageHeaders = append(messageHeaders, headerKeyVal{Key: ":method", Value: request.Method})
messageHeaders = append(messageHeaders, headerKeyVal{Key: ":path", Value: request.RequestURI})
messageHeaders = append(messageHeaders, headerKeyVal{Key: ":authority", Value: request.Host})
messageHeaders = append(messageHeaders, headerKeyVal{Key: ":scheme", Value: "http"})
}
messageHeaders = append(messageHeaders, headerKeyVal{Key: "x-request-start", Value: fmt.Sprintf("%.3f", float64(captureTime.UnixNano()) / float64(1000000000))})
messageHeaders = append(messageHeaders, *messageExtraHeaders...)
httpVersion := request.Proto
requestBody := messageBody{Truncated: false, AsBytes: body}
return httpMessage{
IsRequest: true,
Headers: messageHeaders,
HTTPVersion: httpVersion,
Body: requestBody,
captureTime: captureTime,
orig: request,
requestSenderIp: requestSenderIp,
}
}
func responseToMessage(response *http.Response, captureTime time.Time, body string, isHTTP2 bool) httpMessage {
messageHeaders := make([]headerKeyVal, 0)
for key, value := range response.Header {
messageHeaders = append(messageHeaders, headerKeyVal{Key: key, Value: value[0]})
}
if !isHTTP2 {
messageHeaders = append(messageHeaders, headerKeyVal{Key: ":status", Value: strconv.Itoa(response.StatusCode)})
}
httpVersion := response.Proto
requestBody := messageBody{Truncated: false, AsBytes: body}
return httpMessage{
IsRequest: false,
Headers: messageHeaders,
HTTPVersion: httpVersion,
Body: requestBody,
captureTime: captureTime,
orig: response,
}
}
func (matcher *requestResponseMatcher) addDuration(requestHTTPMessage *httpMessage, responseHTTPMessage *httpMessage) {
durationMs := float64(responseHTTPMessage.captureTime.UnixNano() / 1000000) - float64(requestHTTPMessage.captureTime.UnixNano() / 1000000)
if durationMs < 1 {
durationMs = 1
}
responseHTTPMessage.Headers = append(responseHTTPMessage.Headers, headerKeyVal{Key: "x-up9-duration-ms", Value: fmt.Sprintf("%.0f", durationMs)})
}
func splitIdent(ident string) []string {
ident = strings.Replace(ident, "->", " ", -1)
return strings.Split(ident, " ")
}
func genKey(split []string) string {
key := fmt.Sprintf("%s:%s->%s:%s,%s", split[0], split[2], split[1], split[3], split[4])
return key
}
func (matcher *requestResponseMatcher) deleteOlderThan(t time.Time) int {
keysToPop := make([]string, 0)
for item := range matcher.openMessagesMap.IterBuffered() {
// Map only contains values of type httpMessage
message, _ := item.Val.(*httpMessage)
if message.captureTime.Before(t) {
keysToPop = append(keysToPop, item.Key)
}
}
numDeleted := len(keysToPop)
for _, key := range keysToPop {
_, _ = matcher.openMessagesMap.Pop(key)
}
return numDeleted
}

View File

@@ -3,7 +3,10 @@ package tap
import (
"bufio"
"bytes"
"compress/gzip"
b64 "encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/ioutil"
@@ -24,14 +27,6 @@ type tcpID struct {
dstPort string
}
type ConnectionInfo struct {
ClientIP string
ClientPort string
ServerIP string
ServerPort string
IsOutgoing bool
}
func (tid *tcpID) String() string {
return fmt.Sprintf("%s->%s %s->%s", tid.srcIP, tid.dstIP, tid.srcPort, tid.dstPort)
}
@@ -46,7 +41,6 @@ type httpReader struct {
tcpID tcpID
isClient bool
isHTTP2 bool
isOutgoing bool
msgQueue chan httpReaderDataMsg // Channel of captured reassembled tcp payload
data []byte
captureTime time.Time
@@ -79,7 +73,7 @@ func (h *httpReader) run(wg *sync.WaitGroup) {
b := bufio.NewReader(h)
if isHTTP2, err := checkIsHTTP2Connection(b, h.isClient); err != nil {
SilentError("HTTP/2-Prepare-Connection", "stream %s Failed to check if client is HTTP/2: %s (%v,%+v)", h.ident, err, err, err)
SilentError("HTTP/2-Prepare-Connection", "stream %s Failed to check if client is HTTP/2: %s (%v,%+v)\n", h.ident, err, err, err)
// Do something?
} else {
h.isHTTP2 = isHTTP2
@@ -88,7 +82,7 @@ func (h *httpReader) run(wg *sync.WaitGroup) {
if h.isHTTP2 {
err := prepareHTTP2Connection(b, h.isClient)
if err != nil {
SilentError("HTTP/2-Prepare-Connection-After-Check", "stream %s error: %s (%v,%+v)", h.ident, err, err, err)
SilentError("HTTP/2-Prepare-Connection-After-Check", "stream %s error: %s (%v,%+v)\n", h.ident, err, err, err)
}
h.grpcAssembler = createGrpcAssembler(b)
}
@@ -99,7 +93,7 @@ func (h *httpReader) run(wg *sync.WaitGroup) {
if err == io.EOF || err == io.ErrUnexpectedEOF {
break
} else if err != nil {
SilentError("HTTP/2", "stream %s error: %s (%v,%+v)", h.ident, err, err, err)
SilentError("HTTP/2", "stream %s error: %s (%v,%+v)\n", h.ident, err, err, err)
continue
}
} else if h.isClient {
@@ -107,7 +101,7 @@ func (h *httpReader) run(wg *sync.WaitGroup) {
if err == io.EOF || err == io.ErrUnexpectedEOF {
break
} else if err != nil {
SilentError("HTTP-request", "stream %s Request error: %s (%v,%+v)", h.ident, err, err, err)
SilentError("HTTP-request", "stream %s Request error: %s (%v,%+v)\n", h.ident, err, err, err)
continue
}
} else {
@@ -115,7 +109,7 @@ func (h *httpReader) run(wg *sync.WaitGroup) {
if err == io.EOF || err == io.ErrUnexpectedEOF {
break
} else if err != nil {
SilentError("HTTP-response", "stream %s Response error: %s (%v,%+v)", h.ident, err, err, err)
SilentError("HTTP-response", "stream %s Response error: %s (%v,%+v)\n", h.ident, err, err, err)
continue
}
}
@@ -123,49 +117,38 @@ func (h *httpReader) run(wg *sync.WaitGroup) {
}
func (h *httpReader) handleHTTP2Stream() error {
streamID, messageHTTP1, err := h.grpcAssembler.readMessage()
streamID, messageHTTP1, body, err := h.grpcAssembler.readMessage()
h.messageCount++
if err != nil {
return err
}
var reqResPair *requestResponsePair
var connectionInfo *ConnectionInfo
var reqResPair *envoyMessageWrapper
switch messageHTTP1 := messageHTTP1.(type) {
case http.Request:
ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.srcIP, h.tcpID.dstIP, h.tcpID.srcPort, h.tcpID.dstPort, streamID)
connectionInfo = &ConnectionInfo{
ClientIP: h.tcpID.srcIP,
ClientPort: h.tcpID.srcPort,
ServerIP: h.tcpID.dstIP,
ServerPort: h.tcpID.dstPort,
IsOutgoing: h.isOutgoing,
}
reqResPair = reqResMatcher.registerRequest(ident, &messageHTTP1, h.captureTime)
reqResPair = reqResMatcher.registerRequest(ident, &messageHTTP1, h.captureTime, body, true)
case http.Response:
ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.dstIP, h.tcpID.srcIP, h.tcpID.dstPort, h.tcpID.srcPort, streamID)
connectionInfo = &ConnectionInfo{
ClientIP: h.tcpID.dstIP,
ClientPort: h.tcpID.dstPort,
ServerIP: h.tcpID.srcIP,
ServerPort: h.tcpID.srcPort,
IsOutgoing: h.isOutgoing,
}
reqResPair = reqResMatcher.registerResponse(ident, &messageHTTP1, h.captureTime)
reqResPair = reqResMatcher.registerResponse(ident, &messageHTTP1, h.captureTime, body, true)
}
if reqResPair != nil {
statsTracker.incMatchedMessages()
if h.harWriter != nil {
h.harWriter.WritePair(
reqResPair.Request.orig.(*http.Request),
reqResPair.Request.captureTime,
reqResPair.Response.orig.(*http.Response),
reqResPair.Response.captureTime,
connectionInfo,
reqResPair.HttpBufferedTrace.Request.orig.(*http.Request),
reqResPair.HttpBufferedTrace.Request.captureTime,
reqResPair.HttpBufferedTrace.Response.orig.(*http.Response),
reqResPair.HttpBufferedTrace.Response.captureTime,
reqResPair.HttpBufferedTrace.Request.requestSenderIp,
)
} else {
jsonStr, err := json.Marshal(reqResPair)
if err != nil {
return err
}
broadcastReqResPair(jsonStr)
}
}
@@ -182,35 +165,37 @@ func (h *httpReader) handleHTTP1ClientStream(b *bufio.Reader) error {
req.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind
s := len(body)
if err != nil {
SilentError("HTTP-request-body", "stream %s Got body err: %s", h.ident, err)
SilentError("HTTP-request-body", "stream %s Got body err: %s\n", h.ident, err)
} else if h.hexdump {
Debug("Body(%d/0x%x) - %s", len(body), len(body), hex.Dump(body))
Info("Body(%d/0x%x)\n%s\n", len(body), len(body), hex.Dump(body))
}
if err := req.Body.Close(); err != nil {
SilentError("HTTP-request-body-close", "stream %s Failed to close request body: %s", h.ident, err)
SilentError("HTTP-request-body-close", "stream %s Failed to close request body: %s\n", h.ident, err)
}
encoding := req.Header["Content-Encoding"]
Debug("HTTP/1 Request: %s %s %s (Body:%d) -> %s", h.ident, req.Method, req.URL, s, encoding)
bodyStr, err := readBody(body, encoding)
if err != nil {
SilentError("HTTP-request-body-decode", "stream %s Failed to decode body: %s\n", h.ident, err)
}
Info("HTTP/%s Request: %s %s (Body:%d)\n", h.ident, req.Method, req.URL, s)
ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.srcIP, h.tcpID.dstIP, h.tcpID.srcPort, h.tcpID.dstPort, h.messageCount)
reqResPair := reqResMatcher.registerRequest(ident, req, h.captureTime)
reqResPair := reqResMatcher.registerRequest(ident, req, h.captureTime, bodyStr, false)
if reqResPair != nil {
statsTracker.incMatchedMessages()
if h.harWriter != nil {
h.harWriter.WritePair(
reqResPair.Request.orig.(*http.Request),
reqResPair.Request.captureTime,
reqResPair.Response.orig.(*http.Response),
reqResPair.Response.captureTime,
&ConnectionInfo{
ClientIP: h.tcpID.srcIP,
ClientPort: h.tcpID.srcPort,
ServerIP: h.tcpID.dstIP,
ServerPort: h.tcpID.dstPort,
IsOutgoing: h.isOutgoing,
},
reqResPair.HttpBufferedTrace.Request.orig.(*http.Request),
reqResPair.HttpBufferedTrace.Request.captureTime,
reqResPair.HttpBufferedTrace.Response.orig.(*http.Response),
reqResPair.HttpBufferedTrace.Response.captureTime,
reqResPair.HttpBufferedTrace.Request.requestSenderIp,
)
} else {
jsonStr, err := json.Marshal(reqResPair)
if err != nil {
SilentError("HTTP-marshal", "stream %s Error convert request response to json: %s\n", h.ident, err)
}
broadcastReqResPair(jsonStr)
}
}
@@ -239,13 +224,13 @@ func (h *httpReader) handleHTTP1ServerStream(b *bufio.Reader) error {
res.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind
s := len(body)
if err != nil {
SilentError("HTTP-response-body", "HTTP/%s: failed to get body(parsed len:%d): %s", h.ident, s, err)
SilentError("HTTP-response-body", "HTTP/%s: failed to get body(parsed len:%d): %s\n", h.ident, s, err)
}
if h.hexdump {
Debug("Body(%d/0x%x) - %s", len(body), len(body), hex.Dump(body))
Info("Body(%d/0x%x)\n%s\n", len(body), len(body), hex.Dump(body))
}
if err := res.Body.Close(); err != nil {
SilentError("HTTP-response-body-close", "HTTP/%s: failed to close body(parsed len:%d): %s", h.ident, s, err)
SilentError("HTTP-response-body-close", "HTTP/%s: failed to close body(parsed len:%d): %s\n", h.ident, s, err)
}
sym := ","
if res.ContentLength > 0 && res.ContentLength != int64(s) {
@@ -256,29 +241,54 @@ func (h *httpReader) handleHTTP1ServerStream(b *bufio.Reader) error {
contentType = []string{http.DetectContentType(body)}
}
encoding := res.Header["Content-Encoding"]
Debug("HTTP/1 Response: %s %s URL:%s (%d%s%d%s) -> %s", h.ident, res.Status, req, res.ContentLength, sym, s, contentType, encoding)
Info("HTTP/%s Response: %s URL:%s (%d%s%d%s) -> %s\n", h.ident, res.Status, req, res.ContentLength, sym, s, contentType, encoding)
bodyStr, err := readBody(body, encoding)
if err != nil {
SilentError("HTTP-response-body-decode", "stream %s Failed to decode body: %s\n", h.ident, err)
}
ident := fmt.Sprintf("%s->%s %s->%s %d", h.tcpID.dstIP, h.tcpID.srcIP, h.tcpID.dstPort, h.tcpID.srcPort, h.messageCount)
reqResPair := reqResMatcher.registerResponse(ident, res, h.captureTime)
reqResPair := reqResMatcher.registerResponse(ident, res, h.captureTime, bodyStr, false)
if reqResPair != nil {
statsTracker.incMatchedMessages()
if h.harWriter != nil {
h.harWriter.WritePair(
reqResPair.Request.orig.(*http.Request),
reqResPair.Request.captureTime,
reqResPair.Response.orig.(*http.Response),
reqResPair.Response.captureTime,
&ConnectionInfo{
ClientIP: h.tcpID.dstIP,
ClientPort: h.tcpID.dstPort,
ServerIP: h.tcpID.srcIP,
ServerPort: h.tcpID.srcPort,
IsOutgoing: h.isOutgoing,
},
reqResPair.HttpBufferedTrace.Request.orig.(*http.Request),
reqResPair.HttpBufferedTrace.Request.captureTime,
reqResPair.HttpBufferedTrace.Response.orig.(*http.Response),
reqResPair.HttpBufferedTrace.Response.captureTime,
reqResPair.HttpBufferedTrace.Request.requestSenderIp,
)
} else {
jsonStr, err := json.Marshal(reqResPair)
if err != nil {
SilentError("HTTP-marshal", "stream %s Error convert request response to json: %s\n", h.ident, err)
}
broadcastReqResPair(jsonStr)
}
}
return nil
}
func readBody(bodyBytes []byte, encoding []string) (string, error) {
var bodyBuffer io.Reader
bodyBuffer = bytes.NewBuffer(bodyBytes)
var err error
if len(encoding) > 0 && (encoding[0] == "gzip" || encoding[0] == "deflate") {
bodyBuffer, err = gzip.NewReader(bodyBuffer)
if err != nil {
SilentError("HTTP-gunzip", "Failed to gzip decode: %s\n", err)
return "", err
}
}
if _, ok := bodyBuffer.(*gzip.Reader); ok {
err = bodyBuffer.(*gzip.Reader).Close()
if err != nil {
return "", err
}
}
buf := new(bytes.Buffer)
_, err = buf.ReadFrom(bodyBuffer)
return b64.StdEncoding.EncodeToString(buf.Bytes()), err
}

View File

@@ -10,9 +10,10 @@ package tap
import (
"encoding/hex"
"encoding/json"
"flag"
"fmt"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared"
"log"
"os"
"os/signal"
@@ -32,10 +33,12 @@ import (
)
const AppPortsEnvVar = "APP_PORTS"
const OutPortEnvVar = "WEB_SOCKET_PORT"
const maxHTTP2DataLenEnvVar = "HTTP2_DATA_SIZE_LIMIT"
// default is 1MB, more than the max size accepted by collector and traffic-dumper
const maxHTTP2DataLenDefault = 1 * 1024 * 1024
const cleanPeriod = time.Second * 10
const outboundThrottleCacheExpiryPeriod = time.Minute * 15
var remoteOnlyOutboundPorts = []int { 80, 443 }
func parseAppPorts(appPortsList string) []int {
@@ -43,7 +46,7 @@ func parseAppPorts(appPortsList string) []int {
for _, portStr := range strings.Split(appPortsList, ",") {
parsedInt, parseError := strconv.Atoi(portStr)
if parseError != nil {
log.Printf("Provided app port %v is not a valid number!", portStr)
fmt.Println("Provided app port ", portStr, " is not a valid number!")
} else {
ports = append(ports, parsedInt)
}
@@ -51,6 +54,13 @@ func parseAppPorts(appPortsList string) []int {
return ports
}
func parseHostAppAddresses(hostAppAddressesString string) []string {
if len(hostAppAddressesString) == 0 {
return []string{}
}
return strings.Split(hostAppAddressesString, ",")
}
var maxcount = flag.Int("c", -1, "Only grab this many packets, then exit")
var decoder = flag.String("decoder", "", "Name of the decoder to use (default: guess from capture)")
var statsevery = flag.Int("stats", 60, "Output statistics every N seconds")
@@ -80,6 +90,7 @@ var tstype = flag.String("timestamp_type", "", "Type of timestamps to use")
var promisc = flag.Bool("promisc", true, "Set promiscuous mode")
var anydirection = flag.Bool("anydirection", false, "Capture http requests to other hosts")
var staleTimeoutSeconds = flag.Int("staletimout", 120, "Max time in seconds to keep connections which don't transmit data")
var hostAppAddressesString = flag.String("targets", "", "Comma separated list of ip:ports to tap")
var memprofile = flag.String("memprofile", "", "Write memory profile")
@@ -110,20 +121,24 @@ var stats struct {
overlapPackets int
}
type TapOpts struct {
HostMode bool
type CollectorMessage struct {
MessageType string
Ports *[]int `json:"ports,omitempty"`
Addresses *[]string `json:"addresses,omitempty"`
}
var outputLevel int
var errorsMap map[string]uint
var errorsMapMutex sync.Mutex
var nErrors uint
var ownIps []string // global
var hostMode bool // global
var appPorts []int // global
var ownIps []string //global
var hostMode bool //global
var HostAppAddresses []string //global
/* minOutputLevel: Error will be printed only if outputLevel is above this value
* t: key for errorsMap (counting errors)
* s, a: arguments log.Printf
* s, a: arguments fmt.Printf
* Note: Too bad for perf that a... is evaluated
*/
func logError(minOutputLevel int, t string, s string, a ...interface{}) {
@@ -132,10 +147,9 @@ func logError(minOutputLevel int, t string, s string, a ...interface{}) {
nb, _ := errorsMap[t]
errorsMap[t] = nb + 1
errorsMapMutex.Unlock()
if outputLevel >= minOutputLevel {
formatStr := fmt.Sprintf("%s: %s", t, s)
rlog.Errorf(formatStr, a...)
fmt.Printf(formatStr, a...)
}
}
func Error(t string, s string, a ...interface{}) {
@@ -144,11 +158,15 @@ func Error(t string, s string, a ...interface{}) {
func SilentError(t string, s string, a ...interface{}) {
logError(2, t, s, a...)
}
func Debug(s string, a ...interface{}) {
rlog.Debugf(s, a...)
func Info(s string, a ...interface{}) {
if outputLevel >= 1 {
fmt.Printf(s, a...)
}
}
func Trace(s string, a ...interface{}) {
rlog.Tracef(1, s, a...)
func Debug(s string, a ...interface{}) {
if outputLevel >= 2 {
fmt.Printf(s, a...)
}
}
func inArrayInt(arr []int, valueToCheck int) bool {
@@ -169,8 +187,9 @@ func inArrayString(arr []string, valueToCheck string) bool {
return false
}
// Context
// The assembler context
/*
* The assembler context
*/
type Context struct {
CaptureInfo gopacket.CaptureInfo
}
@@ -179,27 +198,22 @@ func (c *Context) GetCaptureInfo() gopacket.CaptureInfo {
return c.CaptureInfo
}
func StartPassiveTapper(opts *TapOpts) (<-chan *OutputChannelItem, <-chan *OutboundLink) {
hostMode = opts.HostMode
func StartPassiveTapper() <-chan *OutputChannelItem {
var harWriter *HarWriter
if *dumpToHar {
harWriter = NewHarWriter(*HarOutputDir, *harEntriesPerFile)
}
outboundLinkWriter := NewOutboundLinkWriter()
go startPassiveTapper(harWriter, outboundLinkWriter)
go startPassiveTapper(harWriter)
if harWriter != nil {
return harWriter.OutChan, outboundLinkWriter.OutChan
return harWriter.OutChan
}
return nil, outboundLinkWriter.OutChan
return nil
}
func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWriter) {
log.SetFlags(log.LstdFlags | log.LUTC | log.Lshortfile)
func startPassiveTapper(harWriter *HarWriter) {
defer util.Run()()
if *debug {
outputLevel = 2
@@ -212,43 +226,68 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
if localhostIPs, err := getLocalhostIPs(); err != nil {
// TODO: think this over
rlog.Info("Failed to get self IP addresses")
rlog.Errorf("Getting-Self-Address", "Error getting self ip address: %s (%v,%+v)", err, err, err)
fmt.Println("Failed to get self IP addresses")
Error("Getting-Self-Address", "Error getting self ip address: %s (%v,%+v)\n", err, err, err)
ownIps = make([]string, 0)
} else {
ownIps = localhostIPs
}
appPortsStr := os.Getenv(AppPortsEnvVar)
var appPorts []int
if appPortsStr == "" {
rlog.Info("Received empty/no APP_PORTS env var! only listening to http on port 80!")
fmt.Println("Received empty/no APP_PORTS env var! only listening to http on port 80!")
appPorts = make([]int, 0)
} else {
appPorts = parseAppPorts(appPortsStr)
}
SetFilterPorts(appPorts)
tapOutputPort := os.Getenv(OutPortEnvVar)
if tapOutputPort == "" {
fmt.Println("Received empty/no WEB_SOCKET_PORT env var! falling back to port 8080")
tapOutputPort = "8080"
}
envVal := os.Getenv(maxHTTP2DataLenEnvVar)
if envVal == "" {
rlog.Infof("Received empty/no HTTP2_DATA_SIZE_LIMIT env var! falling back to %v", maxHTTP2DataLenDefault)
fmt.Println("Received empty/no HTTP2_DATA_SIZE_LIMIT env var! falling back to", maxHTTP2DataLenDefault)
maxHTTP2DataLen = maxHTTP2DataLenDefault
} else {
if convertedInt, err := strconv.Atoi(envVal); err != nil {
rlog.Infof("Received invalid HTTP2_DATA_SIZE_LIMIT env var! falling back to %v", maxHTTP2DataLenDefault)
fmt.Println("Received invalid HTTP2_DATA_SIZE_LIMIT env var! falling back to", maxHTTP2DataLenDefault)
maxHTTP2DataLen = maxHTTP2DataLenDefault
} else {
rlog.Infof("Received HTTP2_DATA_SIZE_LIMIT env var: %v", maxHTTP2DataLenDefault)
fmt.Println("Received HTTP2_DATA_SIZE_LIMIT env var:", maxHTTP2DataLenDefault)
maxHTTP2DataLen = convertedInt
}
}
hostMode = os.Getenv(shared.HostModeEnvVar) == "1"
log.Printf("App Ports: %v", gSettings.filterPorts)
fmt.Printf("App Ports: %v\n", appPorts)
fmt.Printf("Tap output websocket port: %s\n", tapOutputPort)
var onCollectorMessage = func(message []byte) {
var parsedMessage CollectorMessage
err := json.Unmarshal(message, &parsedMessage)
if err == nil {
if parsedMessage.MessageType == "setPorts" {
Debug("Got message from collector. Type: %s, Ports: %v\n", parsedMessage.MessageType, parsedMessage.Ports)
appPorts = *parsedMessage.Ports
} else if parsedMessage.MessageType == "setAddresses" {
Debug("Got message from collector. Type: %s, IPs: %v\n", parsedMessage.MessageType, parsedMessage.Addresses)
HostAppAddresses = *parsedMessage.Addresses
Info("Filtering for the following addresses: %s\n", HostAppAddresses)
}
} else {
Error("Collector-Message-Parsing", "Error parsing message from collector: %s (%v,%+v)\n", err, err, err)
}
}
go startOutputServer(tapOutputPort, onCollectorMessage)
var handle *pcap.Handle
var err error
if *fname != "" {
if handle, err = pcap.OpenOffline(*fname); err != nil {
log.Fatalf("PCAP OpenOffline error: %v", err)
log.Fatal("PCAP OpenOffline error:", err)
}
} else {
// This is a little complicated because we want to allow all possible options
@@ -274,15 +313,15 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
}
}
if handle, err = inactive.Activate(); err != nil {
log.Fatalf("PCAP Activate error: %v", err)
log.Fatal("PCAP Activate error:", err)
}
defer handle.Close()
}
if len(flag.Args()) > 0 {
bpffilter := strings.Join(flag.Args(), " ")
rlog.Infof("Using BPF filter %q", bpffilter)
Info("Using BPF filter %q\n", bpffilter)
if err = handle.SetBPFFilter(bpffilter); err != nil {
log.Fatalf("BPF filter error: %v", err)
log.Fatal("BPF filter error:", err)
}
}
@@ -290,7 +329,6 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
harWriter.Start()
defer harWriter.Stop()
}
defer outboundLinkWriter.Stop()
var dec gopacket.Decoder
var ok bool
@@ -304,18 +342,13 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
source := gopacket.NewPacketSource(handle, dec)
source.Lazy = *lazy
source.NoCopy = true
rlog.Info("Starting to read packets")
Info("Starting to read packets\n")
count := 0
bytes := int64(0)
start := time.Now()
defragger := ip4defrag.NewIPv4Defragmenter()
streamFactory := &tcpStreamFactory{
doHTTP: !*nohttp,
harWriter: harWriter,
outbountLinkWriter: outboundLinkWriter,
}
streamFactory := &tcpStreamFactory{doHTTP: !*nohttp, harWriter: harWriter}
streamPool := reassembly.NewStreamPool(streamFactory)
assembler := reassembly.NewAssembler(streamPool)
var assemblerMutex sync.Mutex
@@ -345,7 +378,7 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
errorMapLen := len(errorsMap)
errorsSummery := fmt.Sprintf("%v", errorsMap)
errorsMapMutex.Unlock()
log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v) - Errors Summary: %s",
fmt.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)\nErrors Summary: %s\n",
count,
bytes,
time.Since(start),
@@ -357,8 +390,8 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
// At this moment
memStats := runtime.MemStats{}
runtime.ReadMemStats(&memStats)
log.Printf(
"mem: %d, goroutines: %d, unmatched messages: %d",
fmt.Printf(
"mem: %d, goroutines: %d, unmatched messages: %d\n",
memStats.HeapAlloc,
runtime.NumGoroutine(),
reqResMatcher.openMessagesMap.Count(),
@@ -367,8 +400,8 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
// Since the last print
cleanStats := cleaner.dumpStats()
appStats := statsTracker.dumpStats()
log.Printf(
"flushed connections %d, closed connections: %d, deleted messages: %d, matched messages: %d",
fmt.Printf(
"flushed connections %d, closed connections: %d, deleted messages: %d, matched messages: %d\n",
cleanStats.flushed,
cleanStats.closed,
cleanStats.deleted,
@@ -379,11 +412,11 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
for packet := range source.Packets() {
count++
rlog.Debugf("PACKET #%d", count)
Debug("PACKET #%d\n", count)
data := packet.Data()
bytes += int64(len(data))
if *hexdumppkt {
rlog.Debugf("Packet content (%d/0x%x) - %s", len(data), len(data), hex.Dump(data))
Debug("Packet content (%d/0x%x)\n%s\n", len(data), len(data), hex.Dump(data))
}
// defrag the IPv4 packet if required
@@ -398,18 +431,18 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
if err != nil {
log.Fatalln("Error while de-fragmenting", err)
} else if newip4 == nil {
rlog.Debugf("Fragment...")
Debug("Fragment...\n")
continue // packet fragment, we don't have whole packet yet.
}
if newip4.Length != l {
stats.ipdefrag++
rlog.Debugf("Decoding re-assembled packet: %s", newip4.NextLayerType())
Debug("Decoding re-assembled packet: %s\n", newip4.NextLayerType())
pb, ok := packet.(gopacket.PacketBuilder)
if !ok {
log.Panic("Not a PacketBuilder")
panic("Not a PacketBuilder")
}
nextDecoder := newip4.NextLayerType()
_ = nextDecoder.Decode(newip4.Payload, pb)
nextDecoder.Decode(newip4.Payload, pb)
}
}
@@ -426,7 +459,7 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
CaptureInfo: packet.Metadata().CaptureInfo,
}
stats.totalsz += len(tcp.Payload)
rlog.Debugf("%s : %v -> %s : %v", packet.NetworkLayer().NetworkFlow().Src(), tcp.SrcPort, packet.NetworkLayer().NetworkFlow().Dst(), tcp.DstPort)
//fmt.Println(packet.NetworkLayer().NetworkFlow().Src(), ":", tcp.SrcPort, " -> ", packet.NetworkLayer().NetworkFlow().Dst(), ":", tcp.DstPort)
assemblerMutex.Lock()
assembler.AssembleWithContext(packet.NetworkLayer().NetworkFlow(), tcp, &c)
assemblerMutex.Unlock()
@@ -437,11 +470,11 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
errorsMapMutex.Lock()
errorMapLen := len(errorsMap)
errorsMapMutex.Unlock()
log.Printf("Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)", count, bytes, time.Since(start), nErrors, errorMapLen)
fmt.Fprintf(os.Stderr, "Processed %v packets (%v bytes) in %v (errors: %v, errTypes:%v)\n", count, bytes, time.Since(start), nErrors, errorMapLen)
}
select {
case <-signalChan:
log.Printf("Caught SIGINT: aborting")
fmt.Fprintf(os.Stderr, "\nCaught SIGINT: aborting\n")
done = true
default:
// NOP: continue
@@ -454,7 +487,7 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
assemblerMutex.Lock()
closed := assembler.FlushAll()
assemblerMutex.Unlock()
rlog.Debugf("Final flush: %d closed", closed)
Debug("Final flush: %d closed", closed)
if outputLevel >= 2 {
streamPool.Dump()
}
@@ -464,34 +497,34 @@ func startPassiveTapper(harWriter *HarWriter, outboundLinkWriter *OutboundLinkWr
if err != nil {
log.Fatal(err)
}
_ = pprof.WriteHeapProfile(f)
_ = f.Close()
pprof.WriteHeapProfile(f)
f.Close()
}
streamFactory.WaitGoRoutines()
assemblerMutex.Lock()
rlog.Debugf("%s", assembler.Dump())
Debug("%s\n", assembler.Dump())
assemblerMutex.Unlock()
if !*nodefrag {
log.Printf("IPdefrag:\t\t%d", stats.ipdefrag)
fmt.Printf("IPdefrag:\t\t%d\n", stats.ipdefrag)
}
log.Printf("TCP stats:")
log.Printf(" missed bytes:\t\t%d", stats.missedBytes)
log.Printf(" total packets:\t\t%d", stats.pkt)
log.Printf(" rejected FSM:\t\t%d", stats.rejectFsm)
log.Printf(" rejected Options:\t%d", stats.rejectOpt)
log.Printf(" reassembled bytes:\t%d", stats.sz)
log.Printf(" total TCP bytes:\t%d", stats.totalsz)
log.Printf(" conn rejected FSM:\t%d", stats.rejectConnFsm)
log.Printf(" reassembled chunks:\t%d", stats.reassembled)
log.Printf(" out-of-order packets:\t%d", stats.outOfOrderPackets)
log.Printf(" out-of-order bytes:\t%d", stats.outOfOrderBytes)
log.Printf(" biggest-chunk packets:\t%d", stats.biggestChunkPackets)
log.Printf(" biggest-chunk bytes:\t%d", stats.biggestChunkBytes)
log.Printf(" overlap packets:\t%d", stats.overlapPackets)
log.Printf(" overlap bytes:\t\t%d", stats.overlapBytes)
log.Printf("Errors: %d", nErrors)
fmt.Printf("TCP stats:\n")
fmt.Printf(" missed bytes:\t\t%d\n", stats.missedBytes)
fmt.Printf(" total packets:\t\t%d\n", stats.pkt)
fmt.Printf(" rejected FSM:\t\t%d\n", stats.rejectFsm)
fmt.Printf(" rejected Options:\t%d\n", stats.rejectOpt)
fmt.Printf(" reassembled bytes:\t%d\n", stats.sz)
fmt.Printf(" total TCP bytes:\t%d\n", stats.totalsz)
fmt.Printf(" conn rejected FSM:\t%d\n", stats.rejectConnFsm)
fmt.Printf(" reassembled chunks:\t%d\n", stats.reassembled)
fmt.Printf(" out-of-order packets:\t%d\n", stats.outOfOrderPackets)
fmt.Printf(" out-of-order bytes:\t%d\n", stats.outOfOrderBytes)
fmt.Printf(" biggest-chunk packets:\t%d\n", stats.biggestChunkPackets)
fmt.Printf(" biggest-chunk bytes:\t%d\n", stats.biggestChunkBytes)
fmt.Printf(" overlap packets:\t%d\n", stats.overlapPackets)
fmt.Printf(" overlap bytes:\t\t%d\n", stats.overlapBytes)
fmt.Printf("Errors: %d\n", nErrors)
for e := range errorsMap {
log.Printf(" %s:\t\t%d", e, errorsMap[e])
fmt.Printf(" %s:\t\t%d\n", e, errorsMap[e])
}
}

239
api/pkg/tap/tap_output.go Normal file
View File

@@ -0,0 +1,239 @@
package tap
import (
"bytes"
"encoding/json"
"fmt"
"log"
"net/http"
"time"
"github.com/gorilla/websocket"
"github.com/patrickmn/go-cache"
)
const (
// Time allowed to write a message to the peer.
writeWait = 10 * time.Second
// Time allowed to read the next pong message from the peer.
pongWait = 60 * time.Second
// Send pings to peer with this period. Must be less than pongWait.
pingPeriod = (pongWait * 9) / 10
// Maximum message size allowed from peer.
maxMessageSize = 512
)
var (
newline = []byte{'\n'}
space = []byte{' '}
hub *Hub
outboundSocketNotifyExpiringCache = cache.New(outboundThrottleCacheExpiryPeriod, outboundThrottleCacheExpiryPeriod)
)
var upgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
CheckOrigin: func (_ *http.Request) bool { return true },
}
// Client is a middleman between the websocket connection and the hub.
type Client struct {
hub *Hub
// The websocket connection.
conn *websocket.Conn
// Buffered channel of outbound messages.
send chan []byte
}
type OutBoundLinkMessage struct {
SourceIP string `json:"sourceIP"`
IP string `json:"ip"`
Port int `json:"port"`
Type string `json:"type"`
}
// readPump pumps messages from the websocket connection to the hub.
//
// The application runs readPump in a per-connection goroutine. The application
// ensures that there is at most one reader on a connection by executing all
// reads from this goroutine.
func (c *Client) readPump() {
defer func() {
c.hub.unregister <- c
c.conn.Close()
}()
c.conn.SetReadLimit(maxMessageSize)
c.conn.SetReadDeadline(time.Now().Add(pongWait))
c.conn.SetPongHandler(func(string) error { c.conn.SetReadDeadline(time.Now().Add(pongWait)); return nil })
for {
_, message, err := c.conn.ReadMessage()
if err != nil {
if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) {
log.Printf("error: %v", err)
}
break
}
message = bytes.TrimSpace(bytes.Replace(message, newline, space, -1))
c.hub.onMessageCallback(message)
}
}
// writePump pumps messages from the hub to the websocket connection.
//
// A goroutine running writePump is started for each connection. The
// application ensures that there is at most one writer to a connection by
// executing all writes from this goroutine.
func (c *Client) writePump() {
ticker := time.NewTicker(pingPeriod)
defer func() {
ticker.Stop()
c.conn.Close()
}()
for {
select {
case message, ok := <-c.send:
c.conn.SetWriteDeadline(time.Now().Add(writeWait))
if !ok {
// The hub closed the channel.
c.conn.WriteMessage(websocket.CloseMessage, []byte{})
return
}
w, err := c.conn.NextWriter(websocket.TextMessage)
if err != nil {
return
}
w.Write(message)
if err := w.Close(); err != nil {
return
}
case <-ticker.C:
c.conn.SetWriteDeadline(time.Now().Add(writeWait))
if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil {
return
}
}
}
}
type Hub struct {
// Registered clients.
clients map[*Client]bool
// Inbound messages from the clients.
broadcast chan []byte
// Register requests from the clients.
register chan *Client
// Unregister requests from clients.
unregister chan *Client
// Handle messages from client
onMessageCallback func([]byte)
}
func newHub(onMessageCallback func([]byte)) *Hub {
return &Hub{
broadcast: make(chan []byte),
register: make(chan *Client),
unregister: make(chan *Client),
clients: make(map[*Client]bool),
onMessageCallback: onMessageCallback,
}
}
func (h *Hub) run() {
for {
select {
case client := <-h.register:
h.clients[client] = true
case client := <-h.unregister:
if _, ok := h.clients[client]; ok {
delete(h.clients, client)
close(client.send)
}
case message := <-h.broadcast:
// matched messages counter is incremented in this thread instead of in multiple http reader
// threads in order to reduce contention.
statsTracker.incMatchedMessages()
for client := range h.clients {
select {
case client.send <- message:
default:
close(client.send)
delete(h.clients, client)
}
}
}
}
}
// serveWs handles websocket requests from the peer.
func serveWs(hub *Hub, w http.ResponseWriter, r *http.Request) {
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
log.Println(err)
return
}
client := &Client{hub: hub, conn: conn, send: make(chan []byte, 256)}
client.hub.register <- client
// Allow collection of memory referenced by the caller by doing all work in
// new goroutines.
go client.writePump()
go client.readPump()
}
func startOutputServer(port string, messageCallback func([]byte)) {
hub = newHub(messageCallback)
go hub.run()
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
serveWs(hub, w, r)
})
err := http.ListenAndServe("0.0.0.0:" + port, nil)
if err != nil {
log.Fatal("Output server error: ", err)
}
}
func broadcastReqResPair(reqResJson []byte) {
hub.broadcast <- reqResJson
}
func broadcastOutboundLink(srcIP string, dstIP string, dstPort int) {
cacheKey := fmt.Sprintf("%s -> %s:%d", srcIP, dstIP, dstPort)
_, isInCache := outboundSocketNotifyExpiringCache.Get(cacheKey)
if isInCache {
return
} else {
outboundSocketNotifyExpiringCache.SetDefault(cacheKey, true)
}
socketMessage := OutBoundLinkMessage{
SourceIP: srcIP,
IP: dstIP,
Port: dstPort,
Type: "outboundSocketDetected",
}
jsonStr, err := json.Marshal(socketMessage)
if err != nil {
log.Printf("error marshalling outbound socket detection object: %v", err)
} else {
hub.broadcast <- jsonStr
}
}

View File

@@ -34,7 +34,7 @@ type tcpStream struct {
func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassembly.TCPFlowDirection, nextSeq reassembly.Sequence, start *bool, ac reassembly.AssemblerContext) bool {
// FSM
if !t.tcpstate.CheckState(tcp, dir) {
SilentError("FSM-rejection", "%s: Packet rejected by FSM (state:%s)", t.ident, t.tcpstate.String())
//SilentError("FSM", "%s: Packet rejected by FSM (state:%s)\n", t.ident, t.tcpstate.String())
stats.rejectFsm++
if !t.fsmerr {
t.fsmerr = true
@@ -47,7 +47,7 @@ func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassem
// Options
err := t.optchecker.Accept(tcp, ci, dir, nextSeq, start)
if err != nil {
SilentError("OptionChecker-rejection", "%s: Packet rejected by OptionChecker: %s", t.ident, err)
//SilentError("OptionChecker", "%s: Packet rejected by OptionChecker: %s\n", t.ident, err)
stats.rejectOpt++
if !*nooptcheck {
return false
@@ -58,10 +58,10 @@ func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassem
if *checksum {
c, err := tcp.ComputeChecksum()
if err != nil {
SilentError("ChecksumCompute", "%s: Got error computing checksum: %s", t.ident, err)
SilentError("ChecksumCompute", "%s: Got error computing checksum: %s\n", t.ident, err)
accept = false
} else if c != 0x0 {
SilentError("Checksum", "%s: Invalid checksum: 0x%x", t.ident, c)
SilentError("Checksum", "%s: Invalid checksum: 0x%x\n", t.ident, c)
accept = false
}
}
@@ -95,7 +95,7 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass
if sgStats.OverlapBytes != 0 && sgStats.OverlapPackets == 0 {
// In the original example this was handled with panic().
// I don't know what this error means or how to handle it properly.
SilentError("Invalid-Overlap", "bytes:%d, pkts:%d", sgStats.OverlapBytes, sgStats.OverlapPackets)
SilentError("Invalid-Overlap", "bytes:%d, pkts:%d\n", sgStats.OverlapBytes, sgStats.OverlapPackets)
}
stats.overlapBytes += sgStats.OverlapBytes
stats.overlapPackets += sgStats.OverlapPackets
@@ -106,7 +106,7 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass
} else {
ident = fmt.Sprintf("%v %v(%s): ", t.net.Reverse(), t.transport.Reverse(), dir)
}
Trace("%s: SG reassembled packet with %d bytes (start:%v,end:%v,skip:%d,saved:%d,nb:%d,%d,overlap:%d,%d)", ident, length, start, end, skip, saved, sgStats.Packets, sgStats.Chunks, sgStats.OverlapBytes, sgStats.OverlapPackets)
Debug("%s: SG reassembled packet with %d bytes (start:%v,end:%v,skip:%d,saved:%d,nb:%d,%d,overlap:%d,%d)\n", ident, length, start, end, skip, saved, sgStats.Packets, sgStats.Chunks, sgStats.OverlapBytes, sgStats.OverlapPackets)
if skip == -1 && *allowmissinginit {
// this is allowed
} else if skip != 0 {
@@ -125,18 +125,18 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass
}
dnsSize := binary.BigEndian.Uint16(data[:2])
missing := int(dnsSize) - len(data[2:])
Trace("dnsSize: %d, missing: %d", dnsSize, missing)
Debug("dnsSize: %d, missing: %d\n", dnsSize, missing)
if missing > 0 {
Debug("Missing some bytes: %d", missing)
Info("Missing some bytes: %d\n", missing)
sg.KeepFrom(0)
return
}
p := gopacket.NewDecodingLayerParser(layers.LayerTypeDNS, dns)
err := p.DecodeLayers(data[2:], &decoded)
if err != nil {
SilentError("DNS-parser", "Failed to decode DNS: %v", err)
SilentError("DNS-parser", "Failed to decode DNS: %v\n", err)
} else {
Trace("DNS: %s", gopacket.LayerDump(dns))
Debug("DNS: %s\n", gopacket.LayerDump(dns))
}
if len(data) > 2+int(dnsSize) {
sg.KeepFrom(2 + int(dnsSize))
@@ -144,7 +144,7 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass
} else if t.isHTTP {
if length > 0 {
if *hexdump {
Trace("Feeding http with:%s", hex.Dump(data))
Debug("Feeding http with:\n%s", hex.Dump(data))
}
// This is where we pass the reassembled information onwards
// This channel is read by an httpReader object
@@ -158,7 +158,7 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass
}
func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool {
Trace("%s: Connection closed", t.ident)
Debug("%s: Connection closed\n", t.ident)
if t.isHTTP {
close(t.client.msgQueue)
close(t.server.msgQueue)

View File

@@ -2,7 +2,6 @@ package tap
import (
"fmt"
"github.com/romana/rlog"
"sync"
"github.com/google/gopacket"
@@ -16,27 +15,24 @@ import (
* Generates a new tcp stream for each new tcp connection. Closes the stream when the connection closes.
*/
type tcpStreamFactory struct {
wg sync.WaitGroup
doHTTP bool
harWriter *HarWriter
outbountLinkWriter *OutboundLinkWriter
wg sync.WaitGroup
doHTTP bool
harWriter *HarWriter
}
func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.TCP, ac reassembly.AssemblerContext) reassembly.Stream {
rlog.Debugf("* NEW: %s %s", net, transport)
Debug("* NEW: %s %s\n", net, transport)
fsmOptions := reassembly.TCPSimpleFSMOptions{
SupportMissingEstablishment: *allowmissinginit,
}
rlog.Debugf("Current App Ports: %v", gSettings.filterPorts)
srcIp := net.Src().String()
Debug("Current App Ports: %v\n", appPorts)
dstIp := net.Dst().String()
dstPort := int(tcp.DstPort)
if factory.shouldNotifyOnOutboundLink(dstIp, dstPort) {
factory.outbountLinkWriter.WriteOutboundLink(net.Src().String(), dstIp, dstPort)
broadcastOutboundLink(net.Src().String(), dstIp, dstPort)
}
props := factory.getStreamProps(srcIp, dstIp, dstPort)
isHTTP := props.isTapTarget
isHTTP := factory.shouldTap(dstIp, dstPort)
stream := &tcpStream{
net: net,
transport: transport,
@@ -60,7 +56,6 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T
hexdump: *hexdump,
parent: stream,
isClient: true,
isOutgoing: props.isOutgoing,
harWriter: factory.harWriter,
}
stream.server = httpReader{
@@ -74,7 +69,6 @@ func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.T
},
hexdump: *hexdump,
parent: stream,
isOutgoing: props.isOutgoing,
harWriter: factory.harWriter,
}
factory.wg.Add(2)
@@ -89,35 +83,28 @@ func (factory *tcpStreamFactory) WaitGoRoutines() {
factory.wg.Wait()
}
func (factory *tcpStreamFactory) getStreamProps(srcIP string, dstIP string, dstPort int) *streamProps {
func (factory *tcpStreamFactory) shouldTap(dstIP string, dstPort int) bool {
if hostMode {
if inArrayString(gSettings.filterAuthorities, fmt.Sprintf("%s:%d", dstIP, dstPort)) == true {
rlog.Debugf("getStreamProps %s", fmt.Sprintf("+ host1 %s:%d", dstIP, dstPort))
return &streamProps{isTapTarget: true, isOutgoing: false}
} else if inArrayString(gSettings.filterAuthorities, dstIP) == true {
rlog.Debugf("getStreamProps %s", fmt.Sprintf("+ host2 %s", dstIP))
return &streamProps{isTapTarget: true, isOutgoing: false}
} else if *anydirection && inArrayString(gSettings.filterAuthorities, srcIP) == true {
rlog.Debugf("getStreamProps %s", fmt.Sprintf("+ host3 %s", srcIP))
return &streamProps{isTapTarget: true, isOutgoing: true}
if inArrayString(HostAppAddresses, fmt.Sprintf("%s:%d", dstIP, dstPort)) == true {
return true
} else if inArrayString(HostAppAddresses, dstIP) == true {
return true
}
return &streamProps{isTapTarget: false}
return false
} else {
isTappedPort := dstPort == 80 || (gSettings.filterPorts != nil && (inArrayInt(gSettings.filterPorts, dstPort)))
isTappedPort := dstPort == 80 || (appPorts != nil && (inArrayInt(appPorts, dstPort)))
if !isTappedPort {
rlog.Debugf("getStreamProps %s", fmt.Sprintf("- notHost1 %d", dstPort))
return &streamProps{isTapTarget: false, isOutgoing: false}
return false
}
isOutgoing := !inArrayString(ownIps, dstIP)
if !*anydirection && isOutgoing {
rlog.Debugf("getStreamProps %s", fmt.Sprintf("- notHost2"))
return &streamProps{isTapTarget: false, isOutgoing: isOutgoing}
if !*anydirection {
isDirectedHere := inArrayString(ownIps, dstIP)
if !isDirectedHere {
return false
}
}
rlog.Debugf("getStreamProps %s", fmt.Sprintf("+ notHost3 %s -> %s:%d", srcIP, dstIP, dstPort))
return &streamProps{isTapTarget: true}
return true
}
}
@@ -128,9 +115,3 @@ func (factory *tcpStreamFactory) shouldNotifyOnOutboundLink(dstIP string, dstPor
}
return true
}
type streamProps struct {
isTapTarget bool
isOutgoing bool
}

View File

@@ -1,201 +0,0 @@
package up9
import (
"bytes"
"compress/zlib"
"encoding/json"
"fmt"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared"
"io/ioutil"
"log"
"mizuserver/pkg/database"
"mizuserver/pkg/models"
"net/http"
"net/url"
"strings"
"time"
)
const (
AnalyzeCheckSleepTime = 5 * time.Second
)
type GuestToken struct {
Token string `json:"token"`
Model string `json:"model"`
}
type ModelStatus struct {
LastMajorGeneration float64 `json:"lastMajorGeneration"`
}
func getGuestToken(url string, target *GuestToken) error {
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
rlog.Infof("Got token from the server, starting to json decode... status code: %v", resp.StatusCode)
return json.NewDecoder(resp.Body).Decode(target)
}
func CreateAnonymousToken(envPrefix string) (*GuestToken, error) {
tokenUrl := fmt.Sprintf("https://trcc.%s/anonymous/token", envPrefix)
if strings.HasPrefix(envPrefix, "http") {
tokenUrl = fmt.Sprintf("%s/api/token", envPrefix)
}
token := &GuestToken{}
if err := getGuestToken(tokenUrl, token); err != nil {
rlog.Infof("Failed to get token, %s", err)
return nil, err
}
return token, nil
}
func GetRemoteUrl(analyzeDestination string, analyzeToken string) string {
return fmt.Sprintf("https://%s/share/%s", analyzeDestination, analyzeToken)
}
func CheckIfModelReady(analyzeDestination string, analyzeModel string, analyzeToken string) bool {
statusUrl, _ := url.Parse(fmt.Sprintf("https://trcc.%s/models/%s/status", analyzeDestination, analyzeModel))
req := &http.Request{
Method: http.MethodGet,
URL: statusUrl,
Header: map[string][]string{
"Content-Type": {"application/json"},
"Guest-Auth": {analyzeToken},
},
}
statusResp, err := http.DefaultClient.Do(req)
if err != nil {
return false
}
defer statusResp.Body.Close()
target := &ModelStatus{}
_ = json.NewDecoder(statusResp.Body).Decode(&target)
return target.LastMajorGeneration > 0
}
func GetTrafficDumpUrl(analyzeDestination string, analyzeModel string) *url.URL {
strUrl := fmt.Sprintf("https://traffic.%s/dumpTrafficBulk/%s", analyzeDestination, analyzeModel)
if strings.HasPrefix(analyzeDestination, "http") {
strUrl = fmt.Sprintf("%s/api/workspace/dumpTrafficBulk", analyzeDestination)
}
postUrl, _ := url.Parse(strUrl)
return postUrl
}
type AnalyzeInformation struct {
IsAnalyzing bool
SentCount int
AnalyzedModel string
AnalyzeToken string
AnalyzeDestination string
}
func (info *AnalyzeInformation) Reset() {
info.IsAnalyzing = false
info.AnalyzedModel = ""
info.AnalyzeToken = ""
info.AnalyzeDestination = ""
info.SentCount = 0
}
var analyzeInformation = &AnalyzeInformation{}
func GetAnalyzeInfo() *shared.AnalyzeStatus {
return &shared.AnalyzeStatus{
IsAnalyzing: analyzeInformation.IsAnalyzing,
RemoteUrl: GetRemoteUrl(analyzeInformation.AnalyzeDestination, analyzeInformation.AnalyzeToken),
IsRemoteReady: CheckIfModelReady(analyzeInformation.AnalyzeDestination, analyzeInformation.AnalyzedModel, analyzeInformation.AnalyzeToken),
SentCount: analyzeInformation.SentCount,
}
}
func UploadEntriesImpl(token string, model string, envPrefix string, sleepIntervalSec int) {
analyzeInformation.IsAnalyzing = true
analyzeInformation.AnalyzedModel = model
analyzeInformation.AnalyzeToken = token
analyzeInformation.AnalyzeDestination = envPrefix
analyzeInformation.SentCount = 0
sleepTime := time.Second * time.Duration(sleepIntervalSec)
var timestampFrom int64 = 0
for {
timestampTo := time.Now().UnixNano() / int64(time.Millisecond)
rlog.Infof("Getting entries from %v, to %v\n", timestampFrom, timestampTo)
entriesArray := database.GetEntriesFromDb(timestampFrom, timestampTo)
if len(entriesArray) > 0 {
fullEntriesExtra := make([]models.FullEntryDetailsExtra, 0)
for _, data := range entriesArray {
harEntry := models.FullEntryDetailsExtra{}
if err := models.GetEntry(&data, &harEntry); err != nil {
continue
}
fullEntriesExtra = append(fullEntriesExtra, harEntry)
}
rlog.Infof("About to upload %v entries\n", len(fullEntriesExtra))
body, jMarshalErr := json.Marshal(fullEntriesExtra)
if jMarshalErr != nil {
analyzeInformation.Reset()
rlog.Infof("Stopping analyzing")
log.Fatal(jMarshalErr)
}
var in bytes.Buffer
w := zlib.NewWriter(&in)
_, _ = w.Write(body)
_ = w.Close()
reqBody := ioutil.NopCloser(bytes.NewReader(in.Bytes()))
req := &http.Request{
Method: http.MethodPost,
URL: GetTrafficDumpUrl(envPrefix, model),
Header: map[string][]string{
"Content-Encoding": {"deflate"},
"Content-Type": {"application/octet-stream"},
"Guest-Auth": {token},
},
Body: reqBody,
}
if _, postErr := http.DefaultClient.Do(req); postErr != nil {
analyzeInformation.Reset()
rlog.Info("Stopping analyzing")
log.Fatal(postErr)
}
analyzeInformation.SentCount += len(entriesArray)
rlog.Infof("Finish uploading %v entries to %s\n", len(entriesArray), GetTrafficDumpUrl(envPrefix, model))
} else {
rlog.Infof("Nothing to upload")
}
rlog.Infof("Sleeping for %v...\n", sleepTime)
time.Sleep(sleepTime)
timestampFrom = timestampTo
}
}
func UpdateAnalyzeStatus(callback func(data []byte)) {
for {
if !analyzeInformation.IsAnalyzing {
time.Sleep(AnalyzeCheckSleepTime)
continue
}
analyzeStatus := GetAnalyzeInfo()
socketMessage := shared.CreateWebSocketMessageTypeAnalyzeStatus(*analyzeStatus)
jsonMessage, _ := json.Marshal(socketMessage)
callback(jsonMessage)
time.Sleep(AnalyzeCheckSleepTime)
}
}

View File

@@ -1,58 +0,0 @@
package utils
import (
"context"
"fmt"
"gorm.io/gorm/logger"
"gorm.io/gorm/utils"
"time"
)
// TruncatingLogger implements the gorm logger.Interface interface. Its purpose is to act as gorm's logger while truncating logs to a max of 50 characters to minimise the performance impact
type TruncatingLogger struct {
LogLevel logger.LogLevel
SlowThreshold time.Duration
}
func (truncatingLogger *TruncatingLogger) LogMode(logLevel logger.LogLevel) logger.Interface {
truncatingLogger.LogLevel = logLevel
return truncatingLogger
}
func (truncatingLogger *TruncatingLogger) Info(_ context.Context, message string, __ ...interface{}) {
if truncatingLogger.LogLevel < logger.Info {
return
}
fmt.Printf("gorm info: %.150s\n", message)
}
func (truncatingLogger *TruncatingLogger) Warn(_ context.Context, message string, __ ...interface{}) {
if truncatingLogger.LogLevel < logger.Warn {
return
}
fmt.Printf("gorm warning: %.150s\n", message)
}
func (truncatingLogger *TruncatingLogger) Error(_ context.Context, message string, __ ...interface{}) {
if truncatingLogger.LogLevel < logger.Error {
return
}
fmt.Printf("gorm error: %.150s\n", message)
}
func (truncatingLogger *TruncatingLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) {
if truncatingLogger.LogLevel == logger.Silent {
return
}
elapsed := time.Since(begin)
if err != nil {
sql, rows := fc() // copied into every condition as this is a potentially heavy operation best done only when necessary
truncatingLogger.Error(ctx, fmt.Sprintf("Error in %s: %v - elapsed: %fs affected rows: %d, sql: %s", utils.FileWithLineNum(), err, elapsed.Seconds(), rows, sql))
} else if truncatingLogger.LogLevel >= logger.Warn && elapsed > truncatingLogger.SlowThreshold {
sql, rows := fc()
truncatingLogger.Warn(ctx, fmt.Sprintf("Slow sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql))
} else if truncatingLogger.LogLevel >= logger.Info {
sql, rows := fc()
truncatingLogger.Info(ctx, fmt.Sprintf("Sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql))
}
}

View File

@@ -1,9 +1,11 @@
package utils
import (
"encoding/json"
"fmt"
"github.com/gofiber/fiber/v2"
"github.com/romana/rlog"
"log"
"mizuserver/pkg/models"
"net/url"
"os"
"os/signal"
@@ -21,7 +23,7 @@ func StartServer(app *fiber.App) {
go func() {
_ = <-signals
rlog.Infof("Shutting down...")
fmt.Println("Shutting down...")
_ = app.Shutdown()
}()
@@ -59,3 +61,27 @@ func SetHostname(address, newHostname string) string {
return replacedUrl.String()
}
func GetResolvedBaseEntry(entry models.MizuEntry) models.BaseEntryDetails {
entryUrl := entry.Url
service := entry.Service
if entry.ResolvedDestination != "" {
entryUrl = SetHostname(entryUrl, entry.ResolvedDestination)
service = SetHostname(service, entry.ResolvedDestination)
}
return models.BaseEntryDetails{
Id: entry.EntryId,
Url: entryUrl,
Service: service,
Path: entry.Path,
StatusCode: entry.Status,
Method: entry.Method,
Timestamp: entry.Timestamp,
RequestSenderIp: entry.RequestSenderIp,
}
}
func GetBytesFromStruct(v interface{}) []byte{
a, _ := json.Marshal(v)
return a
}

View File

@@ -1,8 +0,0 @@
package version
var (
SemVer = "0.0.1"
Branch = "develop"
GitCommitHash = "" // this var is overridden using ldflags in makefile when building
BuildTimestamp = "" // this var is overridden using ldflags in makefile when building
)

View File

@@ -14,7 +14,7 @@ then
fi
echo "building $DOCKER_TAGGED_BUILD"
docker build -t "$DOCKER_TAGGED_BUILD" --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} .
docker build -t "$DOCKER_TAGGED_BUILD" .
echo pushing to "$REPOSITORY"
docker push "$DOCKER_TAGGED_BUILD"

View File

@@ -26,10 +26,10 @@ build-all: ## build for all supported platforms
@mkdir -p bin && echo "SHA256 checksums available for compiled binaries \n\nRun \`shasum -a 256 -c mizu_OS_ARCH.sha256\` to verify\n\n" > bin/README.md
@$(MAKE) build GOOS=darwin GOARCH=amd64
@$(MAKE) build GOOS=linux GOARCH=amd64
@# $(MAKE) build GOOS=darwin GOARCH=arm64
@# $(MAKE) GOOS=windows GOARCH=amd64
@# $(MAKE) GOOS=linux GOARCH=386
@# $(MAKE) GOOS=windows GOARCH=386
@# $(MAKE) GOOS=darwin GOARCH=arm64
@# $(MAKE) GOOS=linux GOARCH=arm64
@# $(MAKE) GOOS=windows GOARCH=arm64
@echo "---------"

View File

@@ -2,14 +2,11 @@ package cmd
import (
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
)
type MizuFetchOptions struct {
FromTimestamp int64
ToTimestamp int64
Directory string
MizuPort uint16
Limit uint16
Directory string
}
var mizuFetchOptions = MizuFetchOptions{}
@@ -18,11 +15,6 @@ var fetchCmd = &cobra.Command{
Use: "fetch",
Short: "Download recorded traffic to files",
RunE: func(cmd *cobra.Command, args []string) error {
if isCompatible, err := mizu.CheckVersionCompatibility(mizuFetchOptions.MizuPort); err != nil {
return err
} else if !isCompatible {
return nil
}
RunMizuFetch(&mizuFetchOptions)
return nil
},
@@ -31,8 +23,6 @@ var fetchCmd = &cobra.Command{
func init() {
rootCmd.AddCommand(fetchCmd)
fetchCmd.Flags().Uint16VarP(&mizuFetchOptions.Limit, "limit", "l", 1000, "Provide a custom limit for entries to fetch")
fetchCmd.Flags().StringVarP(&mizuFetchOptions.Directory, "directory", "d", ".", "Provide a custom directory for fetched entries")
fetchCmd.Flags().Int64Var(&mizuFetchOptions.FromTimestamp, "from", 0, "Custom start timestamp for fetched entries")
fetchCmd.Flags().Int64Var(&mizuFetchOptions.ToTimestamp, "to", 0, "Custom end timestamp fetched entries")
fetchCmd.Flags().Uint16VarP(&mizuFetchOptions.MizuPort, "port", "p", 8899, "Custom port for mizu")
}

View File

@@ -4,7 +4,6 @@ import (
"archive/zip"
"bytes"
"fmt"
"github.com/up9inc/mizu/cli/kubernetes"
"io"
"io/ioutil"
"log"
@@ -15,8 +14,7 @@ import (
)
func RunMizuFetch(fetch *MizuFetchOptions) {
mizuProxiedUrl := kubernetes.GetMizuCollectorProxiedHostAndPath(fetch.MizuPort)
resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, fetch.FromTimestamp, fetch.ToTimestamp))
resp, err := http.Get(fmt.Sprintf("http://localhost:8899/api/har?limit=%v", fetch.Limit))
if err != nil {
log.Fatal(err)
}
@@ -55,7 +53,7 @@ func Unzip(reader *zip.Reader, dest string) error {
path := filepath.Join(dest, f.Name)
// Check for ZipSlip (Directory traversal)
if !strings.HasPrefix(path, filepath.Clean(dest)+string(os.PathSeparator)) {
if !strings.HasPrefix(path, filepath.Clean(dest) + string(os.PathSeparator)) {
return fmt.Errorf("illegal file path: %s", path)
}
@@ -63,7 +61,7 @@ func Unzip(reader *zip.Reader, dest string) error {
_ = os.MkdirAll(path, f.Mode())
} else {
_ = os.MkdirAll(filepath.Dir(path), f.Mode())
fmt.Print("writing HAR file [ ", path, " ] .. ")
fmt.Print("writing HAR file [ ", path, " ] .. ")
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
if err != nil {
return err
@@ -72,7 +70,7 @@ func Unzip(reader *zip.Reader, dest string) error {
if err := f.Close(); err != nil {
panic(err)
}
fmt.Println(" done")
fmt.Println(" done")
}()
_, err = io.Copy(f, rc)
@@ -92,3 +90,5 @@ func Unzip(reader *zip.Reader, dest string) error {
return nil
}

View File

@@ -3,86 +3,43 @@ package cmd
import (
"errors"
"fmt"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared/units"
"os"
"regexp"
"strings"
"github.com/spf13/cobra"
)
type MizuTapOptions struct {
GuiPort uint16
Namespace string
AllNamespaces bool
Analysis bool
AnalysisDestination string
KubeConfigPath string
MizuImage string
MizuPodPort uint16
PlainTextFilterRegexes []string
TapOutgoing bool
HideHealthChecks bool
MaxEntriesDBSizeBytes int64
SleepIntervalSec uint16
}
var mizuTapOptions = &MizuTapOptions{}
var direction string
var humanMaxEntriesDBSize string
var regex *regexp.Regexp
const maxEntriesDBSizeFlagName = "max-entries-db-size"
const analysisMessageToConfirm = `NOTE: running mizu with --analysis flag will upload recorded traffic
to UP9 cloud for further analysis and enriched presentation options.
`
var tapCmd = &cobra.Command{
Use: "tap [POD REGEX]",
Short: "Record ingoing traffic of a kubernetes pod",
Long: `Record the ingoing traffic of a kubernetes pod.
Supported protocols are HTTP and gRPC.`,
Supported protocols are HTTP and gRPC.`,
RunE: func(cmd *cobra.Command, args []string) error {
RunMizuTap(regex, mizuTapOptions)
return nil
},
PreRunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
return errors.New("POD REGEX argument is required")
} else if len(args) > 1 {
return errors.New("unexpected number of arguments")
}
var compileErr error
regex, compileErr = regexp.Compile(args[0])
if compileErr != nil {
return errors.New(fmt.Sprintf("%s is not a valid regex %s", args[0], compileErr))
regex, err := regexp.Compile(args[0])
if err != nil {
return errors.New(fmt.Sprintf("%s is not a valid regex %s", args[0], err))
}
var parseHumanDataSizeErr error
mizuTapOptions.MaxEntriesDBSizeBytes, parseHumanDataSizeErr = units.HumanReadableToBytes(humanMaxEntriesDBSize)
if parseHumanDataSizeErr != nil {
return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize))
}
fmt.Printf("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes))
directionLowerCase := strings.ToLower(direction)
if directionLowerCase == "any" {
mizuTapOptions.TapOutgoing = true
} else if directionLowerCase == "in" {
mizuTapOptions.TapOutgoing = false
} else {
return errors.New(fmt.Sprintf("%s is not a valid value for flag --direction. Acceptable values are in/any.", direction))
}
if mizuTapOptions.Analysis {
fmt.Printf(analysisMessageToConfirm)
if !uiUtils.AskForConfirmation("Would you like to proceed [y/n]: ") {
fmt.Println("You can always run mizu without analysis, aborting")
os.Exit(0)
}
}
RunMizuTap(regex, mizuTapOptions)
return nil
},
}
@@ -92,14 +49,9 @@ func init() {
tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector")
tapCmd.Flags().BoolVar(&mizuTapOptions.Analysis, "analysis", false, "Uploads traffic to UP9 for further analysis (Beta)")
tapCmd.Flags().StringVar(&mizuTapOptions.AnalysisDestination, "dest", "up9.app", "Destination environment")
tapCmd.Flags().Uint16VarP(&mizuTapOptions.SleepIntervalSec, "upload-interval", "", 10, "Interval in seconds for uploading data to UP9")
tapCmd.Flags().BoolVarP(&mizuTapOptions.AllNamespaces, "all-namespaces", "A", false, "Tap all namespaces")
tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file")
tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", mizu.Branch, mizu.SemVer), "Custom image for mizu collector")
tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:latest", mizu.Branch), "Custom image for mizu collector")
tapCmd.Flags().Uint16VarP(&mizuTapOptions.MizuPodPort, "mizu-port", "", 8899, "Port which mizu cli will attempt to forward from the mizu collector pod")
tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "regex-masking", "r", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies")
tapCmd.Flags().StringVarP(&direction, "direction", "", "in", "Record traffic that goes in this direction (relative to the tapped pod): in/any")
tapCmd.Flags().BoolVar(&mizuTapOptions.HideHealthChecks, "hide-healthchecks", false, "hides requests with kube-probe or prometheus user-agent headers")
tapCmd.Flags().StringVarP(&humanMaxEntriesDBSize, maxEntriesDBSizeFlagName, "", "200MB", "override the default max entries db size of 200mb")
}

View File

@@ -3,20 +3,18 @@ package cmd
import (
"context"
"fmt"
"github.com/romana/rlog"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/debounce"
core "k8s.io/api/core/v1"
"log"
"net/http"
"net/url"
"os"
"os/signal"
"regexp"
"syscall"
"time"
core "k8s.io/api/core/v1"
"github.com/up9inc/mizu/cli/debounce"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
)
var mizuServiceAccountExists bool
@@ -40,29 +38,12 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel() // cancel will be called when this function exits
targetNamespace := getNamespace(tappingOptions, kubernetesProvider)
if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegexQuery, targetNamespace); err != nil {
if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegexQuery); err != nil {
return
} else {
currentlyTappedPods = matchingPods
}
var namespacesStr string
if targetNamespace != mizu.K8sAllNamespaces {
namespacesStr = fmt.Sprintf("namespace \"%s\"", targetNamespace)
} else {
namespacesStr = "all namespaces"
}
fmt.Printf("Tapping pods in %s\n", namespacesStr)
if len(currentlyTappedPods) == 0 {
var suggestionStr string
if targetNamespace != mizu.K8sAllNamespaces {
suggestionStr = "\nSelect a different namespace with -n or tap all namespaces with -A"
}
fmt.Printf("Did not find any pods matching the regex argument%s\n", suggestionStr)
}
nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods)
if err != nil {
return
@@ -78,6 +59,8 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
//block until exit signal or error
waitForFinish(ctx, cancel)
// TODO handle incoming traffic from tapper using a channel
}
func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
@@ -85,7 +68,7 @@ func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Pro
return err
}
if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
if err := createMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
return err
}
@@ -96,7 +79,7 @@ func createMizuAggregator(ctx context.Context, kubernetesProvider *kubernetes.Pr
var err error
mizuServiceAccountExists = createRBACIfNecessary(ctx, kubernetesProvider)
_, err = kubernetesProvider.CreateMizuAggregatorPod(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.MizuImage, mizuServiceAccountExists, mizuApiFilteringOptions, tappingOptions.MaxEntriesDBSizeBytes)
_, err = kubernetesProvider.CreateMizuAggregatorPod(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.MizuImage, mizuServiceAccountExists, mizuApiFilteringOptions)
if err != nil {
fmt.Printf("Error creating mizu collector pod: %v\n", err)
return err
@@ -112,44 +95,36 @@ func createMizuAggregator(ctx context.Context, kubernetesProvider *kubernetes.Pr
}
func getMizuApiFilteringOptions(tappingOptions *MizuTapOptions) (*shared.TrafficFilteringOptions, error) {
var compiledRegexSlice []*shared.SerializableRegexp
if tappingOptions.PlainTextFilterRegexes != nil && len(tappingOptions.PlainTextFilterRegexes) > 0 {
compiledRegexSlice = make([]*shared.SerializableRegexp, 0)
for _, regexStr := range tappingOptions.PlainTextFilterRegexes {
compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr)
if err != nil {
fmt.Printf("Regex %s is invalid: %v", regexStr, err)
return nil, err
}
compiledRegexSlice = append(compiledRegexSlice, compiledRegex)
}
if tappingOptions.PlainTextFilterRegexes == nil || len(tappingOptions.PlainTextFilterRegexes) == 0 {
return nil, nil
}
return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: tappingOptions.HideHealthChecks}, nil
compiledRegexSlice := make([]*shared.SerializableRegexp, 0)
for _, regexStr := range tappingOptions.PlainTextFilterRegexes {
compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr)
if err != nil {
fmt.Printf("Regex %s is invalid: %v", regexStr, err)
return nil, err
}
compiledRegexSlice = append(compiledRegexSlice, compiledRegex)
}
return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice}, nil
}
func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error {
if len(nodeToTappedPodIPMap) > 0 {
if err := kubernetesProvider.ApplyMizuTapperDaemonSet(
ctx,
mizu.ResourcesNamespace,
mizu.TapperDaemonSetName,
tappingOptions.MizuImage,
mizu.TapperPodName,
fmt.Sprintf("%s.%s.svc.cluster.local", aggregatorService.Name, aggregatorService.Namespace),
nodeToTappedPodIPMap,
mizuServiceAccountExists,
tappingOptions.TapOutgoing,
); err != nil {
fmt.Printf("Error creating mizu tapper daemonset: %v\n", err)
return err
}
} else {
if err := kubernetesProvider.RemoveDaemonSet(ctx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
fmt.Printf("Error deleting mizu tapper daemonset: %v\n", err)
return err
}
func createMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error {
if err := kubernetesProvider.ApplyMizuTapperDaemonSet(
ctx,
mizu.ResourcesNamespace,
mizu.TapperDaemonSetName,
tappingOptions.MizuImage,
mizu.TapperPodName,
fmt.Sprintf("%s.%s.svc.cluster.local", aggregatorService.Name, aggregatorService.Namespace),
nodeToTappedPodIPMap,
mizuServiceAccountExists,
); err != nil {
fmt.Printf("Error creating mizu tapper daemonset: %v\n", err)
return err
}
return nil
@@ -171,12 +146,10 @@ func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
}
func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, podRegex *regexp.Regexp, tappingOptions *MizuTapOptions) {
targetNamespace := getNamespace(tappingOptions, kubernetesProvider)
added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, targetNamespace), podRegex)
added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, getNamespace(tappingOptions, kubernetesProvider)), podRegex)
restartTappers := func() {
if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegex, targetNamespace); err != nil {
if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegex); err != nil {
fmt.Printf("Error getting pods by regex: %s (%v,%+v)\n", err, err, err)
cancel()
} else {
@@ -189,7 +162,7 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
cancel()
}
if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
if err := createMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
fmt.Printf("Error updating daemonset: %s (%v,%+v)\n", err, err, err)
cancel()
}
@@ -199,10 +172,10 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
for {
select {
case newTarget := <-added:
fmt.Printf(mizu.Green, fmt.Sprintf("+%s\n", newTarget.Name))
fmt.Printf("+%s\n", newTarget.Name)
case removedTarget := <-removed:
fmt.Printf(mizu.Red, fmt.Sprintf("-%s\n", removedTarget.Name))
fmt.Printf("-%s\n", removedTarget.Name)
restartTappersDebouncer.SetOn()
case modifiedTarget := <-modified:
@@ -230,6 +203,7 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.AggregatorPodName))
added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, mizu.ResourcesNamespace), podExactRegex)
isPodReady := false
var portForward *kubernetes.PortForward
for {
select {
case <-added:
@@ -241,31 +215,12 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
case modifiedPod := <-modified:
if modifiedPod.Status.Phase == "Running" && !isPodReady {
isPodReady = true
go func() {
err := kubernetes.StartProxy(kubernetesProvider, tappingOptions.GuiPort, mizu.ResourcesNamespace, mizu.AggregatorPodName)
if err != nil {
fmt.Printf("Error occured while running k8s proxy %v\n", err)
cancel()
}
}()
mizuProxiedUrl := kubernetes.GetMizuCollectorProxiedHostAndPath(tappingOptions.GuiPort)
fmt.Printf("Mizu is available at http://%s\n", mizuProxiedUrl)
time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready
if tappingOptions.Analysis {
urlPath := fmt.Sprintf("http://%s/api/uploadEntries?dest=%s&interval=%v", mizuProxiedUrl, url.QueryEscape(tappingOptions.AnalysisDestination), tappingOptions.SleepIntervalSec)
u, err := url.ParseRequestURI(urlPath)
if err != nil {
log.Fatal(fmt.Sprintf("Failed parsing the URL %v\n", err))
}
rlog.Debugf("Sending get request to %v\n", u.String())
if response, err := http.Get(u.String()); err != nil || response.StatusCode != 200 {
fmt.Printf("error sending upload entries req, status code: %v, err: %v\n", response.StatusCode, err)
} else {
fmt.Printf(mizu.Purple, "Traffic is uploading to UP9 for further analsys")
fmt.Println()
}
var err error
portForward, err = kubernetes.NewPortForward(kubernetesProvider, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.GuiPort, tappingOptions.MizuPodPort, cancel)
fmt.Printf("Web interface is now available at http://localhost:%d\n", tappingOptions.GuiPort)
if err != nil {
fmt.Printf("error forwarding port to pod %s\n", err)
cancel()
}
}
@@ -279,6 +234,9 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
cancel()
case <-ctx.Done():
if portForward != nil {
portForward.Stop()
}
return
}
}
@@ -327,8 +285,7 @@ func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
}
func syncApiStatus(ctx context.Context, cancel context.CancelFunc, tappingOptions *MizuTapOptions) {
controlSocketStr := fmt.Sprintf("ws://%s/ws", kubernetes.GetMizuCollectorProxiedHostAndPath(tappingOptions.GuiPort))
controlSocket, err := mizu.CreateControlSocket(controlSocketStr)
controlSocket, err := mizu.CreateControlSocket(fmt.Sprintf("ws://localhost:%d/ws", tappingOptions.GuiPort))
if err != nil {
fmt.Printf("error establishing control socket connection %s\n", err)
cancel()
@@ -341,7 +298,7 @@ func syncApiStatus(ctx context.Context, cancel context.CancelFunc, tappingOption
default:
err = controlSocket.SendNewTappedPodsListMessage(currentlyTappedPods)
if err != nil {
rlog.Debugf("error Sending message via control socket %v, error: %s\n", controlSocketStr, err)
fmt.Printf("error Sending message via control socket %s\n", err)
}
time.Sleep(10 * time.Second)
}

View File

@@ -2,33 +2,17 @@ package cmd
import (
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
)
type MizuViewOptions struct {
GuiPort uint16
}
var mizuViewOptions = &MizuViewOptions{}
var viewCmd = &cobra.Command{
Use: "view",
Short: "Open GUI in browser",
RunE: func(cmd *cobra.Command, args []string) error {
if isCompatible, err := mizu.CheckVersionCompatibility(mizuFetchOptions.MizuPort); err != nil {
return err
} else if !isCompatible {
return nil
}
runMizuView(mizuViewOptions)
runMizuView()
return nil
},
}
func init() {
rootCmd.AddCommand(viewCmd)
viewCmd.Flags().Uint16VarP(&mizuViewOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
}

View File

@@ -8,7 +8,7 @@ import (
"net/http"
)
func runMizuView(mizuViewOptions *MizuViewOptions) {
func runMizuView() {
kubernetesProvider := kubernetes.NewProvider("")
ctx, cancel := context.WithCancel(context.Background())
@@ -23,17 +23,11 @@ func runMizuView(mizuViewOptions *MizuViewOptions) {
return
}
mizuProxiedUrl := kubernetes.GetMizuCollectorProxiedHostAndPath(mizuViewOptions.GuiPort)
_, err = http.Get(fmt.Sprintf("http://%s/", mizuProxiedUrl))
_, err = http.Get("http://localhost:8899/")
if err == nil {
fmt.Printf("Found a running service %s and open port %d\n", mizu.AggregatorPodName, mizuViewOptions.GuiPort)
fmt.Printf("Found a running service %s and open port 8899\n", mizu.AggregatorPodName)
return
}
fmt.Printf("Found service %s, creating k8s proxy\n", mizu.AggregatorPodName)
fmt.Printf("Mizu is available at http://%s\n", kubernetes.GetMizuCollectorProxiedHostAndPath(mizuViewOptions.GuiPort))
err = kubernetes.StartProxy(kubernetesProvider, mizuViewOptions.GuiPort, mizu.ResourcesNamespace, mizu.AggregatorPodName)
if err != nil {
fmt.Printf("Error occured while running k8s proxy %v\n", err)
}
fmt.Printf("Found service %s, creating port forwarding to 8899\n", mizu.AggregatorPodName)
portForwardApiPod(ctx, kubernetesProvider, cancel, &MizuTapOptions{GuiPort: 8899, MizuPodPort: 8899})
}

View File

@@ -4,13 +4,11 @@ go 1.16
require (
github.com/gorilla/websocket v1.4.2
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
github.com/spf13/cobra v1.1.3
github.com/up9inc/mizu/shared v0.0.0
k8s.io/api v0.21.2
k8s.io/apimachinery v0.21.2
k8s.io/client-go v0.21.2
k8s.io/kubectl v0.21.2
k8s.io/api v0.21.0
k8s.io/apimachinery v0.21.0
k8s.io/client-go v0.21.0
)
replace github.com/up9inc/mizu/shared v0.0.0 => ../shared

View File

@@ -23,7 +23,6 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
github.com/Azure/go-autorest/autorest v0.11.12 h1:gI8ytXbxMfI+IVbI9mP2JGCTXIuhHLgRlvQ9X4PsnHE=
@@ -40,132 +39,67 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E=
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU=
github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs=
github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk=
github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA=
github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64=
github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4=
github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY=
github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk=
github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
github.com/go-openapi/validate v0.19.8/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -191,10 +125,8 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -202,9 +134,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -215,8 +146,6 @@ github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
@@ -224,7 +153,6 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
@@ -277,56 +205,41 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA=
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -337,35 +250,21 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI=
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
@@ -373,8 +272,6 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
@@ -382,7 +279,6 @@ github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzu
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -393,35 +289,24 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -457,11 +342,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -469,11 +352,9 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -487,7 +368,6 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -512,9 +392,7 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -522,12 +400,9 @@ golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -535,13 +410,10 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M=
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
@@ -561,7 +433,6 @@ golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxb
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -572,9 +443,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -595,10 +464,8 @@ golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapK
golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -641,7 +508,6 @@ google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfG
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
@@ -674,50 +540,33 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
k8s.io/api v0.21.2 h1:vz7DqmRsXTCSa6pNxXwQ1IYeAZgdIsua+DZU+o+SX3Y=
k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU=
k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc=
k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM=
k8s.io/cli-runtime v0.21.2/go.mod h1:8u/jFcM0QpoI28f6sfrAAIslLCXUYKD5SsPPMWiHYrI=
k8s.io/client-go v0.21.2 h1:Q1j4L/iMN4pTw6Y4DWppBoUxgKO8LbffEMVEV00MUp0=
k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA=
k8s.io/code-generator v0.21.2/go.mod h1:8mXJDCB7HcRo1xiEQstcguZkbxZaqeUOrO9SsicWs3U=
k8s.io/component-base v0.21.2/go.mod h1:9lvmIThzdlrJj5Hp8Z/TOgIkdfsNARQ1pT+3PByuiuc=
k8s.io/component-helpers v0.21.2/go.mod h1:DbyFt/A0p6Cv+R5+QOGSJ5f5t4xDfI8Yb89a57DgJlQ=
k8s.io/api v0.21.0 h1:gu5iGF4V6tfVCQ/R+8Hc0h7H1JuEhzyEi9S4R5LM8+Y=
k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU=
k8s.io/apimachinery v0.21.0 h1:3Fx+41if+IRavNcKOz09FwEXDBG6ORh6iMsTSelhkMA=
k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY=
k8s.io/client-go v0.21.0 h1:n0zzzJsAQmJngpC0IhgFcApZyoGXPrDIAD601HD09ag=
k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA=
k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
k8s.io/kubectl v0.21.2 h1:9XPCetvOMDqrIZZXb1Ei+g8t6KrIp9ENJaysQjUuLiE=
k8s.io/kubectl v0.21.2/go.mod h1:PgeUclpG8VVmmQIl8zpLar3IQEpFc9mrmvlwY3CK1xo=
k8s.io/metrics v0.21.2/go.mod h1:wzlOINZMCtWq8dR9gHlyaOemmYlOpAoldEIXE82gAhI=
k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
sigs.k8s.io/kustomize/api v0.8.8/go.mod h1:He1zoK0nk43Pc6NlV085xDXDXTNprtcyKZVm3swsdNY=
sigs.k8s.io/kustomize/cmd/config v0.9.10/go.mod h1:Mrby0WnRH7hA6OwOYnYpfpiY0WJIMgYrEDfwOeFdMK0=
sigs.k8s.io/kustomize/kustomize/v4 v4.1.2/go.mod h1:PxBvo4WGYlCLeRPL+ziT64wBXqbgfcalOS/SXa/tcyo=
sigs.k8s.io/kustomize/kyaml v0.10.17/go.mod h1:mlQFagmkm1P+W4lZJbJ/yaxMd8PqMRSC4cPcfUVt5Hg=
sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=

View File

@@ -0,0 +1,52 @@
package kubernetes
import (
"bytes"
"context"
"fmt"
"k8s.io/apimachinery/pkg/util/httpstream"
"k8s.io/client-go/tools/portforward"
"k8s.io/client-go/transport/spdy"
"net/http"
"net/url"
"strings"
)
type PortForward struct {
stopChan chan struct{}
}
func NewPortForward(kubernetesProvider *Provider, namespace string, podName string, localPort uint16, podPort uint16, cancel context.CancelFunc) (*PortForward, error) {
dialer := getHttpDialer(kubernetesProvider, namespace, podName)
stopChan, readyChan := make(chan struct{}, 1), make(chan struct{}, 1)
out, errOut := new(bytes.Buffer), new(bytes.Buffer)
forwarder, err := portforward.New(dialer, []string{fmt.Sprintf("%d:%d", localPort, podPort)}, stopChan, readyChan, out, errOut)
if err != nil {
return nil, err
}
go func() {
err = forwarder.ForwardPorts() // this is blocking
if err != nil {
fmt.Printf("kubernetes port-forwarding error: %s", err)
cancel()
}
}()
return &PortForward{stopChan: stopChan}, nil
}
func (portForward *PortForward) Stop() {
close(portForward.stopChan)
}
func getHttpDialer(kubernetesProvider *Provider, namespace string, podName string) httpstream.Dialer {
roundTripper, upgrader, err := spdy.RoundTripperFor(&kubernetesProvider.clientConfig)
if err != nil {
panic(err)
}
path := fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/portforward", namespace, podName)
hostIP := strings.TrimLeft(kubernetesProvider.clientConfig.Host, "htps:/")
serverURL := url.URL{Scheme: "https", Path: path, Host: hostIP}
return spdy.NewDialer(upgrader, &http.Client{Transport: roundTripper}, http.MethodPost, &serverURL)
}

View File

@@ -6,16 +6,15 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/up9inc/mizu/cli/mizu"
"path/filepath"
"regexp"
"strconv"
"github.com/up9inc/mizu/shared"
core "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
resource "k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/watch"
applyconfapp "k8s.io/client-go/applyconfigurations/apps/v1"
@@ -72,29 +71,11 @@ func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) w
return watcher
}
func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace string, podName string, podImage string, linkServiceAccount bool, mizuApiFilteringOptions *shared.TrafficFilteringOptions, maxEntriesDBSizeBytes int64) (*core.Pod, error) {
func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace string, podName string, podImage string, linkServiceAccount bool, mizuApiFilteringOptions *shared.TrafficFilteringOptions) (*core.Pod, error) {
marshaledFilteringOptions, err := json.Marshal(mizuApiFilteringOptions)
if err != nil {
return nil, err
}
cpuLimit, err := resource.ParseQuantity("750m")
if err != nil {
return nil, errors.New("invalid cpu limit for aggregator container")
}
memLimit, err := resource.ParseQuantity("512Mi")
if err != nil {
return nil, errors.New("invalid memory limit for aggregator container")
}
cpuRequests, err := resource.ParseQuantity("50m")
if err != nil {
return nil, errors.New("invalid cpu request for aggregator container")
}
memRequests, err := resource.ParseQuantity("50Mi")
if err != nil {
return nil, errors.New("invalid memory request for aggregator container")
}
pod := &core.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: podName,
@@ -117,25 +98,12 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace
Name: shared.MizuFilteringOptionsEnvVar,
Value: string(marshaledFilteringOptions),
},
{
Name: shared.MaxEntriesDBSizeByteSEnvVar,
Value: strconv.FormatInt(maxEntriesDBSizeBytes, 10),
},
},
Resources: core.ResourceRequirements{
Limits: core.ResourceList{
"cpu": cpuLimit,
"memory": memLimit,
},
Requests: core.ResourceList{
"cpu": cpuRequests,
"memory": memRequests,
},
},
},
},
DNSPolicy: core.DNSClusterFirstWithHostNet,
TerminationGracePeriodSeconds: new(int64),
// Affinity: TODO: define node selector for all relevant nodes for this mizu instance
},
}
//define the service account only when it exists to prevent pod crash
@@ -248,117 +216,30 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string,
}
func (provider *Provider) RemovePod(ctx context.Context, namespace string, podName string) error {
if isFound, err := provider.CheckPodExists(ctx, namespace, podName);
err != nil {
return err
} else if !isFound {
return nil
}
return provider.clientSet.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{})
}
func (provider *Provider) RemoveService(ctx context.Context, namespace string, serviceName string) error {
if isFound, err := provider.CheckServiceExists(ctx, namespace, serviceName);
err != nil {
return err
} else if !isFound {
return nil
}
return provider.clientSet.CoreV1().Services(namespace).Delete(ctx, serviceName, metav1.DeleteOptions{})
}
func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, daemonSetName string) error {
if isFound, err := provider.CheckDaemonSetExists(ctx, namespace, daemonSetName);
err != nil {
return err
} else if !isFound {
return nil
}
return provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{})
}
func (provider *Provider) CheckPodExists(ctx context.Context, namespace string, name string) (bool, error) {
listOptions := metav1.ListOptions{
FieldSelector: fmt.Sprintf("metadata.name=%s", name),
Limit: 1,
}
resourceList, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, listOptions)
if err != nil {
return false, err
}
if len(resourceList.Items) > 0 {
return true, nil
}
return false, nil
}
func (provider *Provider) CheckServiceExists(ctx context.Context, namespace string, name string) (bool, error) {
listOptions := metav1.ListOptions{
FieldSelector: fmt.Sprintf("metadata.name=%s", name),
Limit: 1,
}
resourceList, err := provider.clientSet.CoreV1().Services(namespace).List(ctx, listOptions)
if err != nil {
return false, err
}
if len(resourceList.Items) > 0 {
return true, nil
}
return false, nil
}
func (provider *Provider) CheckDaemonSetExists(ctx context.Context, namespace string, name string) (bool, error) {
listOptions := metav1.ListOptions{
FieldSelector: fmt.Sprintf("metadata.name=%s", name),
Limit: 1,
}
resourceList, err := provider.clientSet.AppsV1().DaemonSets(namespace).List(ctx, listOptions)
if err != nil {
return false, err
}
if len(resourceList.Items) > 0 {
return true, nil
}
return false, nil
}
func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, aggregatorPodIp string, nodeToTappedPodIPMap map[string][]string, linkServiceAccount bool, tapOutgoing bool) error {
if len(nodeToTappedPodIPMap) == 0 {
return fmt.Errorf("Daemon set %s must tap at least 1 pod", daemonSetName)
}
func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, aggregatorPodIp string, nodeToTappedPodIPMap map[string][]string, linkServiceAccount bool) error {
nodeToTappedPodIPMapJsonStr, err := json.Marshal(nodeToTappedPodIPMap)
if err != nil {
return err
}
mizuCmd := []string{
"./mizuagent",
"-i", "any",
"--tap",
"--hardump",
"--aggregator-address", fmt.Sprintf("ws://%s/wsTapper", aggregatorPodIp),
}
if tapOutgoing {
mizuCmd = append(mizuCmd, "--anydirection")
}
privileged := true
agentContainer := applyconfcore.Container()
agentContainer.WithName(tapperPodName)
agentContainer.WithImage(podImage)
agentContainer.WithImagePullPolicy(core.PullAlways)
agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(privileged))
agentContainer.WithCommand(mizuCmd...)
agentContainer.WithCommand("./mizuagent", "-i", "any", "--tap", "--hardump", "--aggregator-address", fmt.Sprintf("ws://%s/wsTapper", aggregatorPodIp))
agentContainer.WithEnv(
applyconfcore.EnvVar().WithName(shared.HostModeEnvVar).WithValue("1"),
applyconfcore.EnvVar().WithName(shared.TappedAddressesPerNodeDictEnvVar).WithValue(string(nodeToTappedPodIPMapJsonStr)),
@@ -370,32 +251,6 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
),
),
)
cpuLimit, err := resource.ParseQuantity("500m")
if err != nil {
return errors.New("invalid cpu limit for tapper container")
}
memLimit, err := resource.ParseQuantity("1Gi")
if err != nil {
return errors.New("invalid memory limit for tapper container")
}
cpuRequests, err := resource.ParseQuantity("50m")
if err != nil {
return errors.New("invalid cpu request for tapper container")
}
memRequests, err := resource.ParseQuantity("50Mi")
if err != nil {
return errors.New("invalid memory request for tapper container")
}
agentResourceLimits := core.ResourceList{
"cpu": cpuLimit,
"memory": memLimit,
}
agentResourceRequests := core.ResourceList{
"cpu": cpuRequests,
"memory": memRequests,
}
agentResources := applyconfcore.ResourceRequirements().WithRequests(agentResourceRequests).WithLimits(agentResourceLimits)
agentContainer.WithResources(agentResources)
nodeNames := make([]string, 0, len(nodeToTappedPodIPMap))
for nodeName := range nodeToTappedPodIPMap {
@@ -446,8 +301,8 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
return err
}
func (provider *Provider) GetAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespace string) ([]core.Pod, error) {
pods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
func (provider *Provider) GetAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp) ([]core.Pod, error) {
pods, err := provider.clientSet.CoreV1().Pods(mizu.K8sAllNamespaces).List(ctx, metav1.ListOptions{})
if err != nil {
return nil, err
}

View File

@@ -1,63 +0,0 @@
package kubernetes
import (
"fmt"
"k8s.io/kubectl/pkg/proxy"
"net"
"net/http"
"strings"
"time"
)
const k8sProxyApiPrefix = "/"
const mizuServicePort = 80
func StartProxy(kubernetesProvider *Provider, mizuPort uint16, mizuNamespace string, mizuServiceName string) error {
filter := &proxy.FilterServer{
AcceptPaths: proxy.MakeRegexpArrayOrDie(proxy.DefaultPathAcceptRE),
RejectPaths: proxy.MakeRegexpArrayOrDie(proxy.DefaultPathRejectRE),
AcceptHosts: proxy.MakeRegexpArrayOrDie(proxy.DefaultHostAcceptRE),
RejectMethods: proxy.MakeRegexpArrayOrDie(proxy.DefaultMethodRejectRE),
}
proxyHandler, err := proxy.NewProxyHandler(k8sProxyApiPrefix, filter, &kubernetesProvider.clientConfig, time.Second*2)
if err != nil {
return err
}
mux := http.NewServeMux()
mux.Handle(k8sProxyApiPrefix, proxyHandler)
mux.Handle("/static/", getRerouteHttpHandlerMizuStatic(proxyHandler, mizuNamespace, mizuServiceName))
mux.Handle("/mizu/", getRerouteHttpHandlerMizuAPI(proxyHandler, mizuNamespace, mizuServiceName))
l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", "127.0.0.1", int(mizuPort)))
if err != nil {
return err
}
server := http.Server{
Handler: mux,
}
return server.Serve(l)
}
func getMizuCollectorProxiedHostAndPath(mizuNamespace string, mizuServiceName string) string {
return fmt.Sprintf("/api/v1/namespaces/%s/services/%s:%d/proxy/", mizuNamespace, mizuServiceName, mizuServicePort)
}
func GetMizuCollectorProxiedHostAndPath(mizuPort uint16) string {
return fmt.Sprintf("localhost:%d/mizu", mizuPort)
}
func getRerouteHttpHandlerMizuAPI(proxyHandler http.Handler, mizuNamespace string, mizuServiceName string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r.URL.Path = strings.Replace(r.URL.Path, "/mizu/", getMizuCollectorProxiedHostAndPath(mizuNamespace, mizuServiceName), 1)
proxyHandler.ServeHTTP(w, r)
})
}
func getRerouteHttpHandlerMizuStatic(proxyHandler http.Handler, mizuNamespace string, mizuServiceName string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r.URL.Path = strings.Replace(r.URL.Path, "/static/", fmt.Sprintf("%s/static/", getMizuCollectorProxiedHostAndPath(mizuNamespace, mizuServiceName)), 1)
proxyHandler.ServeHTTP(w, r)
})
}

View File

@@ -15,14 +15,3 @@ const (
TapperPodName = "mizu-tapper"
K8sAllNamespaces = ""
)
const (
Black = "\033[1;30m%s\033[0m"
Red = "\033[1;31m%s\033[0m"
Green = "\033[1;32m%s\033[0m"
Yellow = "\033[1;33m%s\033[0m"
Purple = "\033[1;34m%s\033[0m"
Magenta = "\033[1;35m%s\033[0m"
Teal = "\033[1;36m%s\033[0m"
White = "\033[1;37m%s\033[0m"
)

View File

@@ -1,46 +0,0 @@
package mizu
import (
"encoding/json"
"fmt"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/semver"
"net/http"
"net/url"
)
func getApiVersion(port uint16) (string, error) {
versionUrl, _ := url.Parse(fmt.Sprintf("http://localhost:%d/mizu/metadata/version", port))
req := &http.Request{
Method: http.MethodGet,
URL: versionUrl,
}
statusResp, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer statusResp.Body.Close()
versionResponse := &shared.VersionResponse{}
if err := json.NewDecoder(statusResp.Body).Decode(&versionResponse); err != nil {
return "", err
}
return versionResponse.SemVer, nil
}
func CheckVersionCompatibility(port uint16) (bool, error) {
apiSemVer, err := getApiVersion(port)
if err != nil {
return false, err
}
if semver.SemVersion(apiSemVer).Major() == semver.SemVersion(SemVer).Major() &&
semver.SemVersion(apiSemVer).Minor() == semver.SemVersion(SemVer).Minor() {
return true, nil
}
fmt.Printf(Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)\n", SemVer, apiSemVer))
return false, nil
}

View File

@@ -1,26 +0,0 @@
package uiUtils
import (
"bufio"
"fmt"
"github.com/up9inc/mizu/cli/mizu"
"log"
"os"
"strings"
)
func AskForConfirmation(s string) bool {
reader := bufio.NewReader(os.Stdin)
fmt.Printf(mizu.Magenta, s)
response, err := reader.ReadString('\n')
if err != nil {
log.Fatal(err)
}
response = strings.ToLower(strings.TrimSpace(response))
if response == "" || response == "y" || response == "yes" {
return true
}
return false
}

View File

@@ -1,9 +1,8 @@
package shared
const (
MizuFilteringOptionsEnvVar = "SENSITIVE_DATA_FILTERING_OPTIONS"
HostModeEnvVar = "HOST_MODE"
NodeNameEnvVar = "NODE_NAME"
MizuFilteringOptionsEnvVar = "SENSITIVE_DATA_FILTERING_OPTIONS"
HostModeEnvVar = "HOST_MODE"
NodeNameEnvVar = "NODE_NAME"
TappedAddressesPerNodeDictEnvVar = "TAPPED_ADDRESSES_PER_HOST"
MaxEntriesDBSizeByteSEnvVar = "MAX_ENTRIES_DB_BYTES"
)

View File

@@ -2,8 +2,4 @@ module github.com/up9inc/mizu/shared
go 1.16
require (
github.com/gorilla/websocket v1.4.2
github.com/docker/go-units v0.4.0
)
require github.com/gorilla/websocket v1.4.2

View File

@@ -1,4 +1,2 @@
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=

View File

@@ -1,42 +1,28 @@
package shared
type WebSocketMessageType string
const (
WebSocketMessageTypeEntry WebSocketMessageType = "entry"
WebSocketMessageTypeTappedEntry WebSocketMessageType = "tappedEntry"
WebSocketMessageTypeUpdateStatus WebSocketMessageType = "status"
WebSocketMessageTypeAnalyzeStatus WebSocketMessageType = "analyzeStatus"
WebSocketMessageTypeEntry WebSocketMessageType = "entry"
WebSocketMessageTypeTappedEntry WebSocketMessageType = "tappedEntry"
WebSocketMessageTypeUpdateStatus WebSocketMessageType = "status"
)
type WebSocketMessageMetadata struct {
MessageType WebSocketMessageType `json:"messageType,omitempty"`
}
type WebSocketAnalyzeStatusMessage struct {
*WebSocketMessageMetadata
AnalyzeStatus AnalyzeStatus `json:"analyzeStatus"`
}
type AnalyzeStatus struct {
IsAnalyzing bool `json:"isAnalyzing"`
RemoteUrl string `json:"remoteUrl"`
IsRemoteReady bool `json:"isRemoteReady"`
SentCount int `json:"sentCount"`
}
type WebSocketStatusMessage struct {
*WebSocketMessageMetadata
TappingStatus TapStatus `json:"tappingStatus"`
}
type TapStatus struct {
Pods []PodInfo `json:"pods"`
Pods []PodInfo `json:"pods"`
}
type PodInfo struct {
Namespace string `json:"namespace"`
Name string `json:"name"`
Name string `json:"name"`
}
func CreateWebSocketStatusMessage(tappingStatus TapStatus) WebSocketStatusMessage {
@@ -48,21 +34,6 @@ func CreateWebSocketStatusMessage(tappingStatus TapStatus) WebSocketStatusMessag
}
}
func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSocketAnalyzeStatusMessage {
return WebSocketAnalyzeStatusMessage{
WebSocketMessageMetadata: &WebSocketMessageMetadata{
MessageType: WebSocketMessageTypeAnalyzeStatus,
},
AnalyzeStatus: analyzeStatus,
}
}
type TrafficFilteringOptions struct {
PlainTextMaskingRegexes []*SerializableRegexp
HideHealthChecks bool
}
type VersionResponse struct {
SemVer string `json:"semver"`
}

View File

@@ -1,28 +0,0 @@
package semver
import (
"regexp"
)
type SemVersion string
func (v SemVersion) Breakdown() (string, string, string) {
re := regexp.MustCompile(`\d+`)
breakdown := re.FindAllString(string(v), 3)
return breakdown[0], breakdown[1], breakdown[2]
}
func (v SemVersion) Major() string {
major, _, _ := v.Breakdown()
return major
}
func (v SemVersion) Minor() string {
_, minor, _ := v.Breakdown()
return minor
}
func (v SemVersion) Patch() string {
_, _, patch := v.Breakdown()
return patch
}

View File

@@ -7,7 +7,7 @@ import (
)
const (
DEFAULT_SOCKET_RETRIES = 3
DEFAULT_SOCKET_RETRIES = 3
DEFAULT_SOCKET_RETRY_SLEEP_TIME = time.Second * 10
)

View File

@@ -1,11 +0,0 @@
package units
import "github.com/docker/go-units"
func BytesToHumanReadable(bytes int64) string {
return units.HumanSize(float64(bytes))
}
func HumanReadableToBytes(humanReadableSize string) (int64, error) {
return units.FromHumanSize(humanReadableSize)
}

View File

@@ -1,11 +0,0 @@
module github.com/up9inc/mizu/tap
go 1.16
require (
github.com/google/gopacket v1.1.19
github.com/google/martian v2.1.0+incompatible
github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
golang.org/x/net v0.0.0-20210421230115-4e50805a0758
)

View File

@@ -1,29 +0,0 @@
github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1pzKW+1SsBnJDOHppJN9stOEwS+CRWyUtyYGU=
github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI=
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210421230115-4e50805a0758 h1:aEpZnXcAmXkd6AvLb2OPt+EN1Zu/8Ne3pCqPjja5PXY=
golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe h1:WdX7u8s3yOigWAhHEaDl8r9G+4XwFQEQFtBMYyN+kXQ=
golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

View File

@@ -1,122 +0,0 @@
package tap
import (
"fmt"
"net/http"
"strings"
"time"
"github.com/orcaman/concurrent-map"
)
type requestResponsePair struct {
Request httpMessage `json:"request"`
Response httpMessage `json:"response"`
}
type httpMessage struct {
isRequest bool
captureTime time.Time
orig interface{}
}
// Key is {client_addr}:{client_port}->{dest_addr}:{dest_port}
type requestResponseMatcher struct {
openMessagesMap cmap.ConcurrentMap
}
func createResponseRequestMatcher() requestResponseMatcher {
newMatcher := &requestResponseMatcher{openMessagesMap: cmap.New()}
return *newMatcher
}
func (matcher *requestResponseMatcher) registerRequest(ident string, request *http.Request, captureTime time.Time) *requestResponsePair {
split := splitIdent(ident)
key := genKey(split)
requestHTTPMessage := httpMessage{
isRequest: true,
captureTime: captureTime,
orig: request,
}
if response, found := matcher.openMessagesMap.Pop(key); found {
// Type assertion always succeeds because all of the map's values are of httpMessage type
responseHTTPMessage := response.(*httpMessage)
if responseHTTPMessage.isRequest {
SilentError("Request-Duplicate", "Got duplicate request with same identifier")
return nil
}
Trace("Matched open Response for %s", key)
return matcher.preparePair(&requestHTTPMessage, responseHTTPMessage)
}
matcher.openMessagesMap.Set(key, &requestHTTPMessage)
Trace("Registered open Request for %s", key)
return nil
}
func (matcher *requestResponseMatcher) registerResponse(ident string, response *http.Response, captureTime time.Time) *requestResponsePair {
split := splitIdent(ident)
key := genKey(split)
responseHTTPMessage := httpMessage{
isRequest: false,
captureTime: captureTime,
orig: response,
}
if request, found := matcher.openMessagesMap.Pop(key); found {
// Type assertion always succeeds because all of the map's values are of httpMessage type
requestHTTPMessage := request.(*httpMessage)
if !requestHTTPMessage.isRequest {
SilentError("Response-Duplicate", "Got duplicate response with same identifier")
return nil
}
Trace("Matched open Request for %s", key)
return matcher.preparePair(requestHTTPMessage, &responseHTTPMessage)
}
matcher.openMessagesMap.Set(key, &responseHTTPMessage)
Trace("Registered open Response for %s", key)
return nil
}
func (matcher *requestResponseMatcher) preparePair(requestHTTPMessage *httpMessage, responseHTTPMessage *httpMessage) *requestResponsePair {
return &requestResponsePair{
Request: *requestHTTPMessage,
Response: *responseHTTPMessage,
}
}
func splitIdent(ident string) []string {
ident = strings.Replace(ident, "->", " ", -1)
return strings.Split(ident, " ")
}
func genKey(split []string) string {
key := fmt.Sprintf("%s:%s->%s:%s,%s", split[0], split[2], split[1], split[3], split[4])
return key
}
func (matcher *requestResponseMatcher) deleteOlderThan(t time.Time) int {
keysToPop := make([]string, 0)
for item := range matcher.openMessagesMap.IterBuffered() {
// Map only contains values of type httpMessage
message, _ := item.Val.(*httpMessage)
if message.captureTime.Before(t) {
keysToPop = append(keysToPop, item.Key)
}
}
numDeleted := len(keysToPop)
for _, key := range keysToPop {
_, _ = matcher.openMessagesMap.Pop(key)
}
return numDeleted
}

View File

@@ -1,29 +0,0 @@
package tap
type OutboundLink struct {
Src string
DstIP string
DstPort int
}
func NewOutboundLinkWriter() *OutboundLinkWriter {
return &OutboundLinkWriter{
OutChan: make(chan *OutboundLink),
}
}
type OutboundLinkWriter struct {
OutChan chan *OutboundLink
}
func (olw *OutboundLinkWriter) WriteOutboundLink(src string, DstIP string, DstPort int) {
olw.OutChan <- &OutboundLink{
Src: src,
DstIP: DstIP,
DstPort: DstPort,
}
}
func (olw *OutboundLinkWriter) Stop() {
close(olw.OutChan)
}

View File

@@ -1,31 +0,0 @@
package tap
type globalSettings struct {
filterPorts []int
filterAuthorities []string
}
var gSettings = &globalSettings{
filterPorts: []int{},
filterAuthorities: []string{},
}
func SetFilterPorts(ports []int) {
gSettings.filterPorts = ports
}
func GetFilterPorts() []int {
ports := make([]int, len(gSettings.filterPorts))
copy(ports, gSettings.filterPorts)
return ports
}
func SetFilterAuthorities(ipAddresses []string) {
gSettings.filterAuthorities = ipAddresses
}
func GetFilterIPs() []string {
addresses := make([]string, len(gSettings.filterAuthorities))
copy(addresses, gSettings.filterAuthorities)
return addresses
}

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 11 KiB

BIN
ui/public/favicon.ico Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.8 KiB

View File

@@ -2,7 +2,7 @@
<html lang="en">
<head>
<meta charset="utf-8" />
<link rel="icon" href="%PUBLIC_URL%/fav.svg" />
<link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="theme-color" content="#000000" />
<meta

View File

@@ -6,20 +6,21 @@
width: 100%
.header
height: 60px
height: 80px
display: flex
align-items: center
padding: 5px 24px
justify-content: space-between
padding-left: 24px
.title
font-size: 45px
letter-spacing: 2px
img
height: 45px
height: 40px
.description
margin-left: 10px
font-size: 11px
padding-top: 10px
font-size: 14px
font-weight: bold
color: $light-blue-color

View File

@@ -1,96 +1,18 @@
import React, {useState} from 'react';
import './App.sass';
import logo from './components/assets/Mizu-logo.svg';
import {Button} from "@material-ui/core";
import React from 'react';
import {HarPage} from "./components/HarPage";
import Tooltip from "./components/Tooltip";
import {makeStyles} from "@material-ui/core/styles";
const useStyles = makeStyles(() => ({
tooltip: {
backgroundColor: "#3868dc",
color: "white",
fontSize: 13,
},
}));
import './App.sass';
import logo from './components/assets/Mizu.svg';
const App = () => {
const classes = useStyles();
const [analyzeStatus, setAnalyzeStatus] = useState(null);
const analysisMessage = analyzeStatus?.isRemoteReady ?
<span>
<table>
<tr>
<td>Status</td>
<td><b>Available</b></td>
</tr>
<tr>
<td>Messages</td>
<td><b>{analyzeStatus?.sentCount}</b></td>
</tr>
</table>
</span> :
analyzeStatus?.sentCount > 0 ?
<span>
<table>
<tr>
<td>Status</td>
<td><b>Processing</b></td>
</tr>
<tr>
<td>Messages</td>
<td><b>{analyzeStatus?.sentCount}</b></td>
</tr>
<tr>
<td colSpan={2}> Please allow a few minutes for the analysis to complete</td>
</tr>
</table>
</span> :
<span>
<table>
<tr>
<td>Status</td>
<td><b>Waiting for traffic</b></td>
</tr>
<tr>
<td>Messages</td>
<td><b>{analyzeStatus?.sentCount}</b></td>
</tr>
</table>
</span>
return (
<div className="mizuApp">
<div className="header">
<div style={{display: "flex", alignItems: "center"}}>
<div className="title"><img src={logo} alt="logo"/></div>
<div className="description">Traffic viewer for Kubernetes</div>
</div>
{analyzeStatus?.isAnalyzing &&
<Tooltip title={analysisMessage} isSimple classes={classes}>
<div>
<Button
variant="contained"
color="primary"
disabled={!analyzeStatus?.isRemoteReady}
onClick={() => {
window.open(analyzeStatus?.remoteUrl)
}}>
Analysis
</Button>
</div>
</Tooltip>
}
</div>
<HarPage setAnalyzeStatus={setAnalyzeStatus}/>
return (
<div className="mizuApp">
<div className="header">
<div className="title"><img src={logo} alt="logo"/></div>
<div className="description">Traffic viewer for Kubernetes</div>
</div>
);
<HarPage/>
</div>
);
}
export default App;

View File

@@ -1,13 +1,7 @@
import React from "react";
import styles from './style/HarEntry.module.sass';
import StatusCode, {getClassification, StatusCodeClassification} from "./StatusCode";
import StatusCode from "./StatusCode";
import {EndpointPath} from "./EndpointPath";
import ingoingIconSuccess from "./assets/ingoing-traffic-success.svg"
import ingoingIconFailure from "./assets/ingoing-traffic-failure.svg"
import ingoingIconNeutral from "./assets/ingoing-traffic-neutral.svg"
import outgoingIconSuccess from "./assets/outgoing-traffic-success.svg"
import outgoingIconFailure from "./assets/outgoing-traffic-failure.svg"
import outgoingIconNeutral from "./assets/outgoing-traffic-neutral.svg"
interface HAREntry {
method?: string,
@@ -18,7 +12,6 @@ interface HAREntry {
url?: string;
isCurrentRevision?: boolean;
timestamp: Date;
isOutgoing?: boolean;
}
interface HAREntryProps {
@@ -28,26 +21,6 @@ interface HAREntryProps {
}
export const HarEntry: React.FC<HAREntryProps> = ({entry, setFocusedEntryId, isSelected}) => {
const classification = getClassification(entry.statusCode)
let ingoingIcon;
let outgoingIcon;
switch(classification) {
case StatusCodeClassification.SUCCESS: {
ingoingIcon = ingoingIconSuccess;
outgoingIcon = outgoingIconSuccess;
break;
}
case StatusCodeClassification.FAILURE: {
ingoingIcon = ingoingIconFailure;
outgoingIcon = outgoingIconFailure;
break;
}
case StatusCodeClassification.NEUTRAL: {
ingoingIcon = ingoingIconNeutral;
outgoingIcon = outgoingIconNeutral;
break;
}
}
return <>
<div id={entry.id} className={`${styles.row} ${isSelected ? styles.rowSelected : ''}`} onClick={() => setFocusedEntryId(entry.id)}>
@@ -60,14 +33,7 @@ export const HarEntry: React.FC<HAREntryProps> = ({entry, setFocusedEntryId, isS
{entry.service}
</div>
</div>
<div className={styles.directionContainer}>
{entry.isOutgoing ?
<img src={outgoingIcon} alt="outgoing traffic" title="outgoing"/>
:
<img src={ingoingIcon} alt="ingoing traffic" title="ingoing"/>
}
</div>
<div className={styles.timestamp}>{new Date(+entry.timestamp)?.toLocaleString()}</div>
</div>
</>
};
};

View File

@@ -5,7 +5,7 @@ import {makeStyles} from "@material-ui/core";
import "./style/HarPage.sass";
import styles from './style/HarEntriesList.module.sass';
import {HAREntryDetailed} from "./HarEntryDetailed";
import playIcon from './assets/run.svg';
import playIcon from './assets/play.svg';
import pauseIcon from './assets/pause.svg';
import variables from './style/variables.module.scss';
import {StatusBar} from "./StatusBar";
@@ -35,27 +35,7 @@ enum ConnectionStatus {
Paused
}
interface HarPageProps {
setAnalyzeStatus: (status: any) => void;
}
const mizuAPIPathPrefix = "/mizu";
// When working locally (with npm run start) we need to change the PORT
const getMizuApiUrl = () => {
return `${window.location.origin}${mizuAPIPathPrefix}`;
};
const getMizuWebsocketUrl = () => {
return `ws://${window.location.host}${mizuAPIPathPrefix}/ws`;
}
const mizuApiUrl = getMizuApiUrl();
const mizuWebsocketUrl = getMizuWebsocketUrl();
export const HarPage: React.FC<HarPageProps> = ({setAnalyzeStatus}) => {
export const HarPage: React.FC = () => {
const classes = useLayoutStyles();
@@ -75,26 +55,26 @@ export const HarPage: React.FC<HarPageProps> = ({setAnalyzeStatus}) => {
const ws = useRef(null);
const openWebSocket = () => {
ws.current = new WebSocket(mizuWebsocketUrl);
ws.current = new WebSocket("ws://localhost:8899/ws");
ws.current.onopen = () => setConnection(ConnectionStatus.Connected);
ws.current.onclose = () => setConnection(ConnectionStatus.Closed);
}
if (ws.current) {
if(ws.current) {
ws.current.onmessage = e => {
if (!e?.data) return;
if(!e?.data) return;
const message = JSON.parse(e.data);
switch (message.messageType) {
case "entry":
const entry = message.data
if (connection === ConnectionStatus.Paused) {
if(connection === ConnectionStatus.Paused) {
setNoMoreDataBottom(false)
return;
}
if (!focusedEntryId) setFocusedEntryId(entry.id)
if(!focusedEntryId) setFocusedEntryId(entry.id)
let newEntries = [...entries];
if (entries.length === 1000) {
if(entries.length === 1000) {
newEntries = newEntries.splice(1);
setNoMoreDataTop(false);
}
@@ -103,9 +83,6 @@ export const HarPage: React.FC<HarPageProps> = ({setAnalyzeStatus}) => {
case "status":
setTappingStatus(message.tappingStatus);
break
case "analyzeStatus":
setAnalyzeStatus(message.analyzeStatus);
break
default:
console.error(`unsupported websocket message type, Got: ${message.messageType}`)
}
@@ -114,27 +91,22 @@ export const HarPage: React.FC<HarPageProps> = ({setAnalyzeStatus}) => {
useEffect(() => {
openWebSocket();
fetch(`${mizuApiUrl}/api/tapStatus`)
fetch(`http://localhost:8899/api/tapStatus`)
.then(response => response.json())
.then(data => setTappingStatus(data));
fetch(`${mizuApiUrl}/api/analyzeStatus`)
.then(response => response.json())
.then(data => setAnalyzeStatus(data));
// eslint-disable-next-line
}, []);
useEffect(() => {
if (!focusedEntryId) return;
if(!focusedEntryId) return;
setSelectedHarEntry(null)
fetch(`${mizuApiUrl}/api/entries/${focusedEntryId}`)
fetch(`http://localhost:8899/api/entries/${focusedEntryId}`)
.then(response => response.json())
.then(data => setSelectedHarEntry(data));
}, [focusedEntryId])
},[focusedEntryId])
const toggleConnection = () => {
setConnection(connection === ConnectionStatus.Connected ? ConnectionStatus.Paused : ConnectionStatus.Connected);
setConnection(connection === ConnectionStatus.Connected ? ConnectionStatus.Paused : ConnectionStatus.Connected );
}
const getConnectionStatusClass = (isContainer) => {
@@ -163,12 +135,11 @@ export const HarPage: React.FC<HarPageProps> = ({setAnalyzeStatus}) => {
return (
<div className="HarPage">
<div className="harPageHeader">
<img style={{cursor: "pointer", marginRight: 15, height: 30}} alt="pause"
src={connection === ConnectionStatus.Connected ? pauseIcon : playIcon} onClick={toggleConnection}/>
<img style={{cursor: "pointer", marginRight: 15, height: 30}} alt="pause" src={connection === ConnectionStatus.Connected ? pauseIcon : playIcon} onClick={toggleConnection}/>
<div className="connectionText">
{getConnectionTitle()}
<div className={"indicatorContainer " + getConnectionStatusClass(true)}>
<div className={"indicator " + getConnectionStatusClass(false)}/>
<div className={"indicator " + getConnectionStatusClass(false)} />
</div>
</div>
</div>
@@ -198,8 +169,7 @@ export const HarPage: React.FC<HarPageProps> = ({setAnalyzeStatus}) => {
</div>
</div>
<div className={classes.details}>
{selectedHarEntry &&
<HAREntryDetailed harEntry={selectedHarEntry} classes={{root: classes.harViewer}}/>}
{selectedHarEntry && <HAREntryDetailed harEntry={selectedHarEntry} classes={{root: classes.harViewer}}/>}
</div>
</div>}
{tappingStatus?.pods != null && <StatusBar tappingStatus={tappingStatus}/>}

View File

@@ -1,7 +1,7 @@
import React from "react";
import styles from './style/StatusCode.module.sass';
export enum StatusCodeClassification {
enum StatusCodeClassification {
SUCCESS = "success",
FAILURE = "failure",
NEUTRAL = "neutral"
@@ -14,12 +14,6 @@ interface HAREntryProps {
const StatusCode: React.FC<HAREntryProps> = ({statusCode}) => {
const classification = getClassification(statusCode)
return <span className={`${styles[classification]} ${styles.base}`}>{statusCode}</span>
};
export function getClassification(statusCode: number): string {
let classification = StatusCodeClassification.NEUTRAL;
if (statusCode >= 200 && statusCode <= 399) {
@@ -28,7 +22,7 @@ export function getClassification(statusCode: number): string {
classification = StatusCodeClassification.FAILURE;
}
return classification
}
return <span className={`${styles[classification]} ${styles.base}`}>{statusCode}</span>
};
export default StatusCode;
export default StatusCode;

View File

@@ -4,7 +4,6 @@ import React from "react";
export interface TooltipProps extends MUITooltipProps {
variant?: 'default' | 'wide' | 'fit';
isSimple?: boolean;
classes?: any;
}
export type TooltipPlacement = 'bottom-end' | 'bottom-start' | 'bottom' | 'left-end' | 'left-start' | 'left' | 'right-end' | 'right-start' | 'right' | 'top-end' | 'top-start' | 'top';

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 44 KiB

View File

@@ -1,5 +0,0 @@
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M16.5175 11.1465C16.8392 10.8869 17 10.4434 17 10C17 9.55657 16.8392 9.11314 16.5175 8.85348L12.5425 5.64459C13.2682 5.23422 14.1067 5 15 5C17.7614 5 20 7.23858 20 10C20 12.7614 17.7614 15 15 15C14.1067 15 13.2682 14.7658 12.5425 14.3554L16.5175 11.1465Z" fill="#BCCEFD"/>
<path d="M16 10C16 10.3167 15.8749 10.6335 15.6247 10.8189L10.1706 14.8624C9.65543 15.2444 9 14.7858 9 14.0435V5.95652C9 5.21417 9.65543 4.75564 10.1706 5.13758L15.6247 9.18106C15.8749 9.36653 16 9.68326 16 10Z" fill="#EB5757"/>
<path d="M0 10C0 8.89543 0.895431 8 2 8H10C11.1046 8 12 8.89543 12 10C12 11.1046 11.1046 12 10 12H2C0.895431 12 0 11.1046 0 10Z" fill="#EB5757"/>
</svg>

Before

Width:  |  Height:  |  Size: 800 B

View File

@@ -1,5 +0,0 @@
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M16.5175 11.1465C16.8392 10.8869 17 10.4434 17 10C17 9.55657 16.8392 9.11314 16.5175 8.85348L12.5425 5.64459C13.2682 5.23422 14.1067 5 15 5C17.7614 5 20 7.23858 20 10C20 12.7614 17.7614 15 15 15C14.1067 15 13.2682 14.7658 12.5425 14.3554L16.5175 11.1465Z" fill="#BCCEFD"/>
<path d="M16 10C16 10.3167 15.8749 10.6335 15.6247 10.8189L10.1706 14.8624C9.65543 15.2444 9 14.7858 9 14.0435V5.95652C9 5.21417 9.65543 4.75564 10.1706 5.13758L15.6247 9.18106C15.8749 9.36653 16 9.68326 16 10Z" fill="gray"/>
<path d="M0 10C0 8.89543 0.895431 8 2 8H10C11.1046 8 12 8.89543 12 10C12 11.1046 11.1046 12 10 12H2C0.895431 12 0 11.1046 0 10Z" fill="gray"/>
</svg>

Before

Width:  |  Height:  |  Size: 794 B

View File

@@ -1,5 +0,0 @@
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M16.5175 11.1465C16.8392 10.8869 17 10.4434 17 10C17 9.55657 16.8392 9.11314 16.5175 8.85348L12.5425 5.64459C13.2682 5.23422 14.1067 5 15 5C17.7614 5 20 7.23858 20 10C20 12.7614 17.7614 15 15 15C14.1067 15 13.2682 14.7658 12.5425 14.3554L16.5175 11.1465Z" fill="#BCCEFD"/>
<path d="M16 10C16 10.3167 15.8749 10.6335 15.6247 10.8189L10.1706 14.8624C9.65543 15.2444 9 14.7858 9 14.0435V5.95652C9 5.21417 9.65543 4.75564 10.1706 5.13758L15.6247 9.18106C15.8749 9.36653 16 9.68326 16 10Z" fill="#27AE60"/>
<path d="M0 10C0 8.89543 0.895431 8 2 8H10C11.1046 8 12 8.89543 12 10C12 11.1046 11.1046 12 10 12H2C0.895431 12 0 11.1046 0 10Z" fill="#27AE60"/>
</svg>

Before

Width:  |  Height:  |  Size: 800 B

View File

@@ -1,5 +0,0 @@
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M15 15C17.7614 15 20 12.7615 20 10C20 7.23861 17.7614 5.00003 15 5.00003C13.3642 5.00003 11.9118 5.78558 10.9996 7.00003H14C15.6569 7.00003 17 8.34318 17 10C17 11.6569 15.6569 13 14 13H10.9996C11.9118 14.2145 13.3642 15 15 15Z" fill="#BCCEFD"/>
<rect x="4" y="8.00003" width="12" height="4" rx="2" fill="#EB5757"/>
<path d="M5.96244e-08 10C6.34015e-08 9.68329 0.125088 9.36656 0.375266 9.18109L5.82939 5.13761C6.34457 4.75567 7 5.2142 7 5.95655L7 14.0435C7 14.7859 6.34457 15.2444 5.82939 14.8625L0.375266 10.819C0.125088 10.6335 5.58474e-08 10.3168 5.96244e-08 10Z" fill="#EB5757"/>
</svg>

Before

Width:  |  Height:  |  Size: 736 B

View File

@@ -1,5 +0,0 @@
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M15 15C17.7614 15 20 12.7615 20 10C20 7.23861 17.7614 5.00003 15 5.00003C13.3642 5.00003 11.9118 5.78558 10.9996 7.00003H14C15.6569 7.00003 17 8.34318 17 10C17 11.6569 15.6569 13 14 13H10.9996C11.9118 14.2145 13.3642 15 15 15Z" fill="#BCCEFD"/>
<rect x="4" y="8.00003" width="12" height="4" rx="2" fill="gray"/>
<path d="M5.96244e-08 10C6.34015e-08 9.68329 0.125088 9.36656 0.375266 9.18109L5.82939 5.13761C6.34457 4.75567 7 5.2142 7 5.95655L7 14.0435C7 14.7859 6.34457 15.2444 5.82939 14.8625L0.375266 10.819C0.125088 10.6335 5.58474e-08 10.3168 5.96244e-08 10Z" fill="gray"/>
</svg>

Before

Width:  |  Height:  |  Size: 730 B

View File

@@ -1,5 +0,0 @@
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M15 15C17.7614 15 20 12.7615 20 10C20 7.23861 17.7614 5.00003 15 5.00003C13.3642 5.00003 11.9118 5.78558 10.9996 7.00003H14C15.6569 7.00003 17 8.34318 17 10C17 11.6569 15.6569 13 14 13H10.9996C11.9118 14.2145 13.3642 15 15 15Z" fill="#BCCEFD"/>
<rect x="4" y="8.00003" width="12" height="4" rx="2" fill="#27AE60"/>
<path d="M5.96244e-08 10C6.34015e-08 9.68329 0.125088 9.36656 0.375266 9.18109L5.82939 5.13761C6.34457 4.75567 7 5.2142 7 5.95655L7 14.0435C7 14.7859 6.34457 15.2444 5.82939 14.8625L0.375266 10.819C0.125088 10.6335 5.58474e-08 10.3168 5.96244e-08 10Z" fill="#27AE60"/>
</svg>

Before

Width:  |  Height:  |  Size: 736 B

View File

@@ -0,0 +1,4 @@
<svg width="30" height="30" viewBox="0 0 30 30" fill="none" xmlns="http://www.w3.org/2000/svg">
<rect width="30" height="30" rx="15" fill="#205CF5"/>
<path d="M17.0747 15L12.9876 12.6433V17.3567L17.0747 15ZM20 15C20 15.3167 19.8392 15.6335 19.5175 15.8189L12.5051 19.8624C11.8427 20.2444 11 19.7858 11 19.0435V10.9565C11 10.2142 11.8427 9.75564 12.5051 10.1376L19.5175 14.1811C19.8392 14.3665 20 14.6833 20 15Z" fill="white"/>
</svg>

After

Width:  |  Height:  |  Size: 434 B

View File

@@ -1,4 +0,0 @@
<svg width="30" height="30" viewBox="0 0 30 30" fill="none" xmlns="http://www.w3.org/2000/svg">
<circle cx="15" cy="15" r="13.5" stroke="#205CF5" stroke-width="3"/>
<path d="M20 15C20 15.3167 19.8392 15.6335 19.5175 15.8189L12.5051 19.8624C11.8427 20.2444 11 19.7858 11 19.0435V10.9565C11 10.2142 11.8427 9.75564 12.5051 10.1376L19.5175 14.1811C19.8392 14.3665 20 14.6833 20 15Z" fill="#205CF5"/>
</svg>

Before

Width:  |  Height:  |  Size: 404 B

View File

@@ -37,10 +37,9 @@
.timestamp
font-size: 12px
color: $secondary-font-color
padding-left: 12px
padding-left: 8px
padding-right: 8px
flex-shrink: 0
width: 145px
text-align: left
.endpointServiceContainer
display: flex
@@ -48,10 +47,4 @@
overflow: hidden
padding-right: 10px
padding-left: 10px
flex-grow: 1
.directionContainer
display: flex
border-right: 1px solid $data-background-color
padding: 4px
padding-right: 12px
flex-grow: 1

View File

@@ -6,13 +6,7 @@
flex-direction: column
overflow: hidden
flex-grow: 1
height: calc(100vh - 70px)
.harPageHeader
padding: 20px 24px
display: flex
align-items: center
background-color: $header-background-color
height: calc(100vh - 80px)
.harPageHeader
padding: 20px 24px

View File

@@ -19,4 +19,4 @@ $blue-gray: #494677;
successColor: $success-color;
failureColor: $failure-color;
blueGray: $blue-gray;
}
}

View File

@@ -14,7 +14,6 @@ body
-moz-osx-font-smoothing: grayscale
margin: 0
padding: 0
overflow: hidden
code
font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New", monospace