Compare commits

...

51 Commits

Author SHA1 Message Date
gadotroee
8d8310ee02 Revert "feature/TRA_3427_demo_mode (#150)" (#151)
This reverts commit 71eff5ea04.
2021-07-29 21:06:41 +03:00
RoyUP9
0824524d62 Add telemetry to config (#152) 2021-07-29 11:02:09 +03:00
Selton Fiuza
71eff5ea04 feature/TRA_3427_demo_mode (#150)
* Demo Mode MVP

* messages improve

* downloading based on the OS

* downloading based on the OS

* downloading based on the OS

* Modeler keep running

* A lot of revisions comes now

* Fix color
2021-07-28 14:50:15 -03:00
Alon Girmonsky
50e404f51e Create mizu-ui.png (#140)
* Create mizu-ui.png
2021-07-27 19:59:24 +03:00
Igor Gov
ffa34039b1 Fix fetch telemetry report (#145) 2021-07-27 15:10:22 +03:00
nimrod-up9
d888706e1e Show that the default answer to analysis prompt is yes. (#144) 2021-07-27 14:52:43 +03:00
nimrod-up9
1ef17542dd Format socket ID as int in logs. (#143) 2021-07-27 14:49:08 +03:00
Igor Gov
0566f63d72 Remove redundant '\n' at the end of each log (#142) 2021-07-27 13:10:40 +03:00
gadotroee
6d49339e29 Fix log to debug (#139) 2021-07-26 14:04:48 +03:00
gadotroee
58f0de4d4e WriteFile fix(#138) 2021-07-26 12:00:36 +03:00
gadotroee
f175480f65 Adding (basic) configuration (#135) 2021-07-26 11:23:35 +03:00
RamiBerm
6dd2bf705b TRA-3437 switch fiber and ikisocket with gin-gonic and gorilla websocket (#136)
* WIP

* WIP

* WIP

* Update socket_server_handlers.go and socket_routes.go

* Fix stuck sockets

* Update go.mod, go.sum, and 5 more files...

* Update socket_routes.go

* Update Dockerfile, go.sum, and fiber_middleware.go

* fix analyze

Co-authored-by: RamiBerm <rami.berman@up9.com>
2021-07-25 13:08:29 +03:00
Igor Gov
f64ee23c74 Introducing new logger, logging debug to file and info to stderr (#134)
* Introducing new logger to file debug and info to stderr
2021-07-25 10:08:37 +03:00
nimrod-up9
803681a239 Renamed collector, aggregator to api server, api folder to agent (#133)
* Renamed aggregator -> apiServer.

* Format errors with container names.

* Renamed collector -> apiServer.

* Rephrased help messages.

* Moved api -> agent.

* Continue renameing api -> agent in Makefile and Dockerfiles.
2021-07-22 17:17:17 +03:00
lirazyehezkel
a2150b4a78 UI api helper (#131)
* Api helper

* cr fixes
2021-07-22 15:41:09 +03:00
Igor Gov
ac358be877 Making kube config errors more user friendly (#132)
* Making kube config errors more user friendly
2021-07-22 15:26:28 +03:00
nimrod-up9
2996c1a4bc Install Mizu in a dedicated namespace (#123)
* Use "mizu" namespace instead of "default". Create and delete as necessary.

* Wait until namespace is deleted.

* Distinguish between timeout and other errors.

* Sorted consts.

* k8s provider gets the names of Mizu serviceaccount, clusterrole and clusterrolebindings from caller.

* Renames.

* Remove non-namespaced mizu resources when finished: clusterrole and clusterrolebindings.

* Don't wait for namespace deletion if it was already deleted.

* When watching pods, check for cancellation before reading from channels.

* Allow user to cancel resource deletion and to cancel the wait.

* Increased cleanup timeout.

* go mod tidy.

* Ignore cli build products.

* Print err.

* Don't delete clusterrole and clusterrolebinding if we do not have permissions.

* Added roles list in README.

* Added clusterrole and clusterrolebindings examples.
2021-07-22 14:26:12 +03:00
Igor Gov
e42c4f8648 Cleaning unused code (#129)
* Cleaning unused code
2021-07-21 17:07:32 +03:00
Igor Gov
7d5ed601df Check if newer version exists in Github (#128)
* Check if newer version exists in Github
2021-07-21 16:36:47 +03:00
Igor Gov
30651c0f75 Merge pull request #127 from up9inc/version_cli_bin
Adding version as part of cli/bin
2021-07-21 11:25:48 +03:00
Igor Gov
594f1b973a . 2021-07-21 11:23:49 +03:00
RamiBerm
77ced2a46d Merge pull request #126 from up9inc/disable_redact_flag
Add option to disable sensitive data redaction by flag and fix CPU leaks
2021-07-21 11:20:36 +03:00
Igor Gov
b195ed9905 Adding version as part of cli/bin 2021-07-21 11:18:37 +03:00
Igor Gov
59ef3a4244 Merge pull request #125 from up9inc/version_artifcat
Add version artifact
2021-07-21 11:12:32 +03:00
Igor Gov
360a4ea562 . 2021-07-21 11:06:42 +03:00
Igor Gov
e883358cd6 . 2021-07-21 11:05:20 +03:00
RamiBerm
efb1a0b58b Update main.go 2021-07-21 11:04:12 +03:00
Igor Gov
eb67f76e2b . 2021-07-21 11:03:30 +03:00
RamiBerm
5b9c134ab2 Update go.mod and go.sum 2021-07-21 10:59:14 +03:00
RamiBerm
8db12a4b1a Update go.mod 2021-07-21 10:53:25 +03:00
Igor Gov
7fb85df3ac Add version artifact 2021-07-21 10:52:37 +03:00
RamiBerm
edc3d04d59 Update go.mod and go.sum 2021-07-21 10:52:14 +03:00
RamiBerm
679bf35ce3 Update go.mod 2021-07-21 10:51:32 +03:00
RamiBerm
17e1ccf9ef Update size_enforcer.go 2021-07-21 10:49:33 +03:00
RamiBerm
80e97e7f7e Update main.go, tap.go, and 4 more files... 2021-07-21 10:49:27 +03:00
Igor Gov
5c86e20c92 Merge pull request #124 from up9inc/mizu_telemetry
Adding telemetry reports
2021-07-20 16:58:50 +03:00
Igor Gov
4a030c02f7 revert 2021-07-20 16:57:38 +03:00
Igor Gov
ea5054866d . 2021-07-20 16:55:32 +03:00
Igor Gov
a11e8f730e . 2021-07-20 16:54:51 +03:00
Igor Gov
1e66ebd8b3 . 2021-07-20 16:51:52 +03:00
Igor Gov
3be0c9ecd9 Adding telemetry reports 2021-07-20 16:47:20 +03:00
RamiBerm
ef8314b554 Update debug.Dockerfile 2021-07-20 09:58:21 +03:00
RamiBerm
b57cb0e615 Update main.go 2021-07-20 09:58:18 +03:00
RamiBerm
1d24188a02 Update tapRunner.go 2021-07-20 09:58:08 +03:00
RamiBerm
9d9f64098e Update go.mod and go.sum 2021-07-19 17:39:59 +03:00
nimrod-up9
fbf3d1729e Missing request body (#120)
* Never use harRequest.PostData.Params. Always use harRequest.PostData.Text.

* Comment.
2021-07-19 12:38:28 +03:00
RamiBerm
eb02ecda20 Update main.go 2021-07-19 09:08:21 +03:00
Igor Gov
dc62195a8f Merge pull request #118 from ksudhir007/update-readme
Update readme, add prerequisites section
2021-07-15 20:20:28 +03:00
Sudhir Kasanavesi
38b58dba69 Update readme, add prerequisites section 2021-07-15 09:17:58 -07:00
Igor Gov
765feafbcc Merge pull request #114 from ksudhir007/fix-kubeconfig
Support KUBECONFIG environment variable.
2021-07-15 18:09:40 +03:00
Sudhir Kasanavesi
cef0e01cf6 Support KUBECONFIG environment variable.
If KUBECONFIG environment variable is set, use it. Otherwise default to ~/.kube/config
2021-07-14 17:42:46 -07:00
85 changed files with 1693 additions and 21835 deletions

View File

@@ -66,6 +66,9 @@ jobs:
COMMIT_HASH=${{ github.sha }}
- name: Build and Push CLI
run: make push-cli SEM_VER='${{ steps.versioning.outputs.version }}' BUILD_TIMESTAMP='${{ steps.version_parameters.outputs.build_timestamp }}'
- shell: bash
run: |
echo '${{ steps.versioning.outputs.version }}' >> cli/bin/version.txt
- name: publish
uses: ncipollo/release-action@v1
with:
@@ -75,3 +78,4 @@ jobs:
tag: ${{ steps.versioning.outputs.version }}
prerelease: ${{ github.ref != 'refs/heads/main' }}
bodyFile: 'cli/bin/README.md'

View File

@@ -13,10 +13,10 @@ ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
RUN apk add libpcap-dev gcc g++ make
# Move to api working directory (/api-build).
WORKDIR /app/api-build
# Move to agent working directory (/agent-build).
WORKDIR /app/agent-build
COPY api/go.mod api/go.sum ./
COPY agent/go.mod agent/go.sum ./
COPY shared/go.mod shared/go.mod ../shared/
COPY tap/go.mod tap/go.mod ../tap/
RUN go mod download
@@ -28,10 +28,10 @@ ARG GIT_BRANCH
ARG BUILD_TIMESTAMP
ARG SEM_VER
# Copy and build api code
# Copy and build agent code
COPY shared ../shared
COPY tap ../tap
COPY api .
COPY agent .
RUN go build -ldflags="-s -w \
-X 'mizuserver/pkg/version.GitCommitHash=${COMMIT_HASH}' \
-X 'mizuserver/pkg/version.Branch=${GIT_BRANCH}' \
@@ -45,10 +45,13 @@ RUN apk add bash libpcap-dev tcpdump
WORKDIR /app
# Copy binary and config files from /build to root folder of scratch container.
COPY --from=builder ["/app/api-build/mizuagent", "."]
COPY --from=builder ["/app/agent-build/mizuagent", "."]
COPY --from=site-build ["/app/ui-build/build", "site"]
COPY api/start.sh .
COPY agent/start.sh .
# gin-gonic runs in debug mode without this
ENV GIN_MODE=release
# this script runs both apiserver and passivetapper and exits either if one of them exits, preventing a scenario where the container runs without one process
ENTRYPOINT "/app/mizuagent"

View File

@@ -8,7 +8,7 @@ SHELL=/bin/bash
# HELP
# This will output the help for each task
# thanks to https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.PHONY: help ui api cli tap docker
.PHONY: help ui agent cli tap docker
help: ## This help.
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -27,10 +27,10 @@ ui: ## build UI
cli: # build CLI
@echo "building cli"; cd cli && $(MAKE) build
api: ## build API server
@(echo "building API server .." )
@(cd api; go build -o build/apiserver main.go)
@ls -l api/build
agent: ## build mizuagent server
@(echo "building mizu agent .." )
@(cd agent; go build -o build/mizuagent main.go)
@ls -l agent/build
#tap: ## build tap binary
# @(cd tap; go build -o build/tap ./src)
@@ -55,13 +55,13 @@ push-cli:
gsutil setmeta -r -h "Cache-Control:public, max-age=30" gs://${BUCKET_PATH}/\*
clean: clean-ui clean-api clean-cli clean-docker ## Clean all build artifacts
clean: clean-ui clean-agent clean-cli clean-docker ## Clean all build artifacts
clean-ui:
@(rm -rf ui/build ; echo "UI cleanup done" )
clean-api:
@(rm -rf api/build ; echo "api cleanup done" )
clean-agent:
@(rm -rf agent/build ; echo "agent cleanup done" )
clean-cli:
@(cd cli; make clean ; echo "CLI cleanup done" )

133
README.md
View File

@@ -1,6 +1,17 @@
# 水 mizu
![Mizu: The API Traffic Viewer for Kubernetes](assets/mizu-logo.svg)
# The API Traffic Viewer for Kubernetes
A simple-yet-powerful API traffic viewer for Kubernetes to help you troubleshoot and debug your microservices. Think TCPDump and Chrome Dev Tools combined.
![Simple UI](assets/mizu-ui.png)
## Features
- Simple and powerful CLI
- Real time view of all HTTP requests, REST and gRPC API calls
- No installation or code instrumentation
- Works completely on premises (on-prem)
## Download
Download `mizu` for your platform and operating system
@@ -26,12 +37,126 @@ SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/
### Development (unstable) build
Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page.
## Prerequisites
1. Set `KUBECONFIG` environment variable to your kubernetes configuration. If this is not set, mizu assumes that configuration is at `${HOME}/.kube/config`
2. mizu needs following permissions on your kubernetes cluster to run
```yaml
- apiGroups:
- ""
resources:
- pods
verbs:
- list
- watch
- create
- apiGroups:
- ""
resources:
- services
verbs:
- create
- apiGroups:
- apps
resources:
- daemonsets
verbs:
- create
- patch
- apiGroups:
- ""
resources:
- namespaces
verbs:
- list
- watch
- create
- delete
- apiGroups:
- ""
resources:
- services/proxy
verbs:
- get
```
3. Optionally, for resolving traffic ip to kubernetes service name, mizu needs below permissions
```yaml
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- apps
- extensions
resources:
- pods
verbs:
- get
- list
- watch
- apiGroups:
- apps
- extensions
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- ""
- apps
- extensions
resources:
- endpoints
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- serviceaccounts
verbs:
- get
- create
- apiGroups:
- rbac.authorization.k8s.io
resources:
- clusterroles
verbs:
- list
- create
- delete
- apiGroups:
- rbac.authorization.k8s.io
resources:
- clusterrolebindings
verbs:
- list
- create
- delete
```
See `examples/roles` for example `clusterroles`.
## How to run
1. Find pod you'd like to tap to in your Kubernetes cluster
1. Find pods you'd like to tap to in your Kubernetes cluster
2. Run `mizu tap PODNAME` or `mizu tap REGEX`
3. Open browser on `http://localhost:8899` as instructed ..
4. Watch the WebAPI traffic flowing ..
3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI ..
4. Watch the API traffic flowing ..
5. Type ^C to stop
## Examples

View File

@@ -1,5 +1,5 @@
# mizu API server
API server for MIZU
# mizu agent
Agent for MIZU (API server and tapper)
Basic APIs:
* /fetch - retrieve traffic data
* /stats - retrieve statistics of collected data
@@ -14,7 +14,7 @@ Basic APIs:
### Connecting
1. Start mizu using the cli with the debug image `mizu tap --mizu-image gcr.io/up9-docker-hub/mizu/debug:latest {tapped_pod_name}`
2. Forward the debug port using `kubectl port-forward -n default mizu-collector 2345:2345`
2. Forward the debug port using `kubectl port-forward -n default mizu-api-server 2345:2345`
3. Run the run/debug configuration you've created earlier in Intellij.
<small>Do note that dlv won't start the api until a debugger connects to it.</small>

View File

@@ -3,17 +3,16 @@ module mizuserver
go 1.16
require (
github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a
github.com/beevik/etree v1.1.0
github.com/djherbis/atime v1.0.0
github.com/fasthttp/websocket v1.4.3-beta.1 // indirect
github.com/fsnotify/fsnotify v1.4.9
github.com/gin-contrib/static v0.0.1
github.com/gin-gonic/gin v1.7.2
github.com/go-playground/locales v0.13.0
github.com/go-playground/universal-translator v0.17.0
github.com/go-playground/validator/v10 v10.5.0
github.com/gofiber/fiber/v2 v2.8.0
github.com/google/martian v2.1.0+incompatible
github.com/gorilla/websocket v1.4.2
github.com/leodido/go-urn v1.2.1 // indirect
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
github.com/up9inc/mizu/shared v0.0.0
github.com/up9inc/mizu/tap v0.0.0
@@ -23,7 +22,6 @@ require (
k8s.io/api v0.21.0
k8s.io/apimachinery v0.21.0
k8s.io/client-go v0.21.0
github.com/fsnotify/fsnotify v1.4.9
)
replace github.com/up9inc/mizu/shared v0.0.0 => ../shared

View File

@@ -41,11 +41,6 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/andybalholm/brotli v1.0.1 h1:KqhlKozYbRtJvsPrrEeXcO+N2l6NYT5A2QAFmSULpEc=
github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a h1:76llBleIE3fkdqaJFDzdirtiYhQPdIQem8H8r2iwA1Q=
github.com/antoniodipinto/ikisocket v0.0.0-20210417133349-f1502512d69a/go.mod h1:QvDfsDQDmGxUsvEeWabVZ5pp2FMXpOkwQV0L6SE6cp0=
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48=
github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
@@ -69,14 +64,18 @@ github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/fasthttp/websocket v1.4.2/go.mod h1:smsv/h4PBEBaU0XDTY5UwJTpZv69fQ0FfcLJr21mA6Y=
github.com/fasthttp/websocket v1.4.3-beta.1 h1:stc4P2aoxYKsdmbe1AJ5mAm73Fxc1NOgrZpPftvZIXQ=
github.com/fasthttp/websocket v1.4.3-beta.1/go.mod h1:JGrgLaT02bL9NuJkZbHN8mVV2tkCJZQh7yJ5/XCXO2g=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-contrib/static v0.0.1 h1:JVxuvHPuUfkoul12N7dtQw7KRn/pSMq7Ue1Va9Swm1U=
github.com/gin-contrib/static v0.0.1/go.mod h1:CSxeF+wep05e0kCOsqWdAWbSszmc31zTIbD8TvWl7Hs=
github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
github.com/gin-gonic/gin v1.7.2 h1:Tg03T9yM2xa8j6I3Z3oqLaQRSmKvxPd6g/2HJ6zICFA=
github.com/gin-gonic/gin v1.7.2/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -96,6 +95,8 @@ github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8c
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-playground/validator/v10 v10.5.0 h1:X9rflw/KmpACwT8zdrm1upefpvdy6ur8d1kWyq6sg3E=
github.com/go-playground/validator/v10 v10.5.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
@@ -124,12 +125,6 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe
github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
github.com/gofiber/fiber/v2 v2.1.3/go.mod h1:MMiSv1HrDkN8Pv7NeVDYK+T/lwXOEKAvPBbLvJPCEfA=
github.com/gofiber/fiber/v2 v2.7.1/go.mod h1:f8BRRIMjMdRyt2qmJ/0Sea3j3rwwfufPrh9WNBRiVZ0=
github.com/gofiber/fiber/v2 v2.8.0 h1:BdWvZmg/WY/Vjtjm38aXOp1Lks1BhuyS2b7lSWSPAzk=
github.com/gofiber/fiber/v2 v2.8.0/go.mod h1:Ah3IJikrKNRepl/HuVawppS25X7FWohwfCSRn7kJG28=
github.com/gofiber/websocket/v2 v2.0.3 h1:nqPGHB4LQhxKX5KJUjayOd2xiiENieS/dn6TPfCL8uk=
github.com/gofiber/websocket/v2 v2.0.3/go.mod h1:/OTEImCxORKE5unw0dWqJYovid6vZF+wB1W0aaMKs2M=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -190,7 +185,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
@@ -202,6 +196,7 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -210,13 +205,7 @@ github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaR
github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4=
github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -226,13 +215,14 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ=
github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -255,7 +245,6 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1pzKW+1SsBnJDOHppJN9stOEwS+CRWyUtyYGU=
github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -269,9 +258,6 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI=
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0=
github.com/savsgio/gotils v0.0.0-20200117113501-90175b0fbe3f/go.mod h1:lHhJedqxCoHN+zMtwGNTXWmF0u9Jt363FYRhV6g0CdY=
github.com/savsgio/gotils v0.0.0-20200616100644-13ff1fd2c28c h1:KKqhycXW1WVNkX7r4ekTV2gFkbhdyihlWD8c0/FiWmk=
github.com/savsgio/gotils v0.0.0-20200616100644-13ff1fd2c28c/go.mod h1:TWNAOTaVzGOXq8RbEvHnhzA/A2sLZzgn0m6URjnukY8=
github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@@ -283,25 +269,17 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
github.com/valyala/fasthttp v1.15.1/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
github.com/valyala/fasthttp v1.18.0/go.mod h1:jjraHZVbKOXftJfsOYoAjaeygpj5hr8ermTRJNroD7A=
github.com/valyala/fasthttp v1.23.0 h1:0ufwSD9BhWa6f8HWdmdq4FHQ23peRo3Ng/Qs8m5NcFs=
github.com/valyala/fasthttp v1.23.0/go.mod h1:0mw2RjXGOzxf4NL2jni3gUQ7LfjjUSiG5sskOUUSEpU=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
@@ -377,11 +355,8 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226101413-39120d07d75e/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210421230115-4e50805a0758 h1:aEpZnXcAmXkd6AvLb2OPt+EN1Zu/8Ne3pCqPjja5PXY=
golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -420,17 +395,15 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201210223839-7e3030f88018/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe h1:WdX7u8s3yOigWAhHEaDl8r9G+4XwFQEQFtBMYyN+kXQ=
golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

View File

@@ -4,34 +4,34 @@ import (
"encoding/json"
"flag"
"fmt"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/cors"
"github.com/gin-contrib/static"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"
"mizuserver/pkg/api"
"mizuserver/pkg/middleware"
"mizuserver/pkg/models"
"mizuserver/pkg/routes"
"mizuserver/pkg/sensitiveDataFiltering"
"mizuserver/pkg/utils"
"net/http"
"os"
"os/signal"
"strings"
)
var shouldTap = flag.Bool("tap", false, "Run in tapper mode without API")
var aggregator = flag.Bool("aggregator", false, "Run in aggregator mode with API")
var apiServer = flag.Bool("api-server", false, "Run in API server mode with API")
var standalone = flag.Bool("standalone", false, "Run in standalone tapper and API mode")
var aggregatorAddress = flag.String("aggregator-address", "", "Address of mizu collector for tapping")
var apiServerAddress = flag.String("api-server-address", "", "Address of mizu API server")
func main() {
flag.Parse()
hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
tapOpts := &tap.TapOpts{HostMode: hostMode}
if !*shouldTap && !*aggregator && !*standalone {
if !*shouldTap && !*apiServer && !*standalone {
panic("One of the flags --tap, --api or --standalone must be provided")
}
@@ -45,8 +45,8 @@ func main() {
hostApi(nil)
} else if *shouldTap {
if *aggregatorAddress == "" {
panic("Aggregator address must be provided with --aggregator-address when using --tap")
if *apiServerAddress == "" {
panic("API server address must be provided with --api-server-address when using --tap")
}
tapTargets := getTapTargets()
@@ -57,14 +57,14 @@ func main() {
harOutputChannel, outboundLinkOutputChannel := tap.StartPassiveTapper(tapOpts)
socketConnection, err := shared.ConnectToSocketServer(*aggregatorAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false)
socketConnection, err := shared.ConnectToSocketServer(*apiServerAddress, shared.DEFAULT_SOCKET_RETRIES, shared.DEFAULT_SOCKET_RETRY_SLEEP_TIME, false)
if err != nil {
panic(fmt.Sprintf("Error connecting to socket server at %s %v", *aggregatorAddress, err))
panic(fmt.Sprintf("Error connecting to socket server at %s %v", *apiServerAddress, err))
}
go pipeChannelToSocket(socketConnection, harOutputChannel)
go api.StartReadingOutbound(outboundLinkOutputChannel)
} else if *aggregator {
} else if *apiServer {
socketHarOutChannel := make(chan *tap.OutputChannelItem, 1000)
filteredHarChannel := make(chan *tap.OutputChannelItem)
@@ -82,23 +82,19 @@ func main() {
}
func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) {
app := fiber.New()
app := gin.Default()
app.Use(cors.New(cors.Config{
AllowOrigins: "*",
AllowMethods: "*",
AllowHeaders: "*",
}))
middleware.FiberMiddleware(app) // Register Fiber's middleware for app.
app.Static("/", "./site")
//Simple route to know server is running
app.Get("/echo", func(c *fiber.Ctx) error {
return c.SendString("Hello, World 👋!")
app.GET("/echo", func(c *gin.Context) {
c.String(http.StatusOK, "Hello, World 👋!")
})
eventHandlers := api.RoutesEventHandlers{
SocketHarOutChannel: socketHarOutputChannel,
}
app.Use(static.ServeRoot("/", "./site"))
app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
routes.WebSocketRoutes(app, &eventHandlers)
routes.EntriesRoutes(app)
routes.MetadataRoutes(app)
@@ -107,6 +103,22 @@ func hostApi(socketHarOutputChannel chan<- *tap.OutputChannelItem) {
utils.StartServer(app)
}
func CORSMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT")
if c.Request.Method == "OPTIONS" {
c.AbortWithStatus(204)
return
}
c.Next()
}
}
func getTapTargets() []string {
nodeName := os.Getenv(shared.NodeNameEnvVar)
var tappedAddressesPerNodeDict map[string][]string
@@ -143,7 +155,9 @@ func filterHarItems(inChannel <-chan *tap.OutputChannelItem, outChannel chan *ta
continue
}
sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions)
if !filterOptions.DisableRedaction {
sensitiveDataFiltering.FilterSensitiveInfoFromHarRequest(message, filterOptions)
}
outChannel <- message
}

View File

@@ -0,0 +1,100 @@
package api
import (
"encoding/json"
"fmt"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"
"mizuserver/pkg/controllers"
"mizuserver/pkg/models"
"mizuserver/pkg/routes"
"mizuserver/pkg/up9"
"sync"
)
var browserClientSocketUUIDs = make([]int, 0)
var socketListLock = sync.Mutex{}
type RoutesEventHandlers struct {
routes.EventHandlers
SocketHarOutChannel chan<- *tap.OutputChannelItem
}
func init() {
go up9.UpdateAnalyzeStatus(broadcastToBrowserClients)
}
func (h *RoutesEventHandlers) WebSocketConnect(socketId int, isTapper bool) {
if isTapper {
rlog.Infof("Websocket event - Tapper connected, socket ID: %d", socketId)
} else {
rlog.Infof("Websocket event - Browser socket connected, socket ID: %d", socketId)
socketListLock.Lock()
browserClientSocketUUIDs = append(browserClientSocketUUIDs, socketId)
socketListLock.Unlock()
}
}
func (h *RoutesEventHandlers) WebSocketDisconnect(socketId int, isTapper bool) {
if isTapper {
rlog.Infof("Websocket event - Tapper disconnected, socket ID: %d", socketId)
} else {
rlog.Infof("Websocket event - Browser socket disconnected, socket ID: %d", socketId)
socketListLock.Lock()
removeSocketUUIDFromBrowserSlice(socketId)
socketListLock.Unlock()
}
}
func broadcastToBrowserClients(message []byte) {
for _, socketId := range browserClientSocketUUIDs {
go func(socketId int) {
err := routes.SendToSocket(socketId, message)
if err != nil {
fmt.Printf("error sending message to socket ID %d: %v", socketId, err)
}
}(socketId)
}
}
func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
var socketMessageBase shared.WebSocketMessageMetadata
err := json.Unmarshal(message, &socketMessageBase)
if err != nil {
rlog.Infof("Could not unmarshal websocket message %v\n", err)
} else {
switch socketMessageBase.MessageType {
case shared.WebSocketMessageTypeTappedEntry:
var tappedEntryMessage models.WebSocketTappedEntryMessage
err := json.Unmarshal(message, &tappedEntryMessage)
if err != nil {
rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
} else {
h.SocketHarOutChannel <- tappedEntryMessage.Data
}
case shared.WebSocketMessageTypeUpdateStatus:
var statusMessage shared.WebSocketStatusMessage
err := json.Unmarshal(message, &statusMessage)
if err != nil {
rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
} else {
controllers.TapStatus = statusMessage.TappingStatus
broadcastToBrowserClients(message)
}
default:
rlog.Infof("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
}
}
}
func removeSocketUUIDFromBrowserSlice(uuidToRemove int) {
newUUIDSlice := make([]int, 0, len(browserClientSocketUUIDs))
for _, uuid := range browserClientSocketUUIDs {
if uuid != uuidToRemove {
newUUIDSlice = append(newUUIDSlice, uuid)
}
}
browserClientSocketUUIDs = newUUIDSlice
}

View File

@@ -3,7 +3,7 @@ package controllers
import (
"encoding/json"
"fmt"
"github.com/gofiber/fiber/v2"
"github.com/gin-gonic/gin"
"github.com/google/martian/har"
"github.com/romana/rlog"
"mizuserver/pkg/database"
@@ -11,19 +11,20 @@ import (
"mizuserver/pkg/up9"
"mizuserver/pkg/utils"
"mizuserver/pkg/validation"
"net/http"
"strings"
"time"
)
func GetEntries(c *fiber.Ctx) error {
func GetEntries(c *gin.Context) {
entriesFilter := &models.EntriesFilter{}
if err := c.QueryParser(entriesFilter); err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
if err := c.BindQuery(entriesFilter); err != nil {
c.JSON(http.StatusBadRequest, err)
}
err := validation.Validate(entriesFilter)
if err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
c.JSON(http.StatusBadRequest, err)
}
order := database.OperatorToOrderMapping[entriesFilter.Operator]
@@ -50,18 +51,18 @@ func GetEntries(c *fiber.Ctx) error {
baseEntries = append(baseEntries, harEntry)
}
return c.Status(fiber.StatusOK).JSON(baseEntries)
c.JSON(http.StatusOK, baseEntries)
}
func GetHARs(c *fiber.Ctx) error {
func GetHARs(c *gin.Context) {
entriesFilter := &models.HarFetchRequestBody{}
order := database.OrderDesc
if err := c.QueryParser(entriesFilter); err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
if err := c.BindQuery(entriesFilter); err != nil {
c.JSON(http.StatusBadRequest, err)
}
err := validation.Validate(entriesFilter)
if err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
c.JSON(http.StatusBadRequest, err)
}
var timestampFrom, timestampTo int64
@@ -137,40 +138,45 @@ func GetHARs(c *fiber.Ctx) error {
retObj[k] = bytesData
}
buffer := utils.ZipData(retObj)
return c.Status(fiber.StatusOK).SendStream(buffer)
c.Data(http.StatusOK, "application/octet-stream", buffer.Bytes())
}
func UploadEntries(c *fiber.Ctx) error {
func UploadEntries(c *gin.Context) {
rlog.Infof("Upload entries - started\n")
uploadRequestBody := &models.UploadEntriesRequestBody{}
if err := c.QueryParser(uploadRequestBody); err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
if err := c.BindQuery(uploadRequestBody); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
if err := validation.Validate(uploadRequestBody); err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
c.JSON(http.StatusBadRequest, err)
return
}
if up9.GetAnalyzeInfo().IsAnalyzing {
return c.Status(fiber.StatusBadRequest).SendString("Cannot analyze, mizu is already analyzing")
c.String(http.StatusBadRequest, "Cannot analyze, mizu is already analyzing")
return
}
rlog.Infof("Upload entries - creating token. dest %s\n", uploadRequestBody.Dest)
token, err := up9.CreateAnonymousToken(uploadRequestBody.Dest)
if err != nil {
return c.Status(fiber.StatusServiceUnavailable).SendString("Can't get token")
c.String(http.StatusServiceUnavailable, "Cannot analyze, mizu is already analyzing")
return
}
rlog.Infof("Upload entries - uploading. token: %s model: %s\n", token.Token, token.Model)
go up9.UploadEntriesImpl(token.Token, token.Model, uploadRequestBody.Dest, uploadRequestBody.SleepIntervalSec)
return c.Status(fiber.StatusOK).SendString("OK")
c.String(http.StatusOK, "OK")
}
func GetFullEntries(c *fiber.Ctx) error {
func GetFullEntries(c *gin.Context) {
entriesFilter := &models.HarFetchRequestBody{}
if err := c.QueryParser(entriesFilter); err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
if err := c.BindQuery(entriesFilter); err != nil {
c.JSON(http.StatusBadRequest, err)
}
err := validation.Validate(entriesFilter)
if err != nil {
return c.Status(fiber.StatusBadRequest).JSON(err)
c.JSON(http.StatusBadRequest, err)
}
var timestampFrom, timestampTo int64
@@ -195,38 +201,37 @@ func GetFullEntries(c *fiber.Ctx) error {
}
result = append(result, harEntry)
}
return c.Status(fiber.StatusOK).JSON(result)
c.JSON(http.StatusOK, result)
}
func GetEntry(c *fiber.Ctx) error {
func GetEntry(c *gin.Context) {
var entryData models.MizuEntry
database.GetEntriesTable().
Where(map[string]string{"entryId": c.Params("entryId")}).
Where(map[string]string{"entryId": c.Param("entryId")}).
First(&entryData)
fullEntry := models.FullEntryDetails{}
if err := models.GetEntry(&entryData, &fullEntry); err != nil {
return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
c.JSON(http.StatusInternalServerError, map[string]interface{}{
"error": true,
"msg": "Can't get entry details",
})
}
return c.Status(fiber.StatusOK).JSON(fullEntry)
c.JSON(http.StatusOK, fullEntry)
}
func DeleteAllEntries(c *fiber.Ctx) error {
func DeleteAllEntries(c *gin.Context) {
database.GetEntriesTable().
Where("1 = 1").
Delete(&models.MizuEntry{})
return c.Status(fiber.StatusOK).JSON(fiber.Map{
c.JSON(http.StatusOK, map[string]string{
"msg": "Success",
})
}
func GetGeneralStats(c *fiber.Ctx) error {
func GetGeneralStats(c *gin.Context) {
sqlQuery := "SELECT count(*) as count, min(timestamp) as min, max(timestamp) as max from mizu_entries"
var result struct {
Count int
@@ -234,5 +239,5 @@ func GetGeneralStats(c *fiber.Ctx) error {
Max int
}
database.GetEntriesTable().Raw(sqlQuery).Scan(&result)
return c.Status(fiber.StatusOK).JSON(&result)
c.JSON(http.StatusOK, result)
}

View File

@@ -1,12 +1,13 @@
package controllers
import (
"github.com/gofiber/fiber/v2"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/shared"
"mizuserver/pkg/version"
"net/http"
)
func GetVersion(c *fiber.Ctx) error {
func GetVersion(c *gin.Context) {
resp := shared.VersionResponse{SemVer: version.SemVer}
return c.Status(fiber.StatusOK).JSON(resp)
c.JSON(http.StatusOK, resp)
}

View File

@@ -0,0 +1,12 @@
package controllers
import (
"github.com/gin-gonic/gin"
"mizuserver/pkg/holder"
"net/http"
)
func GetCurrentResolvingInformation(c *gin.Context) {
c.JSON(http.StatusOK, holder.GetResolver().GetMap())
}

View File

@@ -0,0 +1,18 @@
package controllers
import (
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/shared"
"mizuserver/pkg/up9"
"net/http"
)
var TapStatus shared.TapStatus
func GetTappingStatus(c *gin.Context) {
c.JSON(http.StatusOK, TapStatus)
}
func AnalyzeInformation(c *gin.Context) {
c.JSON(http.StatusOK, up9.GetAnalyzeInfo())
}

View File

@@ -62,7 +62,7 @@ func getMaxEntriesDBByteSize() (int64, error) {
maxEntriesDBByteSize := defaultMaxDatabaseSizeBytes
var err error
maxEntriesDBSizeByteSEnvVarValue := os.Getenv(shared.MaxEntriesDBSizeByteSEnvVar)
maxEntriesDBSizeByteSEnvVarValue := os.Getenv(shared.MaxEntriesDBSizeBytesEnvVar)
if maxEntriesDBSizeByteSEnvVarValue != "" {
maxEntriesDBByteSize, err = strconv.ParseInt(maxEntriesDBSizeByteSEnvVarValue, 10, 64)
}

View File

@@ -113,8 +113,8 @@ type EntriesFilter struct {
}
type UploadEntriesRequestBody struct {
Dest string `query:"dest"`
SleepIntervalSec int `query:"interval"`
Dest string `form:"dest"`
SleepIntervalSec int `form:"interval"`
}
type HarFetchRequestBody struct {

View File

@@ -0,0 +1,22 @@
package resolver
import (
"k8s.io/client-go/kubernetes"
_ "k8s.io/client-go/plugin/pkg/client/auth/azure"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
restclient "k8s.io/client-go/rest"
)
func NewFromInCluster(errOut chan error) (*Resolver, error) {
config, err := restclient.InClusterConfig()
if err != nil {
return nil, err
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, err
}
return &Resolver{clientConfig: config, clientSet: clientset, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}, nil
}

View File

@@ -0,0 +1,25 @@
package routes
import (
"github.com/gin-gonic/gin"
"mizuserver/pkg/controllers"
)
// EntriesRoutes defines the group of har entries routes.
func EntriesRoutes(ginApp *gin.Engine) {
routeGroup := ginApp.Group("/api")
routeGroup.GET("/entries", controllers.GetEntries) // get entries (base/thin entries)
routeGroup.GET("/entries/:entryId", controllers.GetEntry) // get single (full) entry
routeGroup.GET("/exportEntries", controllers.GetFullEntries)
routeGroup.GET("/uploadEntries", controllers.UploadEntries)
routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation)
routeGroup.GET("/har", controllers.GetHARs)
routeGroup.GET("/resetDB", controllers.DeleteAllEntries) // get single (full) entry
routeGroup.GET("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB
routeGroup.GET("/tapStatus", controllers.GetTappingStatus) // get tapping status
routeGroup.GET("/analyzeStatus", controllers.AnalyzeInformation)
}

View File

@@ -0,0 +1,13 @@
package routes
import (
"github.com/gin-gonic/gin"
"mizuserver/pkg/controllers"
)
// MetadataRoutes defines the group of metadata routes.
func MetadataRoutes(app *gin.Engine) {
routeGroup := app.Group("/metadata")
routeGroup.GET("/version", controllers.GetVersion)
}

View File

@@ -0,0 +1,18 @@
package routes
import (
"github.com/gin-gonic/gin"
"net/http"
)
// NotFoundRoute defines the 404 Error route.
func NotFoundRoute(app *gin.Engine) {
app.Use(
func(c *gin.Context) {
c.JSON(http.StatusNotFound, map[string]interface{}{
"error": true,
"msg": "sorry, endpoint is not found",
})
},
)
}

View File

@@ -0,0 +1,118 @@
package routes
import (
"errors"
"fmt"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
"github.com/up9inc/mizu/shared/debounce"
"net/http"
"sync"
"time"
)
type EventHandlers interface {
WebSocketConnect(socketId int, isTapper bool)
WebSocketDisconnect(socketId int, isTapper bool)
WebSocketMessage(socketId int, message []byte)
}
type SocketConnection struct {
connection *websocket.Conn
lock *sync.Mutex
eventHandlers EventHandlers
isTapper bool
}
var websocketUpgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
}
var websocketIdsLock = sync.Mutex{}
var connectedWebsockets map[int]*SocketConnection
var connectedWebsocketIdCounter = 0
func init() {
websocketUpgrader.CheckOrigin = func(r *http.Request) bool { return true } // like cors for web socket
connectedWebsockets = make(map[int]*SocketConnection, 0)
}
func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
app.GET("/ws", func(c *gin.Context) {
websocketHandler(c.Writer, c.Request, eventHandlers, false)
})
app.GET("/wsTapper", func(c *gin.Context) {
websocketHandler(c.Writer, c.Request, eventHandlers, true)
})
}
func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool) {
conn, err := websocketUpgrader.Upgrade(w, r, nil)
if err != nil {
fmt.Println("Failed to set websocket upgrade: %+v", err)
return
}
websocketIdsLock.Lock()
connectedWebsocketIdCounter++
socketId := connectedWebsocketIdCounter
connectedWebsockets[socketId] = &SocketConnection{connection: conn, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper}
websocketIdsLock.Unlock()
defer func() {
socketCleanup(socketId, connectedWebsockets[socketId])
}()
eventHandlers.WebSocketConnect(socketId, isTapper)
for {
_, msg, err := conn.ReadMessage()
if err != nil {
fmt.Printf("Conn err: %v\n", err)
break
}
eventHandlers.WebSocketMessage(socketId, msg)
}
}
func socketCleanup(socketId int, socketConnection *SocketConnection) {
err := socketConnection.connection.Close()
if err != nil {
fmt.Printf("Error closing socket connection for socket id %d: %v\n", socketId, err)
}
websocketIdsLock.Lock()
connectedWebsockets[socketId] = nil
websocketIdsLock.Unlock()
socketConnection.eventHandlers.WebSocketDisconnect(socketId, socketConnection.isTapper)
}
var db = debounce.NewDebouncer(time.Second * 5, func() {
fmt.Println("Successfully sent to socket")
})
func SendToSocket(socketId int, message []byte) error {
socketObj := connectedWebsockets[socketId]
if socketObj == nil {
return errors.New("Socket is disconnected")
}
var sent = false
time.AfterFunc(time.Second * 5, func() {
if !sent {
fmt.Println("Socket timed out")
socketCleanup(socketId, socketObj)
}
})
socketObj.lock.Lock() // gorilla socket panics from concurrent writes to a single socket
err := socketObj.connection.WriteMessage(1, message)
socketObj.lock.Unlock()
sent = true
return err
}

View File

@@ -1,32 +1,42 @@
package utils
import (
"github.com/gofiber/fiber/v2"
"context"
"github.com/gin-gonic/gin"
"github.com/romana/rlog"
"log"
"net/http"
"net/url"
"os"
"os/signal"
"reflect"
"syscall"
"time"
)
// StartServer starts the server with a graceful shutdown
func StartServer(app *fiber.App) {
func StartServer(app *gin.Engine) {
signals := make(chan os.Signal, 2)
signal.Notify(signals,
os.Interrupt, // this catch ctrl + c
syscall.SIGTSTP, // this catch ctrl + z
)
srv := &http.Server{
Addr: ":8080",
Handler: app,
}
go func() {
_ = <-signals
rlog.Infof("Shutting down...")
_ = app.Shutdown()
ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
_ = srv.Shutdown(ctx)
os.Exit(0)
}()
// Run server.
if err := app.Listen(":8899"); err != nil {
if err := app.Run(":8899"); err != nil {
log.Printf("Oops... Server is not running! Reason: %v", err)
}
}

View File

@@ -1,99 +0,0 @@
package api
import (
"encoding/json"
"github.com/antoniodipinto/ikisocket"
"github.com/romana/rlog"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"
"mizuserver/pkg/controllers"
"mizuserver/pkg/models"
"mizuserver/pkg/routes"
"mizuserver/pkg/up9"
)
var browserClientSocketUUIDs = make([]string, 0)
type RoutesEventHandlers struct {
routes.EventHandlers
SocketHarOutChannel chan<- *tap.OutputChannelItem
}
func init() {
go up9.UpdateAnalyzeStatus(broadcastToBrowserClients)
}
func (h *RoutesEventHandlers) WebSocketConnect(ep *ikisocket.EventPayload) {
if ep.Kws.GetAttribute("is_tapper") == true {
rlog.Infof("Websocket Connection event - Tapper connected: %s", ep.SocketUUID)
} else {
rlog.Infof("Websocket Connection event - Browser socket connected: %s", ep.SocketUUID)
browserClientSocketUUIDs = append(browserClientSocketUUIDs, ep.SocketUUID)
}
}
func (h *RoutesEventHandlers) WebSocketDisconnect(ep *ikisocket.EventPayload) {
if ep.Kws.GetAttribute("is_tapper") == true {
rlog.Infof("Disconnection event - Tapper connected: %s", ep.SocketUUID)
} else {
rlog.Infof("Disconnection event - Browser socket connected: %s", ep.SocketUUID)
removeSocketUUIDFromBrowserSlice(ep.SocketUUID)
}
}
func broadcastToBrowserClients(message []byte) {
ikisocket.EmitToList(browserClientSocketUUIDs, message)
}
func (h *RoutesEventHandlers) WebSocketClose(ep *ikisocket.EventPayload) {
if ep.Kws.GetAttribute("is_tapper") == true {
rlog.Infof("Websocket Close event - Tapper connected: %s", ep.SocketUUID)
} else {
rlog.Infof("Websocket Close event - Browser socket connected: %s", ep.SocketUUID)
removeSocketUUIDFromBrowserSlice(ep.SocketUUID)
}
}
func (h *RoutesEventHandlers) WebSocketError(ep *ikisocket.EventPayload) {
rlog.Infof("Socket error - Socket uuid : %s %v", ep.SocketUUID, ep.Error)
}
func (h *RoutesEventHandlers) WebSocketMessage(ep *ikisocket.EventPayload) {
var socketMessageBase shared.WebSocketMessageMetadata
err := json.Unmarshal(ep.Data, &socketMessageBase)
if err != nil {
rlog.Infof("Could not unmarshal websocket message %v\n", err)
} else {
switch socketMessageBase.MessageType {
case shared.WebSocketMessageTypeTappedEntry:
var tappedEntryMessage models.WebSocketTappedEntryMessage
err := json.Unmarshal(ep.Data, &tappedEntryMessage)
if err != nil {
rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
} else {
h.SocketHarOutChannel <- tappedEntryMessage.Data
}
case shared.WebSocketMessageTypeUpdateStatus:
var statusMessage shared.WebSocketStatusMessage
err := json.Unmarshal(ep.Data, &statusMessage)
if err != nil {
rlog.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
} else {
controllers.TapStatus = statusMessage.TappingStatus
broadcastToBrowserClients(ep.Data)
}
default:
rlog.Infof("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
}
}
}
func removeSocketUUIDFromBrowserSlice(uuidToRemove string) {
newUUIDSlice := make([]string, 0, len(browserClientSocketUUIDs))
for _, uuid := range browserClientSocketUUIDs {
if uuid != uuidToRemove {
newUUIDSlice = append(newUUIDSlice, uuid)
}
}
browserClientSocketUUIDs = newUUIDSlice
}

View File

@@ -1,11 +0,0 @@
package controllers
import (
"github.com/gofiber/fiber/v2"
"mizuserver/pkg/holder"
)
func GetCurrentResolvingInformation(c *fiber.Ctx) error {
return c.Status(fiber.StatusOK).JSON(holder.GetResolver().GetMap())
}

View File

@@ -1,17 +0,0 @@
package controllers
import (
"github.com/gofiber/fiber/v2"
"github.com/up9inc/mizu/shared"
"mizuserver/pkg/up9"
)
var TapStatus shared.TapStatus
func GetTappingStatus(c *fiber.Ctx) error {
return c.Status(fiber.StatusOK).JSON(TapStatus)
}
func AnalyzeInformation(c *fiber.Ctx) error {
return c.Status(fiber.StatusOK).JSON(up9.GetAnalyzeInfo())
}

View File

@@ -1,18 +0,0 @@
package middleware
import (
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/cors"
"github.com/gofiber/fiber/v2/middleware/logger"
)
// FiberMiddleware provide Fiber's built-in middlewares.
// See: https://docs.gofiber.io/api/middleware
func FiberMiddleware(a *fiber.App) {
a.Use(
// Add CORS to each route.
cors.New(),
// Add simple logger.
logger.New(),
)
}

View File

@@ -1,61 +0,0 @@
package resolver
import (
"k8s.io/client-go/kubernetes"
_ "k8s.io/client-go/plugin/pkg/client/auth/azure"
_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/homedir"
"path/filepath"
)
func NewFromInCluster(errOut chan error) (*Resolver, error) {
config, err := restclient.InClusterConfig()
if err != nil {
return nil, err
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, err
}
return &Resolver{clientConfig: config, clientSet: clientset, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}, nil
}
func NewFromOutOfCluster(kubeConfigPath string, errOut chan error) (*Resolver, error) {
if kubeConfigPath == "" {
home := homedir.HomeDir()
kubeConfigPath = filepath.Join(home, ".kube", "config")
}
configPathList := filepath.SplitList(kubeConfigPath)
configLoadingRules := &clientcmd.ClientConfigLoadingRules{}
if len(configPathList) <= 1 {
configLoadingRules.ExplicitPath = kubeConfigPath
} else {
configLoadingRules.Precedence = configPathList
}
contextName := ""
clientConfigLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
configLoadingRules,
&clientcmd.ConfigOverrides{
CurrentContext: contextName,
},
)
clientConfig, err := clientConfigLoader.ClientConfig()
if err != nil {
return nil, err
}
clientset, err := kubernetes.NewForConfig(clientConfig)
if err != nil {
return nil, err
}
return &Resolver{clientConfig: clientConfig, clientSet: clientset, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}, nil
}
func NewFromExisting(clientConfig *restclient.Config, clientSet *kubernetes.Clientset, errOut chan error) *Resolver {
return &Resolver{clientConfig: clientConfig, clientSet: clientSet, nameMap: make(map[string]string), serviceMap: make(map[string]string), errOut: errOut}
}

View File

@@ -1,25 +0,0 @@
package routes
import (
"github.com/gofiber/fiber/v2"
"mizuserver/pkg/controllers"
)
// EntriesRoutes defines the group of har entries routes.
func EntriesRoutes(fiberApp *fiber.App) {
routeGroup := fiberApp.Group("/api")
routeGroup.Get("/entries", controllers.GetEntries) // get entries (base/thin entries)
routeGroup.Get("/entries/:entryId", controllers.GetEntry) // get single (full) entry
routeGroup.Get("/exportEntries", controllers.GetFullEntries)
routeGroup.Get("/uploadEntries", controllers.UploadEntries)
routeGroup.Get("/resolving", controllers.GetCurrentResolvingInformation)
routeGroup.Get("/har", controllers.GetHARs)
routeGroup.Get("/resetDB", controllers.DeleteAllEntries) // get single (full) entry
routeGroup.Get("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB
routeGroup.Get("/tapStatus", controllers.GetTappingStatus) // get tapping status
routeGroup.Get("/analyzeStatus", controllers.AnalyzeInformation)
}

View File

@@ -1,13 +0,0 @@
package routes
import (
"github.com/gofiber/fiber/v2"
"mizuserver/pkg/controllers"
)
// MetadataRoutes defines the group of metadata routes.
func MetadataRoutes(fiberApp *fiber.App) {
routeGroup := fiberApp.Group("/metadata")
routeGroup.Get("/version", controllers.GetVersion)
}

View File

@@ -1,15 +0,0 @@
package routes
import "github.com/gofiber/fiber/v2"
// NotFoundRoute defines the 404 Error route.
func NotFoundRoute(fiberApp *fiber.App) {
fiberApp.Use(
func(c *fiber.Ctx) error {
return c.Status(fiber.StatusNotFound).JSON(fiber.Map{
"error": true,
"msg": "sorry, endpoint is not found",
})
},
)
}

View File

@@ -1,31 +0,0 @@
package routes
import (
"github.com/antoniodipinto/ikisocket"
"github.com/gofiber/fiber/v2"
)
type EventHandlers interface {
WebSocketConnect(ep *ikisocket.EventPayload)
WebSocketDisconnect(ep *ikisocket.EventPayload)
WebSocketClose(ep *ikisocket.EventPayload)
WebSocketError(ep *ikisocket.EventPayload)
WebSocketMessage(ep *ikisocket.EventPayload)
}
func WebSocketRoutes(app *fiber.App, eventHandlers EventHandlers) {
app.Get("/ws", ikisocket.New(func(kws *ikisocket.Websocket) {
kws.SetAttribute("is_tapper", false)
}))
app.Get("/wsTapper", ikisocket.New(func(kws *ikisocket.Websocket) {
// Tapper clients are handled differently, they don't need to receive new message broadcasts.
kws.SetAttribute("is_tapper", true)
}))
ikisocket.On(ikisocket.EventMessage, eventHandlers.WebSocketMessage)
ikisocket.On(ikisocket.EventConnect, eventHandlers.WebSocketConnect)
ikisocket.On(ikisocket.EventDisconnect, eventHandlers.WebSocketDisconnect)
ikisocket.On(ikisocket.EventClose, eventHandlers.WebSocketClose) // This event is called when the server disconnects the user actively with .Close() method
ikisocket.On(ikisocket.EventError, eventHandlers.WebSocketError) // On error event
}

BIN
assets/mizu-example.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 811 KiB

24
assets/mizu-logo.svg Normal file

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 44 KiB

BIN
assets/mizu-ui.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 491 KiB

1
cli/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
bin

34
cli/cmd/config.go Normal file
View File

@@ -0,0 +1,34 @@
package cmd
import (
"fmt"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/uiUtils"
"io/ioutil"
)
var outputFileName string
var configCmd = &cobra.Command{
Use: "config",
Short: "Generate example config file to stdout",
RunE: func(cmd *cobra.Command, args []string) error {
template := mizu.GetTemplateConfig()
if outputFileName != "" {
data := []byte(template)
_ = ioutil.WriteFile(outputFileName, data, 0644)
mizu.Log.Infof(fmt.Sprintf("Template File written to %s", fmt.Sprintf(uiUtils.Purple, outputFileName)))
} else {
mizu.Log.Debugf("Writing template config.\n%v", template)
fmt.Printf("%v", template)
}
return nil
},
}
func init() {
rootCmd.AddCommand(configCmd)
configCmd.Flags().StringVarP(&outputFileName, "file", "f", "", "Save content to local file")
}

View File

@@ -18,6 +18,7 @@ var fetchCmd = &cobra.Command{
Use: "fetch",
Short: "Download recorded traffic to files",
RunE: func(cmd *cobra.Command, args []string) error {
go mizu.ReportRun("fetch", mizuTapOptions)
if isCompatible, err := mizu.CheckVersionCompatibility(mizuFetchOptions.MizuPort); err != nil {
return err
} else if !isCompatible {

View File

@@ -5,6 +5,7 @@ import (
"bytes"
"fmt"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
"io"
"io/ioutil"
"log"
@@ -15,7 +16,7 @@ import (
)
func RunMizuFetch(fetch *MizuFetchOptions) {
mizuProxiedUrl := kubernetes.GetMizuCollectorProxiedHostAndPath(fetch.MizuPort)
mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(fetch.MizuPort)
resp, err := http.Get(fmt.Sprintf("http://%s/api/har?from=%v&to=%v", mizuProxiedUrl, fetch.FromTimestamp, fetch.ToTimestamp))
if err != nil {
log.Fatal(err)
@@ -63,7 +64,7 @@ func Unzip(reader *zip.Reader, dest string) error {
_ = os.MkdirAll(path, f.Mode())
} else {
_ = os.MkdirAll(filepath.Dir(path), f.Mode())
fmt.Print("writing HAR file [ ", path, " ] .. ")
mizu.Log.Infof("writing HAR file [ %v ]", path)
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
if err != nil {
return err
@@ -72,7 +73,7 @@ func Unzip(reader *zip.Reader, dest string) error {
if err := f.Close(); err != nil {
panic(err)
}
fmt.Println(" done")
mizu.Log.Info(" done")
}()
_, err = io.Copy(f, rc)

View File

@@ -1,14 +1,32 @@
package cmd
import (
"errors"
"fmt"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
)
var commandLineFlags []string
var rootCmd = &cobra.Command{
Use: "mizu",
Short: "A web traffic viewer for kubernetes",
Long: `A web traffic viewer for kubernetes
Further info is available at https://github.com/up9inc/mizu`,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := mizu.InitConfig(commandLineFlags); err != nil {
mizu.Log.Errorf("Invalid config, Exit %s", err)
return errors.New(fmt.Sprintf("%v", err))
}
prettifiedConfig := mizu.GetConfigStr()
mizu.Log.Debugf("Final Config: %s", prettifiedConfig)
return nil
},
}
func init() {
rootCmd.PersistentFlags().StringSliceVar(&commandLineFlags, "set", []string{}, "Override values using --set")
}
// Execute adds all child commands to the root command and sets flags appropriately.

View File

@@ -25,18 +25,17 @@ type MizuTapOptions struct {
HideHealthChecks bool
MaxEntriesDBSizeBytes int64
SleepIntervalSec uint16
DisableRedaction bool
}
var mizuTapOptions = &MizuTapOptions{}
var direction string
var humanMaxEntriesDBSize string
var regex *regexp.Regexp
const maxEntriesDBSizeFlagName = "max-entries-db-size"
const analysisMessageToConfirm = `NOTE: running mizu with --analysis flag will upload recorded traffic
to UP9 cloud for further analysis and enriched presentation options.
`
const analysisMessageToConfirm = `NOTE: running mizu with --analysis flag will upload recorded traffic for further analysis and enriched presentation options.`
var tapCmd = &cobra.Command{
Use: "tap [POD REGEX]",
@@ -44,10 +43,18 @@ var tapCmd = &cobra.Command{
Long: `Record the ingoing traffic of a kubernetes pod.
Supported protocols are HTTP and gRPC.`,
RunE: func(cmd *cobra.Command, args []string) error {
go mizu.ReportRun("tap", mizuTapOptions)
RunMizuTap(regex, mizuTapOptions)
return nil
},
PreRunE: func(cmd *cobra.Command, args []string) error {
mizu.Log.Debugf("Getting params")
mizuTapOptions.AnalysisDestination = mizu.GetString(mizu.ConfigurationKeyAnalyzingDestination)
mizuTapOptions.SleepIntervalSec = uint16(mizu.GetInt(mizu.ConfigurationKeyUploadInterval))
mizuTapOptions.MizuImage = mizu.GetString(mizu.ConfigurationKeyMizuImage)
mizu.Log.Debugf(uiUtils.PrettyJson(mizuTapOptions))
if len(args) == 0 {
return errors.New("POD REGEX argument is required")
} else if len(args) > 1 {
@@ -65,7 +72,7 @@ Supported protocols are HTTP and gRPC.`,
if parseHumanDataSizeErr != nil {
return errors.New(fmt.Sprintf("Could not parse --max-entries-db-size value %s", humanMaxEntriesDBSize))
}
fmt.Printf("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.\n", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes))
mizu.Log.Infof("Mizu will store up to %s of traffic, old traffic will be cleared once the limit is reached.", units.BytesToHumanReadable(mizuTapOptions.MaxEntriesDBSizeBytes))
directionLowerCase := strings.ToLower(direction)
if directionLowerCase == "any" {
@@ -77,9 +84,9 @@ Supported protocols are HTTP and gRPC.`,
}
if mizuTapOptions.Analysis {
fmt.Printf(analysisMessageToConfirm)
if !uiUtils.AskForConfirmation("Would you like to proceed [y/n]: ") {
fmt.Println("You can always run mizu without analysis, aborting")
mizu.Log.Infof(analysisMessageToConfirm)
if !uiUtils.AskForConfirmation("Would you like to proceed [Y/n]: ") {
mizu.Log.Infof("You can always run mizu without analysis, aborting")
os.Exit(0)
}
}
@@ -93,13 +100,11 @@ func init() {
tapCmd.Flags().Uint16VarP(&mizuTapOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
tapCmd.Flags().StringVarP(&mizuTapOptions.Namespace, "namespace", "n", "", "Namespace selector")
tapCmd.Flags().BoolVar(&mizuTapOptions.Analysis, "analysis", false, "Uploads traffic to UP9 for further analysis (Beta)")
tapCmd.Flags().StringVar(&mizuTapOptions.AnalysisDestination, "dest", "up9.app", "Destination environment")
tapCmd.Flags().Uint16VarP(&mizuTapOptions.SleepIntervalSec, "upload-interval", "", 10, "Interval in seconds for uploading data to UP9")
tapCmd.Flags().BoolVarP(&mizuTapOptions.AllNamespaces, "all-namespaces", "A", false, "Tap all namespaces")
tapCmd.Flags().StringVarP(&mizuTapOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file")
tapCmd.Flags().StringVarP(&mizuTapOptions.MizuImage, "mizu-image", "", fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", mizu.Branch, mizu.SemVer), "Custom image for mizu collector")
tapCmd.Flags().StringArrayVarP(&mizuTapOptions.PlainTextFilterRegexes, "regex-masking", "r", nil, "List of regex expressions that are used to filter matching values from text/plain http bodies")
tapCmd.Flags().StringVarP(&direction, "direction", "", "in", "Record traffic that goes in this direction (relative to the tapped pod): in/any")
tapCmd.Flags().BoolVar(&mizuTapOptions.HideHealthChecks, "hide-healthchecks", false, "hides requests with kube-probe or prometheus user-agent headers")
tapCmd.Flags().StringVarP(&humanMaxEntriesDBSize, maxEntriesDBSizeFlagName, "", "200MB", "override the default max entries db size of 200mb")
tapCmd.Flags().BoolVar(&mizuTapOptions.DisableRedaction, "no-redact", false, "Disables redaction of potentially sensitive request/response headers and body values")
}

View File

@@ -3,12 +3,14 @@ package cmd
import (
"context"
"fmt"
"github.com/romana/rlog"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/debounce"
core "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/clientcmd"
"log"
"net/http"
"net/url"
@@ -20,10 +22,11 @@ import (
)
var mizuServiceAccountExists bool
var aggregatorService *core.Service
var apiServerService *core.Service
const (
updateTappersDelay = 5 * time.Second
cleanupTimeout = time.Minute
)
var currentlyTappedPods []core.Pod
@@ -34,7 +37,17 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
return
}
kubernetesProvider := kubernetes.NewProvider(tappingOptions.KubeConfigPath)
kubernetesProvider, err := kubernetes.NewProvider(tappingOptions.KubeConfigPath)
if err != nil {
if clientcmd.IsEmptyConfig(err) {
mizu.Log.Infof(uiUtils.Red, "Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'\n")
return
}
if clientcmd.IsConfigurationInvalid(err) {
mizu.Log.Infof(uiUtils.Red, "Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'\n")
return
}
}
defer cleanUpMizuResources(kubernetesProvider)
ctx, cancel := context.WithCancel(context.Background())
@@ -42,6 +55,7 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
targetNamespace := getNamespace(tappingOptions, kubernetesProvider)
if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegexQuery, targetNamespace); err != nil {
mizu.Log.Infof("Error listing pods: %v", err)
return
} else {
currentlyTappedPods = matchingPods
@@ -53,14 +67,14 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
} else {
namespacesStr = "all namespaces"
}
fmt.Printf("Tapping pods in %s\n", namespacesStr)
mizu.Log.Infof("Tapping pods in %s", namespacesStr)
if len(currentlyTappedPods) == 0 {
var suggestionStr string
if targetNamespace != mizu.K8sAllNamespaces {
suggestionStr = "\nSelect a different namespace with -n or tap all namespaces with -A"
}
fmt.Printf("Did not find any pods matching the regex argument%s\n", suggestionStr)
mizu.Log.Infof("Did not find any pods matching the regex argument%s", suggestionStr)
}
nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods)
@@ -72,6 +86,7 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
return
}
mizu.CheckNewerVersion()
go portForwardApiPod(ctx, kubernetesProvider, cancel, tappingOptions) // TODO convert this to job for built in pod ttl or have the running app handle this
go watchPodsForTapping(ctx, kubernetesProvider, cancel, podRegexQuery, tappingOptions)
go syncApiStatus(ctx, cancel, tappingOptions)
@@ -81,7 +96,11 @@ func RunMizuTap(podRegexQuery *regexp.Regexp, tappingOptions *MizuTapOptions) {
}
func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
if err := createMizuAggregator(ctx, kubernetesProvider, tappingOptions, mizuApiFilteringOptions); err != nil {
if err := createMizuNamespace(ctx, kubernetesProvider); err != nil {
return err
}
if err := createMizuApiServer(ctx, kubernetesProvider, tappingOptions, mizuApiFilteringOptions); err != nil {
return err
}
@@ -92,19 +111,34 @@ func createMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Pro
return nil
}
func createMizuAggregator(ctx context.Context, kubernetesProvider *kubernetes.Provider, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Provider) error {
_, err := kubernetesProvider.CreateNamespace(ctx, mizu.ResourcesNamespace)
if err != nil {
mizu.Log.Infof("Error creating Namespace %s: %v", mizu.ResourcesNamespace, err)
}
return err
}
func createMizuApiServer(ctx context.Context, kubernetesProvider *kubernetes.Provider, tappingOptions *MizuTapOptions, mizuApiFilteringOptions *shared.TrafficFilteringOptions) error {
var err error
mizuServiceAccountExists = createRBACIfNecessary(ctx, kubernetesProvider)
_, err = kubernetesProvider.CreateMizuAggregatorPod(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, tappingOptions.MizuImage, mizuServiceAccountExists, mizuApiFilteringOptions, tappingOptions.MaxEntriesDBSizeBytes)
var serviceAccountName string
if mizuServiceAccountExists {
serviceAccountName = mizu.ServiceAccountName
} else {
serviceAccountName = ""
}
_, err = kubernetesProvider.CreateMizuApiServerPod(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName, tappingOptions.MizuImage, serviceAccountName, mizuApiFilteringOptions, tappingOptions.MaxEntriesDBSizeBytes)
if err != nil {
fmt.Printf("Error creating mizu collector pod: %v\n", err)
mizu.Log.Infof("Error creating mizu %s pod: %v", mizu.ApiServerPodName, err)
return err
}
aggregatorService, err = kubernetesProvider.CreateService(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName, mizu.AggregatorPodName)
apiServerService, err = kubernetesProvider.CreateService(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName, mizu.ApiServerPodName)
if err != nil {
fmt.Printf("Error creating mizu collector service: %v\n", err)
mizu.Log.Infof("Error creating mizu %s service: %v", mizu.ApiServerPodName, err)
return err
}
@@ -119,35 +153,42 @@ func getMizuApiFilteringOptions(tappingOptions *MizuTapOptions) (*shared.Traffic
for _, regexStr := range tappingOptions.PlainTextFilterRegexes {
compiledRegex, err := shared.CompileRegexToSerializableRegexp(regexStr)
if err != nil {
fmt.Printf("Regex %s is invalid: %v", regexStr, err)
mizu.Log.Infof("Regex %s is invalid: %v", regexStr, err)
return nil, err
}
compiledRegexSlice = append(compiledRegexSlice, compiledRegex)
}
}
return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: tappingOptions.HideHealthChecks}, nil
return &shared.TrafficFilteringOptions{PlainTextMaskingRegexes: compiledRegexSlice, HideHealthChecks: tappingOptions.HideHealthChecks, DisableRedaction: tappingOptions.DisableRedaction}, nil
}
func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provider, nodeToTappedPodIPMap map[string][]string, tappingOptions *MizuTapOptions) error {
if len(nodeToTappedPodIPMap) > 0 {
var serviceAccountName string
if mizuServiceAccountExists {
serviceAccountName = mizu.ServiceAccountName
} else {
serviceAccountName = ""
}
if err := kubernetesProvider.ApplyMizuTapperDaemonSet(
ctx,
mizu.ResourcesNamespace,
mizu.TapperDaemonSetName,
tappingOptions.MizuImage,
mizu.TapperPodName,
fmt.Sprintf("%s.%s.svc.cluster.local", aggregatorService.Name, aggregatorService.Namespace),
fmt.Sprintf("%s.%s.svc.cluster.local", apiServerService.Name, apiServerService.Namespace),
nodeToTappedPodIPMap,
mizuServiceAccountExists,
serviceAccountName,
tappingOptions.TapOutgoing,
); err != nil {
fmt.Printf("Error creating mizu tapper daemonset: %v\n", err)
mizu.Log.Infof("Error creating mizu tapper daemonset: %v", err)
return err
}
} else {
if err := kubernetesProvider.RemoveDaemonSet(ctx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
fmt.Printf("Error deleting mizu tapper daemonset: %v\n", err)
mizu.Log.Infof("Error deleting mizu tapper daemonset: %v", err)
return err
}
}
@@ -156,17 +197,37 @@ func updateMizuTappers(ctx context.Context, kubernetesProvider *kubernetes.Provi
}
func cleanUpMizuResources(kubernetesProvider *kubernetes.Provider) {
fmt.Printf("\nRemoving mizu resources\n")
mizu.Log.Infof("\nRemoving mizu resources\n")
removalCtx, _ := context.WithTimeout(context.Background(), 5*time.Second)
if err := kubernetesProvider.RemovePod(removalCtx, mizu.ResourcesNamespace, mizu.AggregatorPodName); err != nil {
fmt.Printf("Error removing Pod %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err)
removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
defer cancel()
if err := kubernetesProvider.RemoveNamespace(removalCtx, mizu.ResourcesNamespace); err != nil {
mizu.Log.Infof("Error removing Namespace %s: %s (%v,%+v)", mizu.ResourcesNamespace, err, err, err)
return
}
if err := kubernetesProvider.RemoveService(removalCtx, mizu.ResourcesNamespace, mizu.AggregatorPodName); err != nil {
fmt.Printf("Error removing Service %s in namespace %s: %s (%v,%+v)\n", mizu.AggregatorPodName, mizu.ResourcesNamespace, err, err, err)
if mizuServiceAccountExists {
if err := kubernetesProvider.RemoveNonNamespacedResources(removalCtx, mizu.ClusterRoleName, mizu.ClusterRoleBindingName); err != nil {
mizu.Log.Infof("Error removing non-namespaced resources: %s (%v,%+v)", err, err, err)
return
}
}
if err := kubernetesProvider.RemoveDaemonSet(removalCtx, mizu.ResourcesNamespace, mizu.TapperDaemonSetName); err != nil {
fmt.Printf("Error removing DaemonSet %s in namespace %s: %s (%v,%+v)\n", mizu.TapperDaemonSetName, mizu.ResourcesNamespace, err, err, err)
// Call cancel if a terminating signal was received. Allows user to skip the wait.
go func() {
waitForFinish(removalCtx, cancel)
}()
if err := kubernetesProvider.WaitUtilNamespaceDeleted(removalCtx, mizu.ResourcesNamespace); err != nil {
switch {
case removalCtx.Err() == context.Canceled:
// Do nothing. User interrupted the wait.
case err == wait.ErrWaitTimeout:
mizu.Log.Infof("Timeout while removing Namespace %s", mizu.ResourcesNamespace)
default:
mizu.Log.Infof("Error while waiting for Namespace %s to be deleted: %s (%v,%+v)", mizu.ResourcesNamespace, err, err, err)
}
}
}
@@ -177,7 +238,7 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
restartTappers := func() {
if matchingPods, err := kubernetesProvider.GetAllPodsMatchingRegex(ctx, podRegex, targetNamespace); err != nil {
fmt.Printf("Error getting pods by regex: %s (%v,%+v)\n", err, err, err)
mizu.Log.Infof("Error getting pods by regex: %s (%v,%+v)", err, err, err)
cancel()
} else {
currentlyTappedPods = matchingPods
@@ -185,12 +246,12 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
nodeToTappedPodIPMap, err := getNodeHostToTappedPodIpsMap(currentlyTappedPods)
if err != nil {
fmt.Printf("Error building node to ips map: %s (%v,%+v)\n", err, err, err)
mizu.Log.Infof("Error building node to ips map: %s (%v,%+v)", err, err, err)
cancel()
}
if err := updateMizuTappers(ctx, kubernetesProvider, nodeToTappedPodIPMap, tappingOptions); err != nil {
fmt.Printf("Error updating daemonset: %s (%v,%+v)\n", err, err, err)
mizu.Log.Infof("Error updating daemonset: %s (%v,%+v)", err, err, err)
cancel()
}
}
@@ -199,10 +260,10 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
for {
select {
case newTarget := <-added:
fmt.Printf(mizu.Green, fmt.Sprintf("+%s\n", newTarget.Name))
mizu.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", newTarget.Name))
case removedTarget := <-removed:
fmt.Printf(mizu.Red, fmt.Sprintf("-%s\n", removedTarget.Name))
mizu.Log.Infof(uiUtils.Red, fmt.Sprintf("-%s", removedTarget.Name))
restartTappersDebouncer.SetOn()
case modifiedTarget := <-modified:
@@ -227,29 +288,33 @@ func watchPodsForTapping(ctx context.Context, kubernetesProvider *kubernetes.Pro
}
func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, tappingOptions *MizuTapOptions) {
podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.AggregatorPodName))
podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", mizu.ApiServerPodName))
added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider.GetPodWatcher(ctx, mizu.ResourcesNamespace), podExactRegex)
isPodReady := false
timeAfter := time.After(25 * time.Second)
for {
select {
case <-ctx.Done():
return
case <-added:
continue
case <-removed:
fmt.Printf("%s removed\n", mizu.AggregatorPodName)
mizu.Log.Infof("%s removed", mizu.ApiServerPodName)
cancel()
return
case modifiedPod := <-modified:
if modifiedPod.Status.Phase == "Running" && !isPodReady {
isPodReady = true
go func() {
err := kubernetes.StartProxy(kubernetesProvider, tappingOptions.GuiPort, mizu.ResourcesNamespace, mizu.AggregatorPodName)
err := kubernetes.StartProxy(kubernetesProvider, tappingOptions.GuiPort, mizu.ResourcesNamespace, mizu.ApiServerPodName)
if err != nil {
fmt.Printf("Error occured while running k8s proxy %v\n", err)
mizu.Log.Infof("Error occured while running k8s proxy %v", err)
cancel()
}
}()
mizuProxiedUrl := kubernetes.GetMizuCollectorProxiedHostAndPath(tappingOptions.GuiPort)
fmt.Printf("Mizu is available at http://%s\n", mizuProxiedUrl)
mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(tappingOptions.GuiPort)
mizu.Log.Infof("Mizu is available at http://%s", mizuProxiedUrl)
time.Sleep(time.Second * 5) // Waiting to be sure the proxy is ready
if tappingOptions.Analysis {
@@ -259,41 +324,37 @@ func portForwardApiPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
if err != nil {
log.Fatal(fmt.Sprintf("Failed parsing the URL %v\n", err))
}
rlog.Debugf("Sending get request to %v\n", u.String())
mizu.Log.Debugf("Sending get request to %v", u.String())
if response, err := http.Get(u.String()); err != nil || response.StatusCode != 200 {
fmt.Printf("error sending upload entries req, status code: %v, err: %v\n", response.StatusCode, err)
mizu.Log.Infof("error sending upload entries req, status code: %v, err: %v", response.StatusCode, err)
} else {
fmt.Printf(mizu.Purple, "Traffic is uploading to UP9 for further analsys")
fmt.Println()
mizu.Log.Infof(uiUtils.Purple, "Traffic is uploading to UP9 for further analysis")
}
}
}
case <-time.After(25 * time.Second):
case <-timeAfter:
if !isPodReady {
fmt.Printf("error: %s pod was not ready in time", mizu.AggregatorPodName)
mizu.Log.Infof("error: %s pod was not ready in time", mizu.ApiServerPodName)
cancel()
}
case <-errorChan:
cancel()
case <-ctx.Done():
return
}
}
}
func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool {
mizuRBACExists, err := kubernetesProvider.DoesMizuRBACExist(ctx, mizu.ResourcesNamespace)
mizuRBACExists, err := kubernetesProvider.DoesServiceAccountExist(ctx, mizu.ResourcesNamespace, mizu.ServiceAccountName)
if err != nil {
fmt.Printf("warning: could not ensure mizu rbac resources exist %v\n", err)
mizu.Log.Infof("warning: could not ensure mizu rbac resources exist %v", err)
return false
}
if !mizuRBACExists {
err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.ResourcesNamespace, mizu.RBACVersion)
err := kubernetesProvider.CreateMizuRBAC(ctx, mizu.ResourcesNamespace, mizu.ServiceAccountName, mizu.ClusterRoleName, mizu.ClusterRoleBindingName, mizu.RBACVersion)
if err != nil {
fmt.Printf("warning: could not create mizu rbac resources %v\n", err)
mizu.Log.Infof("warning: could not create mizu rbac resources %v", err)
return false
}
}
@@ -327,10 +388,10 @@ func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
}
func syncApiStatus(ctx context.Context, cancel context.CancelFunc, tappingOptions *MizuTapOptions) {
controlSocketStr := fmt.Sprintf("ws://%s/ws", kubernetes.GetMizuCollectorProxiedHostAndPath(tappingOptions.GuiPort))
controlSocketStr := fmt.Sprintf("ws://%s/ws", kubernetes.GetMizuApiServerProxiedHostAndPath(tappingOptions.GuiPort))
controlSocket, err := mizu.CreateControlSocket(controlSocketStr)
if err != nil {
fmt.Printf("error establishing control socket connection %s\n", err)
mizu.Log.Infof("error establishing control socket connection %s", err)
cancel()
}
@@ -341,7 +402,7 @@ func syncApiStatus(ctx context.Context, cancel context.CancelFunc, tappingOption
default:
err = controlSocket.SendNewTappedPodsListMessage(currentlyTappedPods)
if err != nil {
rlog.Debugf("error Sending message via control socket %v, error: %s\n", controlSocketStr, err)
mizu.Log.Debugf("error Sending message via control socket %v, error: %s", controlSocketStr, err)
}
time.Sleep(10 * time.Second)
}

View File

@@ -1,32 +1,30 @@
package cmd
import (
"fmt"
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/mizu"
"strconv"
"time"
"github.com/spf13/cobra"
)
type MizuVersionOptions struct {
DebugInfo bool
DebugInfo bool
}
var mizuVersionOptions = &MizuVersionOptions{}
var versionCmd = &cobra.Command{
Use: "version",
Short: "Print version info",
RunE: func(cmd *cobra.Command, args []string) error {
go mizu.ReportRun("version", mizuVersionOptions)
if mizuVersionOptions.DebugInfo {
timeStampInt, _ := strconv.ParseInt(mizu.BuildTimestamp, 10, 0)
fmt.Printf("Version: %s \nBranch: %s (%s) \n", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)
fmt.Printf("Build Time: %s (%s)\n", mizu.BuildTimestamp, time.Unix(timeStampInt, 0))
mizu.Log.Infof("Version: %s \nBranch: %s (%s)", mizu.SemVer, mizu.Branch, mizu.GitCommitHash)
mizu.Log.Infof("Build Time: %s (%s)", mizu.BuildTimestamp, time.Unix(timeStampInt, 0))
} else {
fmt.Printf("Version: %s (%s)\n", mizu.SemVer, mizu.Branch)
mizu.Log.Infof("Version: %s (%s)", mizu.SemVer, mizu.Branch)
}
return nil
},

View File

@@ -6,7 +6,8 @@ import (
)
type MizuViewOptions struct {
GuiPort uint16
GuiPort uint16
KubeConfigPath string
}
var mizuViewOptions = &MizuViewOptions{}
@@ -15,7 +16,8 @@ var viewCmd = &cobra.Command{
Use: "view",
Short: "Open GUI in browser",
RunE: func(cmd *cobra.Command, args []string) error {
if isCompatible, err := mizu.CheckVersionCompatibility(mizuFetchOptions.MizuPort); err != nil {
go mizu.ReportRun("view", mizuViewOptions)
if isCompatible, err := mizu.CheckVersionCompatibility(mizuViewOptions.GuiPort); err != nil {
return err
} else if !isCompatible {
return nil
@@ -23,12 +25,11 @@ var viewCmd = &cobra.Command{
runMizuView(mizuViewOptions)
return nil
},
}
func init() {
rootCmd.AddCommand(viewCmd)
viewCmd.Flags().Uint16VarP(&mizuViewOptions.GuiPort, "gui-port", "p", 8899, "Provide a custom port for the web interface webserver")
viewCmd.Flags().StringVarP(&mizuViewOptions.KubeConfigPath, "kube-config", "k", "", "Path to kube-config file")
}

View File

@@ -5,35 +5,47 @@ import (
"fmt"
"github.com/up9inc/mizu/cli/kubernetes"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/uiUtils"
"k8s.io/client-go/tools/clientcmd"
"net/http"
)
func runMizuView(mizuViewOptions *MizuViewOptions) {
kubernetesProvider := kubernetes.NewProvider("")
kubernetesProvider, err := kubernetes.NewProvider(mizuViewOptions.KubeConfigPath)
if err != nil {
if clientcmd.IsEmptyConfig(err) {
mizu.Log.Infof("Couldn't find the kube config file, or file is empty. Try adding '--kube-config=<path to kube config file>'")
return
}
if clientcmd.IsConfigurationInvalid(err) {
mizu.Log.Infof(uiUtils.Red, "Invalid kube config file. Try using a different config with '--kube-config=<path to kube config file>'")
return
}
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
exists, err := kubernetesProvider.DoesServicesExist(ctx, mizu.ResourcesNamespace, mizu.AggregatorPodName)
exists, err := kubernetesProvider.DoesServicesExist(ctx, mizu.ResourcesNamespace, mizu.ApiServerPodName)
if err != nil {
panic(err)
}
if !exists {
fmt.Printf("The %s service not found\n", mizu.AggregatorPodName)
mizu.Log.Infof("The %s service not found", mizu.ApiServerPodName)
return
}
mizuProxiedUrl := kubernetes.GetMizuCollectorProxiedHostAndPath(mizuViewOptions.GuiPort)
mizuProxiedUrl := kubernetes.GetMizuApiServerProxiedHostAndPath(mizuViewOptions.GuiPort)
_, err = http.Get(fmt.Sprintf("http://%s/", mizuProxiedUrl))
if err == nil {
fmt.Printf("Found a running service %s and open port %d\n", mizu.AggregatorPodName, mizuViewOptions.GuiPort)
mizu.Log.Infof("Found a running service %s and open port %d", mizu.ApiServerPodName, mizuViewOptions.GuiPort)
return
}
fmt.Printf("Found service %s, creating k8s proxy\n", mizu.AggregatorPodName)
mizu.Log.Infof("Found service %s, creating k8s proxy", mizu.ApiServerPodName)
fmt.Printf("Mizu is available at http://%s\n", kubernetes.GetMizuCollectorProxiedHostAndPath(mizuViewOptions.GuiPort))
err = kubernetes.StartProxy(kubernetesProvider, mizuViewOptions.GuiPort, mizu.ResourcesNamespace, mizu.AggregatorPodName)
mizu.Log.Infof("Mizu is available at http://%s\n", kubernetes.GetMizuApiServerProxiedHostAndPath(mizuViewOptions.GuiPort))
err = kubernetes.StartProxy(kubernetesProvider, mizuViewOptions.GuiPort, mizu.ResourcesNamespace, mizu.ApiServerPodName)
if err != nil {
fmt.Printf("Error occured while running k8s proxy %v\n", err)
mizu.Log.Infof("Error occured while running k8s proxy %v", err)
}
}

View File

@@ -3,10 +3,12 @@ module github.com/up9inc/mizu/cli
go 1.16
require (
github.com/google/go-github/v37 v37.0.0
github.com/gorilla/websocket v1.4.2
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
github.com/spf13/cobra v1.1.3
github.com/up9inc/mizu/shared v0.0.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
k8s.io/api v0.21.2
k8s.io/apimachinery v0.21.2
k8s.io/client-go v0.21.2

View File

@@ -99,6 +99,7 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
@@ -174,6 +175,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -203,8 +205,13 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github/v37 v37.0.0 h1:rCspN8/6kB1BAJWZfuafvHhyfIo5fkAulaP/3bOQ/tM=
github.com/google/go-github/v37 v37.0.0/go.mod h1:LM7in3NmXDrX58GbEHy7FtNLbI2JijX93RnMKvWG3m4=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -218,6 +225,7 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
@@ -245,6 +253,7 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
@@ -324,6 +333,8 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
@@ -331,6 +342,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -355,8 +367,6 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI=
github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -679,8 +689,9 @@ gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -705,6 +716,7 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
k8s.io/kubectl v0.21.2 h1:9XPCetvOMDqrIZZXb1Ei+g8t6KrIp9ENJaysQjUuLiE=
k8s.io/kubectl v0.21.2/go.mod h1:PgeUclpG8VVmmQIl8zpLar3IQEpFc9mrmvlwY3CK1xo=

View File

@@ -6,6 +6,11 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/up9inc/mizu/cli/mizu"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/homedir"
"os"
"path/filepath"
"regexp"
"strconv"
@@ -14,8 +19,8 @@ import (
core "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
resource "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/watch"
applyconfapp "k8s.io/client-go/applyconfigurations/apps/v1"
@@ -29,7 +34,7 @@ import (
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
_ "k8s.io/client-go/tools/portforward"
"k8s.io/client-go/util/homedir"
watchtools "k8s.io/client-go/tools/watch"
)
type Provider struct {
@@ -40,15 +45,14 @@ type Provider struct {
}
const (
serviceAccountName = "mizu-service-account"
fieldManagerName = "mizu-manager"
fieldManagerName = "mizu-manager"
)
func NewProvider(kubeConfigPath string) *Provider {
func NewProvider(kubeConfigPath string) (*Provider, error) {
kubernetesConfig := loadKubernetesConfiguration(kubeConfigPath)
restClientConfig, err := kubernetesConfig.ClientConfig()
if err != nil {
panic(err.Error())
return nil, err
}
clientSet := getClientSet(restClientConfig)
@@ -56,7 +60,7 @@ func NewProvider(kubeConfigPath string) *Provider {
clientSet: clientSet,
kubernetesConfig: kubernetesConfig,
clientConfig: *restClientConfig,
}
}, nil
}
func (provider *Provider) CurrentNamespace() string {
@@ -64,6 +68,46 @@ func (provider *Provider) CurrentNamespace() string {
return ns
}
func (provider *Provider) WaitUtilNamespaceDeleted(ctx context.Context, name string) error {
fieldSelector := fmt.Sprintf("metadata.name=%s", name)
var limit int64 = 1
lw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
options.FieldSelector = fieldSelector
options.Limit = limit
return provider.clientSet.CoreV1().Namespaces().List(ctx, options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
options.FieldSelector = fieldSelector
options.Limit = limit
return provider.clientSet.CoreV1().Namespaces().Watch(ctx, options)
},
}
var preconditionFunc watchtools.PreconditionFunc = func(store cache.Store) (bool, error) {
_, exists, err := store.Get(&core.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}})
if err != nil {
return false, err
}
if exists {
return false, nil
}
return true, nil
}
conditionFunc := func(e watch.Event) (bool, error) {
if e.Type == watch.Deleted {
return true, nil
}
return false, nil
}
obj := &core.Namespace{}
_, err := watchtools.UntilWithSync(ctx, lw, obj, preconditionFunc, conditionFunc)
return err
}
func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) watch.Interface {
watcher, err := provider.clientSet.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{Watch: true})
if err != nil {
@@ -72,7 +116,16 @@ func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) w
return watcher
}
func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace string, podName string, podImage string, linkServiceAccount bool, mizuApiFilteringOptions *shared.TrafficFilteringOptions, maxEntriesDBSizeBytes int64) (*core.Pod, error) {
func (provider *Provider) CreateNamespace(ctx context.Context, name string) (*core.Namespace, error) {
namespaceSpec := &core.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
}
return provider.clientSet.CoreV1().Namespaces().Create(ctx, namespaceSpec, metav1.CreateOptions{})
}
func (provider *Provider) CreateMizuApiServerPod(ctx context.Context, namespace string, podName string, podImage string, serviceAccountName string, mizuApiFilteringOptions *shared.TrafficFilteringOptions, maxEntriesDBSizeBytes int64) (*core.Pod, error) {
marshaledFilteringOptions, err := json.Marshal(mizuApiFilteringOptions)
if err != nil {
return nil, err
@@ -80,19 +133,19 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace
cpuLimit, err := resource.ParseQuantity("750m")
if err != nil {
return nil, errors.New("invalid cpu limit for aggregator container")
return nil, errors.New(fmt.Sprintf("invalid cpu limit for %s container", podName))
}
memLimit, err := resource.ParseQuantity("512Mi")
if err != nil {
return nil, errors.New("invalid memory limit for aggregator container")
return nil, errors.New(fmt.Sprintf("invalid memory limit for %s container", podName))
}
cpuRequests, err := resource.ParseQuantity("50m")
if err != nil {
return nil, errors.New("invalid cpu request for aggregator container")
return nil, errors.New(fmt.Sprintf("invalid cpu request for %s container", podName))
}
memRequests, err := resource.ParseQuantity("50Mi")
if err != nil {
return nil, errors.New("invalid memory request for aggregator container")
return nil, errors.New(fmt.Sprintf("invalid memory request for %s container", podName))
}
pod := &core.Pod{
@@ -107,7 +160,7 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace
Name: podName,
Image: podImage,
ImagePullPolicy: core.PullAlways,
Command: []string{"./mizuagent", "--aggregator"},
Command: []string{"./mizuagent", "--api-server"},
Env: []core.EnvVar{
{
Name: shared.HostModeEnvVar,
@@ -118,17 +171,17 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace
Value: string(marshaledFilteringOptions),
},
{
Name: shared.MaxEntriesDBSizeByteSEnvVar,
Name: shared.MaxEntriesDBSizeBytesEnvVar,
Value: strconv.FormatInt(maxEntriesDBSizeBytes, 10),
},
},
Resources: core.ResourceRequirements{
Limits: core.ResourceList{
"cpu": cpuLimit,
"cpu": cpuLimit,
"memory": memLimit,
},
Requests: core.ResourceList{
"cpu": cpuRequests,
"cpu": cpuRequests,
"memory": memRequests,
},
},
@@ -139,7 +192,7 @@ func (provider *Provider) CreateMizuAggregatorPod(ctx context.Context, namespace
},
}
//define the service account only when it exists to prevent pod crash
if linkServiceAccount {
if serviceAccountName != "" {
pod.Spec.ServiceAccountName = serviceAccountName
}
return provider.clientSet.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{})
@@ -160,7 +213,7 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s
return provider.clientSet.CoreV1().Services(namespace).Create(ctx, &service, metav1.CreateOptions{})
}
func (provider *Provider) DoesMizuRBACExist(ctx context.Context, namespace string) (bool, error) {
func (provider *Provider) DoesServiceAccountExist(ctx context.Context, namespace string, serviceAccountName string) (bool, error) {
serviceAccount, err := provider.clientSet.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccountName, metav1.GetOptions{})
var statusError *k8serrors.StatusError
@@ -191,9 +244,7 @@ func (provider *Provider) DoesServicesExist(ctx context.Context, namespace strin
return service != nil, nil
}
func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, version string) error {
clusterRoleName := "mizu-cluster-role"
func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, serviceAccountName string, clusterRoleName string, clusterRoleBindingName string, version string) error {
serviceAccount := &core.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: serviceAccountName,
@@ -216,7 +267,7 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string,
}
clusterRoleBinding := &rbac.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "mizu-cluster-role-binding",
Name: clusterRoleBindingName,
Labels: map[string]string{"mizu-cli-version": version},
},
RoleRef: rbac.RoleRef{
@@ -247,9 +298,50 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string,
return nil
}
func (provider *Provider) RemoveNamespace(ctx context.Context, name string) error {
if isFound, err := provider.CheckNamespaceExists(ctx, name); err != nil {
return err
} else if !isFound {
return nil
}
return provider.clientSet.CoreV1().Namespaces().Delete(ctx, name, metav1.DeleteOptions{})
}
func (provider *Provider) RemoveNonNamespacedResources(ctx context.Context, clusterRoleName string, clusterRoleBindingName string) error {
if err := provider.RemoveClusterRole(ctx, clusterRoleName); err != nil {
return err
}
if err := provider.RemoveClusterRoleBinding(ctx, clusterRoleBindingName); err != nil {
return err
}
return nil
}
func (provider *Provider) RemoveClusterRole(ctx context.Context, name string) error {
if isFound, err := provider.CheckClusterRoleExists(ctx, name); err != nil {
return err
} else if !isFound {
return nil
}
return provider.clientSet.RbacV1().ClusterRoles().Delete(ctx, name, metav1.DeleteOptions{})
}
func (provider *Provider) RemoveClusterRoleBinding(ctx context.Context, name string) error {
if isFound, err := provider.CheckClusterRoleBindingExists(ctx, name); err != nil {
return err
} else if !isFound {
return nil
}
return provider.clientSet.RbacV1().ClusterRoleBindings().Delete(ctx, name, metav1.DeleteOptions{})
}
func (provider *Provider) RemovePod(ctx context.Context, namespace string, podName string) error {
if isFound, err := provider.CheckPodExists(ctx, namespace, podName);
err != nil {
if isFound, err := provider.CheckPodExists(ctx, namespace, podName); err != nil {
return err
} else if !isFound {
return nil
@@ -259,8 +351,7 @@ func (provider *Provider) RemovePod(ctx context.Context, namespace string, podNa
}
func (provider *Provider) RemoveService(ctx context.Context, namespace string, serviceName string) error {
if isFound, err := provider.CheckServiceExists(ctx, namespace, serviceName);
err != nil {
if isFound, err := provider.CheckServiceExists(ctx, namespace, serviceName); err != nil {
return err
} else if !isFound {
return nil
@@ -270,8 +361,7 @@ func (provider *Provider) RemoveService(ctx context.Context, namespace string, s
}
func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string, daemonSetName string) error {
if isFound, err := provider.CheckDaemonSetExists(ctx, namespace, daemonSetName);
err != nil {
if isFound, err := provider.CheckDaemonSetExists(ctx, namespace, daemonSetName); err != nil {
return err
} else if !isFound {
return nil
@@ -280,10 +370,61 @@ func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string,
return provider.clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, daemonSetName, metav1.DeleteOptions{})
}
func (provider *Provider) CheckNamespaceExists(ctx context.Context, name string) (bool, error) {
listOptions := metav1.ListOptions{
FieldSelector: fmt.Sprintf("metadata.name=%s", name),
Limit: 1,
}
resourceList, err := provider.clientSet.CoreV1().Namespaces().List(ctx, listOptions)
if err != nil {
return false, err
}
if len(resourceList.Items) > 0 {
return true, nil
}
return false, nil
}
func (provider *Provider) CheckClusterRoleExists(ctx context.Context, name string) (bool, error) {
listOptions := metav1.ListOptions{
FieldSelector: fmt.Sprintf("metadata.name=%s", name),
Limit: 1,
}
resourceList, err := provider.clientSet.RbacV1().ClusterRoles().List(ctx, listOptions)
if err != nil {
return false, err
}
if len(resourceList.Items) > 0 {
return true, nil
}
return false, nil
}
func (provider *Provider) CheckClusterRoleBindingExists(ctx context.Context, name string) (bool, error) {
listOptions := metav1.ListOptions{
FieldSelector: fmt.Sprintf("metadata.name=%s", name),
Limit: 1,
}
resourceList, err := provider.clientSet.RbacV1().ClusterRoleBindings().List(ctx, listOptions)
if err != nil {
return false, err
}
if len(resourceList.Items) > 0 {
return true, nil
}
return false, nil
}
func (provider *Provider) CheckPodExists(ctx context.Context, namespace string, name string) (bool, error) {
listOptions := metav1.ListOptions{
FieldSelector: fmt.Sprintf("metadata.name=%s", name),
Limit: 1,
Limit: 1,
}
resourceList, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, listOptions)
if err != nil {
@@ -300,7 +441,7 @@ func (provider *Provider) CheckPodExists(ctx context.Context, namespace string,
func (provider *Provider) CheckServiceExists(ctx context.Context, namespace string, name string) (bool, error) {
listOptions := metav1.ListOptions{
FieldSelector: fmt.Sprintf("metadata.name=%s", name),
Limit: 1,
Limit: 1,
}
resourceList, err := provider.clientSet.CoreV1().Services(namespace).List(ctx, listOptions)
if err != nil {
@@ -317,7 +458,7 @@ func (provider *Provider) CheckServiceExists(ctx context.Context, namespace stri
func (provider *Provider) CheckDaemonSetExists(ctx context.Context, namespace string, name string) (bool, error) {
listOptions := metav1.ListOptions{
FieldSelector: fmt.Sprintf("metadata.name=%s", name),
Limit: 1,
Limit: 1,
}
resourceList, err := provider.clientSet.AppsV1().DaemonSets(namespace).List(ctx, listOptions)
if err != nil {
@@ -331,7 +472,7 @@ func (provider *Provider) CheckDaemonSetExists(ctx context.Context, namespace st
return false, nil
}
func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, aggregatorPodIp string, nodeToTappedPodIPMap map[string][]string, linkServiceAccount bool, tapOutgoing bool) error {
func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool) error {
if len(nodeToTappedPodIPMap) == 0 {
return fmt.Errorf("Daemon set %s must tap at least 1 pod", daemonSetName)
}
@@ -346,7 +487,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
"-i", "any",
"--tap",
"--hardump",
"--aggregator-address", fmt.Sprintf("ws://%s/wsTapper", aggregatorPodIp),
"--api-server-address", fmt.Sprintf("ws://%s/wsTapper", apiServerPodIp),
}
if tapOutgoing {
mizuCmd = append(mizuCmd, "--anydirection")
@@ -372,26 +513,26 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
)
cpuLimit, err := resource.ParseQuantity("500m")
if err != nil {
return errors.New("invalid cpu limit for tapper container")
return errors.New(fmt.Sprintf("invalid cpu limit for %s container", tapperPodName))
}
memLimit, err := resource.ParseQuantity("1Gi")
if err != nil {
return errors.New("invalid memory limit for tapper container")
return errors.New(fmt.Sprintf("invalid memory limit for %s container", tapperPodName))
}
cpuRequests, err := resource.ParseQuantity("50m")
if err != nil {
return errors.New("invalid cpu request for tapper container")
return errors.New(fmt.Sprintf("invalid cpu request for %s container", tapperPodName))
}
memRequests, err := resource.ParseQuantity("50Mi")
if err != nil {
return errors.New("invalid memory request for tapper container")
return errors.New(fmt.Sprintf("invalid memory request for %s container", tapperPodName))
}
agentResourceLimits := core.ResourceList{
"cpu": cpuLimit,
"cpu": cpuLimit,
"memory": memLimit,
}
agentResourceRequests := core.ResourceList{
"cpu": cpuRequests,
"cpu": cpuRequests,
"memory": memRequests,
}
agentResources := applyconfcore.ResourceRequirements().WithRequests(agentResourceRequests).WithLimits(agentResourceLimits)
@@ -425,7 +566,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
podSpec.WithHostNetwork(true)
podSpec.WithDNSPolicy(core.DNSClusterFirstWithHostNet)
podSpec.WithTerminationGracePeriodSeconds(0)
if linkServiceAccount {
if serviceAccountName != "" {
podSpec.WithServiceAccountName(serviceAccountName)
}
podSpec.WithContainers(agentContainer)
@@ -469,11 +610,16 @@ func getClientSet(config *restclient.Config) *kubernetes.Clientset {
}
func loadKubernetesConfiguration(kubeConfigPath string) clientcmd.ClientConfig {
if kubeConfigPath == "" {
kubeConfigPath = os.Getenv("KUBECONFIG")
}
if kubeConfigPath == "" {
home := homedir.HomeDir()
kubeConfigPath = filepath.Join(home, ".kube", "config")
}
mizu.Log.Debugf("Using kube config %s", kubeConfigPath)
configPathList := filepath.SplitList(kubeConfigPath)
configLoadingRules := &clientcmd.ClientConfigLoadingRules{}
if len(configPathList) <= 1 {

View File

@@ -40,24 +40,24 @@ func StartProxy(kubernetesProvider *Provider, mizuPort uint16, mizuNamespace str
return server.Serve(l)
}
func getMizuCollectorProxiedHostAndPath(mizuNamespace string, mizuServiceName string) string {
func getMizuApiServerProxiedHostAndPath(mizuNamespace string, mizuServiceName string) string {
return fmt.Sprintf("/api/v1/namespaces/%s/services/%s:%d/proxy/", mizuNamespace, mizuServiceName, mizuServicePort)
}
func GetMizuCollectorProxiedHostAndPath(mizuPort uint16) string {
func GetMizuApiServerProxiedHostAndPath(mizuPort uint16) string {
return fmt.Sprintf("localhost:%d/mizu", mizuPort)
}
func getRerouteHttpHandlerMizuAPI(proxyHandler http.Handler, mizuNamespace string, mizuServiceName string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r.URL.Path = strings.Replace(r.URL.Path, "/mizu/", getMizuCollectorProxiedHostAndPath(mizuNamespace, mizuServiceName), 1)
r.URL.Path = strings.Replace(r.URL.Path, "/mizu/", getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName), 1)
proxyHandler.ServeHTTP(w, r)
})
}
func getRerouteHttpHandlerMizuStatic(proxyHandler http.Handler, mizuNamespace string, mizuServiceName string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r.URL.Path = strings.Replace(r.URL.Path, "/static/", fmt.Sprintf("%s/static/", getMizuCollectorProxiedHostAndPath(mizuNamespace, mizuServiceName)), 1)
r.URL.Path = strings.Replace(r.URL.Path, "/static/", fmt.Sprintf("%s/static/", getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName)), 1)
proxyHandler.ServeHTTP(w, r)
})
}

View File

@@ -1,7 +1,11 @@
package main
import "github.com/up9inc/mizu/cli/cmd"
import (
"github.com/up9inc/mizu/cli/cmd"
"github.com/up9inc/mizu/cli/mizu"
)
func main() {
mizu.InitLogger()
cmd.Execute()
}

221
cli/mizu/config.go Normal file
View File

@@ -0,0 +1,221 @@
package mizu
import (
"errors"
"fmt"
"github.com/up9inc/mizu/cli/uiUtils"
"gopkg.in/yaml.v3"
"io/ioutil"
"os"
"path"
"reflect"
"strconv"
"strings"
)
const separator = "="
var configObj = map[string]interface{}{}
type CommandLineFlag struct {
CommandLineName string
YamlHierarchyName string
DefaultValue interface{}
}
const (
ConfigurationKeyAnalyzingDestination = "tap.dest"
ConfigurationKeyUploadInterval = "tap.uploadInterval"
ConfigurationKeyMizuImage = "mizuImage"
ConfigurationKeyTelemetry = "telemetry"
)
var allowedSetFlags = []CommandLineFlag{
{
CommandLineName: "dest",
YamlHierarchyName: ConfigurationKeyAnalyzingDestination,
DefaultValue: "up9.app",
// TODO: maybe add short description that we can show
},
{
CommandLineName: "uploadInterval",
YamlHierarchyName: ConfigurationKeyUploadInterval,
DefaultValue: 10,
},
{
CommandLineName: "mizuImage",
YamlHierarchyName: ConfigurationKeyMizuImage,
DefaultValue: fmt.Sprintf("gcr.io/up9-docker-hub/mizu/%s:%s", Branch, SemVer),
},
{
CommandLineName: "telemetry",
YamlHierarchyName: ConfigurationKeyTelemetry,
DefaultValue: true,
},
}
func GetString(key string) string {
return fmt.Sprintf("%v", getValueFromMergedConfig(key))
}
func GetBool(key string) bool {
stringVal := GetString(key)
Log.Debugf("Found string value %v", stringVal)
val, err := strconv.ParseBool(stringVal)
if err != nil {
Log.Warningf(uiUtils.Red, fmt.Sprintf( "Invalid value %v for key %s, expected bool", stringVal, key))
os.Exit(1)
}
return val
}
func GetInt(key string) int {
stringVal := GetString(key)
Log.Debugf("Found string value %v", stringVal)
val, err := strconv.Atoi(stringVal)
if err != nil {
Log.Warningf(uiUtils.Red, fmt.Sprintf("Invalid value %v for key %s, expected int", stringVal, key))
os.Exit(1)
}
return val
}
func InitConfig(commandLineValues []string) error {
Log.Debugf("Merging default values")
mergeDefaultValues()
Log.Debugf("Merging config file values")
if err1 := mergeConfigFile(); err1 != nil {
Log.Infof(fmt.Sprintf(uiUtils.Red, "Invalid config file\n"))
return err1
}
Log.Debugf("Merging command line values")
if err2 := mergeCommandLineFlags(commandLineValues); err2 != nil {
Log.Infof(fmt.Sprintf(uiUtils.Red, "Invalid commanad argument\n"))
return err2
}
finalConfigPrettified, _ := uiUtils.PrettyJson(configObj)
Log.Debugf("Merged all config successfully\n Final config: %v", finalConfigPrettified)
return nil
}
func GetTemplateConfig() string {
templateConfig := map[string]interface{}{}
for _, allowedFlag := range allowedSetFlags {
addToConfigObj(allowedFlag.YamlHierarchyName, allowedFlag.DefaultValue, templateConfig)
}
prettifiedConfig, _ := uiUtils.PrettyYaml(templateConfig)
return prettifiedConfig
}
func GetConfigStr() string {
val, _ := uiUtils.PrettyYaml(configObj)
return val
}
func getValueFromMergedConfig(key string) interface{} {
if a, ok := configObj[key]; ok {
return a
}
return nil
}
func mergeDefaultValues() {
for _, allowedFlag := range allowedSetFlags {
Log.Debugf("Setting %v to %v", allowedFlag.YamlHierarchyName, allowedFlag.DefaultValue)
configObj[allowedFlag.YamlHierarchyName] = allowedFlag.DefaultValue
}
}
func mergeConfigFile() error {
Log.Debugf("Merging mizu config file values")
home, homeDirErr := os.UserHomeDir()
if homeDirErr != nil {
return nil
}
reader, openErr := os.Open(path.Join(home, ".mizu", "config.yaml"))
if openErr != nil {
return nil
}
buf, readErr := ioutil.ReadAll(reader)
if readErr != nil {
return readErr
}
m := make(map[string]interface{})
if err := yaml.Unmarshal(buf, &m); err != nil {
return err
}
for k, v := range m {
addToConfig(k, v)
}
return nil
}
func addToConfig(prefix string, value interface{}) {
typ := reflect.TypeOf(value).Kind()
if typ == reflect.Map {
for k1, v1 := range value.(map[string]interface{}) {
addToConfig(fmt.Sprintf("%s.%s", prefix, k1), v1)
}
} else {
validateConfigFileKey(prefix)
configObj[prefix] = value
}
}
func mergeCommandLineFlags(commandLineValues []string) error {
Log.Debugf("Merging Command line flags")
for _, e := range commandLineValues {
if !strings.Contains(e, separator) {
return errors.New(fmt.Sprintf("invalid set argument %s", e))
}
split := strings.SplitN(e, separator, 2)
if len(split) != 2 {
return errors.New(fmt.Sprintf("invalid set argument %s", e))
}
setFlagKey, argumentValue := split[0], split[1]
argumentNameInConfig, err := flagFromAllowed(setFlagKey)
if err != nil {
return err
}
configObj[argumentNameInConfig] = argumentValue
}
return nil
}
func flagFromAllowed(setFlagKey string) (string, error) {
for _, allowedFlag := range allowedSetFlags {
if strings.ToLower(allowedFlag.CommandLineName) == strings.ToLower(setFlagKey) {
return allowedFlag.YamlHierarchyName, nil
}
}
return "", errors.New(fmt.Sprintf("invalid set argument %s", setFlagKey))
}
func validateConfigFileKey(configFileKey string) {
for _, allowedFlag := range allowedSetFlags {
if allowedFlag.YamlHierarchyName == configFileKey {
return
}
}
Log.Info(fmt.Sprintf("Unknown argument: %s. Exit", configFileKey))
os.Exit(1)
}
func addToConfigObj(key string, value interface{}, configObj map[string]interface{}) {
typ := reflect.TypeOf(value).Kind()
if typ != reflect.Map {
if strings.Contains(key, ".") {
split := strings.SplitN(key, ".", 2)
firstLevelKey := split[0]
if _, ok := configObj[firstLevelKey]; !ok {
configObj[firstLevelKey] = map[string]interface{}{}
}
addToConfigObj(split[1], value, configObj[firstLevelKey].(map[string]interface{}))
} else {
configObj[key] = value
}
}
}

View File

@@ -9,20 +9,12 @@ var (
)
const (
ResourcesNamespace = "default"
TapperDaemonSetName = "mizu-tapper-daemon-set"
AggregatorPodName = "mizu-collector"
TapperPodName = "mizu-tapper"
K8sAllNamespaces = ""
)
const (
Black = "\033[1;30m%s\033[0m"
Red = "\033[1;31m%s\033[0m"
Green = "\033[1;32m%s\033[0m"
Yellow = "\033[1;33m%s\033[0m"
Purple = "\033[1;34m%s\033[0m"
Magenta = "\033[1;35m%s\033[0m"
Teal = "\033[1;36m%s\033[0m"
White = "\033[1;37m%s\033[0m"
ApiServerPodName = "mizu-api-server"
ClusterRoleBindingName = "mizu-cluster-role-binding"
ClusterRoleName = "mizu-cluster-role"
K8sAllNamespaces = ""
ResourcesNamespace = "mizu"
ServiceAccountName = "mizu-service-account"
TapperDaemonSetName = "mizu-tapper-daemon-set"
TapperPodName = "mizu-tapper"
)

39
cli/mizu/logger.go Normal file
View File

@@ -0,0 +1,39 @@
package mizu
import (
"fmt"
"github.com/op/go-logging"
"os"
"path"
)
var Log = logging.MustGetLogger("mizu_cli")
var format = logging.MustStringFormatter(
`%{time} %{level:.5s} ▶ %{pid} %{shortfile} %{shortfunc} ▶ %{message}`,
)
func InitLogger() {
homeDirPath, _ := os.UserHomeDir()
mizuDirPath := path.Join(homeDirPath, ".mizu")
if err := os.MkdirAll(mizuDirPath, os.ModePerm); err != nil {
panic(fmt.Sprintf("Failed creating .mizu dir: %v, err %v", mizuDirPath, err))
}
logPath := path.Join(mizuDirPath, "log.log")
f, err := os.OpenFile(logPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
panic(fmt.Sprintf("Failed mizu log file: %v, err %v", logPath, err))
}
fileLog := logging.NewLogBackend(f, "", 0)
consoleLog := logging.NewLogBackend(os.Stderr, "", 0)
backend2Formatter := logging.NewBackendFormatter(fileLog, format)
backend1Leveled := logging.AddModuleLevel(consoleLog)
backend1Leveled.SetLevel(logging.INFO, "")
logging.SetBackend(backend1Leveled, backend2Formatter)
Log.Debugf("Running mizu version %v", SemVer)
}

39
cli/mizu/telemetry.go Normal file
View File

@@ -0,0 +1,39 @@
package mizu
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
)
const telemetryUrl = "https://us-east4-up9-prod.cloudfunctions.net/mizu-telemetry"
func ReportRun(cmd string, args interface{}) {
if !GetBool(ConfigurationKeyTelemetry) {
Log.Debugf("not reporting due to config value")
return
}
if Branch != "main" {
Log.Debugf("reporting only on main branch")
return
}
argsBytes, _ := json.Marshal(args)
argsMap := map[string]string{
"telemetry_type": "execution",
"cmd": cmd,
"args": string(argsBytes),
"component": "mizu_cli",
"BuildTimestamp": BuildTimestamp,
"version": SemVer}
argsMap["message"] = fmt.Sprintf("mizu %v - %v", argsMap["cmd"], string(argsBytes))
jsonValue, _ := json.Marshal(argsMap)
if resp, err := http.Post(telemetryUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
Log.Debugf("error sending telemetry err: %v, response %v", err, resp)
} else {
Log.Debugf("Successfully reported telemetry")
}
}

View File

@@ -1,15 +1,19 @@
package mizu
import (
"context"
"encoding/json"
"fmt"
"github.com/google/go-github/v37/github"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/semver"
"io/ioutil"
"net/http"
"net/url"
"time"
)
func getApiVersion(port uint16) (string, error) {
versionUrl, _ := url.Parse(fmt.Sprintf("http://localhost:%d/mizu/metadata/version", port))
req := &http.Request{
@@ -41,6 +45,48 @@ func CheckVersionCompatibility(port uint16) (bool, error) {
return true, nil
}
fmt.Printf(Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)\n", SemVer, apiSemVer))
Log.Infof(uiUtils.Red, fmt.Sprintf("cli version (%s) is not compatible with api version (%s)", SemVer, apiSemVer))
return false, nil
}
func CheckNewerVersion() {
Log.Debugf("Checking for newer version...")
start := time.Now()
client := github.NewClient(nil)
latestRelease, _, err := client.Repositories.GetLatestRelease(context.Background(), "up9inc", "mizu")
if err != nil {
Log.Debugf("Failed to get latest release")
return
}
versionFileUrl := ""
for _, asset := range latestRelease.Assets {
if *asset.Name == "version.txt" {
versionFileUrl = *asset.BrowserDownloadURL
break
}
}
if versionFileUrl == "" {
Log.Debugf("Version file not found in the latest release")
return
}
res, err := http.Get(versionFileUrl)
if err != nil {
Log.Debugf("http.Get version asset -> %v", err)
return
}
data, err := ioutil.ReadAll(res.Body)
res.Body.Close()
if err != nil {
Log.Debugf("ioutil.ReadAll -> %v", err)
return
}
gitHubVersion := string(data)
gitHubVersion = gitHubVersion[:len(gitHubVersion)-1]
Log.Debugf("Finished version validation, took %v", time.Since(start))
if SemVer < gitHubVersion {
Log.Infof(uiUtils.Yellow, fmt.Sprintf("Update available! %v -> %v (%v)", SemVer, gitHubVersion, *latestRelease.HTMLURL))
}
}

13
cli/uiUtils/colors.go Normal file
View File

@@ -0,0 +1,13 @@
package uiUtils
const (
Black = "\033[1;30m%s\033[0m"
Red = "\033[1;31m%s\033[0m"
Green = "\033[1;32m%s\033[0m"
Yellow = "\033[1;33m%s\033[0m"
Purple = "\033[1;34m%s\033[0m"
Magenta = "\033[1;35m%s\033[0m"
Teal = "\033[1;36m%s\033[0m"
White = "\033[1;37m%s\033[0m"
)

View File

@@ -3,7 +3,6 @@ package uiUtils
import (
"bufio"
"fmt"
"github.com/up9inc/mizu/cli/mizu"
"log"
"os"
"strings"
@@ -12,7 +11,7 @@ import (
func AskForConfirmation(s string) bool {
reader := bufio.NewReader(os.Stdin)
fmt.Printf(mizu.Magenta, s)
fmt.Printf(Magenta, s)
response, err := reader.ReadString('\n')
if err != nil {

View File

@@ -0,0 +1,36 @@
package uiUtils
import (
"bytes"
"encoding/json"
"gopkg.in/yaml.v3"
)
const (
empty = ""
tab = "\t"
)
func PrettyJson(data interface{}) (string, error) {
buffer := new(bytes.Buffer)
encoder := json.NewEncoder(buffer)
encoder.SetIndent(empty, tab)
err := encoder.Encode(data)
if err != nil {
return empty, err
}
return buffer.String(), nil
}
func PrettyYaml(data interface{}) (string, error) {
buffer := new(bytes.Buffer)
encoder := yaml.NewEncoder(buffer)
encoder.SetIndent(0)
err := encoder.Encode(data)
if err != nil {
return empty, err
}
return buffer.String(), nil
}

View File

@@ -1,4 +1,4 @@
# creates image in which mizu api is remotely debuggable using delve
# creates image in which mizu agent is remotely debuggable using delve
FROM node:14-slim AS site-build
WORKDIR /app/ui-build
@@ -14,18 +14,21 @@ ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
RUN apk add libpcap-dev gcc g++ make
# Move to api working directory (/api-build).
WORKDIR /app/api-build
# Move to agent working directory (/agent-build).
WORKDIR /app/agent-build
COPY api/go.mod api/go.sum ./
COPY agent/go.mod agent/go.sum ./
COPY shared/go.mod shared/go.mod ../shared/
COPY tap/go.mod tap/go.mod ../tap/
RUN go mod download
# cheap trick to make the build faster (As long as go.mod wasn't changes)
RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
# Copy and build api code
# Copy and build agent code
COPY shared ../shared
COPY api .
COPY tap ../tap
COPY agent .
RUN go build -gcflags="all=-N -l" -o mizuagent .
@@ -35,10 +38,11 @@ RUN apk add bash libpcap-dev tcpdump
WORKDIR /app
# Copy binary and config files from /build to root folder of scratch container.
COPY --from=builder ["/app/api-build/mizuagent", "."]
COPY --from=builder ["/app/agent-build/mizuagent", "."]
COPY --from=site-build ["/app/ui-build/build", "site"]
# install remote debugging tool
RUN go get github.com/go-delve/delve/cmd/dlv
CMD ["sh", "-c", "dlv --headless=true --listen=:2345 --log --api-version=2 --accept-multiclient exec ./mizuagent -- --aggregator"]
ENTRYPOINT "/app/mizuagent"
#CMD ["sh", "-c", "dlv --headless=true --listen=:2345 --log --api-version=2 --accept-multiclient exec ./mizuagent -- --api-server"]

View File

@@ -0,0 +1,35 @@
# This example shows the roles required for a user to be able to use Mizu in all namespaces with IP resolution disabled.
# (Traffic will be recorded, but Mizu will not translate IP addresses to names)
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mizu-runner-clusterrole
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["list", "watch", "create"]
- apiGroups: [""]
resources: ["services"]
verbs: ["create"]
- apiGroups: ["apps"]
resources: ["daemonsets"]
verbs: ["create", "patch"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["services/proxy"]
verbs: ["get"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mizu-runner-clusterrolebindings
subjects:
- kind: User
name: user1
apiGroup: rbac.authorization.k8s.io
roleRef:
kind: ClusterRole
name: mizu-runner-clusterrole
apiGroup: rbac.authorization.k8s.io

View File

@@ -0,0 +1,52 @@
# This example shows the roles required for a user to be able to use Mizu in all namespaces.
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mizu-runner-clusterrole
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch", "create"]
- apiGroups: [""]
resources: ["services"]
verbs: ["get", "list", "watch", "create"]
- apiGroups: ["apps"]
resources: ["daemonsets"]
verbs: ["create", "patch"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["services/proxy"]
verbs: ["get"]
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["get", "create"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["clusterroles"]
verbs: ["list", "create", "delete"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["clusterrolebindings"]
verbs: ["list", "create", "delete"]
- apiGroups: ["apps", "extensions"]
resources: ["pods"]
verbs: ["get", "list", "watch"]
- apiGroups: ["apps", "extensions"]
resources: ["services"]
verbs: ["get", "list", "watch"]
- apiGroups: ["", "apps", "extensions"]
resources: ["endpoints"]
verbs: ["get", "list", "watch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: mizu-runner-clusterrolebindings
subjects:
- kind: User
name: user1
apiGroup: rbac.authorization.k8s.io
roleRef:
kind: ClusterRole
name: mizu-runner-clusterrole
apiGroup: rbac.authorization.k8s.io

View File

@@ -5,5 +5,5 @@ const (
HostModeEnvVar = "HOST_MODE"
NodeNameEnvVar = "NODE_NAME"
TappedAddressesPerNodeDictEnvVar = "TAPPED_ADDRESSES_PER_HOST"
MaxEntriesDBSizeByteSEnvVar = "MAX_ENTRIES_DB_BYTES"
MaxEntriesDBSizeBytesEnvVar = "MAX_ENTRIES_DB_BYTES"
)

View File

@@ -60,6 +60,7 @@ func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSoc
type TrafficFilteringOptions struct {
PlainTextMaskingRegexes []*SerializableRegexp
HideHealthChecks bool
DisableRedaction bool
}
type VersionResponse struct {

View File

@@ -1,9 +1,11 @@
package tap
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
@@ -46,12 +48,27 @@ type HarFile struct {
}
func NewEntry(request *http.Request, requestTime time.Time, response *http.Response, responseTime time.Time) (*har.Entry, error) {
harRequest, err := har.NewRequest(request, true)
harRequest, err := har.NewRequest(request, false)
if err != nil {
SilentError("convert-request-to-har", "Failed converting request to HAR %s (%v,%+v)", err, err, err)
return nil, errors.New("Failed converting request to HAR")
}
// For requests with multipart/form-data or application/x-www-form-urlencoded Content-Type,
// martian/har will parse the request body and place the parameters in harRequest.PostData.Params
// instead of harRequest.PostData.Text (as the HAR spec requires it).
// Mizu currently only looks at PostData.Text. Therefore, instead of letting martian/har set the content of
// PostData, always copy the request body to PostData.Text.
if (request.ContentLength > 0) {
reqBody, err := ioutil.ReadAll(request.Body)
if err != nil {
SilentError("read-request-body", "Failed converting request to HAR %s (%v,%+v)", err, err, err)
return nil, errors.New("Failed reading request body")
}
request.Body = ioutil.NopCloser(bytes.NewReader(reqBody))
harRequest.PostData.Text = string(reqBody)
}
harResponse, err := har.NewResponse(response, true)
if err != nil {
SilentError("convert-response-to-har", "Failed converting response to HAR %s (%v,%+v)", err, err, err)

View File

@@ -33,8 +33,7 @@ import (
const AppPortsEnvVar = "APP_PORTS"
const maxHTTP2DataLenEnvVar = "HTTP2_DATA_SIZE_LIMIT"
// default is 1MB, more than the max size accepted by collector and traffic-dumper
const maxHTTP2DataLenDefault = 1 * 1024 * 1024
const maxHTTP2DataLenDefault = 1 * 1024 * 1024 // 1MB
const cleanPeriod = time.Second * 10
var remoteOnlyOutboundPorts = []int { 80, 443 }

21229
ui/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -11,6 +11,7 @@
"@types/node": "^12.20.10",
"@types/react": "^17.0.3",
"@types/react-dom": "^17.0.3",
"axios": "^0.21.1",
"node-sass": "^5.0.0",
"numeral": "^2.0.6",
"protobuf-decoder": "^0.1.0",

View File

@@ -4,6 +4,7 @@ import styles from './style/HarEntriesList.module.sass';
import spinner from './assets/spinner.svg';
import ScrollableFeed from "react-scrollable-feed";
import {StatusType} from "./HarFilters";
import Api from "../helpers/api";
interface HarEntriesListProps {
entries: any[];
@@ -25,6 +26,8 @@ enum FetchOperator {
GT = "gt"
}
const api = new Api();
export const HarEntriesList: React.FC<HarEntriesListProps> = ({entries, setEntries, focusedEntryId, setFocusedEntryId, connectionOpen, noMoreDataTop, setNoMoreDataTop, noMoreDataBottom, setNoMoreDataBottom, methodsFilter, statusFilter, pathFilter}) => {
const [loadMoreTop, setLoadMoreTop] = useState(false);
@@ -54,14 +57,9 @@ export const HarEntriesList: React.FC<HarEntriesListProps> = ({entries, setEntri
return entries.filter(filterEntries);
},[entries, filterEntries])
const fetchData = async (operator, timestamp) => {
const response = await fetch(`http://localhost:8899/api/entries?limit=50&operator=${operator}&timestamp=${timestamp}`);
return await response.json();
}
const getOldEntries = useCallback(async () => {
setIsLoadingTop(true);
const data = await fetchData(FetchOperator.LT, entries[0].timestamp);
const data = await api.fetchEntries(FetchOperator.LT, entries[0].timestamp);
setLoadMoreTop(false);
let scrollTo;
@@ -89,7 +87,7 @@ export const HarEntriesList: React.FC<HarEntriesListProps> = ({entries, setEntri
}, [loadMoreTop, connectionOpen, noMoreDataTop, getOldEntries]);
const getNewEntries = async () => {
const data = await fetchData(FetchOperator.GT, entries[entries.length - 1].timestamp);
const data = await api.fetchEntries(FetchOperator.GT, entries[entries.length - 1].timestamp);
let scrollTo;
if(data.length === 0) {
setNoMoreDataBottom(true);

View File

@@ -9,6 +9,7 @@ import playIcon from './assets/run.svg';
import pauseIcon from './assets/pause.svg';
import variables from './style/variables.module.scss';
import {StatusBar} from "./StatusBar";
import Api, {MizuWebsocketURL} from "../helpers/api";
const useLayoutStyles = makeStyles(() => ({
details: {
@@ -39,21 +40,7 @@ interface HarPageProps {
setAnalyzeStatus: (status: any) => void;
}
const mizuAPIPathPrefix = "/mizu";
// When working locally (with npm run start) we need to change the PORT
const getMizuApiUrl = () => {
return `${window.location.origin}${mizuAPIPathPrefix}`;
};
const getMizuWebsocketUrl = () => {
return `ws://${window.location.host}${mizuAPIPathPrefix}/ws`;
}
const mizuApiUrl = getMizuApiUrl();
const mizuWebsocketUrl = getMizuWebsocketUrl();
const api = new Api();
export const HarPage: React.FC<HarPageProps> = ({setAnalyzeStatus}) => {
@@ -75,7 +62,7 @@ export const HarPage: React.FC<HarPageProps> = ({setAnalyzeStatus}) => {
const ws = useRef(null);
const openWebSocket = () => {
ws.current = new WebSocket(mizuWebsocketUrl);
ws.current = new WebSocket(MizuWebsocketURL);
ws.current.onopen = () => setConnection(ConnectionStatus.Connected);
ws.current.onclose = () => setConnection(ConnectionStatus.Closed);
}
@@ -113,24 +100,32 @@ export const HarPage: React.FC<HarPageProps> = ({setAnalyzeStatus}) => {
}
useEffect(() => {
openWebSocket();
fetch(`${mizuApiUrl}/api/tapStatus`)
.then(response => response.json())
.then(data => setTappingStatus(data));
fetch(`${mizuApiUrl}/api/analyzeStatus`)
.then(response => response.json())
.then(data => setAnalyzeStatus(data));
(async () => {
openWebSocket();
try{
const tapStatusResponse = await api.tapStatus();
setTappingStatus(tapStatusResponse);
const analyzeStatusResponse = await api.analyzeStatus();
setAnalyzeStatus(analyzeStatusResponse);
} catch (error) {
console.error(error);
}
})()
// eslint-disable-next-line
}, []);
useEffect(() => {
if (!focusedEntryId) return;
setSelectedHarEntry(null)
fetch(`${mizuApiUrl}/api/entries/${focusedEntryId}`)
.then(response => response.json())
.then(data => setSelectedHarEntry(data));
setSelectedHarEntry(null);
(async () => {
try {
const entryData = await api.getEntry(focusedEntryId);
setSelectedHarEntry(entryData);
} catch (error) {
console.error(error);
}
})()
}, [focusedEntryId])
const toggleConnection = () => {

View File

@@ -29,12 +29,14 @@ export const StatusBar: React.FC<Props> = ({tappingStatus}) => {
<div className="podsCount">{`Tapping ${amountOfPods} ${pluralize('pod', amountOfPods)} in ${pluralize('namespace', uniqueNamespaces.length)} ${uniqueNamespaces.join(", ")}`}</div>
{expandedBar && <div style={{marginTop: 20}}>
<table>
<tr>
<th>Pod name</th>
<th>Namespace</th>
</tr>
<thead>
<tr>
<th>Pod name</th>
<th>Namespace</th>
</tr>
</thead>
<tbody>
{tappingStatus.pods.map(pod => <tr>
{tappingStatus.pods.map(pod => <tr key={pod.name}>
<td>{pod.name}</td>
<td>{pod.namespace}</td>
</tr>)}

45
ui/src/helpers/api.js Normal file
View File

@@ -0,0 +1,45 @@
import * as axios from "axios";
const mizuAPIPathPrefix = "/mizu";
// When working locally (with npm run start) change to:
// export const MizuWebsocketURL = `ws://localhost:8899${mizuAPIPathPrefix}/ws`;
export const MizuWebsocketURL = `ws://${window.location.host}${mizuAPIPathPrefix}/ws`;
export default class Api {
constructor() {
// When working locally (with npm run start) change to:
// const apiURL = `http://localhost:8899/${mizuAPIPathPrefix}/api/`;
const apiURL = `${window.location.origin}${mizuAPIPathPrefix}/api/`;
this.client = axios.create({
baseURL: apiURL,
timeout: 31000,
headers: {
Accept: "application/json",
}
});
}
tapStatus = async () => {
const response = await this.client.get("/tapStatus");
return response.data;
}
analyzeStatus = async () => {
const response = await this.client.get("/analyzeStatus");
return response.data;
}
getEntry = async (entryId) => {
const response = await this.client.get(`/entries/${entryId}`);
return response.data;
}
fetchEntries = async (operator, timestamp) => {
const response = await this.client.get(`/entries?limit=50&operator=${operator}&timestamp=${timestamp}`);
return response.data;
}
}