Compare commits

...

15 Commits

Author SHA1 Message Date
Igor Gov
1fcc22c356 Develop -> main (release 32.0)
Merge pull request #1073 from up9inc/develop #major
2022-05-09 15:08:28 +03:00
RoyUP9
4c0aeb8146 Fixed default mime type to empty string instead of text/html (#1071) 2022-05-08 16:52:50 +03:00
lirazyehezkel
0dc0459dff Disable tapping status when streaming is paused (#1070) 2022-05-08 15:45:33 +03:00
lirazyehezkel
81f06003d5 Outgoing filter remove redundant code (#1069) 2022-05-08 14:58:13 +03:00
lirazyehezkel
c7d068748a TRA-4553 Fix outgoing filter (#1068)
* Fix outgoing filter

* fix labels
2022-05-08 13:44:56 +03:00
gadotroee
8102c49138 Improve acceptance tests setup script (#887) 2022-05-04 11:13:18 +03:00
AmitUp9
65fb2a4fe4 update close icon in oas and service map modals (#1066)
* changed icon

* close icon postion

* linter fixes

* make onSelectedOASService as callback function

* removing lint warning

* small fix

Co-authored-by: Leon <>
2022-05-03 13:59:28 +03:00
David Levanon
57f8a8dca9 Feature/fix tls not listening (#1046)
* avoid chunks with invalid address

* tls tapper should distict between pids

* prettfy tls verbose log and tls key

* support tls from multi threads + duplicate calls to the same target

* introduce fdCache and user address pair as tls key

* remove unused comment

* fix merge conflicts

* use lru for fdcache

* pr fixes - renaming

* fix conflict issue
2022-05-02 21:33:26 +03:00
M. Mert Yıldıran
aead6cbc19 Bring back debug.FreeOSMemory() call to prevent OOMKilled(s) (#1063)
* Revert "Revert "Bring back `debug.FreeOSMemory()` call""

This reverts commit 69538cb7e3.

* Revert "Revert "Revert "Remove `GOGC` environment variable from tapper"""

This reverts commit 3d9f37ab4b.

* Revert "Revert "Revert "Revert "Remove `GOGC` environment variable from tapper""""

This reverts commit 58ff0607ff.

Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com>
2022-05-02 14:13:04 +03:00
gadotroee
2f89690da6 Remove go-cache unused dependency (#1065) 2022-05-02 13:24:28 +03:00
M. Mert Yıldıran
3e47abf208 Add from ARM64 to AMD64 cross-compilation support to the agent image (#910)
* Add from ARM64 to AMD64 cross-compilation support to the agent image

* Replace `RUN`s with `&&`

Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com>
2022-05-02 11:16:05 +03:00
M. Mert Yıldıran
3cbccccb8b Add GraphQL detection (#980)
* Add GraphQL detection

* Change the abbreviation to `GQL`

* Fix the `TestMacros` test case

* Try to fix the security issues reported by Snyk

* Run `go mod tidy` in `agent` directory

* Upgrade `golang.org/x/crypto`

* Downgrade `golang.org/x/crypto`

* Downgrade `golang.org/x/crypto`

* Downgrade `golang.org/x/crypto`

* Upgrade to `github.com/vektah/gqlparser/v2 v2.4.2`

* Run `go mod tidy`

* Replace `github.com/vektah/gqlparser/v2` with its fork `github.com/mertyildiran/gqlparser/v2` to fix the vulns

* Upgrade the `github.com/mertyildiran/gqlparser/v2 v2.4.6`

* Don't upgrade `golang.org/x/net`
2022-05-02 11:06:01 +03:00
gadotroee
08ae2bf6d7 Move "es Lint" to different step and add go lint on logger module (#1064) 2022-05-02 10:01:41 +03:00
Igor Gov
960d39f27d Develop -> main (Patch release 31.1)
Merge pull request #1041 from up9inc/develop #patch
2022-04-25 12:02:38 +03:00
gadotroee
0709b861d6 Develop -> main (Release 31.0)
Merge pull request #1036 from up9inc/develop #major
2022-04-24 15:06:44 +03:00
50 changed files with 407 additions and 43967 deletions

View File

@@ -10,7 +10,7 @@ permissions:
contents: read
jobs:
golangci:
go-lint:
name: Go lint
runs-on: ubuntu-latest
steps:
@@ -141,7 +141,26 @@ jobs:
with:
version: latest
working-directory: tap/extensions/redis
- name: Check logger modified files
id: logger_modified_files
run: devops/check_modified_files.sh logger/
- name: Go lint - logger
uses: golangci/golangci-lint-action@v2
if: steps.logger_modified_files.outputs.matched == 'true'
with:
version: latest
working-directory: logger
es-lint:
name: ES lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 2
- uses: actions/setup-node@v2
with:
node-version: 16

5
.gitignore vendored
View File

@@ -52,4 +52,7 @@ tap/extensions/*/expect
# UI folders to ignore
**/node_modules/**
**/dist/**
*.editorconfig
*.editorconfig
# Ignore *.log files
*.log

View File

@@ -44,11 +44,18 @@ ENV CGO_ENABLED=1 GOOS=linux
ENV GOARCH=arm64 CGO_CFLAGS="-I/work/libpcap"
### Builder image for AArch64 to x86-64 cross-compilation
FROM up9inc/linux-x86_64-musl-go-libpcap AS builder-from-arm64v8-to-amd64
ENV CGO_ENABLED=1 GOOS=linux
ENV GOARCH=amd64 CGO_CFLAGS="-I/libpcap"
### Final builder image where the build happens
# Possible build strategies:
# BUILDARCH=amd64 TARGETARCH=amd64
# BUILDARCH=arm64v8 TARGETARCH=arm64v8
# BUILDARCH=amd64 TARGETARCH=arm64v8
# BUILDARCH=arm64v8 TARGETARCH=amd64
ARG BUILDARCH=amd64
ARG TARGETARCH=amd64
FROM builder-from-${BUILDARCH}-to-${TARGETARCH} AS builder
@@ -66,9 +73,6 @@ COPY tap/extensions/http/go.mod ../tap/extensions/http/
COPY tap/extensions/kafka/go.mod ../tap/extensions/kafka/
COPY tap/extensions/redis/go.mod ../tap/extensions/redis/
RUN go mod download
# cheap trick to make the build faster (as long as go.mod did not change)
RUN go get github.com/patrickmn/go-cache
RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
# Copy and build agent code
COPY shared ../shared

View File

@@ -1,7 +1,10 @@
#!/bin/bash
set -e
PREFIX=$HOME/local/bin
VERSION=v1.22.0
TUNNEL_LOG="tunnel.log"
PROXY_LOG="proxy.log"
echo "Attempting to install minikube and assorted tools to $PREFIX"
@@ -11,7 +14,7 @@ if ! [ -x "$(command -v kubectl)" ]; then
chmod +x kubectl
mv kubectl "$PREFIX"
else
echo "kubetcl is already installed"
echo "kubectl is already installed"
fi
if ! [ -x "$(command -v minikube)" ]; then
@@ -27,35 +30,39 @@ echo "Starting minikube..."
minikube start
echo "Creating mizu tests namespaces"
kubectl create namespace mizu-tests
kubectl create namespace mizu-tests2
kubectl create namespace mizu-tests --dry-run=client -o yaml | kubectl apply -f -
kubectl create namespace mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
echo "Creating httpbin deployments"
kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests
kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests
kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
kubectl create deployment httpbin2 --image=kennethreitz/httpbin -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2
kubectl create deployment httpbin --image=kennethreitz/httpbin -n mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
echo "Creating redis deployment"
kubectl create deployment redis --image=redis -n mizu-tests
kubectl create deployment redis --image=redis -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
echo "Creating rabbitmq deployment"
kubectl create deployment rabbitmq --image=rabbitmq -n mizu-tests
kubectl create deployment rabbitmq --image=rabbitmq -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
echo "Creating httpbin services"
kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests
kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests
kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
kubectl expose deployment httpbin2 --type=NodePort --port=80 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2
kubectl expose deployment httpbin --type=NodePort --port=80 -n mizu-tests2 --dry-run=client -o yaml | kubectl apply -f -
echo "Creating redis service"
kubectl expose deployment redis --type=LoadBalancer --port=6379 -n mizu-tests
kubectl expose deployment redis --type=LoadBalancer --port=6379 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
echo "Creating rabbitmq service"
kubectl expose deployment rabbitmq --type=LoadBalancer --port=5672 -n mizu-tests
kubectl expose deployment rabbitmq --type=LoadBalancer --port=5672 -n mizu-tests --dry-run=client -o yaml | kubectl apply -f -
# TODO: need to understand how to fail if address already in use
echo "Starting proxy"
kubectl proxy --port=8080 &
rm -f ${PROXY_LOG}
kubectl proxy --port=8080 > ${PROXY_LOG} &
PID1=$!
echo "kubectl proxy process id is ${PID1} and log of proxy in ${PROXY_LOG}"
if [[ -z "${CI}" ]]; then
echo "Setting env var of mizu ci image"
@@ -71,5 +78,9 @@ minikube image load "${MIZU_CI_IMAGE}"
echo "Build cli"
cd cli && make build GIT_BRANCH=ci SUFFIX=ci
# TODO: need to understand how to fail if password is asked (sudo)
echo "Starting tunnel"
minikube tunnel &
rm -f ${TUNNEL_LOG}
minikube tunnel > ${TUNNEL_LOG} &
PID2=$!
echo "Minikube tunnel process id is ${PID2} and log of tunnel in ${TUNNEL_LOG}"

View File

@@ -76,6 +76,7 @@ require (
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
@@ -85,6 +86,7 @@ require (
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mertyildiran/gqlparser/v2 v2.4.6 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
@@ -110,7 +112,7 @@ require (
github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
github.com/xlab/treeprint v1.1.0 // indirect
go.starlark.net v0.0.0-20220203230714-bb14e151c28f // indirect
golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab // indirect
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b // indirect
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
golang.org/x/sys v0.0.0-20220207234003-57398862261d // indirect

View File

@@ -83,6 +83,7 @@ github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tN
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -404,6 +405,7 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
@@ -493,6 +495,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/mertyildiran/gqlparser/v2 v2.4.6 h1:enAq4F5PYgW/rYExPNzYt7IYrrZnzrfqdywMA1QdFtM=
github.com/mertyildiran/gqlparser/v2 v2.4.6/go.mod h1:XZId58F+XqRSmoLrdsOLgqA918oNvBzuOORruJWBjDo=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
@@ -751,8 +755,9 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab h1:lnZ4LoV0UMdibeCUfIB2a4uFwRu491WX/VB2reB8xNc=
golang.org/x/crypto v0.0.0-20220208050332-20e1d8d225ab/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b h1:Qwe1rC8PSniVfAFPFJeyUkB+zcysC3RgJBAGk7eqBEU=
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=

View File

@@ -124,8 +124,8 @@ func NewRequest(request map[string]interface{}) (harRequest *Request, err error)
postData, _ := request["postData"].(map[string]interface{})
mimeType := postData["mimeType"]
if mimeType == nil || len(mimeType.(string)) == 0 {
mimeType = "text/html"
if mimeType == nil {
mimeType = ""
}
text := postData["text"]
postDataText := ""
@@ -177,8 +177,8 @@ func NewResponse(response map[string]interface{}) (harResponse *Response, err er
content, _ := response["content"].(map[string]interface{})
mimeType := content["mimeType"]
if mimeType == nil || len(mimeType.(string)) == 0 {
mimeType = "text/html"
if mimeType == nil {
mimeType = ""
}
encoding := content["encoding"]
text := content["text"]

View File

@@ -219,8 +219,6 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

View File

@@ -0,0 +1,31 @@
FROM messense/rust-musl-cross:x86_64-musl AS builder-from-arm64v8-to-amd64
ENV CROSS_TRIPLE x86_64-unknown-linux-musl
ENV CROSS_ROOT /usr/local/musl
ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \
AR=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ar \
CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gcc \
CPP=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-cpp \
CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ \
LD=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ld \
FC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gfortran
# Install Go
WORKDIR /
RUN curl https://go.dev/dl/go1.17.6.linux-arm64.tar.gz -Lo ./go.linux-arm64.tar.gz \
&& curl https://go.dev/dl/go1.17.6.linux-arm64.tar.gz.asc -Lo ./go.linux-arm64.tar.gz.asc \
&& curl https://dl.google.com/dl/linux/linux_signing_key.pub -Lo linux_signing_key.pub \
&& gpg --import linux_signing_key.pub && gpg --verify ./go.linux-arm64.tar.gz.asc ./go.linux-arm64.tar.gz \
&& rm -rf /usr/local/go && tar -C /usr/local -xzf go.linux-arm64.tar.gz
ENV PATH "$PATH:/usr/local/go/bin"
# Compile libpcap from source
RUN curl https://www.tcpdump.org/release/libpcap-1.10.1.tar.gz -Lo ./libpcap.tar.gz \
&& curl https://www.tcpdump.org/release/libpcap-1.10.1.tar.gz.sig -Lo ./libpcap.tar.gz.sig \
&& curl https://www.tcpdump.org/release/signing-key.asc -Lo ./signing-key.asc \
&& gpg --import signing-key.asc && gpg --verify libpcap.tar.gz.sig libpcap.tar.gz \
&& tar -xzf libpcap.tar.gz && mv ./libpcap-* ./libpcap
WORKDIR /libpcap
RUN ./configure --host=x86_64 && make \
&& cp /libpcap/libpcap.a /usr/local/musl/lib/gcc/x86_64-unknown-linux-musl/*/

View File

@@ -0,0 +1,4 @@
#!/bin/bash
set -e
docker build . -t up9inc/linux-x86_64-musl-go-libpcap && docker push up9inc/linux-x86_64-musl-go-libpcap

View File

@@ -13,4 +13,4 @@ test-pull-bin:
test-pull-expect:
@mkdir -p expect
@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5' -m cp -r gs://static.up9.io/mizu/test-pcap/expect8/http/\* expect
@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5' -m cp -r gs://static.up9.io/mizu/test-pcap/expect9/http/\* expect

View File

@@ -4,6 +4,7 @@ go 1.17
require (
github.com/beevik/etree v1.1.0
github.com/mertyildiran/gqlparser/v2 v2.4.6
github.com/stretchr/testify v1.7.0
github.com/up9inc/mizu/tap/api v0.0.0
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd

View File

@@ -1,3 +1,4 @@
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -5,6 +6,13 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mertyildiran/gqlparser/v2 v2.4.6 h1:enAq4F5PYgW/rYExPNzYt7IYrrZnzrfqdywMA1QdFtM=
github.com/mertyildiran/gqlparser/v2 v2.4.6/go.mod h1:XZId58F+XqRSmoLrdsOLgqA918oNvBzuOORruJWBjDo=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -18,8 +26,10 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -0,0 +1,31 @@
package http
import (
"encoding/json"
"github.com/mertyildiran/gqlparser/v2/ast"
"github.com/mertyildiran/gqlparser/v2/parser"
)
func isGraphQL(request map[string]interface{}) bool {
if postData, ok := request["postData"].(map[string]interface{}); ok {
if postData["mimeType"] == "application/json" {
if text, ok := postData["text"].(string); ok {
var data map[string]interface{}
if err := json.Unmarshal([]byte(text), &data); err != nil {
return false
}
if query, ok := data["query"].(string); ok {
_, err := parser.ParseQuery(&ast.Source{Name: "ff", Input: query})
if err == nil {
return true
}
}
}
}
}
return false
}

View File

@@ -71,6 +71,34 @@ var grpcProtocol api.Protocol = api.Protocol{
Priority: 0,
}
var graphQL1Protocol api.Protocol = api.Protocol{
Name: "http",
LongName: "Hypertext Transfer Protocol -- HTTP/1.1 [ GraphQL over HTTP/1.1 ]",
Abbreviation: "GQL",
Macro: "gql",
Version: "1.1",
BackgroundColor: "#e10098",
ForegroundColor: "#ffffff",
FontSize: 12,
ReferenceLink: "https://graphql.org/learn/serving-over-http/",
Ports: []string{"80", "443", "8080"},
Priority: 0,
}
var graphQL2Protocol api.Protocol = api.Protocol{
Name: "http",
LongName: "Hypertext Transfer Protocol Version 2 (HTTP/2) [ GraphQL over HTTP/2 ]",
Abbreviation: "GQL",
Macro: "gql",
Version: "2.0",
BackgroundColor: "#e10098",
ForegroundColor: "#ffffff",
FontSize: 12,
ReferenceLink: "https://graphql.org/learn/serving-over-http/",
Ports: []string{"80", "443", "8080", "50051"},
Priority: 0,
}
const (
TypeHttpRequest = iota
TypeHttpResponse
@@ -208,6 +236,14 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string,
}
}
if isGraphQL(reqDetails) {
if item.Protocol.Version == "2.0" {
item.Protocol = graphQL2Protocol
} else {
item.Protocol = graphQL1Protocol
}
}
if resDetails["bodySize"].(float64) < 0 {
resDetails["bodySize"] = 0
}
@@ -365,8 +401,8 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
postData, _ := request["postData"].(map[string]interface{})
mimeType := postData["mimeType"]
if mimeType == nil || len(mimeType.(string)) == 0 {
mimeType = "text/html"
if mimeType == nil {
mimeType = ""
}
text := postData["text"]
if text != nil {
@@ -447,8 +483,8 @@ func representResponse(response map[string]interface{}) (repResponse []interface
content, _ := response["content"].(map[string]interface{})
mimeType := content["mimeType"]
if mimeType == nil || len(mimeType.(string)) == 0 {
mimeType = "text/html"
if mimeType == nil {
mimeType = ""
}
encoding := content["encoding"]
text := content["text"]
@@ -481,6 +517,7 @@ func (d dissecting) Macros() map[string]string {
`http`: fmt.Sprintf(`proto.name == "%s" and proto.version.startsWith("%c")`, http11protocol.Name, http11protocol.Version[0]),
`http2`: fmt.Sprintf(`proto.name == "%s" and proto.version == "%s"`, http11protocol.Name, http2Protocol.Version),
`grpc`: fmt.Sprintf(`proto.name == "%s" and proto.version == "%s" and proto.macro == "%s"`, http11protocol.Name, grpcProtocol.Version, grpcProtocol.Macro),
`gql`: fmt.Sprintf(`proto.name == "%s" and proto.macro == "%s"`, graphQL1Protocol.Name, graphQL1Protocol.Macro),
}
}

View File

@@ -47,6 +47,7 @@ func TestMacros(t *testing.T) {
"http": `proto.name == "http" and proto.version.startsWith("1")`,
"http2": `proto.name == "http" and proto.version == "2.0"`,
"grpc": `proto.name == "http" and proto.version == "2.0" and proto.macro == "grpc"`,
"gql": `proto.name == "http" and proto.macro == "gql"`,
}
dissector := NewDissector()
macros := dissector.Macros()

View File

@@ -18,6 +18,7 @@ require (
github.com/google/go-cmp v0.5.7 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/martian v2.1.0+incompatible // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect

View File

@@ -71,6 +71,8 @@ github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2c
github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=

View File

@@ -2,6 +2,7 @@ package tap
import (
"runtime"
_debug "runtime/debug"
"sync"
"time"
@@ -47,6 +48,7 @@ func (streamMap *tcpStreamMap) CloseTimedoutTcpStreamChannels() {
for {
<-ticker.C
_debug.FreeOSMemory()
streamMap.streams.Range(func(key interface{}, value interface{}) bool {
// `*tlsStream` is not yet applicable to this routine.
// So, we cast into `(*tcpStream)` and ignore `*tlsStream`

View File

@@ -48,14 +48,14 @@ static __always_inline int get_count_bytes(struct pt_regs *ctx, struct ssl_info*
return countBytes;
}
static __always_inline void add_address_to_chunk(struct pt_regs *ctx, struct tlsChunk* chunk, __u64 id, __u32 fd) {
static __always_inline int add_address_to_chunk(struct pt_regs *ctx, struct tlsChunk* chunk, __u64 id, __u32 fd) {
__u32 pid = id >> 32;
__u64 key = (__u64) pid << 32 | fd;
struct fd_info *fdinfo = bpf_map_lookup_elem(&file_descriptor_to_ipv4, &key);
if (fdinfo == NULL) {
return;
return 0;
}
int err = bpf_probe_read(chunk->address, sizeof(chunk->address), fdinfo->ipv4_addr);
@@ -63,7 +63,10 @@ static __always_inline void add_address_to_chunk(struct pt_regs *ctx, struct tls
if (err != 0) {
log_error(ctx, LOG_ERROR_READING_FD_ADDRESS, id, err, 0l);
return 0;
}
return 1;
}
static __always_inline void send_chunk_part(struct pt_regs *ctx, __u8* buffer, __u64 id,
@@ -143,7 +146,12 @@ static __always_inline void output_ssl_chunk(struct pt_regs *ctx, struct ssl_inf
chunk->len = countBytes;
chunk->fd = info->fd;
add_address_to_chunk(ctx, chunk, id, chunk->fd);
if (!add_address_to_chunk(ctx, chunk, id, chunk->fd)) {
// Without an address, we drop the chunk because there is not much to do with it in Go
//
return;
}
send_chunk(ctx, info->buffer, id, chunk);
}

View File

@@ -6,6 +6,7 @@ import (
"net"
"github.com/go-errors/errors"
"github.com/up9inc/mizu/tap/api"
)
const FLAGS_IS_CLIENT_BIT uint32 = (1 << 0)
@@ -73,3 +74,27 @@ func (c *tlsChunk) getRecordedData() []byte {
func (c *tlsChunk) isRequest() bool {
return (c.isClient() && c.isWrite()) || (c.isServer() && c.isRead())
}
func (c *tlsChunk) getAddressPair() (addressPair, error) {
ip, port, err := c.getAddress()
if err != nil {
return addressPair{}, err
}
if c.isRequest() {
return addressPair{
srcIp: api.UnknownIp,
srcPort: api.UnknownPort,
dstIp: ip,
dstPort: port,
}, nil
} else {
return addressPair{
srcIp: ip,
srcPort: port,
dstIp: api.UnknownIp,
dstPort: api.UnknownPort,
}, nil
}
}

View File

@@ -20,10 +20,17 @@ const (
INODE_FILED_INDEX = 9
)
type addressPair struct {
srcIp net.IP
srcPort uint16
dstIp net.IP
dstPort uint16
}
// This file helps to extract Ip and Port out of a Socket file descriptor.
//
//
// The equivalent bash commands are:
//
//
// > ls -l /proc/<pid>/fd/<fd>
// Output something like "socket:[1234]" for sockets - 1234 is the inode of the socket
// > cat /proc/<pid>/net/tcp | grep <inode>
@@ -31,18 +38,18 @@ const (
// The 1st and 2nd fields are the source and dest ip and ports in a Hex format
// 0100007F:50 is 127.0.0.1:80
func getAddressBySockfd(procfs string, pid uint32, fd uint32, src bool) (net.IP, uint16, error) {
func getAddressBySockfd(procfs string, pid uint32, fd uint32) (addressPair, error) {
inode, err := getSocketInode(procfs, pid, fd)
if err != nil {
return nil, 0, err
return addressPair{}, err
}
tcppath := fmt.Sprintf("%s/%d/net/tcp", procfs, pid)
tcp, err := ioutil.ReadFile(tcppath)
if err != nil {
return nil, 0, errors.Wrap(err, 0)
return addressPair{}, errors.Wrap(err, 0)
}
for _, line := range strings.Split(string(tcp), "\n") {
@@ -53,15 +60,28 @@ func getAddressBySockfd(procfs string, pid uint32, fd uint32, src bool) (net.IP,
}
if inode == parts[INODE_FILED_INDEX] {
if src {
return parseHexAddress(parts[SRC_ADDRESS_FILED_INDEX])
} else {
return parseHexAddress(parts[DST_ADDRESS_FILED_INDEX])
srcIp, srcPort, srcErr := parseHexAddress(parts[SRC_ADDRESS_FILED_INDEX])
if srcErr != nil {
return addressPair{}, srcErr
}
dstIp, dstPort, dstErr := parseHexAddress(parts[DST_ADDRESS_FILED_INDEX])
if dstErr != nil {
return addressPair{}, dstErr
}
return addressPair{
srcIp: srcIp,
srcPort: srcPort,
dstIp: dstIp,
dstPort: dstPort,
}, nil
}
}
return nil, 0, errors.Errorf("address not found [pid: %d] [sockfd: %d] [inode: %s]", pid, fd, inode)
return addressPair{}, errors.Errorf("address not found [pid: %d] [sockfd: %d] [inode: %s]", pid, fd, inode)
}
func getSocketInode(procfs string, pid uint32, fd uint32) (string, error) {

View File

@@ -4,7 +4,6 @@ import (
"bufio"
"bytes"
"fmt"
"net"
"sync"
"time"
@@ -16,10 +15,14 @@ import (
"github.com/cilium/ebpf/perf"
"github.com/go-errors/errors"
"github.com/hashicorp/golang-lru/simplelru"
"github.com/up9inc/mizu/logger"
"github.com/up9inc/mizu/tap/api"
)
const fdCachedItemAvgSize = 40
const fdCacheMaxItems = 500000 / fdCachedItemAvgSize
type tlsPoller struct {
tls *TlsTapper
readers map[string]*tlsReader
@@ -29,10 +32,12 @@ type tlsPoller struct {
extension *api.Extension
procfs string
pidToNamespace sync.Map
fdCache *simplelru.LRU // Actual typs is map[string]addressPair
evictedCounter int
}
func newTlsPoller(tls *TlsTapper, extension *api.Extension, procfs string) *tlsPoller {
return &tlsPoller{
func newTlsPoller(tls *TlsTapper, extension *api.Extension, procfs string) (*tlsPoller, error) {
poller := &tlsPoller{
tls: tls,
readers: make(map[string]*tlsReader),
closedReaders: make(chan string, 100),
@@ -41,6 +46,15 @@ func newTlsPoller(tls *TlsTapper, extension *api.Extension, procfs string) *tlsP
chunksReader: nil,
procfs: procfs,
}
fdCache, err := simplelru.NewLRU(fdCacheMaxItems, poller.fdCacheEvictCallback)
if err != nil {
return nil, errors.Wrap(err, 0)
}
poller.fdCache = fdCache
return poller, nil
}
func (p *tlsPoller) init(bpfObjects *tlsTapperObjects, bufferSize int) error {
@@ -117,35 +131,38 @@ func (p *tlsPoller) pollChunksPerfBuffer(chunks chan<- *tlsChunk) {
func (p *tlsPoller) handleTlsChunk(chunk *tlsChunk, extension *api.Extension, emitter api.Emitter,
options *api.TrafficFilteringOptions, streamsMap api.TcpStreamMap) error {
ip, port, err := chunk.getAddress()
address, err := p.getSockfdAddressPair(chunk)
if err != nil {
return err
address, err = chunk.getAddressPair()
if err != nil {
return err
}
}
key := buildTlsKey(chunk, ip, port)
key := buildTlsKey(address)
reader, exists := p.readers[key]
if !exists {
reader = p.startNewTlsReader(chunk, ip, port, key, emitter, extension, options, streamsMap)
reader = p.startNewTlsReader(chunk, &address, key, emitter, extension, options, streamsMap)
p.readers[key] = reader
}
reader.captureTime = time.Now()
reader.chunks <- chunk
reader.newChunk(chunk)
if os.Getenv("MIZU_VERBOSE_TLS_TAPPER") == "true" {
p.logTls(chunk, ip, port)
p.logTls(chunk, key, reader)
}
return nil
}
func (p *tlsPoller) startNewTlsReader(chunk *tlsChunk, ip net.IP, port uint16, key string,
func (p *tlsPoller) startNewTlsReader(chunk *tlsChunk, address *addressPair, key string,
emitter api.Emitter, extension *api.Extension, options *api.TrafficFilteringOptions,
streamsMap api.TcpStreamMap) *tlsReader {
tcpid := p.buildTcpId(chunk, ip, port)
tcpid := p.buildTcpId(chunk, address)
doneHandler := func(r *tlsReader) {
p.closeReader(key, r)
@@ -197,36 +214,52 @@ func (p *tlsPoller) closeReader(key string, r *tlsReader) {
p.closedReaders <- key
}
func buildTlsKey(chunk *tlsChunk, ip net.IP, port uint16) string {
return fmt.Sprintf("%v:%v-%v:%v", chunk.isClient(), chunk.isRead(), ip, port)
}
func (p *tlsPoller) getSockfdAddressPair(chunk *tlsChunk) (addressPair, error) {
address, err := getAddressBySockfd(p.procfs, chunk.Pid, chunk.Fd)
fdCacheKey := fmt.Sprintf("%d:%d", chunk.Pid, chunk.Fd)
func (p *tlsPoller) buildTcpId(chunk *tlsChunk, ip net.IP, port uint16) api.TcpID {
myIp, myPort, err := getAddressBySockfd(p.procfs, chunk.Pid, chunk.Fd, chunk.isClient())
if err != nil {
// May happen if the socket already closed, very likely to happen for localhost
//
myIp = api.UnknownIp
myPort = api.UnknownPort
if err == nil {
if !chunk.isRequest() {
switchedAddress := addressPair{
srcIp: address.dstIp,
srcPort: address.dstPort,
dstIp: address.srcIp,
dstPort: address.srcPort,
}
p.fdCache.Add(fdCacheKey, switchedAddress)
return switchedAddress, nil
} else {
p.fdCache.Add(fdCacheKey, address)
return address, nil
}
}
if chunk.isRequest() {
return api.TcpID{
SrcIP: myIp.String(),
DstIP: ip.String(),
SrcPort: strconv.FormatUint(uint64(myPort), 10),
DstPort: strconv.FormatUint(uint64(port), 10),
Ident: "",
}
} else {
return api.TcpID{
SrcIP: ip.String(),
DstIP: myIp.String(),
SrcPort: strconv.FormatUint(uint64(port), 10),
DstPort: strconv.FormatUint(uint64(myPort), 10),
Ident: "",
}
fromCacheIfc, ok := p.fdCache.Get(fdCacheKey)
if !ok {
return addressPair{}, err
}
fromCache, ok := fromCacheIfc.(addressPair)
if !ok {
return address, errors.Errorf("Unable to cast %T to addressPair", fromCacheIfc)
}
return fromCache, nil
}
func buildTlsKey(address addressPair) string {
return fmt.Sprintf("%s:%d>%s:%d", address.srcIp, address.srcPort, address.dstIp, address.dstPort)
}
func (p *tlsPoller) buildTcpId(chunk *tlsChunk, address *addressPair) api.TcpID {
return api.TcpID{
SrcIP: address.srcIp.String(),
DstIP: address.dstIp.String(),
SrcPort: strconv.FormatUint(uint64(address.srcPort), 10),
DstPort: strconv.FormatUint(uint64(address.dstPort), 10),
Ident: "",
}
}
@@ -257,7 +290,7 @@ func (p *tlsPoller) clearPids() {
})
}
func (p *tlsPoller) logTls(chunk *tlsChunk, ip net.IP, port uint16) {
func (p *tlsPoller) logTls(chunk *tlsChunk, key string, reader *tlsReader) {
var flagsStr string
if chunk.isClient() {
@@ -272,13 +305,18 @@ func (p *tlsPoller) logTls(chunk *tlsChunk, ip net.IP, port uint16) {
flagsStr += "W"
}
srcIp, srcPort, _ := getAddressBySockfd(p.procfs, chunk.Pid, chunk.Fd, true)
dstIp, dstPort, _ := getAddressBySockfd(p.procfs, chunk.Pid, chunk.Fd, false)
str := strings.ReplaceAll(strings.ReplaceAll(string(chunk.Data[0:chunk.Recorded]), "\n", " "), "\r", "")
logger.Log.Infof("PID: %v (tid: %v) (fd: %v) (client: %v) (addr: %v:%v) (fdaddr %v:%v>%v:%v) (recorded %v out of %v starting at %v) - %v - %v",
chunk.Pid, chunk.Tgid, chunk.Fd, flagsStr, ip, port,
srcIp, srcPort, dstIp, dstPort,
chunk.Recorded, chunk.Len, chunk.Start, str, hex.EncodeToString(chunk.Data[0:chunk.Recorded]))
logger.Log.Infof("[%-44s] %s #%-4d (fd: %d) (recorded %d/%d:%d) - %s - %s",
key, flagsStr, reader.seenChunks, chunk.Fd,
chunk.Recorded, chunk.Len, chunk.Start,
str, hex.EncodeToString(chunk.Data[0:chunk.Recorded]))
}
func (p *tlsPoller) fdCacheEvictCallback(key interface{}, value interface{}) {
p.evictedCounter = p.evictedCounter + 1
if p.evictedCounter%1000000 == 0 {
logger.Log.Infof("Tls fdCache evicted %d items", p.evictedCounter)
}
}

View File

@@ -10,6 +10,7 @@ import (
type tlsReader struct {
key string
chunks chan *tlsChunk
seenChunks int
data []byte
doneHandler func(r *tlsReader)
progress *api.ReadProgress
@@ -23,6 +24,12 @@ type tlsReader struct {
reqResMatcher api.RequestResponseMatcher
}
func (r *tlsReader) newChunk(chunk *tlsChunk) {
r.captureTime = time.Now()
r.seenChunks = r.seenChunks + 1
r.chunks <- chunk
}
func (r *tlsReader) Read(p []byte) (int, error) {
var chunk *tlsChunk

View File

@@ -46,7 +46,13 @@ func (t *TlsTapper) Init(chunksBufferSize int, logBufferSize int, procfs string,
return err
}
t.poller = newTlsPoller(t, extension, procfs)
var err error
t.poller, err = newTlsPoller(t, extension, procfs)
if err != nil {
return err
}
return t.poller.init(&t.bpfObjects, chunksBufferSize)
}

Binary file not shown.

Binary file not shown.

View File

@@ -1,5 +0,0 @@
This example was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).
It is linked to the liraz-test package in the parent directory for development purposes.
You can run `npm install` and then `npm start` to test your package.

View File

@@ -1,13 +0,0 @@
module.exports = {
webpack: {
configure: (webpackConfig) => {
const instanceOfMiniCssExtractPlugin = webpackConfig.plugins.find(
(plugin) => plugin.options && plugin.options.ignoreOrder != null,
);
if(instanceOfMiniCssExtractPlugin)
instanceOfMiniCssExtractPlugin.options.ignoreOrder = true;
return webpackConfig;
}
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,48 +0,0 @@
{
"name": "@up9/mizu-common-example",
"homepage": ".",
"version": "0.0.0",
"private": true,
"scripts": {
"start": "craco start",
"comment-start": "node ../node_modules/react-scripts/bin/react-scripts.js start",
"build": "node ../node_modules/react-scripts/bin/react-scripts.js build",
"test": "node ../node_modules/react-scripts/bin/react-scripts.js test",
"eject": "node ../node_modules/react-scripts/bin/react-scripts.js eject"
},
"dependencies": {
"@testing-library/jest-dom": "file:../node_modules/@testing-library/jest-dom",
"@testing-library/react": "file:../node_modules/@testing-library/react",
"@testing-library/user-event": "file:../node_modules/@testing-library/user-event",
"@types/jest": "file:../node_modules/@types/jest",
"@types/node": "file:../node_modules/@types/node",
"@types/react": "file:../node_modules/@types/react",
"@types/react-dom": "file:../node_modules/@types/react-dom",
"@up9/mizu-common": "file:..",
"react": "file:../node_modules/react",
"react-dom": "file:../node_modules/react-dom"
},
"devDependencies": {
"@babel/plugin-syntax-object-rest-spread": "^7.8.3",
"@craco/craco": "^6.4.3",
"eslint": "^7.11.0",
"node-sass": "^6.0.0",
"recoil": "^0.5.2",
"react-scripts": "^4.0.3"
},
"eslintConfig": {
"extends": "react-app"
},
"browserslist": {
"production": [
">0.2%",
"not dead",
"not op_mini all"
],
"development": [
"last 1 chrome version",
"last 1 firefox version",
"last 1 safari version"
]
}
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.8 KiB

View File

@@ -1,48 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<link rel="shortcut icon" href="%PUBLIC_URL%/favicon.ico" />
<meta
name="viewport"
content="width=device-width, initial-scale=1, shrink-to-fit=no"
/>
<meta name="theme-color" content="#000000" />
<!--
manifest.json provides metadata used when your web app is added to the
homescreen on Android. See https://developers.google.com/web/fundamentals/engage-and-retain/web-app-manifest/
-->
<link rel="manifest" href="%PUBLIC_URL%/manifest.json" />
<!--
Notice the use of %PUBLIC_URL% in the tags above.
It will be replaced with the URL of the `public` folder during the build.
Only files inside the `public` folder can be referenced from the HTML.
Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
work correctly both with client-side routing and a non-root public URL.
Learn how to configure a non-root public URL by running `npm run build`.
-->
<title>@up9/mizu-common</title>
</head>
<body>
<noscript>
You need to enable JavaScript to run this app.
</noscript>
<div id="root"></div>
<!--
This HTML file is a template.
If you open it directly in the browser, you will see an empty page.
You can add webfonts, meta tags, or analytics to this file.
The build step will place the bundled scripts into the <body> tag.
To begin the development, run `npm start` or `yarn start`.
To create a production bundle, use `npm run build` or `yarn build`.
-->
</body>
</html>

View File

@@ -1,15 +0,0 @@
{
"short_name": "@up9/mizu-common",
"name": "@up9/mizu-common",
"icons": [
{
"src": "favicon.ico",
"sizes": "64x64 32x32 24x24 16x16",
"type": "image/x-icon"
}
],
"start_url": ".",
"display": "standalone",
"theme_color": "#000000",
"background_color": "#ffffff"
}

View File

@@ -1,9 +0,0 @@
import React from 'react'
import ReactDOM from 'react-dom'
import App from './App'
it('renders without crashing', () => {
const div = document.createElement('div')
ReactDOM.render(<App />, div)
ReactDOM.unmountComponentAtNode(div)
})

View File

@@ -1,24 +0,0 @@
import TrafficViewer,{useWS, DEFAULT_QUERY, OasModal} from '@up9/mizu-common';
import "@up9/mizu-common/dist/index.css"
import {useEffect} from 'react';
import Api, {getWebsocketUrl} from "./api";
const api = Api.getInstance()
const App = () => {
const {message,error,isOpen, openSocket, closeSocket, sendQueryWhenWsOpen} = useWS(getWebsocketUrl())
const trafficViewerApi = {...api, webSocket:{open : openSocket, close: closeSocket, sendQueryWhenWsOpen: sendQueryWhenWsOpen}}
sendQueryWhenWsOpen(DEFAULT_QUERY);
useEffect(() => {
return () =>{
closeSocket()
}
},[])
return <>
</>
}
export default App

View File

@@ -1,119 +0,0 @@
import * as axios from "axios";
export const MizuWebsocketURL = process.env.REACT_APP_OVERRIDE_WS_URL ? process.env.REACT_APP_OVERRIDE_WS_URL :
window.location.protocol === 'https:' ? `wss://${window.location.host}/ws` : `ws://${window.location.host}/ws`;
export const FormValidationErrorType = "formError";
const CancelToken = axios.CancelToken;
const apiURL = process.env.REACT_APP_OVERRIDE_API_URL ? process.env.REACT_APP_OVERRIDE_API_URL : `${window.location.origin}/`;
let token = null
let client = null
let source = null
export default class Api {
static instance;
static getInstance() {
if (!Api.instance) {
Api.instance = new Api();
}
return Api.instance;
}
constructor() {
token = localStorage.getItem("token");
client = this.getAxiosClient();
source = null;
}
tapStatus = async () => {
const response = await client.get("/status/tap");
return response.data;
}
analyzeStatus = async () => {
const response = await client.get("/status/analyze");
return response.data;
}
getEntry = async (id, query) => {
const response = await client.get(`/entries/${id}?query=${query}`);
return response.data;
}
fetchEntries = async (leftOff, direction, query, limit, timeoutMs) => {
const response = await client.get(`/entries/?leftOff=${leftOff}&direction=${direction}&query=${query}&limit=${limit}&timeoutMs=${timeoutMs}`).catch(function (thrown) {
console.error(thrown.message);
return {};
});
return response.data;
}
getOasServices = async () => {
const response = await client.get("/oas");
return response.data;
}
getOasByService = async (selectedService) => {
const response = await client.get(`/oas/${selectedService}`);
return response.data;
}
validateQuery = async (query) => {
if (source) {
source.cancel();
}
source = CancelToken.source();
const form = new FormData();
form.append('query', query)
const response = await client.post(`/query/validate`, form, {
cancelToken: source.token
}).catch(function (thrown) {
if (!axios.isCancel(thrown)) {
console.error('Validate error', thrown.message);
}
});
if (!response) {
return null;
}
return response.data;
}
persistToken = (tk) => {
token = tk;
client = this.getAxiosClient();
localStorage.setItem('token', token);
}
getAxiosClient = () => {
const headers = {
Accept: "application/json"
}
if (token) {
headers['x-session-token'] = `${token}`; // we use `x-session-token` instead of `Authorization` because the latter is reserved by kubectl proxy, making mizu view not work
}
return axios.create({
baseURL: apiURL,
timeout: 31000,
headers
});
}
}
export function getWebsocketUrl(){
let websocketUrl = MizuWebsocketURL;
if (token) {
websocketUrl += `/${token}`;
}
return websocketUrl;
}

View File

@@ -1,14 +0,0 @@
body {
margin: 0;
padding: 0;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
sans-serif;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
}
code {
font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
monospace;
}

View File

@@ -1,7 +0,0 @@
import './index.css'
import ReactDOM from 'react-dom'
import App from './App'
ReactDOM.render(<App />, document.getElementById('root'))

View File

@@ -1 +0,0 @@
/// <reference types="react-scripts" />

View File

@@ -1,5 +0,0 @@
// jest-dom adds custom jest matchers for asserting on DOM nodes.
// allows you to do things like:
// expect(element).toHaveTextContent(/react/i)
// learn more: https://github.com/testing-library/jest-dom
import '@testing-library/jest-dom/extend-expect';

View File

@@ -1,41 +0,0 @@
{
"compilerOptions": {
"outDir": "dist",
"module": "esnext",
"lib": [
"dom",
"esnext"
],
"moduleResolution": "node",
"jsx": "react-jsx",
"sourceMap": true,
"declaration": true,
"esModuleInterop": true,
"noImplicitReturns": true,
"noImplicitThis": true,
"noImplicitAny": true,
"strictNullChecks": false,
"suppressImplicitAnyIndexErrors": true,
"noUnusedLocals": false,
"noUnusedParameters": false,
"allowSyntheticDefaultImports": true,
"target": "es5",
"allowJs": true,
"skipLibCheck": true,
"strict": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"isolatedModules": true,
"noEmit": true,
"noFallthroughCasesInSwitch": true
},
"include": [
"src"
],
"exclude": [
"node_modules",
"build",
"exmaple"
]
}

View File

@@ -1,6 +1,13 @@
@import '../../variables.module.scss'
.boxContainer
.closeIcon
cursor: pointer
user-select: none
margin-top: -12px
margin-right: -12px
float: right
.boxContainer
display: flex
justifyContent: space-between
padding: 10px
@@ -17,7 +24,7 @@
width: 43px
.title
color:#494677
color: #494677
font-family: Source Sans Pro,Lucida Grande,Tahoma,sans-serif
font-size: 28px
font-weight: 600
@@ -37,4 +44,3 @@
.root
width: 100%

View File

@@ -1,5 +1,5 @@
import { Box, Fade, FormControl, MenuItem, Modal, Backdrop } from "@material-ui/core";
import { useEffect, useState } from "react";
import { Box, Fade, FormControl, Modal, Backdrop } from "@material-ui/core";
import { useCallback, useEffect, useState } from "react";
import { RedocStandalone } from "redoc";
import closeIcon from "assets/closeIcon.svg";
import { toast } from 'react-toastify';
@@ -30,10 +30,10 @@ const OasModal = ({ openModal, handleCloseModal, getOasServices, getOasByService
const [oasServices, setOasServices] = useState([] as string[])
const [selectedServiceName, setSelectedServiceName] = useState("");
const [selectedServiceSpec, setSelectedServiceSpec] = useState(null);
const classes = {root: style.root}
const onSelectedOASService = async (selectedService) => {
const classes = { root: style.root }
const onSelectedOASService = useCallback (async (selectedService) => {
if (oasServices.length === 0) {
setSelectedServiceSpec(null);
setSelectedServiceName("");
@@ -49,7 +49,8 @@ const OasModal = ({ openModal, handleCloseModal, getOasServices, getOasByService
toast.error("Error occurred while fetching service OAS spec", { containerId: TOAST_CONTAINER_ID });
console.error(e);
}
};
// eslint-disable-next-line
},[oasServices]);
useEffect(() => {
(async () => {
@@ -60,11 +61,13 @@ const OasModal = ({ openModal, handleCloseModal, getOasServices, getOasByService
console.error(e);
}
})();
// eslint-disable-next-line
}, [openModal]);
useEffect(() => {
onSelectedOASService(null);
}, [oasServices])
}, [oasServices, onSelectedOASService])
return (
<Modal
@@ -80,18 +83,17 @@ const OasModal = ({ openModal, handleCloseModal, getOasServices, getOasByService
>
<Fade in={openModal}>
<Box sx={modalStyle}>
<img src={closeIcon} alt="close" onClick={handleCloseModal} className={style.closeIcon} />
<div className={style.boxContainer}>
<div className={style.selectHeader}>
<div><img src={openApiLogo} alt="openAPI" className={style.openApilogo} /></div>
<div className={style.title}>Service Catalog</div>
</div>
<div style={{ cursor: "pointer" }}>
<img src={closeIcon} alt="close" onClick={handleCloseModal} />
</div>
</div>
<div className={style.selectContainer} >
<FormControl classes={classes}>
<SearchableDropdown
<SearchableDropdown
options={oasServices}
selectedValues={selectedServiceName}
onChange={onSelectedOASService}

View File

@@ -1,4 +1,4 @@
<svg width="22" height="22" viewBox="0 0 22 22" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M20.5 11C20.5 16.2467 16.2467 20.5 11 20.5C5.75329 20.5 1.5 16.2467 1.5 11C1.5 5.75329 5.75329 1.5 11 1.5C16.2467 1.5 20.5 5.75329 20.5 11Z" stroke="#2B3560"/>
<path d="M14.4762 9.05338L13.1448 7.7219L11.1528 9.71382L9.16091 7.7219L7.82943 9.05338L9.82135 11.0453L7.83226 13.0344L9.16374 14.3659L11.1528 12.3768L13.1419 14.3659L14.4734 13.0344L12.4843 11.0453L14.4762 9.05338Z" fill="#627EF7"/>
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M18.591 9.99997C18.591 14.7446 14.7447 18.5909 10.0001 18.5909C5.25546 18.5909 1.40918 14.7446 1.40918 9.99997C1.40918 5.25534 5.25546 1.40906 10.0001 1.40906C14.7447 1.40906 18.591 5.25534 18.591 9.99997Z" fill="#E9EBF8" stroke="#BCCEFD"/>
<path d="M13.1604 8.23038L11.95 7.01994L10.1392 8.83078L8.32832 7.01994L7.11789 8.23038L8.92872 10.0412L7.12046 11.8495L8.33089 13.0599L10.1392 11.2517L11.9474 13.0599L13.1579 11.8495L11.3496 10.0412L13.1604 8.23038Z" fill="#205CF5"/>
</svg>

Before

Width:  |  Height:  |  Size: 507 B

After

Width:  |  Height:  |  Size: 588 B

View File

@@ -56,3 +56,9 @@
.separtorLine
margin-top: 10px
border: 1px solid #E9EBF8
.closeIcon
cursor: pointer
user-select: none
margin-top: -15px
margin-right: 5px

View File

@@ -49,6 +49,7 @@ const protocols = [
{ key: "HTTP", value: "HTTP", component: <LegentLabel color="#205cf5" name="HTTP" /> },
{ key: "HTTP/2", value: "HTTP/2", component: <LegentLabel color='#244c5a' name="HTTP/2" /> },
{ key: "gRPC", value: "gRPC", component: <LegentLabel color='#244c5a' name="gRPC" /> },
{ key: "GQL", value: "GQL", component: <LegentLabel color='#e10098' name="GQL" /> },
{ key: "AMQP", value: "AMQP", component: <LegentLabel color='#ff6600' name="AMQP" /> },
{ key: "KAFKA", value: "KAFKA", component: <LegentLabel color='#000000' name="KAFKA" /> },
{ key: "REDIS", value: "REDIS", component: <LegentLabel color='#a41e11' name="REDIS" /> },]
@@ -202,7 +203,7 @@ export const ServiceMapModal: React.FC<ServiceMapModalProps> = ({ isOpen, onClos
>
Refresh
</Button>
<img src={closeIcon} alt="close" onClick={() => onClose()} style={{ cursor: "pointer", userSelect: "none" }}></img>
<img src={closeIcon} alt="close" onClick={() => onClose()} className={styles.closeIcon}></img>
</div>
{isLoading && <div className={spinnerStyle.spinnerContainer}>
<img alt="spinner" src={spinnerImg} style={{ height: 50 }} />

View File

@@ -274,25 +274,21 @@ export const EntryItem: React.FC<EntryProps> = ({entry, style, headingMode, name
>
<img
src={outgoingIcon}
alt="Ingoing traffic"
title="Ingoing"
alt="Outgoing traffic"
title="Outgoing"
/>
</Queryable>
:
<Queryable
query={`outgoing == true`}
query={`outgoing == false`}
displayIconOnMouseOver={true}
flipped={true}
iconStyle={{marginTop: "28px"}}
>
<img
src={ingoingIcon}
alt="Outgoing traffic"
title="Outgoing"
onClick={() => {
const query = `outgoing == false`;
setQuery(queryState ? `${queryState} and ${query}` : query);
}}
alt="Ingoing traffic"
title="Ingoing"
/>
</Queryable>
}

View File

@@ -240,7 +240,7 @@ export const TrafficViewer: React.FC<TrafficViewerProps> = ({
return (
<div className={TrafficViewerStyles.TrafficPage}>
{tappingStatus && isShowStatusBar && <StatusBar isDemoBannerView={isDemoBannerView}/>}
{tappingStatus && isShowStatusBar && <StatusBar disabled={ws?.current?.readyState !== WebSocket.OPEN} isDemoBannerView={isDemoBannerView}/>}
<div className={TrafficViewerStyles.TrafficPageHeader}>
<div className={TrafficViewerStyles.TrafficPageStreamStatus}>
<img className={TrafficViewerStyles.playPauseIcon}

View File

@@ -5,6 +5,7 @@ import failIcon from 'assets/failed.svg';
import successIcon from 'assets/success.svg';
import {useRecoilValue} from "recoil";
import tappingStatusAtom, {tappingStatusDetails} from "../../recoil/tappingStatus";
import Tooltip from "./Tooltip";
const pluralize = (noun: string, amount: number) => {
return `${noun}${amount !== 1 ? 's' : ''}`
@@ -12,20 +13,22 @@ const pluralize = (noun: string, amount: number) => {
interface StatusBarProps {
isDemoBannerView: boolean;
disabled?: boolean;
}
export const StatusBar: React.FC<StatusBarProps> = ({isDemoBannerView}) => {
export const StatusBar: React.FC<StatusBarProps> = ({isDemoBannerView, disabled}) => {
const tappingStatus = useRecoilValue(tappingStatusAtom);
const [expandedBar, setExpandedBar] = useState(false);
const {uniqueNamespaces, amountOfPods, amountOfTappedPods, amountOfUntappedPods} = useRecoilValue(tappingStatusDetails);
return <div className={`${isDemoBannerView ? `${style.banner}` : ''} ${style.statusBar} ${(expandedBar ? `${style.expandedStatusBar}` : "")}`} onMouseOver={() => setExpandedBar(true)} onMouseLeave={() => setExpandedBar(false)} data-cy="expandedStatusBar">
return <div style={{opacity: disabled ? 0.4 : 1}} className={`${isDemoBannerView ? `${style.banner}` : ''} ${style.statusBar} ${(expandedBar && !disabled ? `${style.expandedStatusBar}` : "")}`} onMouseOver={() => setExpandedBar(true)} onMouseLeave={() => setExpandedBar(false)} data-cy="expandedStatusBar">
<div className={style.podsCount}>
{tappingStatus.some(pod => !pod.isTapped) && <img src={warningIcon} alt="warning"/>}
{disabled && <Tooltip title={"Tapping status is not updated when streaming is paused"} isSimple><img src={warningIcon} alt="warning"/></Tooltip>}
<span className={style.podsCountText} data-cy="podsCountText">
{`Tapping ${amountOfUntappedPods > 0 ? amountOfTappedPods + " / " + amountOfPods : amountOfPods} ${pluralize('pod', amountOfPods)} in ${pluralize('namespace', uniqueNamespaces.length)} ${uniqueNamespaces.join(", ")}`}
</span>
</div>
{expandedBar && <div style={{marginTop: 20}}>
{expandedBar && !disabled && <div style={{marginTop: 20}}>
<table>
<thead>
<tr>