Entries count changes frequently (#1021)

Co-authored-by: leon-up9 <97597983+leon-up9@users.noreply.github.com>

.github/workflows/acceptance_tests_on_pr.yml

@@ -0,0 +1,38 @@
+name: Acceptance tests on PR
+
+on: push
+
+concurrency:
+  group: acceptance-tests-on-pr-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run-tests:
+    name: Run tests
+    runs-on: ubuntu-latest
+    if: ${{ contains(github.event.head_commit.message, '#run_acceptance_tests') }}
+
+    steps:
+      - name: Set up Go 1.17
+        uses: actions/setup-go@v2
+        with:
+          go-version: '^1.17'
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Setup acceptance test
+        run: ./acceptanceTests/setup.sh
+
+      - name: Create k8s users and change context
+        env:
+          USERNAME_UNRESTRICTED: user-with-clusterwide-access
+          USERNAME_RESTRICTED: user-with-restricted-access
+        run: |
+          ./acceptanceTests/create_user.sh "${USERNAME_UNRESTRICTED}"
+          ./acceptanceTests/create_user.sh "${USERNAME_RESTRICTED}"
+          kubectl apply -f cli/cmd/permissionFiles/permissions-all-namespaces-tap.yaml
+          kubectl config use-context ${USERNAME_UNRESTRICTED}
+
+      - name: Test
+        run: make acceptance-test

.github/workflows/test.yml

@@ -56,7 +56,7 @@ jobs:
       
       - name: Check extensions modified files
         id: ext_modified_files
-        run: devops/check_modified_files.sh tap/extensions/
+        run: devops/check_modified_files.sh tap/extensions/ tap/api/
 
       - name: Extensions Test
         if: github.event_name == 'push' || steps.ext_modified_files.outputs.matched == 'true'
@@ -64,4 +64,3 @@ jobs:
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2
-

Dockerfile

@@ -87,8 +87,8 @@ RUN go build -ldflags="-extldflags=-static -s -w \
     -X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
 
 # Download Basenine executable, verify the sha1sum
-ADD https://github.com/up9inc/basenine/releases/download/v0.7.2/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
-ADD https://github.com/up9inc/basenine/releases/download/v0.7.2/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
+ADD https://github.com/up9inc/basenine/releases/download/v0.7.3/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
+ADD https://github.com/up9inc/basenine/releases/download/v0.7.3/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
 
 RUN shasum -a 256 -c basenine_linux_"${GOARCH}".sha256 && \
     chmod +x ./basenine_linux_"${GOARCH}" && \

agent/go.mod

@@ -19,9 +19,9 @@ require (
 	github.com/nav-inc/datetime v0.1.3
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/orcaman/concurrent-map v1.0.0
-	github.com/patrickmn/go-cache v2.1.0+incompatible
+    github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/stretchr/testify v1.7.0
-	github.com/up9inc/basenine/client/go v0.0.0-20220413173135-69508ca741d7
+	github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0

agent/go.sum

@@ -681,8 +681,8 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
 github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
 github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
-github.com/up9inc/basenine/client/go v0.0.0-20220413173135-69508ca741d7 h1:9aciby1Byjn50gVXpOuvWSe48GdSK1uS2bcBKMZYHKI=
-github.com/up9inc/basenine/client/go v0.0.0-20220413173135-69508ca741d7/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
+github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607 h1:UqxUSkOYOmsLZWQtMSk02ttnhdRwBRLOLt2aDiS9tEk=
+github.com/up9inc/basenine/client/go v0.0.0-20220419100955-e2ca51087607/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/wI2L/jsondiff v0.1.1 h1:r2TkoEet7E4JMO5+s1RCY2R0LrNPNHY6hbDeow2hRHw=

agent/pkg/entries/entries_provider.go

@@ -2,6 +2,7 @@ package entries
 
 import (
 	"encoding/json"
+	"errors"
 	"time"
 
 	basenine "github.com/up9inc/basenine/client/go"
@@ -64,7 +65,7 @@ func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntr
 	}
 	err = json.Unmarshal(bytes, &entry)
 	if err != nil {
-		return nil, err
+		return nil, errors.New(string(bytes))
 	}
 
 	extension := app.ExtensionsMap[entry.Protocol.Name]

cli/go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	github.com/spf13/cobra v1.3.0
 	github.com/spf13/pflag v1.0.5
-	github.com/up9inc/basenine/server/lib v0.0.0-20220413173135-69508ca741d7
+	github.com/up9inc/basenine/server/lib v0.0.0-20220419100955-e2ca51087607
 	github.com/up9inc/mizu/shared v0.0.0
 	github.com/up9inc/mizu/tap/api v0.0.0
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8

cli/go.sum

@@ -600,8 +600,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
-github.com/up9inc/basenine/server/lib v0.0.0-20220413173135-69508ca741d7 h1:3Mi+0tQFVHXYcrFhwH/h6/2b0tayLcYeFPXyzDV3rvc=
-github.com/up9inc/basenine/server/lib v0.0.0-20220413173135-69508ca741d7/go.mod h1:v0hIh31iwDGbkkdeSSppdMNm1oIigfCA2mG2XajKnf8=
+github.com/up9inc/basenine/server/lib v0.0.0-20220419100955-e2ca51087607 h1:gCOwbfjsLslDw63yj/3l9d5TH7ikhIWHd7j0lE9U26U=
+github.com/up9inc/basenine/server/lib v0.0.0-20220419100955-e2ca51087607/go.mod h1:v0hIh31iwDGbkkdeSSppdMNm1oIigfCA2mG2XajKnf8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
 github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk=

shared/kubernetes/provider.go

@@ -806,11 +806,11 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	agentContainer.WithResources(agentResources)
 
 	nodeSelectorRequirement := applyconfcore.NodeSelectorRequirement()
-	nodeSelectorRequirement.WithKey("kubernetes.io/hostname")
+	nodeSelectorRequirement.WithKey("metadata.name")
 	nodeSelectorRequirement.WithOperator(core.NodeSelectorOpIn)
 	nodeSelectorRequirement.WithValues(nodeNames...)
 	nodeSelectorTerm := applyconfcore.NodeSelectorTerm()
-	nodeSelectorTerm.WithMatchExpressions(nodeSelectorRequirement)
+	nodeSelectorTerm.WithMatchFields(nodeSelectorRequirement)
 	nodeSelector := applyconfcore.NodeSelector()
 	nodeSelector.WithNodeSelectorTerms(nodeSelectorTerm)
 	nodeAffinity := applyconfcore.NodeAffinity()

tap/api/api.go

@@ -93,8 +93,8 @@ type RequestResponsePair struct {
 	Response GenericMessage `json:"response"`
 }
 
-// `Protocol` is modified in the later stages of data propagation. Therefore it's not a pointer.
 type OutputChannelItem struct {
+	// `Protocol` is modified in later stages of data propagation. Therefore, it's not a pointer.
 	Protocol       Protocol
 	Capture        Capture
 	Timestamp      int64

tap/extensions/amqp/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect7/amqp/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect8/amqp/\* expect

tap/extensions/http/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect7/http/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect8/http/\* expect

tap/extensions/kafka/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect7/kafka/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect8/kafka/\* expect

tap/extensions/redis/Makefile

@@ -13,4 +13,4 @@ test-pull-bin:
 
 test-pull-expect:
 	@mkdir -p expect
-	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect7/redis/\* expect
+	@[ "${skipexpect}" ] && echo "Skipping downloading expected JSONs" || gsutil  -o 'GSUtil:parallel_process_count=5' -o 'GSUtil:parallel_thread_count=5'  -m cp -r gs://static.up9.io/mizu/test-pcap/expect8/redis/\* expect

ui-common/src/components/TrafficViewer/EntriesList.tsx

@@ -12,7 +12,7 @@ import TrafficViewerApiAtom from "../../recoil/TrafficViewerApi";
 import TrafficViewerApi from "./TrafficViewerApi";
 import focusedEntryIdAtom from "../../recoil/focusedEntryId";
 import {toast} from "react-toastify";
-import {TOAST_CONTAINER_ID} from "../../configs/Consts";
+import {MAX_ENTRIES, TOAST_CONTAINER_ID} from "../../configs/Consts";
 import tappingStatusAtom from "../../recoil/tappingStatus";
 import leftOffTopAtom from "../../recoil/leftOffTop";
 
@@ -98,8 +98,8 @@ export const EntriesList: React.FC<EntriesListProps> = ({
     setIsLoadingTop(false);
 
     const newEntries = [...data.data.reverse(), ...entries];
-    if(newEntries.length > 10000) {
-      newEntries.splice(10000, newEntries.length - 10000)
+    if(newEntries.length > MAX_ENTRIES) {
+      newEntries.splice(MAX_ENTRIES, newEntries.length - MAX_ENTRIES)
     }
     setEntries(newEntries);
 
@@ -125,9 +125,9 @@ export const EntriesList: React.FC<EntriesListProps> = ({
 
   useEffect(() => {
     const newEntries = [...entries];
-    if (newEntries.length > 10000) {
+    if (newEntries.length > MAX_ENTRIES) {
       setLeftOffTop(newEntries[0].id);
-      newEntries.splice(0, newEntries.length - 10000)
+      newEntries.splice(0, newEntries.length - MAX_ENTRIES)
       setNoMoreDataTop(false);
       setEntries(newEntries);
     }
@@ -209,7 +209,7 @@ export const EntriesList: React.FC<EntriesListProps> = ({
       </div>
 
       <div className={styles.footer}>
-        <div>Displaying <b id="entries-length">{entries?.length}</b> results out of <b
+        <div>Displaying <b id="entries-length">{entries?.length > MAX_ENTRIES ? MAX_ENTRIES : entries?.length}</b> results out of <b
           id="total-entries">{queriedTotal}</b> total
         </div>
         {startTime !== 0 && <div>Started listening at <span style={{

ui-common/src/components/UI/InformationIcon.tsx

@@ -1,19 +1,22 @@
-import React, { CSSProperties } from "react";
+import React from "react";
 import styles from "./style/InformationIcon.module.sass"
 
 const DEFUALT_LINK = "https://getmizu.io/docs"
 
-export interface InformationIconProps {
+interface LinkProps {
     link?: string,
-    style?: CSSProperties
+    className?: string
+    title?: string
 }
 
-export const InformationIcon: React.FC<InformationIconProps> = ({ link, style }) => {
-    return <React.Fragment>
-        <a href={DEFUALT_LINK ? DEFUALT_LINK : link} style={style} className={styles.linkStyle} title="documentation" target="_blank">
-            <span>Docs</span>
-        </a>
-    </React.Fragment>
+export const Link: React.FC<LinkProps> = ({ link, className, title, children }) => {
+    return <a href={link} className={className} title={title} target="_blank">
+        {children}
+    </a>
 }
 
-
+export const InformationIcon: React.FC<LinkProps> = ({ className }) => {
+    return <Link title="documentation" className={`${styles.linkStyle} ${className}`} link={DEFUALT_LINK}>
+        <span>Docs</span>
+    </Link>
+}

ui-common/src/components/UI/index.ts

@@ -5,10 +5,10 @@ import Tooltip from "./Tooltip";
 import Checkbox from "./Checkbox"
 import { StatusBar } from "./StatusBar";
 import CustomModal from "./CustomModal";
-import { InformationIcon } from "./InformationIcon";
+import { InformationIcon, Link } from "./InformationIcon";
 import SelectList from "./SelectList";
 import NoDataMessage from "./NoDataMessage";
 
 
-export { LoadingOverlay, Select, Tabs, Tooltip, Checkbox, CustomModal, InformationIcon, SelectList, NoDataMessage }
+export { LoadingOverlay, Select, Tabs, Tooltip, Checkbox, CustomModal, InformationIcon, SelectList, NoDataMessage, Link }
 export { StatusBar }

ui-common/src/configs/Consts.ts

@@ -1 +1,2 @@
-export const TOAST_CONTAINER_ID = "Common";
+export const TOAST_CONTAINER_ID = "Common";
+export const MAX_ENTRIES = 10000;