Compare commits

..

120 Commits

Author SHA1 Message Date
RoyUP9
02b2cbaa03 Added update config route for install mode (#581) 2022-01-04 17:27:14 +02:00
Nimrod Gilboa Markevich
2834ae1e85 Support custom RBAC resources (#572)
Motivation: Allow users to change the default RBAC resources (ServiceAccount, ClusterRole, ClusterRoleBinding, Role and RoleBinding) without having Mizu delete them every run.

Adds app.kubernetes.io/created-by and app.kubernetes.io/managed-by labels to all resources.
The value of app.kubernetes.io/created-by is either mizu-cli or mizu-agent.
The value of app.kubernetes.io/managed-by is mizu.
When Mizu cleans resources (ctrl-c in tap cmd or mizu clean cmd) it removes all RBAC resources that have managed-by=mizu, and only those.

A user may have a ClusterRole named mizu-clusterrole. If it doesn't have the label app.kubernetes.io/managed-by=mizu, then Mizu won't overwrite it and won't delete it.
Other resources (deployments, services etc.) are always removed, regardless of their labels.
2022-01-04 09:48:22 +02:00
Igor Gov
9c45166075 Fixing Mizu url in readme (#580) 2022-01-02 10:42:10 +02:00
RoyUP9
482036182b Added setting ui mode - enterprise or normal (#575) 2021-12-30 14:58:53 +02:00
M. Mert Yıldıran
f535719ddd Use wss:// instead of ws:// in case of HTTPS (#573) 2021-12-30 06:59:05 +02:00
Igor Gov
da2aaa9bd8 API server provider readiness check with echo (#570) 2021-12-29 12:21:12 +02:00
lirazyehezkel
9ada330fcf Mizu enterprise preparing (#567) 2021-12-29 11:49:44 +02:00
Igor Gov
84f7107a33 API server provider readiness check with echo (#569) 2021-12-29 10:50:41 +02:00
RoyUP9
de046c65b0 Mizu install mode (#566) 2021-12-28 15:44:37 +02:00
Igor Gov
b039c2abad Fix: [EKS] server could not find the requested resourceevents.events.k8s.io (#565) 2021-12-28 13:24:51 +02:00
RoyUP9
1e1b5f0c0f move mizu agent config from config (#563) 2021-12-28 09:59:21 +02:00
RoyUP9
15895d5947 Changed sync tappers to start only when sync tappers config is true (#562) 2021-12-27 17:19:00 +02:00
RamiBerm
2da7c0f0ed TRA-4065 fix mizu tapping (#561)
* Update main.go and passive_tapper.go

* Update main.go
2021-12-27 15:11:36 +02:00
RoyUP9
06785ec877 fixed acceptance test go.sum (#560) 2021-12-27 14:45:42 +02:00
RamiBerm
cfe9e863b7 TRA-4065 support inflight tap target update (#556)
* WIP

* WIP

* Update main.go

* Update main.go and passive_tapper.go

* Update passive_tapper.go

* Update passive_tapper.go

* Update passive_tapper.go

* Update passive_tapper.go
2021-12-27 11:50:34 +02:00
RoyUP9
fd97a09624 extracted create and clean resources from tap runner (#557) 2021-12-27 11:32:48 +02:00
M. Mert Yıldıran
52ce6044ea Add 4px padding to the top of endpointServiceContainer (#555) 2021-12-26 16:20:06 +03:00
M. Mert Yıldıran
3a83531590 Use react-lowlight to highlight and json-beautify, xml-formatter to prettify the EntryBodySection (#554)
* Use `react-lowlight` to highlight and `json-beautify` to prettify the `EntryBodySection`

* Bring back the line numbers

* Make the Base64 decoding optional but make it `true` by default

* Align line numbers to right and don't have a dot character

* Make line numbers semi transparent

* Make `markers` code more elegant

* Prettify XML as well
2021-12-26 16:12:17 +03:00
M. Mert Yıldıran
e358aa4c8f Remove TCP related logging to reduce the amount of logs on DEBUG level (#553) 2021-12-26 15:10:02 +03:00
M. Mert Yıldıran
03b1313a9f Don't use Queryable for the Mime type and Encoding fields but use it directly in CollapsibleTitle suffixed components and only enable it for EntryBodySection (#550)
Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com>
2021-12-23 17:45:51 +03:00
M. Mert Yıldıran
32dfe40e18 Make EntryItem more responsive (#552) 2021-12-23 10:25:39 +03:00
M. Mert Yıldıran
12aaa762f6 Fix React Hook useEffect has a missing dependency: 'handleQueryChange' warning (#551) 2021-12-22 20:23:21 +03:00
David Levanon
a75bac181d support linkerd (#547)
* support linkerd - initial commit

* renaming readEnvironmentVariable
2021-12-20 13:57:58 +02:00
gadotroee
2d78785558 Fix acceptance tests (after pods status request change) (#545) 2021-12-19 13:46:14 +02:00
Igor Gov
cba0c682e5 Report pods "isTapped" to FE (#535) 2021-12-19 13:03:53 +02:00
RamiBerm
791f762803 TRA-4062 remove duplicate target pod print (#543) 2021-12-19 11:52:43 +02:00
RamiBerm
d316589bda TRA-4060 fix proxying error (#542) 2021-12-19 11:24:17 +02:00
Igor Gov
36828bcc1d Bringing back the pod watch api server events to make acceptance test more stable (#541) 2021-12-19 09:29:09 +02:00
M. Mert Yıldıran
23332639d0 Fix a JavaScript error in case of null attribute and an interface conversion error in the API server (#540) 2021-12-17 17:26:36 +03:00
Igor Gov
3b69508581 Fix: acceptance tests (#539) 2021-12-16 14:38:32 +02:00
Igor Gov
397d3931ad Fix health tapper status count (#538) 2021-12-16 12:39:45 +02:00
Igor Gov
4de795e463 Alert on acceptance tests failure (#537) 2021-12-16 11:04:36 +02:00
David Levanon
ab029f4394 Send pod info to tapper (#532) 2021-12-16 10:51:03 +02:00
M. Mert Yıldıran
a06d5cfbde Update the "Started listening at" timestamp and queriedTotal state based on database truncation (#533) 2021-12-16 10:46:50 +02:00
M. Mert Yıldıran
b565492eba Change the connection status text and the toggle connection behavior (#534) 2021-12-16 10:42:17 +02:00
Igor Gov
f266f32fea Decreased API server boot time (#536) 2021-12-16 10:36:03 +02:00
Igor Gov
90c210452d API server stores tappers status (#531) 2021-12-15 14:52:49 +02:00
M. Mert Yıldıran
0a915b3fe7 Fix the CSS issue in Queryable component for src.name field on heading mode (#530) 2021-12-15 12:28:46 +03:00
M. Mert Yıldıran
a830bbe023 Fix the glitch (#529)
* Fix the glitch

* Bring back the functionality to "Fetch old records" and "Snap to bottom" buttons
2021-12-15 12:26:18 +03:00
Alex Haiut
f1ba397543 make description of mizu config options public (#527) 2021-12-14 20:03:26 +02:00
M. Mert Yıldıran
4e17ac5654 Remove unnecessary fields and split service into src.name and dst.name (#525)
* Remove unnecessary fields and split `service` into `src.name` and `dst.name`

* Don't fall back to IP address but instead display `[Unresolved]` text

* Fix the CSS issues in the plus icon position and replace the separator `->` text with `SwapHorizIcon`
2021-12-14 11:36:02 +03:00
M. Mert Yıldıran
d274db2d87 Fix the CSS issues in queryable vertical protocol element (#526) 2021-12-12 19:38:14 +03:00
M. Mert Yıldıran
0a2aacfb02 Include milliseconds information into the timestamps in the UI (#524)
* Include milliseconds information into the timestamps in the UI

* Upgrade Basenine version from `0.2.16` to `0.2.17`

* Increase the `width` of timestamp
2021-12-10 18:03:17 +03:00
Igor Gov
3c64c1c7ca Report the platform in telemtry (#523)
Co-authored-by: Igor Gov <igor.govorov1@gmail.com>
2021-12-09 13:12:15 +02:00
Igor Gov
005f000ef6 Disable version check for devs (#522) 2021-12-09 12:11:36 +02:00
M. Mert Yıldıran
1ef3778051 Add type switch for Base field of MizuEntry (#520) 2021-12-08 16:53:57 +03:00
M. Mert Yıldıran
9f1e311689 TRA-4017 Bring back getOldEntries method using fetch API and always start streaming from now (#518)
* Bring back `getOldEntries` method using fetch API

* Determine no more data on top based on `leftOff` value

* Remove `entriesBuffer` state

* Always open WebSocket with some `leftOff` value

* Rename `leftOff` state to `leftOffBottom`

* Don't set the `focusedEntryId` through WebSocket if the WebSocket is closed

* Call `setQueriedCurrent` with addition

* Close WebSocket upon reaching to top

* Open WebSocket upon snapping to bottom

* Close the WebSocket on snap broken event instead

* Set queried current value to zero upon filter submit

* Upgrade `react-scrollable-feed-virtualized` version and use `scrollToIndex` function

* Change the footer text format

* Improve no more data top logic

* Fix `closeWebSocket()` call logic in `onSnapBrokenEvent` and handle `data.meta` being `null` in `getOldEntries`

* Fix the issues around fetching old records

* Clean up `EntriesList.module.sass`

* Decrement initial `leftOffTop` value by `2`

* Fix the order of `incomingEntries` in `getOldEntries`

* Request `leftOffTop - 1` from `fetchEntries`

* Limit the front-end total entries fetched through WebSocket count to `10000`

* Lose the UI performance gain that's provided by #452

* Revert "Fix the selected entry behavior by propagating the `focusedEntryId` through WebSocket (before #452) TRA-3983 (#513)"

This reverts commit 873f252544.

* Fix the issues caused by 09371f141f

* Upgrade Basenine version from `0.2.13` to `0.2.14`

* Upgrade Basenine version from `0.2.14` to `0.2.15`

* Fix the condition of "Fetch old records" button visibility

* Upgrade Basenine version from `0.2.15` to `0.2.16` and fix the UI code related to fetching old records

* Make `newEntries` constant
2021-12-08 15:19:35 +03:00
M. Mert Yıldıran
9aaf18842b Fix the CSS issue in Queryable inside EntryViewLine (#521) 2021-12-07 14:15:49 +03:00
M. Mert Yıldıran
880842c39f Fix the styling of Queryable under StatusCode and Summary components (#519) 2021-12-04 20:25:22 +03:00
David Levanon
296e1bb667 Replace privileged with specific CAPABILITIES requests (#514) 2021-12-02 11:41:13 +02:00
Igor Gov
2910611111 Disable telemetry by env var MIZU_DISABLE_TELEMTRY (#517) 2021-12-02 09:20:27 +02:00
M. Mert Yıldıran
c47959dbd8 Bring back GetEntries HTTP endpoint (#515)
* Bring back `GetEntries` HTTP endpoint

* Upgrade Basenine version from `0.2.12` to `0.2.13`

* Accept negative `leftOff` value

* Remove `max`es from the validations

* Make `timeoutMs` optional

* Update the route comment

* Add `EntriesResponse` struct
2021-12-01 11:55:13 +03:00
M. Mert Yıldıran
af557f7052 Add Queryable component to show a green add circle icon for the queryable UI elements (#512)
* Add `Queryable` component to show a green circle and use it in `EntryViewLine`

* Refactor `Queryable` component

* Use the `Queryable` component `EntryDetailed`

* Use the `Queryable` component `Summary`

* Instead of passing the style to `Queryable`, pass the children components directly

* Make `useTooltip = true` by default in `Queryable`

* Refactor a lot of styling to achieve using `Queryable` in `Protocol` component

* Migrate the last queryable elements in `EntryListItem` to `Queryable` component

* Fix some of the styling issues

* Make horizontal `Protocol` `Queryable` too

* Remove unnecessary child constants

* Revert some of the changes in 2a93f365f5

* Fix rest of the styling issues

* Fix one more styling issue

* Update the screenshots and text in the cheatsheet according to the change

* Use `let` not `var`

* Add missing dependencies to the React hook
2021-11-30 17:52:21 +03:00
M. Mert Yıldıran
b745f65971 Handle unexpected socket close and replace the default rlimit(100) filter with leftOff(-1) filter (#508)
* Handle unexpected socket close and replace the default `rlimit(100)` filter with `leftOff(-1)` filter

* Rename `dontClear` parameter to `resetEntriesBuffer` and remove negation
2021-11-30 16:30:18 +03:00
M. Mert Yıldıran
873f252544 Fix the selected entry behavior by propagating the focusedEntryId through WebSocket (before #452) TRA-3983 (#513)
* Revert the select entry behavior into its original state RACING! (before #452) [TRA-3983 alternative 3]

* Remove the remaining `forceSelect`(s)

* Add a missing `focusedEntryId` prop

* Fix the race condition

* Propagate the `focusedEntryId` through WebSocket to prevent racing
2021-11-30 15:27:10 +03:00
M. Mert Yıldıran
9696ad9bad Show the EntryItem as EntrySummary in EntryDetailed (#506) 2021-11-28 10:59:40 +03:00
M. Mert Yıldıran
a1bda0a6c3 Hide Encoding field if it's undefined or empty in the UI (#511) 2021-11-26 09:40:44 +03:00
M. Mert Yıldıran
a62842ac9f Add HTTP2 Over Cleartext (H2C) support (#510)
* Add HTTP2 Over Cleartext (H2C) support

* Remove a parameter which is a remnant of debugging
2021-11-25 20:36:13 +03:00
M. Mert Yıldıran
e667597e6e Rename URL field to Target URI in the UI to prevent confusion (#509) 2021-11-25 20:15:43 +03:00
Igor Gov
86240e4121 Remove local dev instruction from readme (#507) 2021-11-24 10:46:07 +02:00
David Levanon
b0c8c0c192 Add response body to the error in case of failure (#503)
* add response body to the error in case of failure

* fix typo + make inline condition
2021-11-23 20:16:07 +02:00
Nimrod Gilboa Markevich
1c18eb1b84 Use one channel for events instead of three (#495)
Use one channel for events instead of three separate channels by event type
2021-11-23 15:06:27 +02:00
David Levanon
01d6005a7b minor logging changes (#499)
Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com>
2021-11-23 14:21:53 +02:00
Nimrod Gilboa Markevich
4c97316c02 Remove prevPodPhase (#497)
prevPodPhase does not take into account the fact that there may be more
than one tapper pod. Therefore it is not clear what its value
represents. It is only used in a debug print. It is not worth the effort
to fix for that one debug print.

Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com>
2021-11-23 10:03:36 +02:00
M. Mert Yıldıran
d66c7445e6 Remove SetHostname method in HTTP extension (#496) 2021-11-22 19:30:06 +03:00
M. Mert Yıldıran
12ca3d8779 Make the gRPC and HTTP/2 distinction (#492)
* Remove the extra negation on `nodefrag` flag's value

* Support IPv4 fragmentation and IPv6 at the same time

* Set `Method` and `StatusCode` fields correctly for `HTTP/2`

* Replace unnecessary `grpc` naming with `http2`

* Make the `gRPC` and `HTTP/2` distinction

* Fix the macros of `http` extension

* Fix the macros of other protocol extensions

* Update the method signature of `Represent`

* Fix the `HTTP/2` support

* Fix some minor issues

* Upgrade Basenine version from `0.2.10` to `0.2.11`

Sorts macros before expanding them and prioritize the long macros.

* Don't regex split the gRPC method name

* Re-enable `nodefrag` flag
2021-11-22 17:46:35 +03:00
M. Mert Yıldıran
02a125bb86 Disable IPv4 defragmentation and support IPv6 (#487)
* Remove the extra negation on `nodefrag` flag's value

* Support IPv4 fragmentation and IPv6 at the same time

* Re-enable `nodefrag` flag
2021-11-22 17:35:17 +03:00
M. Mert Yıldıran
08d7fa988e Remove tap/tester/ directory (#489)
Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com>
2021-11-22 17:32:38 +03:00
Nimrod Gilboa Markevich
b1ad2efb96 Warn pods not starting (#493)
Print warning event related to mizu k8s resources.
In non-daemon print to CLI. In Daemon print to API-Server logs.
2021-11-22 15:30:10 +02:00
Alon Girmonsky
ed7b754eca Some changes to the doc (#494) 2021-11-22 09:02:33 +02:00
Igor Gov
c026656b5e Improving daemon documentation (#457) 2021-11-21 19:37:02 +02:00
David Levanon
6caa94f08f Add support to auto discover envoy processes (#459)
* discover envoy pids using cluster ips

* add istio flag to cli + rename mtls flag to istio

* add istio.md to docs

* Fixing typos

* Fix minor typos and grammer in docs

Co-authored-by: Nimrod Gilboa Markevich <nimrod@up9.com>
2021-11-21 15:45:07 +02:00
RoyUP9
b77ea63f42 Add token validity check (#483) 2021-11-21 15:14:02 +02:00
gadotroee
2635964a28 Update README (#486) 2021-11-21 14:09:21 +02:00
M. Mert Yıldıran
a16faca5fb Ignore gob files (#488)
* Ignore gob files

* Remove `*.db` from `.gitignore`
2021-11-21 09:29:01 +03:00
M. Mert Yıldıran
8cf6f56a3c Remove unnecessary tcpdump dependency from Dockerfile (#491) 2021-11-21 09:12:51 +03:00
M. Mert Yıldıran
a849aae94c Upgrade Basenine version from 0.2.9 to 0.2.10 (#484)
* Upgrade Basenine version from `0.2.9` to `0.2.10`

Fixes the issues in `limit` and `rlimit` helpers that occur when they are on the left operand of a binary expression.

* Upgrade the client hash to latest
2021-11-19 18:57:19 +03:00
M. Mert Yıldıran
8118569460 Show the source and destination IP in the entry feed (#485) 2021-11-18 20:21:51 +03:00
Nimrod Gilboa Markevich
2e75834dd0 Refactor watch pods to allow reusing watch wrapper (#470)
Currently shared/kubernetes/watch.go:FilteredWatch only watches pods.
This PR makes it reusable for other types of resources.
This is done in preparation for watching k8s events.
2021-11-18 11:53:11 +02:00
M. Mert Yıldıran
dd53a36d5f Prevent the crash on client-side in case of text being undefined in FancyTextDisplay (#481)
* Prevent the crash on client-side in case of `text` being undefined in `FancyTextDisplay`

* Use `String(text)` instead
2021-11-17 18:50:09 +03:00
M. Mert Yıldıran
ad78f1dcd7 Clear focusedEntryId state in case of a filter is applied (#482) 2021-11-17 18:20:23 +03:00
M. Mert Yıldıran
a13fec3dae Sync entries in batches just as before (using uploadIntervalSec parameter) (#477)
* Sync entries in batches just as before (using `uploadIntervalSec` parameter)

* Replace `lastTimeSynced` value with `time.Time{}`

Since it will be overwritten by the very first iteration.
2021-11-17 15:16:49 +03:00
M. Mert Yıldıran
bb85312b9f Don't omit the key-value pair if the value is false in EntryTableSection (#478) 2021-11-17 15:02:23 +03:00
RamiBerm
18be46809e TRA-3903 minor daemon mode refactor (#479)
* Update common.go and tapRunner.go

* Update common.go
2021-11-17 11:18:08 +02:00
RamiBerm
b7f7daa05c TRA-3903 fix daemon mode in permission restricted configs (#473)
* Update tapRunner.go, permissions-all-namespaces-daemon.yaml, and 2 more files...

* Update tapRunner.go

* Update tapRunner.go and permissions-ns-daemon.yaml

* Update tapRunner.go

* Update tapRunner.go

* Update tapRunner.go
2021-11-17 11:14:43 +02:00
M. Mert Yıldıran
95d2a868e1 Update the UI screenshots (#476)
* Update the UI screenshots

* Update `mizu-ui.png`
2021-11-16 22:44:31 +03:00
RamiBerm
36077a9985 TRA-3903 - display targetted pods before waiting for all daemon resources to be created (#475)
* WIP

* Update tapRunner.go

* Update tapRunner.go
2021-11-16 17:53:38 +02:00
RamiBerm
51e0dd8ba9 TRA-3903 add flag to disable pvc creation for daemon mode (#474)
* Update tapRunner.go and tapConfig.go

* Update tapConfig.go

* Revert "Update tapConfig.go"

This reverts commit 5c7c02c4ab.
2021-11-16 17:11:47 +02:00
M. Mert Yıldıran
7f265dc4c5 Return 404 instead of 500 if the entry could not be found and display a toast message (#464) 2021-11-16 17:13:07 +03:00
RoyUP9
1c75ce314b fixed redact acceptance test (#472) 2021-11-16 15:49:08 +02:00
RamiBerm
89836d8d75 TRA-3903 better health endpoint for daemon mode (#471)
* Update main.go, status_controller.go, and 2 more files...

* Update status_controller.go and mizuTapperSyncer.go
2021-11-16 15:44:27 +02:00
RoyUP9
763f72a640 remove newline in logs, fixed logs time format (#469) 2021-11-16 12:07:48 +02:00
Igor Gov
a6ec246dd1 Stop reduction of user agent header (#468) 2021-11-16 11:33:31 +02:00
RoyUP9
3e30815fb4 changes log format to be more readable (#463) 2021-11-16 11:01:40 +02:00
M. Mert Yıldıran
a6bf39fad5 Prevent elapsedTime to be negative (#467)
Also fix the `elapsedTime` for Redis.
2021-11-16 02:52:48 +03:00
M. Mert Yıldıran
58a1eac247 Set response.bodySize to 0 if it's negative (#466) 2021-11-16 01:58:22 +03:00
M. Mert Yıldıran
ad574956df Upgrade Basenine version from 0.2.8 to 0.2.9 (#465)
Fixes `limit` helper being not finished because of lack of meta updates.
2021-11-16 00:53:29 +03:00
M. Mert Yıldıran
618cb3a409 Optimize UI entry feed performance (#452)
* Optimize the React code for feeding the entries

By building `EntryItem` only once and updating the `entries` state on meta query messages.

* Upgrade `react-scrollable-feed-virtualized` version from `1.4.3` to `1.4.8`

* Fix the `isSelected` state

* Set the query text before deciding the background to prevent lags while typing

* Upgrade Basenine version from `0.2.6` to `0.2.7`

* Set the query background color only if the query is same after the HTTP request and use `useEffect` instead

* Upgrade Basenine version from `0.2.7` to `0.2.8`

* Use `CancelToken` of `axios` instead of trying to check the query state

* Turn `updateQuery` function into a state hook

* Update the macro for `http`

* Do the `source.cancel()` call in `axios.CancelToken`

* Reduce client-side logging
2021-11-15 17:32:05 +03:00
M. Mert Yıldıran
2582b7a65c Ignore SNYK-JS-JSONSCHEMA-1920922 (#462)
Dependency tree:
`node-sass@5.0.0 > node-gyp@7.1.2 > request@2.88.2 > http-signature@1.2.0 > jsprim@1.4.1 > json-schema@0.2.3`

`node-sass` should fix it first.
2021-11-15 17:29:20 +03:00
RoyUP9
4641ee7c54 fixed acceptance test go sum (#458) 2021-11-14 13:54:10 +02:00
RoyUP9
14a5fe11e7 changed logger debug mode to log level (#456) 2021-11-14 12:21:48 +02:00
Nimrod Gilboa Markevich
6909e6e657 Add link to exposing mizu wiki page in README (#455) 2021-11-11 16:31:47 +02:00
RoyUP9
3e132905ce extend cleanup timeout to solve context timeout problem in dump logs (#453) 2021-11-11 14:30:35 +02:00
RoyUP9
ea0b3fb34e moved headless to root config, use headless in view (#450) 2021-11-11 12:11:02 +02:00
M. Mert Yıldıran
5382a52025 Fix the CSS issues in the cheatsheet modal (#448)
* Fix the CSS issues in the cheatsheet modal

* Change the Sass variable names
2021-11-11 11:26:20 +03:00
M. Mert Yıldıran
ed8d36cdad Send the message into this WebSocket connection instead of all (#449) 2021-11-11 11:10:42 +03:00
M. Mert Yıldıran
1ee8fb6292 Fix the acceptance tests after the merger of #279 (#443)
* Enable acceptance tests

* Fix the acceptance tests

* Move `--headless` from `getDefaultCommandArgs` to `getDefaultTapCommandArgs`

* Fix rest of the failing acceptance tests

* Revert "Enable acceptance tests"

This reverts commit 3f919e865a.

* Revert "Revert "Enable acceptance tests""

This reverts commit c0bfe54b70.

* Ignore `--headless` in `mizu view`

* Make all non-informative things informative

* Remove `github.com/stretchr/testify` dependency from the acceptance tests

* Move the helper methods `waitTimeout` and `checkDBHasEntries` from `tap_test.go` to `testsUtils.go`

* Split `checkDBHasEntries` method into `getDBEntries` and `assertEntriesAtLeast` methods

* Revert "Revert "Revert "Enable acceptance tests"""

This reverts commit c13342671c.

* Revert "Revert "Revert "Revert "Enable acceptance tests""""

This reverts commit 0f8c436926.

* Make `getDBEntries` and `checkEntriesAtLeast` methods return errors instead

* Revert "Revert "Revert "Revert "Revert "Enable acceptance tests"""""

This reverts commit 643fdde009.
2021-11-10 18:14:04 +03:00
M. Mert Yıldıran
eb61831a2c Fix the Analysis button's style into its original state (#447)
* Fix the `Analysis` button's style into its original state

* Fix the MUI button style into its original state
2021-11-10 17:52:51 +03:00
M. Mert Yıldıran
81c25f0bd4 Upgrade github.com/up9inc/basenine/client/go version (#446) 2021-11-10 17:09:40 +03:00
M. Mert Yıldıran
b970640ebc Remove the Reconnect button (#444) 2021-11-10 17:06:41 +03:00
M. Mert Yıldıran
d2fe3f6620 Migrate from SQLite to Basenine and introduce a new filtering syntax (#279)
* Fix the OOMKilled error by calling `debug.FreeOSMemory` periodically

* Remove `MAX_NUMBER_OF_GOROUTINES` environment variable

* Change the line

* Increase the default value of `TCP_STREAM_CHANNEL_TIMEOUT_MS` to `10000`

* Write the client and integrate to the new real-time database

* Refactor the WebSocket implementaiton for `/ws`

* Adapt the UI to the new filtering system

* Fix the rest of the issues in the UI

* Increase the buffer of the scanner

* Implement accessing single records

* Increase the buffer of another scanner

* Populate `Request` and `Response` fields of `MizuEntry`

* Add syntax highlighting for the query

* Add database to `Dockerfile`

* Fix some issues

* Update the `realtime_dbms` Git module commit hash

* Upgrade Gin version and print the query string

* Revert "Upgrade Gin version and print the query string"

This reverts commit aa09f904ee.

* Use WebSocket's itself to query instead of the query string

* Fix some errors related to conversion to HAR

* Fix the issues caused by the latest merge

* Fix the build error

* Fix PR validation GitHub workflow

* Replace the git submodule with latest Basenine version `0.1.0`

Remove `realtime_client.go` and use the official client library `github.com/up9inc/basenine/client/go` instead.

* Move Basenine host and port constants to `shared` module

* Reliably execute and wait for Basenine to become available

* Upgrade Basenine version

* Properly close WebSocket and data channel

* Fix the issues caused by the recent merge commit

* Clean up the TypeScript code

* Update `.gitignore`

* Limit the database size

* Add `Macros` method signature to `Dissector` interface and set the macros provided by the protocol extensions

* Run `go mod tidy` on `agent`

* Upgrade `github.com/up9inc/basenine/client/go` version

* Implement a mechanism to update the query using click events in the UI and use it for protocol macros

* Update the query on click to timestamps

* Fix some issues in the WebSocket and channel handling

* Update the query on clicks to status code

* Update the query on clicks to method, path and service

* Update the query on clicks to is outgoing, source and destination ports

* Add an API endpoint to validate the query against syntax errors

* Move the query background color state into `TrafficPage`

* Fix the logic in `setQuery`

* Display a toast message in case of a syntax error in the query

* Remove a call to `fmt.Printf`

* Upgrade Basenine version to `0.1.3`

* Fix an issue related to getting `MAX_ENTRIES_DB_BYTES` environment variable

* Have the `path` key in request details, in HTTP

* Rearrange the HTTP headers for the querying

* Do the same thing for `cookies` and `queryString`

* Update the query on click to table elements

Add the selectors for `TABLE` type representations in HTTP extension.

* Update the query on click to `bodySize` and `elapsedTime` in `EntryTitle`

* Add the selectors for `TABLE` type representations in AMQP extension

* Add the selectors for `TABLE` type representations in Kafka extension

* Add the selectors for `TABLE` type representations in Redis extension

* Define a struct in `tap/api.go` for the section representation data

* Add the selectors for `BODY` type representations

* Add `request.path` to the HTTP request details

* Change the summary string's field name from `path` to `summary`

* Introduce `queryable` CSS class for queryable UI elements and underline them on hover

* Instead of `N requests` at the bottom, make it `Displaying N results (queried X/Y)` and live update the values

Upgrade Basenine version to `0.2.0`.

* Verify the sha256sum of Basenine executable inside `Dockerfile`

* Pass the start time to web UI through WebSocket and always show the `EntriesList` footer

* Pipe the `stderr` of Basenine as well

* Fix the layout issues related to `CodeEditor` in the UI

* Use the correct `shasum` command in `Dockerfile`

* Upgrade Basenine version to `0.2.1`

* Limit the height of `CodeEditor` container

* Remove `Paused` enum `ConnectionStatus` in UI

* Fix the issue caused by the recent merge

* Add the filtering guide (cheatsheet)

* Update open cheatsheet button's title

* Update cheatsheet content

* Remove the old SQLite code, adapt the `--analyze` related code to Basenine

* Change the method signature of `NewEntry`

* Change the method signature of `Represent`

* Introduce `HTTPPair` field in `MizuEntry` specific to HTTP

* Remove `Entry`, `EntryId` and `EstimatedSizeBytes` fields from `MizuEntry`

Also remove the `getEstimatedEntrySizeBytes` method.

* Remove `gorm.io/gorm` dependency

* Remove unused `sensitiveDataFiltering` folder

* Increase the left margin of open cheatsheet button

* Add `overflow: auto` to the cheatsheet `Modal`

* Fix `GetEntry` method

* Fix the macro for gRPC

* Fix an interface conversion in case of AMQP

* Fix two more interface conversion errors in AMQP

* Make the `syncEntriesImpl` method blocking

* Fix a grammar mistake in the cheatsheet

* Adapt to the changes in the recent merge commit

* Improve the cheatsheet text

* Always display the timestamp in `en-US`

* Upgrade Basenine version to `0.2.2`

* Fix the order of closing Basenine connections and channels

* Don't close the Basenine channels at all

* Upgrade Basenine version to `0.2.3`

* Set the initial filter to `rlimit(100)`

* Make Basenine persistent

* Upgrade Basenine version to `0.2.4`

* Update `debug.Dockerfile`

* Fix a failing test

* Upgrade Basenine version to `0.2.5`

* Revert "Do not show play icon when disconnected (#428)"

This reverts commit 8af2e562f8.

* Upgrade Basenine version to `0.2.6`

* Make all non-informative things informative

* Make `100` a constant

* Use `===` in JavaScript no matter what

* Remove a forgotten `console.log`

* Add a comment and update the `query` in `syncEntriesImpl`

* Don't call `panic` in `GetEntry`

* Replace `panic` calls in `startBasenineServer` with `logger.Log.Panicf`

* Remove unnecessary `\n` characters in the logs
2021-11-09 19:54:48 +03:00
gadotroee
31d95c6557 Auto close inactive issues (#441) 2021-11-08 14:49:25 +02:00
RamiBerm
67e9cc1099 fix readme titles (#442) 2021-11-08 13:23:36 +02:00
RamiBerm
4cf3c9c6d3 TRA-3913 support mizu via expose service (#440)
* Update README.md, tapRunner.go, and 4 more files...

* Update testsUtils.go

* Update proxy.go

* Update README.md, testsUtils.go, and 3 more files...

* Update testsUtils.go and provider.go
2021-11-08 11:23:03 +02:00
RamiBerm
b7b012539d TRA-3842 daemon acceptance tests (#429)
* Update tap_test.go and testsUtils.go

* Update tap_test.go

* Update testsUtils.go

* Update tap_test.go and testsUtils.go

* Update tap_test.go and testsUtils.go

* Update testsUtils.go

* Update tap_test.go

* gofmt
2021-11-08 10:03:01 +02:00
David Levanon
8a90f02161 Add support of listening to multiple netns (#418)
* multiple netns listen - initial commit

* multiple netns listen - actual work

* remove redundant log line

* map /proc of host to tapper

* changing kubernetes provider again after big conflict

* revert node-sass version back to 5.0.0

* Rename host_source to hostSource

Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com>

* PR fixes - adding comment + typos + naming conventions

* go fmt + making procfs read only

* setns back to the original value after packet source initialized

Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com>
2021-11-07 16:00:59 +02:00
RamiBerm
a866576cfc TRA-3903 more docs (#438)
* Update README.md and PERMISSIONS.md

* Update PERMISSIONS.md

* Update README.md

* Update PERMISSIONS.md

* Update PERMISSIONS.md
2021-11-07 12:23:15 +02:00
M. Mert Yıldıran
6811bd5050 Remove the ERROR level logs in loadOAS and let the caller handle them in INFO level (#434) 2021-11-07 12:40:57 +03:00
M. Mert Yıldıran
4009386d82 Set a default value for SEM_VER build-time variable in Dockerfile (#435)
Also fix a runtime error that happens when the API server's version is not a valid SemVer.
2021-11-07 12:36:02 +03:00
RamiBerm
a6ebc460b0 TRA-3903 add daemon flag readme doc (#437)
* add daemon flag readme doc

* Update README.md
2021-11-07 09:33:19 +02:00
175 changed files with 7959 additions and 4344 deletions

View File

@@ -30,3 +30,15 @@ jobs:
- name: Test
run: make acceptance-test
- name: Slack notification on failure
uses: ravsamhq/notify-slack-action@v1
if: always()
with:
status: ${{ job.status }}
notification_title: 'Mizu {workflow} has {status_message}'
message_format: '{emoji} *{workflow}* {status_message} during <{run_url}|run>, after commit: <{commit_url}|{commit_sha}>'
footer: 'Linked Repo <{repo_url}|{repo}>'
notify_when: 'failure'
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@@ -0,0 +1,22 @@
name: Close inactive issues
on:
schedule:
- cron: "0 0 * * *"
jobs:
close-issues:
runs-on: ubuntu-latest
permissions:
issues: write
pull-requests: write
steps:
- uses: actions/stale@v3
with:
days-before-issue-stale: 30
days-before-issue-close: 14
stale-issue-label: "stale"
stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
days-before-pr-stale: -1
days-before-pr-close: -1
repo-token: ${{ secrets.GITHUB_TOKEN }}

View File

@@ -1,25 +0,0 @@
name: Security validation
on:
pull_request:
branches:
- 'develop'
- 'main'
jobs:
security:
name: Check for vulnerabilities
runs-on: ubuntu-latest
env:
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
steps:
- uses: actions/checkout@v2
- uses: snyk/actions/setup@master
- name: Set up Go 1.16
uses: actions/setup-go@v2
with:
go-version: '1.16'
- name: Run snyl on all projects
run: snyk test --all-projects

8
.gitignore vendored
View File

@@ -15,7 +15,6 @@
# vendor/
.idea/
build
*.db
# Mac OS
.DS_Store
@@ -29,3 +28,10 @@ build
# pprof
pprof/*
# Database Files
*.bin
*.gob
# Nohup Files - https://man7.org/linux/man-pages/man1/nohup.1p.html
nohup.*

View File

@@ -13,7 +13,7 @@ FROM golang:1.16-alpine AS builder
# Set necessary environment variables needed for our image.
ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
RUN apk add libpcap-dev gcc g++ make bash
RUN apk add libpcap-dev gcc g++ make bash perl-utils
# Move to agent working directory (/agent-build).
WORKDIR /app/agent-build
@@ -24,12 +24,12 @@ COPY tap/go.mod tap/go.mod ../tap/
COPY tap/api/go.* ../tap/api/
RUN go mod download
# cheap trick to make the build faster (As long as go.mod wasn't changes)
RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
ARG COMMIT_HASH
ARG GIT_BRANCH
ARG BUILD_TIMESTAMP
ARG SEM_VER
ARG SEM_VER=0.0.0
# Copy and build agent code
COPY shared ../shared
@@ -41,16 +41,24 @@ RUN go build -ldflags="-s -w \
-X 'mizuserver/pkg/version.BuildTimestamp=${BUILD_TIMESTAMP}' \
-X 'mizuserver/pkg/version.SemVer=${SEM_VER}'" -o mizuagent .
# Download Basenine executable, verify the sha1sum and move it to a directory in $PATH
ADD https://github.com/up9inc/basenine/releases/download/v0.2.19/basenine_linux_amd64 ./basenine_linux_amd64
ADD https://github.com/up9inc/basenine/releases/download/v0.2.19/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
RUN shasum -a 256 -c basenine_linux_amd64.sha256
RUN chmod +x ./basenine_linux_amd64
COPY devops/build_extensions.sh ..
RUN cd .. && /bin/bash build_extensions.sh
FROM alpine:3.14
RUN apk add bash libpcap-dev tcpdump
RUN apk add bash libpcap-dev
WORKDIR /app
# Copy binary and config files from /build to root folder of scratch container.
COPY --from=builder ["/app/agent-build/mizuagent", "."]
COPY --from=builder ["/app/agent-build/basenine_linux_amd64", "/usr/local/bin/basenine"]
COPY --from=builder ["/app/agent/build/extensions", "extensions"]
COPY --from=site-build ["/app/ui-build/build", "site"]
RUN mkdir /app/data/

View File

@@ -4,16 +4,21 @@
A simple-yet-powerful API traffic viewer for Kubernetes enabling you to view all API communication between microservices to help your debug and troubleshoot regressions.
Think TCPDump and Chrome Dev Tools combined.
Think TCPDump and Wireshark re-invented for Kubernetes.
![Simple UI](assets/mizu-ui.png)
## Features
- Simple and powerful CLI
- Real-time view of all HTTP requests, REST and gRPC API calls
- No installation or code instrumentation
- Works completely on premises
- Monitoring network traffic in real-time. Supported protocols:
- [HTTP/1.1](https://datatracker.ietf.org/doc/html/rfc2616) (REST, etc.)
- [HTTP/2](https://datatracker.ietf.org/doc/html/rfc7540) (gRPC)
- [AMQP](https://www.rabbitmq.com/amqp-0-9-1-reference.html) (RabbitMQ, Apache Qpid, etc.)
- [Apache Kafka](https://kafka.apache.org/protocol)
- [Redis](https://redis.io/topics/protocol)
- Works with Kubernetes APIs. No installation or code instrumentation
- Rich filtering
## Requirements
@@ -44,20 +49,11 @@ SHA256 checksums are available on the [Releases](https://github.com/up9inc/mizu/
### Development (unstable) Build
Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page
## Kubeconfig & Permissions
While `mizu`most often works out of the box, you can influence its behavior:
1. [OPTIONAL] Set `KUBECONFIG` environment variable to your Kubernetes configuration. If this is not set, Mizu assumes that configuration is at `${HOME}/.kube/config`
2. `mizu` assumes user running the command has permissions to create resources (such as pods, services, namespaces) on your Kubernetes cluster (no worries - `mizu` resources are cleaned up upon termination)
For detailed list of k8s permissions see [PERMISSIONS](docs/PERMISSIONS.md) document
## How to Run
1. Find pods you'd like to tap to in your Kubernetes cluster
2. Run `mizu tap` or `mizu tap PODNAME`
3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI
2. Run `mizu tap` or `mizu tap PODNAME`
3. Open browser on `http://localhost:8899` **or** as instructed in the CLI
4. Watch the API traffic flowing
5. Type ^C to stop
@@ -83,7 +79,7 @@ To tap all pods in current namespace -
```
To tap specific pod -
### To tap specific pod
```bash
$ kubectl get pods
NAME READY STATUS RESTARTS AGE
@@ -96,7 +92,7 @@ To tap specific pod -
^C
```
To tap multiple pods using regex -
### To tap multiple pods using regex
```bash
$ kubectl get pods
NAME READY STATUS RESTARTS AGE
@@ -114,21 +110,22 @@ To tap multiple pods using regex -
## Configuration
Mizu can work with config file which should be stored in ${HOME}/.mizu/config.yaml (macOS: ~/.mizu/config.yaml) <br />
In case no config file found, defaults will be used <br />
Mizu can optionally work with a config file that can be provided as a CLI argument (using `--set config-path=<PATH>`) or if not provided, will be stored at ${HOME}/.mizu/config.yaml
In case of partial configuration defined, all other fields will be used with defaults <br />
You can always override the defaults or config file with CLI flags
To get the default config params run `mizu config` <br />
To generate a new config file with default values use `mizu config -r`
### Telemetry
By default, mizu reports usage telemetry. It can be disabled by adding a line of `telemetry: false` in the `${HOME}/.mizu/config.yaml` file
## Advanced Usage
### Kubeconfig
It is possible to change the kubeconfig path using `KUBECONFIG` environment variable or the command like flag
with `--set kube-config-path=<PATH>`. </br >
If both are not set - Mizu assumes that configuration is at `${HOME}/.kube/config`
### Namespace-Restricted Mode
Some users have permission to only manage resources in one particular namespace assigned to them
@@ -142,6 +139,8 @@ using the `--namespace` flag or by setting `tap.namespaces` in the config file
Setting `mizu-resources-namespace=mizu` resets Mizu to its default behavior
For detailed list of k8s permissions see [PERMISSIONS](docs/PERMISSIONS.md) document
### User agent filtering
User-agent filtering (like health checks) - can be configured using command-line options:
@@ -173,22 +172,14 @@ Please see [CONTRACT MONITORING](docs/CONTRACT_MONITORING.md) page for more deta
### Configure proxy host
By default, mizu will be accessible via local host: 'http://localhost:8899/mizu/', it is possible to change the host,
for instance, to '0.0.0.0' which can grant access via machine IP address.
This setting can be changed via command line flag `--set tap.proxy-host=<value>` or via config file:
tap
proxy-host: 0.0.0.0
and when changed it will support accessing by IP
By default, mizu will be accessible via local host: 'http://localhost:8899', it is possible to change the host, for
instance, to '0.0.0.0' which can grant access via machine IP address. This setting can be changed via command line
flag `--set tap.proxy-host=<value>` or via config file:
tap proxy-host: 0.0.0.0 and when changed it will support accessing by IP
### Install Mizu standalone
## How to Run local UI
Mizu can be run detached from the cli using the install command: `mizu install`. This type of mizu instance will run
indefinitely in the cluster.
- run from mizu/agent `go run main.go --hars-read --hars-dir <folder>`
- copy Har files into the folder from last command
- change `MizuWebsocketURL` and `apiURL` in `api.js` file
- run from mizu/ui - `npm run start`
- open browser on `localhost:3000`
For more information please refer to [INSTALL STANDALONE](docs/INSTALL_STANDALONE.md)

View File

@@ -1,2 +1,2 @@
test: ## Run acceptance tests.
@go test ./... -timeout 1h
@go test ./... -timeout 1h -v

View File

@@ -2,11 +2,12 @@ package acceptanceTests
import (
"fmt"
"gopkg.in/yaml.v3"
"io/ioutil"
"os"
"os/exec"
"testing"
"gopkg.in/yaml.v3"
)
type tapConfig struct {

View File

@@ -3,6 +3,7 @@ module github.com/up9inc/mizu/tests
go 1.16
require (
github.com/gorilla/websocket v1.4.2
github.com/up9inc/mizu/shared v0.0.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
)

View File

@@ -110,6 +110,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
@@ -156,6 +157,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0=
github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
@@ -191,8 +193,10 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -211,6 +215,7 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
@@ -242,6 +247,7 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
@@ -287,8 +293,10 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
@@ -303,6 +311,7 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@@ -460,6 +469,7 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -519,6 +529,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -628,6 +639,7 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8X
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
@@ -639,6 +651,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
@@ -651,7 +664,9 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
k8s.io/api v0.21.2 h1:vz7DqmRsXTCSa6pNxXwQ1IYeAZgdIsua+DZU+o+SX3Y=
k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU=
k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc=
k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM=
k8s.io/cli-runtime v0.21.2/go.mod h1:8u/jFcM0QpoI28f6sfrAAIslLCXUYKD5SsPPMWiHYrI=
k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA=
@@ -662,6 +677,7 @@ k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8
k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
k8s.io/kubectl v0.21.2/go.mod h1:PgeUclpG8VVmmQIl8zpLar3IQEpFc9mrmvlwY3CK1xo=
@@ -675,5 +691,6 @@ sigs.k8s.io/kustomize/cmd/config v0.9.10/go.mod h1:Mrby0WnRH7hA6OwOYnYpfpiY0WJIM
sigs.k8s.io/kustomize/kustomize/v4 v4.1.2/go.mod h1:PxBvo4WGYlCLeRPL+ziT64wBXqbgfcalOS/SXa/tcyo=
sigs.k8s.io/kustomize/kyaml v0.10.17/go.mod h1:mlQFagmkm1P+W4lZJbJ/yaxMd8PqMRSC4cPcfUVt5Hg=
sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

View File

@@ -66,21 +66,18 @@ func TestTap(t *testing.T) {
entriesCheckFunc := func() error {
timestamp := time.Now().UnixNano() / int64(time.Millisecond)
entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, entriesCount, timestamp)
requestResult, requestErr := executeHttpGetRequest(entriesUrl)
if requestErr != nil {
return fmt.Errorf("failed to get entries, err: %v", requestErr)
entries, err := getDBEntries(timestamp, entriesCount, 1*time.Second)
if err != nil {
return err
}
entries := requestResult.([]interface{})
if len(entries) == 0 {
return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
err = checkEntriesAtLeast(entries, 1)
if err != nil {
return err
}
entry := entries[0].(map[string]interface{})
entry := entries[0]
entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entry["id"])
requestResult, requestErr = executeHttpGetRequest(entryUrl)
requestResult, requestErr := executeHttpGetRequest(entryUrl)
if requestErr != nil {
return fmt.Errorf("failed to get entry, err: %v", requestErr)
}
@@ -150,10 +147,7 @@ func TestTapAllNamespaces(t *testing.T) {
t.Skip("ignored acceptance test")
}
expectedPods := []struct{
Name string
Namespace string
}{
expectedPods := []PodDescriptor{
{Name: "httpbin", Namespace: "mizu-tests"},
{Name: "httpbin", Namespace: "mizu-tests2"},
}
@@ -202,19 +196,7 @@ func TestTapAllNamespaces(t *testing.T) {
}
for _, expectedPod := range expectedPods {
podFound := false
for _, pod := range pods {
podNamespace := pod["namespace"].(string)
podName := pod["name"].(string)
if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
podFound = true
break
}
}
if !podFound {
if !isPodDescriptorInPodArray(pods, expectedPod) {
t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
return
}
@@ -226,10 +208,7 @@ func TestTapMultipleNamespaces(t *testing.T) {
t.Skip("ignored acceptance test")
}
expectedPods := []struct{
Name string
Namespace string
}{
expectedPods := []PodDescriptor{
{Name: "httpbin", Namespace: "mizu-tests"},
{Name: "httpbin2", Namespace: "mizu-tests"},
{Name: "httpbin", Namespace: "mizu-tests2"},
@@ -288,19 +267,7 @@ func TestTapMultipleNamespaces(t *testing.T) {
}
for _, expectedPod := range expectedPods {
podFound := false
for _, pod := range pods {
podNamespace := pod["namespace"].(string)
podName := pod["name"].(string)
if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
podFound = true
break
}
}
if !podFound {
if !isPodDescriptorInPodArray(pods, expectedPod) {
t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
return
}
@@ -313,10 +280,7 @@ func TestTapRegex(t *testing.T) {
}
regexPodName := "httpbin2"
expectedPods := []struct{
Name string
Namespace string
}{
expectedPods := []PodDescriptor{
{Name: regexPodName, Namespace: "mizu-tests"},
}
@@ -371,19 +335,7 @@ func TestTapRegex(t *testing.T) {
}
for _, expectedPod := range expectedPods {
podFound := false
for _, pod := range pods {
podNamespace := pod["namespace"].(string)
podName := pod["name"].(string)
if expectedPod.Namespace == podNamespace && strings.Contains(podName, expectedPod.Name) {
podFound = true
break
}
}
if !podFound {
if !isPodDescriptorInPodArray(pods, expectedPod) {
t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name)
return
}
@@ -431,7 +383,7 @@ func TestTapDryRun(t *testing.T) {
resultChannel <- "fail"
}()
testResult := <- resultChannel
testResult := <-resultChannel
if testResult != "success" {
t.Errorf("unexpected result - dry run cmd not done")
}
@@ -475,9 +427,10 @@ func TestTapRedact(t *testing.T) {
}
proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
requestHeaders := map[string]string{"User-Header": "Mizu"}
requestBody := map[string]string{"User": "Mizu"}
for i := 0; i < defaultEntriesCount; i++ {
if _, requestErr := executeHttpPostRequest(fmt.Sprintf("%v/post", proxyUrl), requestBody); requestErr != nil {
if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
t.Errorf("failed to send proxy request, err: %v", requestErr)
return
}
@@ -486,51 +439,39 @@ func TestTapRedact(t *testing.T) {
redactCheckFunc := func() error {
timestamp := time.Now().UnixNano() / int64(time.Millisecond)
entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
requestResult, requestErr := executeHttpGetRequest(entriesUrl)
if requestErr != nil {
return fmt.Errorf("failed to get entries, err: %v", requestErr)
entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
if err != nil {
return err
}
entries := requestResult.([]interface{})
if len(entries) == 0 {
return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
err = checkEntriesAtLeast(entries, 1)
if err != nil {
return err
}
firstEntry := entries[0].(map[string]interface{})
firstEntry := entries[0]
entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
requestResult, requestErr = executeHttpGetRequest(entryUrl)
requestResult, requestErr := executeHttpGetRequest(entryUrl)
if requestErr != nil {
return fmt.Errorf("failed to get entry, err: %v", requestErr)
}
data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
entryJson := data["entry"].(string)
entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
request := entry["request"].(map[string]interface{})
var entry map[string]interface{}
if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
return fmt.Errorf("failed to parse entry, err: %v", parseErr)
}
entryRequest := entry["request"].(map[string]interface{})
entryPayload := entryRequest["payload"].(map[string]interface{})
entryDetails := entryPayload["details"].(map[string]interface{})
headers := entryDetails["headers"].([]interface{})
headers := request["_headers"].([]interface{})
for _, headerInterface := range headers {
header := headerInterface.(map[string]interface{})
if header["name"].(string) != "User-Agent" {
if header["name"].(string) != "User-Header" {
continue
}
userAgent := header["value"].(string)
if userAgent != "[REDACTED]" {
userHeader := header["value"].(string)
if userHeader != "[REDACTED]" {
return fmt.Errorf("unexpected result - user agent is not redacted")
}
}
postData := entryDetails["postData"].(map[string]interface{})
postData := request["postData"].(map[string]interface{})
textDataStr := postData["text"].(string)
var textData map[string]string
@@ -590,9 +531,10 @@ func TestTapNoRedact(t *testing.T) {
}
proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
requestHeaders := map[string]string{"User-Header": "Mizu"}
requestBody := map[string]string{"User": "Mizu"}
for i := 0; i < defaultEntriesCount; i++ {
if _, requestErr := executeHttpPostRequest(fmt.Sprintf("%v/post", proxyUrl), requestBody); requestErr != nil {
if _, requestErr := executeHttpPostRequestWithHeaders(fmt.Sprintf("%v/post", proxyUrl), requestHeaders, requestBody); requestErr != nil {
t.Errorf("failed to send proxy request, err: %v", requestErr)
return
}
@@ -601,51 +543,39 @@ func TestTapNoRedact(t *testing.T) {
redactCheckFunc := func() error {
timestamp := time.Now().UnixNano() / int64(time.Millisecond)
entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
requestResult, requestErr := executeHttpGetRequest(entriesUrl)
if requestErr != nil {
return fmt.Errorf("failed to get entries, err: %v", requestErr)
entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
if err != nil {
return err
}
entries := requestResult.([]interface{})
if len(entries) == 0 {
return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
err = checkEntriesAtLeast(entries, 1)
if err != nil {
return err
}
firstEntry := entries[0].(map[string]interface{})
firstEntry := entries[0]
entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
requestResult, requestErr = executeHttpGetRequest(entryUrl)
requestResult, requestErr := executeHttpGetRequest(entryUrl)
if requestErr != nil {
return fmt.Errorf("failed to get entry, err: %v", requestErr)
}
data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
entryJson := data["entry"].(string)
entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
request := entry["request"].(map[string]interface{})
var entry map[string]interface{}
if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
return fmt.Errorf("failed to parse entry, err: %v", parseErr)
}
entryRequest := entry["request"].(map[string]interface{})
entryPayload := entryRequest["payload"].(map[string]interface{})
entryDetails := entryPayload["details"].(map[string]interface{})
headers := entryDetails["headers"].([]interface{})
headers := request["_headers"].([]interface{})
for _, headerInterface := range headers {
header := headerInterface.(map[string]interface{})
if header["name"].(string) != "User-Agent" {
if header["name"].(string) != "User-Header" {
continue
}
userAgent := header["value"].(string)
if userAgent == "[REDACTED]" {
userHeader := header["value"].(string)
if userHeader == "[REDACTED]" {
return fmt.Errorf("unexpected result - user agent is redacted")
}
}
postData := entryDetails["postData"].(map[string]interface{})
postData := request["postData"].(map[string]interface{})
textDataStr := postData["text"].(string)
var textData map[string]string
@@ -716,38 +646,26 @@ func TestTapRegexMasking(t *testing.T) {
redactCheckFunc := func() error {
timestamp := time.Now().UnixNano() / int64(time.Millisecond)
entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount, timestamp)
requestResult, requestErr := executeHttpGetRequest(entriesUrl)
if requestErr != nil {
return fmt.Errorf("failed to get entries, err: %v", requestErr)
entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
if err != nil {
return err
}
entries := requestResult.([]interface{})
if len(entries) == 0 {
return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
err = checkEntriesAtLeast(entries, 1)
if err != nil {
return err
}
firstEntry := entries[0].(map[string]interface{})
firstEntry := entries[0]
entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, firstEntry["id"])
requestResult, requestErr = executeHttpGetRequest(entryUrl)
requestResult, requestErr := executeHttpGetRequest(entryUrl)
if requestErr != nil {
return fmt.Errorf("failed to get entry, err: %v", requestErr)
}
data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
entryJson := data["entry"].(string)
entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
request := entry["request"].(map[string]interface{})
var entry map[string]interface{}
if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
return fmt.Errorf("failed to parse entry, err: %v", parseErr)
}
entryRequest := entry["request"].(map[string]interface{})
entryPayload := entryRequest["payload"].(map[string]interface{})
entryDetails := entryPayload["details"].(map[string]interface{})
postData := entryDetails["postData"].(map[string]interface{})
postData := request["postData"].(map[string]interface{})
textData := postData["text"].(string)
if textData != "[REDACTED]" {
@@ -805,7 +723,7 @@ func TestTapIgnoredUserAgents(t *testing.T) {
proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName)
ignoredUserAgentCustomHeader := "Ignored-User-Agent"
headers := map[string]string {"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""}
headers := map[string]string{"User-Agent": ignoredUserAgentValue, ignoredUserAgentCustomHeader: ""}
for i := 0; i < defaultEntriesCount; i++ {
if _, requestErr := executeHttpGetRequestWithHeaders(fmt.Sprintf("%v/get", proxyUrl), headers); requestErr != nil {
t.Errorf("failed to send proxy request, err: %v", requestErr)
@@ -823,38 +741,27 @@ func TestTapIgnoredUserAgents(t *testing.T) {
ignoredUserAgentsCheckFunc := func() error {
timestamp := time.Now().UnixNano() / int64(time.Millisecond)
entriesUrl := fmt.Sprintf("%v/entries?limit=%v&operator=lt&timestamp=%v", apiServerUrl, defaultEntriesCount * 2, timestamp)
requestResult, requestErr := executeHttpGetRequest(entriesUrl)
if requestErr != nil {
return fmt.Errorf("failed to get entries, err: %v", requestErr)
entries, err := getDBEntries(timestamp, defaultEntriesCount, 1*time.Second)
if err != nil {
return err
}
entries := requestResult.([]interface{})
if len(entries) == 0 {
return fmt.Errorf("unexpected entries result - Expected more than 0 entries")
err = checkEntriesAtLeast(entries, 1)
if err != nil {
return err
}
for _, entryInterface := range entries {
entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entryInterface.(map[string]interface{})["id"])
requestResult, requestErr = executeHttpGetRequest(entryUrl)
entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entryInterface["id"])
requestResult, requestErr := executeHttpGetRequest(entryUrl)
if requestErr != nil {
return fmt.Errorf("failed to get entry, err: %v", requestErr)
}
data := requestResult.(map[string]interface{})["data"].(map[string]interface{})
entryJson := data["entry"].(string)
entry := requestResult.(map[string]interface{})["data"].(map[string]interface{})
request := entry["request"].(map[string]interface{})
var entry map[string]interface{}
if parseErr := json.Unmarshal([]byte(entryJson), &entry); parseErr != nil {
return fmt.Errorf("failed to parse entry, err: %v", parseErr)
}
entryRequest := entry["request"].(map[string]interface{})
entryPayload := entryRequest["payload"].(map[string]interface{})
entryDetails := entryPayload["details"].(map[string]interface{})
entryHeaders := entryDetails["headers"].([]interface{})
for _, headerInterface := range entryHeaders {
headers := request["_headers"].([]interface{})
for _, headerInterface := range headers {
header := headerInterface.(map[string]interface{})
if header["name"].(string) != ignoredUserAgentCustomHeader {
continue
@@ -922,21 +829,21 @@ func TestTapDumpLogs(t *testing.T) {
return
}
var dumpsLogsPath string
var dumpLogsPath string
for _, file := range files {
fileName := file.Name()
if strings.Contains(fileName, "mizu_logs") {
dumpsLogsPath = path.Join(mizuFolderPath, fileName)
dumpLogsPath = path.Join(mizuFolderPath, fileName)
break
}
}
if dumpsLogsPath == "" {
if dumpLogsPath == "" {
t.Errorf("dump logs file not found")
return
}
zipReader, zipError := zip.OpenReader(dumpsLogsPath)
zipReader, zipError := zip.OpenReader(dumpLogsPath)
if zipError != nil {
t.Errorf("failed to get zip reader, err: %v", zipError)
return

View File

@@ -3,6 +3,7 @@ package acceptanceTests
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
@@ -10,9 +11,11 @@ import (
"os/exec"
"path"
"strings"
"sync"
"syscall"
"time"
"github.com/gorilla/websocket"
"github.com/up9inc/mizu/shared"
)
@@ -24,8 +27,26 @@ const (
defaultServiceName = "httpbin"
defaultEntriesCount = 50
waitAfterTapPodsReady = 3 * time.Second
cleanCommandTimeout = 1 * time.Minute
)
type PodDescriptor struct {
Name string
Namespace string
}
func isPodDescriptorInPodArray(pods []map[string]interface{}, podDescriptor PodDescriptor) bool {
for _, pod := range pods {
podNamespace := pod["namespace"].(string)
podName := pod["name"].(string)
if podDescriptor.Namespace == podNamespace && strings.Contains(podName, podDescriptor.Name) {
return true
}
}
return false
}
func getCliPath() (string, error) {
dir, filePathErr := os.Getwd()
if filePathErr != nil {
@@ -59,7 +80,11 @@ func getProxyUrl(namespace string, service string) string {
}
func getApiServerUrl(port uint16) string {
return fmt.Sprintf("http://localhost:%v/mizu", port)
return fmt.Sprintf("http://localhost:%v", port)
}
func getWebSocketUrl(port uint16) string {
return fmt.Sprintf("ws://localhost:%v/ws", port)
}
func getDefaultCommandArgs() []string {
@@ -67,8 +92,9 @@ func getDefaultCommandArgs() []string {
telemetry := "telemetry=false"
agentImage := "agent-image=gcr.io/up9-docker-hub/mizu/ci:0.0.0"
imagePullPolicy := "image-pull-policy=Never"
headless := "headless=true"
return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy}
return []string{setFlag, telemetry, setFlag, agentImage, setFlag, imagePullPolicy, setFlag, headless}
}
func getDefaultTapCommandArgs() []string {
@@ -103,6 +129,20 @@ func getDefaultConfigCommandArgs() []string {
return append([]string{configCommand}, defaultCmdArgs...)
}
func getDefaultCleanCommandArgs() []string {
cleanCommand := "clean"
defaultCmdArgs := getDefaultCommandArgs()
return append([]string{cleanCommand}, defaultCmdArgs...)
}
func getDefaultViewCommandArgs() []string {
viewCommand := "view"
defaultCmdArgs := getDefaultCommandArgs()
return append([]string{viewCommand}, defaultCmdArgs...)
}
func retriesExecute(retriesCount int, executeFunc func() error) error {
var lastError interface{}
@@ -195,16 +235,57 @@ func executeHttpGetRequest(url string) (interface{}, error) {
return executeHttpRequest(response, requestErr)
}
func executeHttpPostRequest(url string, body interface{}) (interface{}, error) {
func executeHttpPostRequestWithHeaders(url string, headers map[string]string, body interface{}) (interface{}, error) {
requestBody, jsonErr := json.Marshal(body)
if jsonErr != nil {
return nil, jsonErr
}
response, requestErr := http.Post(url, "application/json", bytes.NewBuffer(requestBody))
request, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(requestBody))
if err != nil {
return nil, err
}
request.Header.Add("Content-Type", "application/json")
for headerKey, headerValue := range headers {
request.Header.Add(headerKey, headerValue)
}
client := &http.Client{}
response, requestErr := client.Do(request)
return executeHttpRequest(response, requestErr)
}
func runMizuClean() error {
cliPath, err := getCliPath()
if err != nil {
return err
}
cleanCmdArgs := getDefaultCleanCommandArgs()
cleanCmd := exec.Command(cliPath, cleanCmdArgs...)
commandDone := make(chan error)
go func() {
if err := cleanCmd.Run(); err != nil {
commandDone <- err
}
commandDone <- nil
}()
select {
case err = <-commandDone:
if err != nil {
return err
}
case <-time.After(cleanCommandTimeout):
return errors.New("clean command timed out")
}
return nil
}
func cleanupCommand(cmd *exec.Cmd) error {
if err := cmd.Process.Signal(syscall.SIGQUIT); err != nil {
return err
@@ -218,11 +299,10 @@ func cleanupCommand(cmd *exec.Cmd) error {
}
func getPods(tapStatusInterface interface{}) ([]map[string]interface{}, error) {
tapStatus := tapStatusInterface.(map[string]interface{})
podsInterface := tapStatus["pods"].([]interface{})
tapPodsInterface := tapStatusInterface.([]interface{})
var pods []map[string]interface{}
for _, podInterface := range podsInterface {
for _, podInterface := range tapPodsInterface {
pods = append(pods, podInterface.(map[string]interface{}))
}
@@ -239,6 +319,77 @@ func getLogsPath() (string, error) {
return logsPath, nil
}
// waitTimeout waits for the waitgroup for the specified max timeout.
// Returns true if waiting timed out.
func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
channel := make(chan struct{})
go func() {
defer close(channel)
wg.Wait()
}()
select {
case <-channel:
return false // completed normally
case <-time.After(timeout):
return true // timed out
}
}
// checkEntriesAtLeast checks whether the number of entries greater than or equal to n
func checkEntriesAtLeast(entries []map[string]interface{}, n int) error {
if len(entries) < n {
return fmt.Errorf("Unexpected entries result - Expected more than %d entries", n-1)
}
return nil
}
// getDBEntries retrieves the entries from the database before the given timestamp.
// Also limits the results according to the limit parameter.
// Timeout for the WebSocket connection is defined by the timeout parameter.
func getDBEntries(timestamp int64, limit int, timeout time.Duration) (entries []map[string]interface{}, err error) {
query := fmt.Sprintf("timestamp < %d and limit(%d)", timestamp, limit)
webSocketUrl := getWebSocketUrl(defaultApiServerPort)
var connection *websocket.Conn
connection, _, err = websocket.DefaultDialer.Dial(webSocketUrl, nil)
if err != nil {
return
}
defer connection.Close()
handleWSConnection := func(wg *sync.WaitGroup) {
defer wg.Done()
for {
_, message, err := connection.ReadMessage()
if err != nil {
return
}
var data map[string]interface{}
if err = json.Unmarshal([]byte(message), &data); err != nil {
return
}
if data["messageType"] == "entry" {
entries = append(entries, data)
}
}
}
err = connection.WriteMessage(websocket.TextMessage, []byte(query))
if err != nil {
return
}
var wg sync.WaitGroup
go handleWSConnection(&wg)
wg.Add(1)
waitTimeout(&wg, timeout)
return
}
func Contains(slice []string, containsValue string) bool {
for _, sliceValue := range slice {
if sliceValue == containsValue {

View File

@@ -3,11 +3,11 @@ module mizuserver
go 1.16
require (
github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b
github.com/djherbis/atime v1.0.0
github.com/fsnotify/fsnotify v1.4.9
github.com/getkin/kin-openapi v0.76.0
github.com/gin-contrib/static v0.0.1
github.com/gin-gonic/gin v1.7.2
github.com/gin-gonic/gin v1.7.7
github.com/go-playground/locales v0.13.0
github.com/go-playground/universal-translator v0.17.0
github.com/go-playground/validator/v10 v10.5.0
@@ -16,13 +16,12 @@ require (
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/up9inc/basenine/client/go v0.0.0-20211215185650-10083bb9a1b3
github.com/up9inc/mizu/shared v0.0.0
github.com/up9inc/mizu/tap v0.0.0
github.com/up9inc/mizu/tap/api v0.0.0
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0
go.mongodb.org/mongo-driver v1.7.1
gorm.io/driver/sqlite v1.1.4
gorm.io/gorm v1.21.8
golang.org/x/text v0.3.5 // indirect
k8s.io/api v0.21.2
k8s.io/apimachinery v0.21.2
k8s.io/client-go v0.21.2

View File

@@ -52,6 +52,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b h1:8m+eVxVVDDyJFidv7Ck1OwqnDaQR6pTSRGlCC2Dnw0A=
github.com/antelman107/net-wait-go v0.0.0-20210623112055-cf684aebda7b/go.mod h1:+tQQjzrp2501Nd6JXrb9s/XsNvFK3ZbxOnCdQl/vDRo=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
@@ -94,7 +96,6 @@ github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg=
github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
@@ -111,7 +112,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
github.com/getkin/kin-openapi v0.76.0 h1:j77zg3Ec+k+r+GA3d8hBoXpAc6KX9TbBPrwQGBIy2sY=
@@ -125,6 +125,8 @@ github.com/gin-contrib/static v0.0.1/go.mod h1:CSxeF+wep05e0kCOsqWdAWbSszmc31zTI
github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
github.com/gin-gonic/gin v1.7.2 h1:Tg03T9yM2xa8j6I3Z3oqLaQRSmKvxPd6g/2HJ6zICFA=
github.com/gin-gonic/gin v1.7.2/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs=
github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U=
github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
@@ -195,31 +197,7 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn
github.com/go-playground/validator/v10 v10.5.0 h1:X9rflw/KmpACwT8zdrm1upefpvdy6ur8d1kWyq6sg3E=
github.com/go-playground/validator/v10 v10.5.0/go.mod h1:xm76BBt941f7yWdGnI2DVPFFg1UK3YY04qifoXU3lOk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs=
github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk=
github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28=
github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo=
github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk=
github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw=
github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360=
github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg=
github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE=
github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc=
github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4=
github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -251,7 +229,6 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -321,12 +298,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI=
github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -336,14 +307,10 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -364,16 +331,12 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ=
github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -395,7 +358,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
@@ -419,7 +381,6 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -447,8 +408,6 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -458,8 +417,6 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
@@ -468,7 +425,6 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
@@ -485,8 +441,9 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -495,24 +452,22 @@ github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/up9inc/basenine/client/go v0.0.0-20211215185650-10083bb9a1b3 h1:FeDCVOBFVpZA5/O5hfPdGTn0rdR2jTEYo3iB2htELI4=
github.com/up9inc/basenine/client/go v0.0.0-20211215185650-10083bb9a1b3/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.mongodb.org/mongo-driver v1.7.1 h1:jwqTeEM3x6L9xDXrCxN0Hbg7vdGfPBOTIkr0+/LYZDA=
go.mongodb.org/mongo-driver v1.7.1/go.mod h1:Q4oFMbo1+MSNqICAdYMlC/zSTrwCogR4R8NzkI+yfU8=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -525,13 +480,11 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
@@ -609,7 +562,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -623,13 +575,10 @@ golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -646,6 +595,7 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -654,7 +604,6 @@ golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
@@ -683,13 +632,9 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
@@ -803,11 +748,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM=
gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw=
gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
gorm.io/gorm v1.21.8 h1:2CEwZSzogdhsKPlJ9OvBKTdlWIpELXb6HbfLfMNhSYI=
gorm.io/gorm v1.21.8/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View File

@@ -1,37 +1,39 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"github.com/up9inc/mizu/shared/kubernetes"
"io/ioutil"
v1 "k8s.io/api/core/v1"
"mizuserver/pkg/api"
"mizuserver/pkg/config"
"mizuserver/pkg/controllers"
"mizuserver/pkg/database"
"mizuserver/pkg/models"
"mizuserver/pkg/providers"
"mizuserver/pkg/routes"
"mizuserver/pkg/up9"
"mizuserver/pkg/utils"
"net/http"
"os"
"os/exec"
"os/signal"
"path"
"path/filepath"
"plugin"
"sort"
"strconv"
"strings"
"syscall"
"time"
v1 "k8s.io/api/core/v1"
"github.com/antelman107/net-wait-go/wait"
"github.com/gin-contrib/static"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
"github.com/op/go-logging"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
"github.com/up9inc/mizu/tap"
@@ -49,10 +51,13 @@ var harsDir = flag.String("hars-dir", "", "Directory to read hars from")
var extensions []*tapApi.Extension // global
var extensionsMap map[string]*tapApi.Extension // global
var startTime int64
const (
socketConnectionRetries = 10
socketConnectionRetries = 30
socketConnectionRetryDelay = time.Second * 2
socketHandshakeTimeout = time.Second * 2
socketHandshakeTimeout = time.Second * 2
uiIndexPath = "./site/index.html"
)
func main() {
@@ -89,17 +94,17 @@ func main() {
panic("API server address must be provided with --api-server-address when using --tap")
}
hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
tapOpts := &tap.TapOpts{HostMode: hostMode}
tapTargets := getTapTargets()
if tapTargets != nil {
tap.SetFilterAuthorities(tapTargets)
logger.Log.Infof("Filtering for the following authorities: %v", tap.GetFilterIPs())
tapOpts.FilterAuthorities = tapTargets
logger.Log.Infof("Filtering for the following authorities: %v", tapOpts.FilterAuthorities)
}
filteredOutputItemsChannel := make(chan *tapApi.OutputChannelItem)
filteringOptions := getTrafficFilteringOptions()
hostMode := os.Getenv(shared.HostModeEnvVar) == "1"
tapOpts := &tap.TapOpts{HostMode: hostMode}
tap.StartPassiveTapper(tapOpts, filteredOutputItemsChannel, extensions, filteringOptions)
socketConnection, err := dialSocketWithRetry(*apiServerAddress, socketConnectionRetries, socketConnectionRetryDelay)
if err != nil {
@@ -109,7 +114,8 @@ func main() {
go pipeTapChannelToSocket(socketConnection, filteredOutputItemsChannel)
} else if *apiServerMode {
database.InitDataBase(config.Config.AgentDatabasePath)
startBasenineServer(shared.BasenineHost, shared.BaseninePort)
startTime = time.Now().UnixNano() / int64(time.Millisecond)
api.StartResolving(*namespace)
outputItemsChannel := make(chan *tapApi.OutputChannelItem)
@@ -142,6 +148,53 @@ func main() {
logger.Log.Info("Exiting")
}
func startBasenineServer(host string, port string) {
cmd := exec.Command("basenine", "-addr", host, "-port", port, "-persistent")
cmd.Dir = config.Config.AgentDatabasePath
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Start()
if err != nil {
logger.Log.Panicf("Failed starting Basenine: %v", err)
}
if !wait.New(
wait.WithProto("tcp"),
wait.WithWait(200*time.Millisecond),
wait.WithBreak(50*time.Millisecond),
wait.WithDeadline(5*time.Second),
wait.WithDebug(true),
).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
logger.Log.Panicf("Basenine is not available: %v", err)
}
// Make a channel to gracefully exit Basenine.
channel := make(chan os.Signal)
signal.Notify(channel, os.Interrupt, syscall.SIGTERM)
// Handle the channel.
go func() {
<-channel
cmd.Process.Signal(syscall.SIGTERM)
}()
// Limit the database size to default 200MB
err = basenine.Limit(host, port, config.Config.MaxDBSizeBytes)
if err != nil {
logger.Log.Panicf("Error while limiting database size: %v", err)
}
for _, extension := range extensions {
macros := extension.Dissector.Macros()
for macro, expanded := range macros {
err = basenine.Macro(host, port, macro, expanded)
if err != nil {
logger.Log.Panicf("Error while adding a macro: %v", err)
}
}
}
}
func loadExtensions() {
dir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
extensionsDir := path.Join(dir, "./extensions/")
@@ -154,7 +207,7 @@ func loadExtensions() {
extensionsMap = make(map[string]*tapApi.Extension)
for i, file := range files {
filename := file.Name()
logger.Log.Infof("Loading extension: %s\n", filename)
logger.Log.Infof("Loading extension: %s", filename)
extension := &tapApi.Extension{
Path: path.Join(extensionsDir, filename),
}
@@ -166,7 +219,7 @@ func loadExtensions() {
var ok bool
dissector, ok = symDissector.(tapApi.Dissector)
if err != nil || !ok {
panic(fmt.Sprintf("Failed to load the extension: %s\n", extension.Path))
panic(fmt.Sprintf("Failed to load the extension: %s", extension.Path))
}
dissector.Register(extension)
extension.Dissector = dissector
@@ -179,7 +232,7 @@ func loadExtensions() {
})
for _, extension := range extensions {
logger.Log.Infof("Extension Properties: %+v\n", extension)
logger.Log.Infof("Extension Properties: %+v", extension)
}
controllers.InitExtensionsMap(extensionsMap)
@@ -197,24 +250,26 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) {
}
app.Use(DisableRootStaticCache())
if err := setUIMode(); err != nil {
logger.Log.Errorf("Error setting ui mode, err: %v", err)
}
app.Use(static.ServeRoot("/", "./site"))
app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before
api.WebSocketRoutes(app, &eventHandlers)
api.WebSocketRoutes(app, &eventHandlers, startTime)
if config.Config.StandaloneMode {
routes.StandaloneRoutes(app)
}
routes.QueryRoutes(app)
routes.EntriesRoutes(app)
routes.MetadataRoutes(app)
routes.StatusRoutes(app)
routes.NotFoundRoute(app)
if config.Config.DaemonMode {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if _, err := startMizuTapperSyncer(ctx); err != nil {
logger.Log.Fatalf("error initializing tapper syncer: %+v", err)
}
}
utils.StartServer(app)
}
@@ -245,8 +300,24 @@ func CORSMiddleware() gin.HandlerFunc {
}
}
func parseEnvVar(env string) map[string][]string {
var mapOfList map[string][]string
func setUIMode() error {
read, err := ioutil.ReadFile(uiIndexPath)
if err != nil {
return err
}
replacedContent := strings.Replace(string(read), "__IS_STANDALONE__", strconv.FormatBool(config.Config.StandaloneMode), 1)
err = ioutil.WriteFile(uiIndexPath, []byte(replacedContent), 0)
if err != nil {
return err
}
return nil
}
func parseEnvVar(env string) map[string][]v1.Pod {
var mapOfList map[string][]v1.Pod
val, present := os.LookupEnv(env)
@@ -256,12 +327,12 @@ func parseEnvVar(env string) map[string][]string {
err := json.Unmarshal([]byte(val), &mapOfList)
if err != nil {
panic(fmt.Sprintf("env var %s's value of %s is invalid! must be map[string][]string %v", env, mapOfList, err))
panic(fmt.Sprintf("env var %s's value of %v is invalid! must be map[string][]v1.Pod %v", env, mapOfList, err))
}
return mapOfList
}
func getTapTargets() []string {
func getTapTargets() []v1.Pod {
nodeName := os.Getenv(shared.NodeNameEnvVar)
tappedAddressesPerNodeDict := parseEnvVar(shared.TappedAddressesPerNodeDictEnvVar)
return tappedAddressesPerNodeDict[nodeName]
@@ -344,10 +415,11 @@ func getSyncEntriesConfig() *shared.SyncEntriesConfig {
}
func determineLogLevel() (logLevel logging.Level) {
logLevel = logging.INFO
if os.Getenv(shared.DebugModeEnvVar) == "1" {
logLevel = logging.DEBUG
logLevel, err := logging.LogLevel(os.Getenv(shared.LogLevelEnvVar))
if err != nil {
logLevel = logging.INFO
}
return
}
@@ -361,69 +433,42 @@ func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time.
socketConnection, _, err := dialer.Dial(socketAddress, nil)
if err != nil {
if i < retryAmount {
logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay / time.Second)
logger.Log.Infof("socket connection to %s failed: %v, retrying %d out of %d in %d seconds...", socketAddress, err, i, retryAmount, retryDelay/time.Second)
time.Sleep(retryDelay)
}
} else {
go handleIncomingMessageAsTapper(socketConnection)
return socketConnection, nil
}
}
return nil, lastErr
}
func startMizuTapperSyncer(ctx context.Context) (*kubernetes.MizuTapperSyncer, error){
provider, err := kubernetes.NewProviderInCluster()
if err != nil {
return nil, err
}
tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
TargetNamespaces: config.Config.TargetNamespaces,
PodFilterRegex: config.Config.TapTargetRegex.Regexp,
MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
AgentImage: config.Config.AgentImage,
TapperResources: config.Config.TapperResources,
ImagePullPolicy: v1.PullPolicy(config.Config.PullPolicy),
DumpLogs: config.Config.DumpLogs,
IgnoredUserAgents: config.Config.IgnoredUserAgents,
MizuApiFilteringOptions: config.Config.MizuApiFilteringOptions,
MizuServiceAccountExists: true, //assume service account exists since daemon mode will not function without it anyway
})
if err != nil {
return nil, err
}
// handle tapperSyncer events (pod changes and errors)
go func() {
for {
select {
case syncerErr, ok := <-tapperSyncer.ErrorOut:
if !ok {
logger.Log.Debug("mizuTapperSyncer err channel closed, ending listener loop")
return
}
logger.Log.Fatalf("fatal tap syncer error: %v", syncerErr)
case _, ok := <-tapperSyncer.TapPodChangesOut:
if !ok {
logger.Log.Debug("mizuTapperSyncer pod changes channel closed, ending listener loop")
return
}
tapStatus := shared.TapStatus{Pods: kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods)}
serializedTapStatus, err := json.Marshal(shared.CreateWebSocketStatusMessage(tapStatus))
if err != nil {
logger.Log.Fatalf("error serializing tap status: %v", err)
}
api.BroadcastToBrowserClients(serializedTapStatus)
providers.TapStatus.Pods = tapStatus.Pods
case <-ctx.Done():
logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")
func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) {
for {
if _, message, err := socketConnection.ReadMessage(); err != nil {
logger.Log.Errorf("error reading message from socket connection, err: %s, (%v,%+v)", err, err, err)
if errors.Is(err, syscall.EPIPE) {
// socket has disconnected, we can safely stop this goroutine
return
}
} else {
var socketMessageBase shared.WebSocketMessageMetadata
if err := json.Unmarshal(message, &socketMessageBase); err != nil {
logger.Log.Errorf("Could not unmarshal websocket message %v", err)
} else {
switch socketMessageBase.MessageType {
case shared.WebSocketMessageTypeTapConfig:
var tapConfigMessage *shared.WebSocketTapConfigMessage
if err := json.Unmarshal(message, &tapConfigMessage); err != nil {
logger.Log.Errorf("received unknown message from socket connection: %s, err: %s, (%v,%+v)", string(message), err, err, err)
} else {
tap.UpdateTapTargets(tapConfigMessage.TapTargets)
}
default:
logger.Log.Warningf("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType)
}
}
}
}()
return tapperSyncer, nil
}
}

View File

@@ -13,7 +13,6 @@ import (
legacyrouter "github.com/getkin/kin-openapi/routers/legacy"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
"github.com/up9inc/mizu/tap/api"
)
@@ -27,7 +26,6 @@ func loadOAS(ctx context.Context) (doc *openapi3.T, contractContent string, rout
path := fmt.Sprintf("%s%s", shared.ConfigDirPath, shared.ContractFileName)
bytes, err := ioutil.ReadFile(path)
if err != nil {
logger.Log.Error(err.Error())
return
}
contractContent = string(bytes)
@@ -35,7 +33,6 @@ func loadOAS(ctx context.Context) (doc *openapi3.T, contractContent string, rout
doc, _ = loader.LoadFromData(bytes)
err = doc.Validate(ctx)
if err != nil {
logger.Log.Error(err.Error())
return
}
router, _ = legacyrouter.NewRouter(doc)

View File

@@ -5,7 +5,6 @@ import (
"context"
"encoding/json"
"fmt"
"mizuserver/pkg/database"
"mizuserver/pkg/holder"
"mizuserver/pkg/providers"
"os"
@@ -14,15 +13,16 @@ import (
"strings"
"time"
"go.mongodb.org/mongo-driver/bson/primitive"
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
tapApi "github.com/up9inc/mizu/tap/api"
"mizuserver/pkg/models"
"mizuserver/pkg/resolver"
"mizuserver/pkg/utils"
basenine "github.com/up9inc/basenine/client/go"
)
var k8sResolver *resolver.Resolver
@@ -76,7 +76,7 @@ func startReadingFiles(workingDir string) {
sort.Sort(utils.ByModTime(harFiles))
if len(harFiles) == 0 {
logger.Log.Infof("Waiting for new files\n")
logger.Log.Infof("Waiting for new files")
time.Sleep(3 * time.Second)
continue
}
@@ -99,11 +99,17 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
panic("Channel of captured messages is nil")
}
connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
if err != nil {
panic(err)
}
connection.InsertMode()
disableOASValidation := false
ctx := context.Background()
doc, contractContent, router, err := loadOAS(ctx)
if err != nil {
logger.Log.Infof("Disabled OAS validation: %s\n", err.Error())
logger.Log.Infof("Disabled OAS validation: %s", err.Error())
disableOASValidation = true
}
@@ -112,13 +118,13 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
extension := extensionsMap[item.Protocol.Name]
resolvedSource, resolvedDestionation := resolveIP(item.ConnectionInfo)
mizuEntry := extension.Dissector.Analyze(item, primitive.NewObjectID().Hex(), resolvedSource, resolvedDestionation)
mizuEntry := extension.Dissector.Analyze(item, resolvedSource, resolvedDestionation)
baseEntry := extension.Dissector.Summarize(mizuEntry)
mizuEntry.EstimatedSizeBytes = getEstimatedEntrySizeBytes(mizuEntry)
mizuEntry.Base = baseEntry
if extension.Protocol.Name == "http" {
if !disableOASValidation {
var httpPair tapApi.HTTPRequestResponsePair
json.Unmarshal([]byte(mizuEntry.Entry), &httpPair)
json.Unmarshal([]byte(mizuEntry.HTTPPair), &httpPair)
contract := handleOAS(ctx, doc, router, httpPair.Request.Payload.RawRequest, httpPair.Response.Payload.RawResponse, contractContent)
baseEntry.ContractStatus = contract.Status
@@ -128,18 +134,18 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
mizuEntry.ContractContent = contract.Content
}
var pair tapApi.RequestResponsePair
json.Unmarshal([]byte(mizuEntry.Entry), &pair)
harEntry, err := utils.NewEntry(&pair)
harEntry, err := utils.NewEntry(mizuEntry.Request, mizuEntry.Response, mizuEntry.StartTime, mizuEntry.ElapsedTime)
if err == nil {
rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Service)
rules, _, _ := models.RunValidationRulesState(*harEntry, mizuEntry.Destination.Name)
baseEntry.Rules = rules
}
}
database.CreateEntry(mizuEntry)
baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(baseEntry)
BroadcastToBrowserClients(baseEntryBytes)
data, err := json.Marshal(mizuEntry)
if err != nil {
panic(err)
}
connection.SendText(string(data))
}
}
@@ -148,7 +154,7 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re
unresolvedSource := connectionInfo.ClientIP
resolvedSource = k8sResolver.Resolve(unresolvedSource)
if resolvedSource == "" {
logger.Log.Debugf("Cannot find resolved name to source: %s\n", unresolvedSource)
logger.Log.Debugf("Cannot find resolved name to source: %s", unresolvedSource)
if os.Getenv("SKIP_NOT_RESOLVED_SOURCE") == "1" {
return
}
@@ -156,7 +162,7 @@ func resolveIP(connectionInfo *tapApi.ConnectionInfo) (resolvedSource string, re
unresolvedDestination := fmt.Sprintf("%s:%s", connectionInfo.ServerIP, connectionInfo.ServerPort)
resolvedDestination = k8sResolver.Resolve(unresolvedDestination)
if resolvedDestination == "" {
logger.Log.Debugf("Cannot find resolved name to dest: %s\n", unresolvedDestination)
logger.Log.Debugf("Cannot find resolved name to dest: %s", unresolvedDestination)
if os.Getenv("SKIP_NOT_RESOLVED_DEST") == "1" {
return
}
@@ -171,21 +177,3 @@ func CheckIsServiceIP(address string) bool {
}
return k8sResolver.CheckIsServiceIP(address)
}
// gives a rough estimate of the size this will take up in the db, good enough for maintaining db size limit accurately
func getEstimatedEntrySizeBytes(mizuEntry *tapApi.MizuEntry) int {
sizeBytes := len(mizuEntry.Entry)
sizeBytes += len(mizuEntry.EntryId)
sizeBytes += len(mizuEntry.Service)
sizeBytes += len(mizuEntry.Url)
sizeBytes += len(mizuEntry.Method)
sizeBytes += len(mizuEntry.RequestSenderIp)
sizeBytes += len(mizuEntry.ResolvedDestination)
sizeBytes += len(mizuEntry.ResolvedSource)
sizeBytes += 8 // Status bytes (sqlite integer is always 8 bytes)
sizeBytes += 8 // Timestamp bytes
sizeBytes += 8 // SizeBytes bytes
sizeBytes += 1 // IsOutgoing bytes
return sizeBytes
}

View File

@@ -1,13 +1,18 @@
package api
import (
"encoding/json"
"errors"
"fmt"
"mizuserver/pkg/models"
"net/http"
"sync"
"time"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/debounce"
"github.com/up9inc/mizu/shared/logger"
)
@@ -39,17 +44,17 @@ func init() {
connectedWebsockets = make(map[int]*SocketConnection, 0)
}
func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers) {
func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) {
app.GET("/ws", func(c *gin.Context) {
websocketHandler(c.Writer, c.Request, eventHandlers, false)
websocketHandler(c.Writer, c.Request, eventHandlers, false, startTime)
})
app.GET("/wsTapper", func(c *gin.Context) {
websocketHandler(c.Writer, c.Request, eventHandlers, true)
websocketHandler(c.Writer, c.Request, eventHandlers, true, startTime)
})
}
func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool) {
conn, err := websocketUpgrader.Upgrade(w, r, nil)
func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers EventHandlers, isTapper bool, startTime int64) {
ws, err := websocketUpgrader.Upgrade(w, r, nil)
if err != nil {
logger.Log.Errorf("Failed to set websocket upgrade: %v", err)
return
@@ -59,30 +64,122 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
connectedWebsocketIdCounter++
socketId := connectedWebsocketIdCounter
connectedWebsockets[socketId] = &SocketConnection{connection: conn, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper}
connectedWebsockets[socketId] = &SocketConnection{connection: ws, lock: &sync.Mutex{}, eventHandlers: eventHandlers, isTapper: isTapper}
websocketIdsLock.Unlock()
var connection *basenine.Connection
var isQuerySet bool
// `!isTapper` means it's a connection from the web UI
if !isTapper {
connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
if err != nil {
panic(err)
}
}
data := make(chan []byte)
meta := make(chan []byte)
defer func() {
socketCleanup(socketId, connectedWebsockets[socketId])
data <- []byte(basenine.CloseChannel)
meta <- []byte(basenine.CloseChannel)
connection.Close()
}()
eventHandlers.WebSocketConnect(socketId, isTapper)
startTimeBytes, _ := models.CreateWebsocketStartTimeMessage(startTime)
SendToSocket(socketId, startTimeBytes)
for {
_, msg, err := conn.ReadMessage()
_, msg, err := ws.ReadMessage()
if err != nil {
logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
if _, ok := err.(*websocket.CloseError); ok {
logger.Log.Debugf("Received websocket close message, socket id: %d", socketId)
} else {
logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
}
break
}
eventHandlers.WebSocketMessage(socketId, msg)
if !isTapper && !isQuerySet {
query := string(msg)
err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
if err != nil {
toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
Type: "error",
AutoClose: 5000,
Text: fmt.Sprintf("Syntax error: %s", err.Error()),
})
SendToSocket(socketId, toastBytes)
break
}
isQuerySet = true
handleDataChannel := func(c *basenine.Connection, data chan []byte) {
for {
bytes := <-data
if string(bytes) == basenine.CloseChannel {
return
}
var dataMap map[string]interface{}
err = json.Unmarshal(bytes, &dataMap)
var base map[string]interface{}
switch dataMap["base"].(type) {
case map[string]interface{}:
base = dataMap["base"].(map[string]interface{})
base["id"] = uint(dataMap["id"].(float64))
default:
logger.Log.Debugf("Base field has an unrecognized type: %+v", dataMap)
continue
}
baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base)
SendToSocket(socketId, baseEntryBytes)
}
}
handleMetaChannel := func(c *basenine.Connection, meta chan []byte) {
for {
bytes := <-meta
if string(bytes) == basenine.CloseChannel {
return
}
var metadata *basenine.Metadata
err = json.Unmarshal(bytes, &metadata)
if err != nil {
logger.Log.Debugf("Error recieving metadata: %v", err.Error())
}
metadataBytes, _ := models.CreateWebsocketQueryMetadataMessage(metadata)
SendToSocket(socketId, metadataBytes)
}
}
go handleDataChannel(connection, data)
go handleMetaChannel(connection, meta)
connection.Query(query, data, meta)
} else {
eventHandlers.WebSocketMessage(socketId, msg)
}
}
}
func socketCleanup(socketId int, socketConnection *SocketConnection) {
err := socketConnection.connection.Close()
if err != nil {
logger.Log.Errorf("Error closing socket connection for socket id %d: %v\n", socketId, err)
logger.Log.Errorf("Error closing socket connection for socket id %d: %v", socketId, err)
}
websocketIdsLock.Lock()

View File

@@ -65,14 +65,14 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
var socketMessageBase shared.WebSocketMessageMetadata
err := json.Unmarshal(message, &socketMessageBase)
if err != nil {
logger.Log.Infof("Could not unmarshal websocket message %v\n", err)
logger.Log.Infof("Could not unmarshal websocket message %v", err)
} else {
switch socketMessageBase.MessageType {
case shared.WebSocketMessageTypeTappedEntry:
var tappedEntryMessage models.WebSocketTappedEntryMessage
err := json.Unmarshal(message, &tappedEntryMessage)
if err != nil {
logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
} else {
// NOTE: This is where the message comes back from the intermediate WebSocket to code.
h.SocketOutChannel <- tappedEntryMessage.Data
@@ -81,16 +81,15 @@ func (h *RoutesEventHandlers) WebSocketMessage(_ int, message []byte) {
var statusMessage shared.WebSocketStatusMessage
err := json.Unmarshal(message, &statusMessage)
if err != nil {
logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
} else {
providers.TapStatus.Pods = statusMessage.TappingStatus.Pods
BroadcastToBrowserClients(message)
}
case shared.WebsocketMessageTypeOutboundLink:
var outboundLinkMessage models.WebsocketOutboundLinkMessage
err := json.Unmarshal(message, &outboundLinkMessage)
if err != nil {
logger.Log.Infof("Could not unmarshal message of message type %s %v\n", socketMessageBase.MessageType, err)
logger.Log.Infof("Could not unmarshal message of message type %s %v", socketMessageBase.MessageType, err)
} else {
handleTLSLink(outboundLinkMessage)
}

View File

@@ -4,7 +4,6 @@ import (
"encoding/json"
"fmt"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api"
"io/ioutil"
"os"
)
@@ -12,7 +11,6 @@ import (
// these values are used when the config.json file is not present
const (
defaultMaxDatabaseSizeBytes int64 = 200 * 1000 * 1000
defaultRegexTarget string = ".*"
DefaultDatabasePath string = "./entries"
)
@@ -48,14 +46,8 @@ func applyDefaultConfig() error {
}
func getDefaultConfig() (*shared.MizuAgentConfig, error) {
regex, err := api.CompileRegexToSerializableRegexp(defaultRegexTarget)
if err != nil {
return nil, err
}
return &shared.MizuAgentConfig{
TapTargetRegex: *regex,
MaxDBSizeBytes: defaultMaxDatabaseSizeBytes,
AgentDatabasePath: DefaultDatabasePath,
DaemonMode: false,
}, nil
}

View File

@@ -2,14 +2,19 @@ package controllers
import (
"encoding/json"
"fmt"
"github.com/gin-gonic/gin"
tapApi "github.com/up9inc/mizu/tap/api"
"mizuserver/pkg/database"
"mizuserver/pkg/models"
"mizuserver/pkg/utils"
"mizuserver/pkg/validation"
"net/http"
"strconv"
"time"
"github.com/gin-gonic/gin"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
tapApi "github.com/up9inc/mizu/tap/api"
)
var extensionsMap map[string]*tapApi.Extension // global
@@ -18,78 +23,113 @@ func InitExtensionsMap(ref map[string]*tapApi.Extension) {
extensionsMap = ref
}
func GetEntries(c *gin.Context) {
entriesFilter := &models.EntriesFilter{}
if err := c.BindQuery(entriesFilter); err != nil {
c.JSON(http.StatusBadRequest, err)
}
err := validation.Validate(entriesFilter)
func Error(c *gin.Context, err error) bool {
if err != nil {
logger.Log.Errorf("Error getting entry: %v", err)
c.Error(err)
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
"error": true,
"type": "error",
"autoClose": "5000",
"msg": err.Error(),
})
return true // signal that there was an error and the caller should return
}
return false // no error, can continue
}
func GetEntries(c *gin.Context) {
entriesRequest := &models.EntriesRequest{}
if err := c.BindQuery(entriesRequest); err != nil {
c.JSON(http.StatusBadRequest, err)
}
order := database.OperatorToOrderMapping[entriesFilter.Operator]
operatorSymbol := database.OperatorToSymbolMapping[entriesFilter.Operator]
var entries []tapApi.MizuEntry
database.GetEntriesTable().
Order(fmt.Sprintf("timestamp %s", order)).
Where(fmt.Sprintf("timestamp %s %v", operatorSymbol, entriesFilter.Timestamp)).
Limit(entriesFilter.Limit).
Find(&entries)
if len(entries) > 0 && order == database.OrderDesc {
// the entries always order from oldest to newest - we should reverse
utils.ReverseSlice(entries)
validationError := validation.Validate(entriesRequest)
if validationError != nil {
c.JSON(http.StatusBadRequest, validationError)
}
baseEntries := make([]tapApi.BaseEntryDetails, 0)
for _, entry := range entries {
baseEntryDetails := tapApi.BaseEntryDetails{}
if err := models.GetEntry(&entry, &baseEntryDetails); err != nil {
continue
}
var pair tapApi.RequestResponsePair
json.Unmarshal([]byte(entry.Entry), &pair)
harEntry, err := utils.NewEntry(&pair)
if err == nil {
rules, _, _ := models.RunValidationRulesState(*harEntry, entry.Service)
baseEntryDetails.Rules = rules
}
baseEntries = append(baseEntries, baseEntryDetails)
if entriesRequest.TimeoutMs == 0 {
entriesRequest.TimeoutMs = 3000
}
c.JSON(http.StatusOK, baseEntries)
data, meta, err := basenine.Fetch(shared.BasenineHost, shared.BaseninePort,
entriesRequest.LeftOff, entriesRequest.Direction, entriesRequest.Query,
entriesRequest.Limit, time.Duration(entriesRequest.TimeoutMs)*time.Millisecond)
if err != nil {
c.JSON(http.StatusInternalServerError, validationError)
}
response := &models.EntriesResponse{}
var dataSlice []interface{}
for _, row := range data {
var dataMap map[string]interface{}
err = json.Unmarshal(row, &dataMap)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{
"error": true,
"type": "error",
"autoClose": "5000",
"msg": string(row),
})
return // exit
}
base := dataMap["base"].(map[string]interface{})
base["id"] = uint(dataMap["id"].(float64))
dataSlice = append(dataSlice, base)
}
var metadata *basenine.Metadata
err = json.Unmarshal(meta, &metadata)
if err != nil {
logger.Log.Debugf("Error recieving metadata: %v", err.Error())
}
response.Data = dataSlice
response.Meta = metadata
c.JSON(http.StatusOK, response)
}
func GetEntry(c *gin.Context) {
var entryData tapApi.MizuEntry
database.GetEntriesTable().
Where(map[string]string{"entryId": c.Param("entryId")}).
First(&entryData)
id, _ := strconv.Atoi(c.Param("id"))
var entry tapApi.MizuEntry
bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, id)
if Error(c, err) {
return // exit
}
err = json.Unmarshal(bytes, &entry)
if err != nil {
c.JSON(http.StatusNotFound, gin.H{
"error": true,
"type": "error",
"autoClose": "5000",
"msg": string(bytes),
})
return // exit
}
extension := extensionsMap[entryData.ProtocolName]
protocol, representation, bodySize, _ := extension.Dissector.Represent(&entryData)
extension := extensionsMap[entry.Protocol.Name]
representation, bodySize, _ := extension.Dissector.Represent(entry.Request, entry.Response)
var rules []map[string]interface{}
var isRulesEnabled bool
if entryData.ProtocolName == "http" {
var pair tapApi.RequestResponsePair
json.Unmarshal([]byte(entryData.Entry), &pair)
harEntry, _ := utils.NewEntry(&pair)
_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entryData.Service)
if entry.Protocol.Name == "http" {
harEntry, _ := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
_, rulesMatched, _isRulesEnabled := models.RunValidationRulesState(*harEntry, entry.Destination.Name)
isRulesEnabled = _isRulesEnabled
inrec, _ := json.Marshal(rulesMatched)
json.Unmarshal(inrec, &rules)
}
c.JSON(http.StatusOK, tapApi.MizuEntryWrapper{
Protocol: protocol,
Protocol: entry.Protocol,
Representation: string(representation),
BodySize: bodySize,
Data: entryData,
Data: entry,
Rules: rules,
IsRulesEnabled: isRulesEnabled,
})

View File

@@ -0,0 +1,31 @@
package controllers
import (
"net/http"
"github.com/gin-gonic/gin"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/shared"
)
type ValidateResponse struct {
Valid bool `json:"valid"`
Message string `json:"message"`
}
func PostValidate(c *gin.Context) {
query := c.PostForm("query")
valid := true
message := ""
err := basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
if err != nil {
valid = false
message = err.Error()
}
c.JSON(http.StatusOK, ValidateResponse{
Valid: valid,
Message: message,
})
}

View File

@@ -0,0 +1,133 @@
package controllers
import (
"context"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/kubernetes"
"github.com/up9inc/mizu/shared/logger"
tapApi "github.com/up9inc/mizu/tap/api"
v1 "k8s.io/api/core/v1"
"mizuserver/pkg/config"
"mizuserver/pkg/models"
"mizuserver/pkg/providers"
"net/http"
"regexp"
"time"
)
var globalTapConfig *models.StandaloneTapConfig
var cancelTapperSyncer context.CancelFunc
var kubernetesProvider *kubernetes.Provider
func PostTapConfig(c *gin.Context) {
tapConfig := &models.StandaloneTapConfig{}
if err := c.Bind(tapConfig); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
if cancelTapperSyncer != nil {
cancelTapperSyncer()
providers.TapStatus = shared.TapStatus{}
providers.TappersStatus = make(map[string]shared.TapperStatus)
broadcastTappedPodsStatus()
}
if kubernetesProvider == nil {
var err error
kubernetesProvider, err = kubernetes.NewProviderInCluster()
if err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
}
ctx, cancel := context.WithCancel(context.Background())
var tappedNamespaces []string
for namespace, tapped := range tapConfig.TappedNamespaces {
if tapped {
tappedNamespaces = append(tappedNamespaces, namespace)
}
}
podRegex, _ := regexp.Compile(".*")
if _, err := startMizuTapperSyncer(ctx, kubernetesProvider, tappedNamespaces, *podRegex, []string{} , tapApi.TrafficFilteringOptions{}, false); err != nil {
c.JSON(http.StatusBadRequest, err)
cancel()
return
}
cancelTapperSyncer = cancel
globalTapConfig = tapConfig
c.JSON(http.StatusOK, "OK")
}
func GetTapConfig(c *gin.Context) {
if globalTapConfig != nil {
c.JSON(http.StatusOK, globalTapConfig)
}
c.JSON(http.StatusBadRequest, "Not config found")
}
func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, istio bool) (*kubernetes.MizuTapperSyncer, error) {
tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
TargetNamespaces: targetNamespaces,
PodFilterRegex: podFilterRegex,
MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
AgentImage: config.Config.AgentImage,
TapperResources: config.Config.TapperResources,
ImagePullPolicy: v1.PullPolicy(config.Config.PullPolicy),
LogLevel: config.Config.LogLevel,
IgnoredUserAgents: ignoredUserAgents,
MizuApiFilteringOptions: mizuApiFilteringOptions,
MizuServiceAccountExists: true, //assume service account exists since install mode will not function without it anyway
Istio: istio,
}, time.Now())
if err != nil {
return nil, err
}
// handle tapperSyncer events (pod changes and errors)
go func() {
for {
select {
case syncerErr, ok := <-tapperSyncer.ErrorOut:
if !ok {
logger.Log.Debug("mizuTapperSyncer err channel closed, ending listener loop")
return
}
logger.Log.Fatalf("fatal tap syncer error: %v", syncerErr)
case _, ok := <-tapperSyncer.TapPodChangesOut:
if !ok {
logger.Log.Debug("mizuTapperSyncer pod changes channel closed, ending listener loop")
return
}
providers.TapStatus = shared.TapStatus{Pods: kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods)}
broadcastTappedPodsStatus()
case tapperStatus, ok := <-tapperSyncer.TapperStatusChangedOut:
if !ok {
logger.Log.Debug("mizuTapperSyncer tapper status changed channel closed, ending listener loop")
return
}
addTapperStatus(tapperStatus)
broadcastTappedPodsStatus()
case <-ctx.Done():
logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")
return
}
}
}()
return tapperSyncer, nil
}

View File

@@ -2,27 +2,32 @@ package controllers
import (
"encoding/json"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
"mizuserver/pkg/api"
"mizuserver/pkg/holder"
"mizuserver/pkg/providers"
"mizuserver/pkg/up9"
"mizuserver/pkg/utils"
"mizuserver/pkg/validation"
"net/http"
"github.com/gin-gonic/gin"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
)
func HealthCheck(c *gin.Context) {
tappers := make([]shared.TapperStatus, 0)
for _, value := range providers.TappersStatus {
tappers = append(tappers, value)
}
response := shared.HealthResponse{
TapStatus: providers.TapStatus,
TappersCount: providers.TappersCount,
TapStatus: providers.TapStatus,
TappersCount: providers.TappersCount,
TappersStatus: tappers,
}
c.JSON(http.StatusOK, response)
}
func PostTappedPods(c *gin.Context) {
tapStatus := &shared.TapStatus{}
if err := c.Bind(tapStatus); err != nil {
@@ -35,14 +40,45 @@ func PostTappedPods(c *gin.Context) {
}
logger.Log.Infof("[Status] POST request: %d tapped pods", len(tapStatus.Pods))
providers.TapStatus.Pods = tapStatus.Pods
message := shared.CreateWebSocketStatusMessage(*tapStatus)
broadcastTappedPodsStatus()
}
func broadcastTappedPodsStatus() {
tappedPodsStatus := utils.GetTappedPodsStatus()
message := shared.CreateWebSocketStatusMessage(tappedPodsStatus)
if jsonBytes, err := json.Marshal(message); err != nil {
logger.Log.Errorf("Could not Marshal message %v\n", err)
logger.Log.Errorf("Could not Marshal message %v", err)
} else {
api.BroadcastToBrowserClients(jsonBytes)
}
}
func addTapperStatus(tapperStatus shared.TapperStatus) {
if providers.TappersStatus == nil {
providers.TappersStatus = make(map[string]shared.TapperStatus)
}
providers.TappersStatus[tapperStatus.NodeName] = tapperStatus
}
func PostTapperStatus(c *gin.Context) {
tapperStatus := &shared.TapperStatus{}
if err := c.Bind(tapperStatus); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
if err := validation.Validate(tapperStatus); err != nil {
c.JSON(http.StatusBadRequest, err)
return
}
logger.Log.Infof("[Status] POST request, tapper status: %v", tapperStatus)
addTapperStatus(*tapperStatus)
broadcastTappedPodsStatus()
}
func GetTappersCount(c *gin.Context) {
c.JSON(http.StatusOK, providers.TappersCount)
}
@@ -58,7 +94,8 @@ func GetAuthStatus(c *gin.Context) {
}
func GetTappingStatus(c *gin.Context) {
c.JSON(http.StatusOK, providers.TapStatus)
tappedPodsStatus := utils.GetTappedPodsStatus()
c.JSON(http.StatusOK, tappedPodsStatus)
}
func AnalyzeInformation(c *gin.Context) {

View File

@@ -1,78 +0,0 @@
package database
import (
"fmt"
"mizuserver/pkg/utils"
"time"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
tapApi "github.com/up9inc/mizu/tap/api"
)
const (
OrderDesc = "desc"
OrderAsc = "asc"
LT = "lt"
GT = "gt"
TimeFormat = "2006-01-02 15:04:05.000000000"
)
var (
DB *gorm.DB
IsDBLocked = false
OperatorToSymbolMapping = map[string]string{
LT: "<",
GT: ">",
}
OperatorToOrderMapping = map[string]string{
LT: OrderDesc,
GT: OrderAsc,
}
)
var DBPath string
func GetEntriesTable() *gorm.DB {
return DB.Table("mizu_entries")
}
func CreateEntry(entry *tapApi.MizuEntry) {
if IsDBLocked {
return
}
GetEntriesTable().Create(entry)
}
func InitDataBase(databasePath string) *gorm.DB {
DBPath = databasePath
DB, _ = gorm.Open(sqlite.Open(databasePath), &gorm.Config{
Logger: &utils.TruncatingLogger{LogLevel: logger.Warn, SlowThreshold: 500 * time.Millisecond},
})
_ = DB.AutoMigrate(&tapApi.MizuEntry{}) // this will ensure table is created
go StartEnforcingDatabaseSize()
return DB
}
func GetEntriesFromDb(timeFrom time.Time, timeTo time.Time, protocolName *string) []tapApi.MizuEntry {
order := OrderDesc
protocolNameCondition := "1 = 1"
if protocolName != nil {
protocolNameCondition = fmt.Sprintf("protocolName = '%s'", *protocolName)
}
var entries []tapApi.MizuEntry
GetEntriesTable().
Where(protocolNameCondition).
Where(fmt.Sprintf("created_at BETWEEN '%s' AND '%s'", timeFrom.Format(TimeFormat), timeTo.Format(TimeFormat))).
Order(fmt.Sprintf("timestamp %s", order)).
Find(&entries)
if len(entries) > 0 {
// the entries always order from oldest to newest so we should revers
utils.ReverseSlice(entries)
}
return entries
}

View File

@@ -1,102 +0,0 @@
package database
import (
"mizuserver/pkg/config"
"os"
"time"
"github.com/fsnotify/fsnotify"
"github.com/up9inc/mizu/shared/debounce"
"github.com/up9inc/mizu/shared/logger"
"github.com/up9inc/mizu/shared/units"
tapApi "github.com/up9inc/mizu/tap/api"
)
const percentageOfMaxSizeBytesToPrune = 15
func StartEnforcingDatabaseSize() {
watcher, err := fsnotify.NewWatcher()
if err != nil {
logger.Log.Fatalf("Error creating filesystem watcher for db size enforcement: %v\n", err)
return
}
checkFileSizeDebouncer := debounce.NewDebouncer(5*time.Second, func() {
checkFileSize(config.Config.MaxDBSizeBytes)
})
go func() {
for {
select {
case event, ok := <-watcher.Events:
if !ok {
return // closed channel
}
if event.Op == fsnotify.Write {
checkFileSizeDebouncer.SetOn()
}
case err, ok := <-watcher.Errors:
if !ok {
return // closed channel
}
logger.Log.Errorf("filesystem watcher encountered error:%v", err)
}
}
}()
err = watcher.Add(DBPath)
if err != nil {
logger.Log.Fatalf("Error adding %s to filesystem watcher for db size enforcement: %v\n", DBPath, err)
}
}
func checkFileSize(maxSizeBytes int64) {
fileStat, err := os.Stat(DBPath)
if err != nil {
logger.Log.Errorf("Error checking %s file size: %v", DBPath, err)
} else {
if fileStat.Size() > maxSizeBytes {
pruneOldEntries(fileStat.Size())
}
}
}
func pruneOldEntries(currentFileSize int64) {
// sqlite locks the database while delete or VACUUM are running and sqlite is terrible at handling its own db lock while a lot of inserts are attempted, we prevent a significant bottleneck by handling the db lock ourselves here
IsDBLocked = true
defer func() { IsDBLocked = false }()
amountOfBytesToTrim := currentFileSize / (100 / percentageOfMaxSizeBytesToPrune)
rows, err := GetEntriesTable().Limit(10000).Order("id").Rows()
if err != nil {
logger.Log.Errorf("Error getting 10000 first db rows: %v", err)
return
}
entryIdsToRemove := make([]uint, 0)
bytesToBeRemoved := int64(0)
for rows.Next() {
if bytesToBeRemoved >= amountOfBytesToTrim {
break
}
var entry tapApi.MizuEntry
err = DB.ScanRows(rows, &entry)
if err != nil {
logger.Log.Errorf("Error scanning db row: %v", err)
continue
}
entryIdsToRemove = append(entryIdsToRemove, entry.ID)
bytesToBeRemoved += int64(entry.EstimatedSizeBytes)
}
if len(entryIdsToRemove) > 0 {
GetEntriesTable().Where(entryIdsToRemove).Delete(tapApi.MizuEntry{})
// VACUUM causes sqlite to shrink the db file after rows have been deleted, the db file will not shrink without this
DB.Exec("VACUUM")
logger.Log.Errorf("Removed %d rows and cleared %s", len(entryIdsToRemove), units.BytesToHumanReadable(bytesToBeRemoved))
} else {
logger.Log.Error("Found no rows to remove when pruning")
}
}

View File

@@ -2,12 +2,12 @@ package models
import (
"encoding/json"
"mizuserver/pkg/rules"
tapApi "github.com/up9inc/mizu/tap/api"
"mizuserver/pkg/rules"
"github.com/google/martian/har"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap"
)
@@ -16,15 +16,26 @@ func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error {
return v.UnmarshalData(r)
}
type EntriesFilter struct {
Limit int `form:"limit" validate:"required,min=1,max=200"`
Operator string `form:"operator" validate:"required,oneof='lt' 'gt'"`
Timestamp int64 `form:"timestamp" validate:"required,min=1"`
type StandaloneTapConfig struct {
TappedNamespaces map[string]bool `json:"tappedNamespaces"`
}
type EntriesRequest struct {
LeftOff int `form:"leftOff" validate:"required,min=-1"`
Direction int `form:"direction" validate:"required,oneof='1' '-1'"`
Query string `form:"query"`
Limit int `form:"limit" validate:"required,min=1"`
TimeoutMs int `form:"timeoutMs" validate:"min=1"`
}
type EntriesResponse struct {
Data []interface{} `json:"data"`
Meta *basenine.Metadata `json:"meta"`
}
type WebSocketEntryMessage struct {
*shared.WebSocketMessageMetadata
Data *tapApi.BaseEntryDetails `json:"data,omitempty"`
Data map[string]interface{} `json:"data,omitempty"`
}
type WebSocketTappedEntryMessage struct {
@@ -42,7 +53,28 @@ type AuthStatus struct {
Model string `json:"model"`
}
func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntryDetails) ([]byte, error) {
type ToastMessage struct {
Type string `json:"type"`
AutoClose uint `json:"autoClose"`
Text string `json:"text"`
}
type WebSocketToastMessage struct {
*shared.WebSocketMessageMetadata
Data *ToastMessage `json:"data,omitempty"`
}
type WebSocketQueryMetadataMessage struct {
*shared.WebSocketMessageMetadata
Data *basenine.Metadata `json:"data,omitempty"`
}
type WebSocketStartTimeMessage struct {
*shared.WebSocketMessageMetadata
Data int64 `json:"data"`
}
func CreateBaseEntryWebSocketMessage(base map[string]interface{}) ([]byte, error) {
message := &WebSocketEntryMessage{
WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
MessageType: shared.WebSocketMessageTypeEntry,
@@ -72,6 +104,36 @@ func CreateWebsocketOutboundLinkMessage(base *tap.OutboundLink) ([]byte, error)
return json.Marshal(message)
}
func CreateWebsocketToastMessage(base *ToastMessage) ([]byte, error) {
message := &WebSocketToastMessage{
WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
MessageType: shared.WebSocketMessageTypeToast,
},
Data: base,
}
return json.Marshal(message)
}
func CreateWebsocketQueryMetadataMessage(base *basenine.Metadata) ([]byte, error) {
message := &WebSocketQueryMetadataMessage{
WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
MessageType: shared.WebSocketMessageTypeQueryMetadata,
},
Data: base,
}
return json.Marshal(message)
}
func CreateWebsocketStartTimeMessage(base int64) ([]byte, error) {
message := &WebSocketStartTimeMessage{
WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
MessageType: shared.WebSocketMessageTypeStartTime,
},
Data: base,
}
return json.Marshal(message)
}
// ExtendedHAR is the top level object of a HAR log.
type ExtendedHAR struct {
Log *ExtendedLog `json:"log"`

View File

@@ -2,16 +2,10 @@ package providers_test
import (
"fmt"
"mizuserver/pkg/config"
"mizuserver/pkg/database"
"mizuserver/pkg/providers"
"testing"
)
func init() {
database.InitDataBase(config.DefaultDatabasePath)
}
func TestNoEntryAddedCount(t *testing.T) {
entriesStats := providers.GetGeneralStats()

View File

@@ -15,12 +15,12 @@ import (
const tlsLinkRetainmentTime = time.Minute * 15
var (
TappersCount int
TapStatus shared.TapStatus
authStatus *models.AuthStatus
RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
tappersCountLock = sync.Mutex{}
TappersCount int
TapStatus shared.TapStatus
TappersStatus map[string]shared.TapperStatus
authStatus *models.AuthStatus
RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime)
tappersCountLock = sync.Mutex{}
)
func GetAuthStatus() (*models.AuthStatus, error) {

View File

@@ -164,10 +164,10 @@ func (resolver *Resolver) watchServices(ctx context.Context) error {
func (resolver *Resolver) saveResolvedName(key string, resolved string, eventType watch.EventType) {
if eventType == watch.Deleted {
resolver.nameMap.Remove(key)
logger.Log.Infof("setting %s=nil\n", key)
logger.Log.Infof("setting %s=nil", key)
} else {
resolver.nameMap.Set(key, resolved)
logger.Log.Infof("setting %s=%s\n", key, resolved)
logger.Log.Infof("setting %s=%s", key, resolved)
}
}
@@ -188,7 +188,7 @@ func (resolver *Resolver) infiniteErrorHandleRetryFunc(ctx context.Context, fun
var statusError *k8serrors.StatusError
if errors.As(err, &statusError) {
if statusError.ErrStatus.Reason == metav1.StatusReasonForbidden {
logger.Log.Infof("Resolver loop encountered permission error, aborting event listening - %v\n", err)
logger.Log.Infof("Resolver loop encountered permission error, aborting event listening - %v", err)
return
}
}

View File

@@ -1,14 +1,15 @@
package routes
import (
"github.com/gin-gonic/gin"
"mizuserver/pkg/controllers"
"github.com/gin-gonic/gin"
)
// EntriesRoutes defines the group of har entries routes.
func EntriesRoutes(ginApp *gin.Engine) {
routeGroup := ginApp.Group("/entries")
routeGroup.GET("/", controllers.GetEntries) // get entries (base/thin entries)
routeGroup.GET("/:entryId", controllers.GetEntry) // get single (full) entry
routeGroup.GET("/", controllers.GetEntries) // get entries (base/thin entries) and metadata
routeGroup.GET("/:id", controllers.GetEntry) // get single (full) entry
}

View File

@@ -0,0 +1,13 @@
package routes
import (
"mizuserver/pkg/controllers"
"github.com/gin-gonic/gin"
)
func QueryRoutes(ginApp *gin.Engine) {
routeGroup := ginApp.Group("/query")
routeGroup.POST("/validate", controllers.PostValidate)
}

View File

@@ -0,0 +1,13 @@
package routes
import (
"github.com/gin-gonic/gin"
"mizuserver/pkg/controllers"
)
func StandaloneRoutes(ginApp *gin.Engine) {
routeGroup := ginApp.Group("/standalone")
routeGroup.POST("/tapConfig", controllers.PostTapConfig)
routeGroup.GET("/tapConfig", controllers.GetTapConfig)
}

View File

@@ -11,6 +11,7 @@ func StatusRoutes(ginApp *gin.Engine) {
routeGroup.GET("/health", controllers.HealthCheck)
routeGroup.POST("/tappedPods", controllers.PostTappedPods)
routeGroup.POST("/tapperStatus", controllers.PostTapperStatus)
routeGroup.GET("/tappersCount", controllers.GetTappersCount)
routeGroup.GET("/tap", controllers.GetTappingStatus)

View File

@@ -1,10 +0,0 @@
package sensitiveDataFiltering
const maskedFieldPlaceholderValue = "[REDACTED]"
//these values MUST be all lower case and contain no `-` or `_` characters
var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password",
"username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt",
"bearer", "clientid", "clientsecret", "redirecturi", "phonenumber",
"zip", "zipcode", "address", "country", "firstname", "lastname",
"middlename", "fname", "lname", "birthdate"}

View File

@@ -7,15 +7,16 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"mizuserver/pkg/database"
"mizuserver/pkg/utils"
"net/http"
"net/url"
"regexp"
"strings"
"sync"
"time"
"github.com/google/martian/har"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
tapApi "github.com/up9inc/mizu/tap/api"
@@ -23,6 +24,7 @@ import (
const (
AnalyzeCheckSleepTime = 5 * time.Second
SentCountLogInterval = 100
)
type GuestToken struct {
@@ -110,14 +112,14 @@ func GetAnalyzeInfo() *shared.AnalyzeStatus {
}
func SyncEntries(syncEntriesConfig *shared.SyncEntriesConfig) error {
logger.Log.Infof("Sync entries - started\n")
logger.Log.Infof("Sync entries - started")
var (
token, model string
guestMode bool
)
if syncEntriesConfig.Token == "" {
logger.Log.Infof("Sync entries - creating anonymous token. env %s\n", syncEntriesConfig.Env)
logger.Log.Infof("Sync entries - creating anonymous token. env %s", syncEntriesConfig.Env)
guestToken, err := createAnonymousToken(syncEntriesConfig.Env)
if err != nil {
return fmt.Errorf("failed creating anonymous token, err: %v", err)
@@ -131,7 +133,7 @@ func SyncEntries(syncEntriesConfig *shared.SyncEntriesConfig) error {
model = syncEntriesConfig.Workspace
guestMode = false
logger.Log.Infof("Sync entries - upserting model. env %s, model %s\n", syncEntriesConfig.Env, model)
logger.Log.Infof("Sync entries - upserting model. env %s, model %s", syncEntriesConfig.Env, model)
if err := upsertModel(token, model, syncEntriesConfig.Env); err != nil {
return fmt.Errorf("failed upserting model, err: %v", err)
}
@@ -142,7 +144,7 @@ func SyncEntries(syncEntriesConfig *shared.SyncEntriesConfig) error {
return fmt.Errorf("invalid model name, model name: %s", model)
}
logger.Log.Infof("Sync entries - syncing. token: %s, model: %s, guest mode: %v\n", token, model, guestMode)
logger.Log.Infof("Sync entries - syncing. token: %s, model: %s, guest mode: %v", token, model, guestMode)
go syncEntriesImpl(token, model, syncEntriesConfig.Env, syncEntriesConfig.UploadIntervalSec, guestMode)
return nil
@@ -204,51 +206,80 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
analyzeInformation.AnalyzeDestination = envPrefix
analyzeInformation.SentCount = 0
sleepTime := time.Second * time.Duration(uploadIntervalSec)
// "http or grpc" filter indicates that we're only interested in HTTP and gRPC entries
query := "http or grpc"
var timeFrom time.Time
protocolFilter := "http"
logger.Log.Infof("Getting entries from the database")
for {
timeTo := time.Now()
logger.Log.Infof("Getting entries from %v, to %v\n", timeFrom.Format(time.RFC3339Nano), timeTo.Format(time.RFC3339Nano))
entriesArray := database.GetEntriesFromDb(timeFrom, timeTo, &protocolFilter)
var connection *basenine.Connection
var err error
connection, err = basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
if err != nil {
panic(err)
}
if len(entriesArray) > 0 {
result := make([]har.Entry, 0)
for _, data := range entriesArray {
var pair tapApi.RequestResponsePair
if err := json.Unmarshal([]byte(data.Entry), &pair); err != nil {
continue
}
harEntry, err := utils.NewEntry(&pair)
if err != nil {
continue
}
if data.ResolvedSource != "" {
harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: data.ResolvedSource})
}
if data.ResolvedDestination != "" {
harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: data.ResolvedDestination})
harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, data.ResolvedDestination)
}
data := make(chan []byte)
meta := make(chan []byte)
// go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64
if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil {
continue
}
defer func() {
data <- []byte(basenine.CloseChannel)
meta <- []byte(basenine.CloseChannel)
connection.Close()
}()
result = append(result, *harEntry)
lastTimeSynced := time.Time{}
batch := make([]har.Entry, 0)
handleDataChannel := func(wg *sync.WaitGroup, connection *basenine.Connection, data chan []byte) {
defer wg.Done()
for {
dataBytes := <-data
if string(dataBytes) == basenine.CloseChannel {
return
}
logger.Log.Infof("About to upload %v entries\n", len(result))
var dataMap map[string]interface{}
err = json.Unmarshal(dataBytes, &dataMap)
body, jMarshalErr := json.Marshal(result)
var entry tapApi.MizuEntry
if err := json.Unmarshal([]byte(dataBytes), &entry); err != nil {
continue
}
harEntry, err := utils.NewEntry(entry.Request, entry.Response, entry.StartTime, entry.ElapsedTime)
if err != nil {
continue
}
if entry.Source.Name != "" {
harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-source", Value: entry.Source.Name})
}
if entry.Destination.Name != "" {
harEntry.Request.Headers = append(harEntry.Request.Headers, har.Header{Name: "x-mizu-destination", Value: entry.Destination.Name})
harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entry.Destination.Name)
}
// go's default marshal behavior is to encode []byte fields to base64, python's default unmarshal behavior is to not decode []byte fields from base64
if harEntry.Response.Content.Text, err = base64.StdEncoding.DecodeString(string(harEntry.Response.Content.Text)); err != nil {
continue
}
batch = append(batch, *harEntry)
now := time.Now()
if lastTimeSynced.Add(time.Duration(uploadIntervalSec) * time.Second).After(now) {
continue
}
lastTimeSynced = now
body, jMarshalErr := json.Marshal(batch)
batchSize := len(batch)
if jMarshalErr != nil {
analyzeInformation.Reset()
logger.Log.Infof("Stopping sync entries")
logger.Log.Fatal(jMarshalErr)
}
batch = make([]har.Entry, 0)
var in bytes.Buffer
w := zlib.NewWriter(&in)
@@ -273,18 +304,33 @@ func syncEntriesImpl(token string, model string, envPrefix string, uploadInterva
logger.Log.Info("Stopping sync entries")
logger.Log.Fatal(postErr)
}
analyzeInformation.SentCount += len(entriesArray)
logger.Log.Infof("Finish uploading %v entries to %s\n", len(entriesArray), GetTrafficDumpUrl(envPrefix, model))
analyzeInformation.SentCount += batchSize
logger.Log.Infof("Uploaded %v entries until now", analyzeInformation.SentCount)
} else {
logger.Log.Infof("Nothing to upload")
if analyzeInformation.SentCount%SentCountLogInterval == 0 {
logger.Log.Infof("Uploaded %v entries until now", analyzeInformation.SentCount)
}
}
logger.Log.Infof("Sleeping for %v...\n", sleepTime)
time.Sleep(sleepTime)
timeFrom = timeTo
}
handleMetaChannel := func(wg *sync.WaitGroup, connection *basenine.Connection, meta chan []byte) {
defer wg.Done()
for {
metaBytes := <-meta
if string(metaBytes) == basenine.CloseChannel {
return
}
}
}
var wg sync.WaitGroup
go handleDataChannel(&wg, connection, data)
go handleMetaChannel(&wg, connection, meta)
wg.Add(2)
connection.Query(query, data, meta)
wg.Wait()
}
func UpdateAnalyzeStatus(callback func(data []byte)) {

View File

@@ -10,7 +10,6 @@ import (
"github.com/google/martian/har"
"github.com/up9inc/mizu/shared/logger"
"github.com/up9inc/mizu/tap/api"
)
// Keep it because we might want cookies in the future
@@ -120,13 +119,11 @@ func BuildPostParams(rawParams []interface{}) []har.Param {
return params
}
func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error) {
reqDetails := request.Payload.(map[string]interface{})["details"].(map[string]interface{})
func NewRequest(request map[string]interface{}) (harRequest *har.Request, err error) {
headers, host, scheme, authority, path, _ := BuildHeaders(request["_headers"].([]interface{}))
cookies := make([]har.Cookie, 0) // BuildCookies(request["_cookies"].([]interface{}))
headers, host, scheme, authority, path, _ := BuildHeaders(reqDetails["headers"].([]interface{}))
cookies := make([]har.Cookie, 0) // BuildCookies(reqDetails["cookies"].([]interface{}))
postData, _ := reqDetails["postData"].(map[string]interface{})
postData, _ := request["postData"].(map[string]interface{})
mimeType, _ := postData["mimeType"]
if mimeType == nil || len(mimeType.(string)) == 0 {
mimeType = "text/html"
@@ -138,7 +135,7 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error
}
queryString := make([]har.QueryString, 0)
for _, _qs := range reqDetails["queryString"].([]interface{}) {
for _, _qs := range request["_queryString"].([]interface{}) {
qs := _qs.(map[string]interface{})
queryString = append(queryString, har.QueryString{
Name: qs["name"].(string),
@@ -146,7 +143,7 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error
})
}
url := fmt.Sprintf("http://%s%s", host, reqDetails["url"].(string))
url := fmt.Sprintf("http://%s%s", host, request["url"].(string))
if strings.HasPrefix(mimeType.(string), "application/grpc") {
url = fmt.Sprintf("%s://%s%s", scheme, authority, path)
}
@@ -157,9 +154,9 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error
}
harRequest = &har.Request{
Method: reqDetails["method"].(string),
Method: request["method"].(string),
URL: url,
HTTPVersion: reqDetails["httpVersion"].(string),
HTTPVersion: request["httpVersion"].(string),
HeadersSize: -1,
BodySize: int64(bytes.NewBufferString(postDataText).Len()),
QueryString: queryString,
@@ -175,13 +172,11 @@ func NewRequest(request *api.GenericMessage) (harRequest *har.Request, err error
return
}
func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err error) {
resDetails := response.Payload.(map[string]interface{})["details"].(map[string]interface{})
func NewResponse(response map[string]interface{}) (harResponse *har.Response, err error) {
headers, _, _, _, _, _status := BuildHeaders(response["_headers"].([]interface{}))
cookies := make([]har.Cookie, 0) // BuildCookies(response["_cookies"].([]interface{}))
headers, _, _, _, _, _status := BuildHeaders(resDetails["headers"].([]interface{}))
cookies := make([]har.Cookie, 0) // BuildCookies(resDetails["cookies"].([]interface{}))
content, _ := resDetails["content"].(map[string]interface{})
content, _ := response["content"].(map[string]interface{})
mimeType, _ := content["mimeType"]
if mimeType == nil || len(mimeType.(string)) == 0 {
mimeType = "text/html"
@@ -200,9 +195,11 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e
Size: int64(len(bodyText)),
}
status := int(resDetails["status"].(float64))
status := int(response["status"].(float64))
if strings.HasPrefix(mimeType.(string), "application/grpc") {
status, err = strconv.Atoi(_status)
if _status != "" {
status, err = strconv.Atoi(_status)
}
if err != nil {
logger.Log.Errorf("Failed converting status to int %s (%v,%+v)", err, err, err)
return nil, errors.New("failed converting response status to int for HAR")
@@ -210,9 +207,9 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e
}
harResponse = &har.Response{
HTTPVersion: resDetails["httpVersion"].(string),
HTTPVersion: response["httpVersion"].(string),
Status: status,
StatusText: resDetails["statusText"].(string),
StatusText: response["statusText"].(string),
HeadersSize: -1,
BodySize: int64(bytes.NewBufferString(bodyText).Len()),
Headers: headers,
@@ -222,34 +219,33 @@ func NewResponse(response *api.GenericMessage) (harResponse *har.Response, err e
return
}
func NewEntry(pair *api.RequestResponsePair) (*har.Entry, error) {
harRequest, err := NewRequest(&pair.Request)
func NewEntry(request map[string]interface{}, response map[string]interface{}, startTime time.Time, elapsedTime int64) (*har.Entry, error) {
harRequest, err := NewRequest(request)
if err != nil {
logger.Log.Errorf("Failed converting request to HAR %s (%v,%+v)", err, err, err)
return nil, errors.New("failed converting request to HAR")
}
harResponse, err := NewResponse(&pair.Response)
harResponse, err := NewResponse(response)
if err != nil {
logger.Log.Errorf("Failed converting response to HAR %s (%v,%+v)", err, err, err)
return nil, errors.New("failed converting response to HAR")
}
totalTime := pair.Response.CaptureTime.Sub(pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
if totalTime < 1 {
totalTime = 1
if elapsedTime < 1 {
elapsedTime = 1
}
harEntry := har.Entry{
StartedDateTime: pair.Request.CaptureTime,
Time: totalTime,
StartedDateTime: startTime,
Time: elapsedTime,
Request: harRequest,
Response: harResponse,
Cache: &har.Cache{},
Timings: &har.Timings{
Send: -1,
Wait: -1,
Receive: totalTime,
Receive: elapsedTime,
},
}

View File

@@ -1,60 +0,0 @@
package utils
import (
"context"
"fmt"
"time"
loggerShared "github.com/up9inc/mizu/shared/logger"
"gorm.io/gorm/logger"
"gorm.io/gorm/utils"
)
// TruncatingLogger implements the gorm logger.Interface interface. Its purpose is to act as gorm's logger while truncating logs to a max of 50 characters to minimise the performance impact
type TruncatingLogger struct {
LogLevel logger.LogLevel
SlowThreshold time.Duration
}
func (truncatingLogger *TruncatingLogger) LogMode(logLevel logger.LogLevel) logger.Interface {
truncatingLogger.LogLevel = logLevel
return truncatingLogger
}
func (truncatingLogger *TruncatingLogger) Info(_ context.Context, message string, __ ...interface{}) {
if truncatingLogger.LogLevel < logger.Info {
return
}
loggerShared.Log.Errorf("gorm info: %.150s", message)
}
func (truncatingLogger *TruncatingLogger) Warn(_ context.Context, message string, __ ...interface{}) {
if truncatingLogger.LogLevel < logger.Warn {
return
}
loggerShared.Log.Errorf("gorm warning: %.150s", message)
}
func (truncatingLogger *TruncatingLogger) Error(_ context.Context, message string, __ ...interface{}) {
if truncatingLogger.LogLevel < logger.Error {
return
}
loggerShared.Log.Errorf("gorm error: %.150s", message)
}
func (truncatingLogger *TruncatingLogger) Trace(ctx context.Context, begin time.Time, fc func() (string, int64), err error) {
if truncatingLogger.LogLevel == logger.Silent {
return
}
elapsed := time.Since(begin)
if err != nil {
sql, rows := fc() // copied into every condition as this is a potentially heavy operation best done only when necessary
truncatingLogger.Error(ctx, fmt.Sprintf("Error in %s: %v - elapsed: %fs affected rows: %d, sql: %s", utils.FileWithLineNum(), err, elapsed.Seconds(), rows, sql))
} else if truncatingLogger.LogLevel >= logger.Warn && elapsed > truncatingLogger.SlowThreshold {
sql, rows := fc()
truncatingLogger.Warn(ctx, fmt.Sprintf("Slow sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql))
} else if truncatingLogger.LogLevel >= logger.Info {
sql, rows := fc()
truncatingLogger.Info(ctx, fmt.Sprintf("Sql query - elapse: %fs rows: %d, sql: %s", elapsed.Seconds(), rows, sql))
}
}

View File

@@ -3,11 +3,12 @@ package utils
import (
"context"
"fmt"
"mizuserver/pkg/providers"
"net/http"
"net/url"
"os"
"os/signal"
"reflect"
"strings"
"syscall"
"time"
@@ -44,15 +45,14 @@ func StartServer(app *gin.Engine) {
}
}
func ReverseSlice(data interface{}) {
value := reflect.ValueOf(data)
valueLen := value.Len()
for i := 0; i <= int((valueLen-1)/2); i++ {
reverseIndex := valueLen - 1 - i
tmp := value.Index(reverseIndex).Interface()
value.Index(reverseIndex).Set(value.Index(i))
value.Index(i).Set(reflect.ValueOf(tmp))
func GetTappedPodsStatus() []shared.TappedPodStatus {
tappedPodsStatus := make([]shared.TappedPodStatus, 0)
for _, pod := range providers.TapStatus.Pods {
status := strings.ToLower(providers.TappersStatus[pod.NodeName].Status)
isTapped := status == "running"
tappedPodsStatus = append(tappedPodsStatus, shared.TappedPodStatus{Name: pod.Name, Namespace: pod.Namespace, IsTapped: isTapped})
}
return tappedPodsStatus
}
func CheckErr(e error) {

Binary file not shown.

Before

Width:  |  Height:  |  Size: 491 KiB

After

Width:  |  Height:  |  Size: 640 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 55 KiB

After

Width:  |  Height:  |  Size: 110 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 43 KiB

After

Width:  |  Height:  |  Size: 53 KiB

View File

@@ -18,8 +18,9 @@ build: ## Build mizu CLI binary (select platform via GOOS / GOARCH env variables
go build -ldflags="-X 'github.com/up9inc/mizu/cli/mizu.GitCommitHash=$(COMMIT_HASH)' \
-X 'github.com/up9inc/mizu/cli/mizu.Branch=$(GIT_BRANCH)' \
-X 'github.com/up9inc/mizu/cli/mizu.BuildTimestamp=$(BUILD_TIMESTAMP)' \
-X 'github.com/up9inc/mizu/cli/mizu.Platform=$(SUFFIX)' \
-X 'github.com/up9inc/mizu/cli/mizu.SemVer=$(SEM_VER)'" \
-o bin/mizu_$(SUFFIX) mizu.go
-o bin/mizu_$(SUFFIX) mizu.go
(cd bin && shasum -a 256 mizu_${SUFFIX} > mizu_${SUFFIX}.sha256)
build-all: ## Build for all supported platforms.

View File

@@ -3,14 +3,14 @@ package apiserver
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"github.com/up9inc/mizu/shared/kubernetes"
"io/ioutil"
"net/http"
"net/url"
"time"
"github.com/up9inc/mizu/shared/kubernetes"
"github.com/up9inc/mizu/cli/config"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
@@ -28,7 +28,7 @@ const DefaultTimeout = 5 * time.Second
func NewProvider(url string, retries int, timeout time.Duration) *Provider {
return &Provider{
url: url,
url: url,
retries: config.GetIntEnvConfig(config.ApiServerRetries, retries),
client: &http.Client{
Timeout: timeout,
@@ -39,8 +39,8 @@ func NewProvider(url string, retries int, timeout time.Duration) *Provider {
func (provider *Provider) TestConnection() error {
retriesLeft := provider.retries
for retriesLeft > 0 {
if _, err := provider.GetHealthStatus(); err != nil {
logger.Log.Debugf("[ERROR] api server not ready yet %v", err)
if isReachable, err := provider.isReachable(); err != nil || !isReachable {
logger.Log.Debugf("api server not ready yet %v", err)
} else {
logger.Log.Debugf("connection test to api server passed successfully")
break
@@ -55,20 +55,31 @@ func (provider *Provider) TestConnection() error {
return nil
}
func (provider *Provider) GetHealthStatus() (*shared.HealthResponse, error) {
healthUrl := fmt.Sprintf("%s/status/health", provider.url)
if response, err := provider.client.Get(healthUrl); err != nil {
return nil, err
} else if response.StatusCode > 299 {
return nil, errors.New(fmt.Sprintf("status code: %d", response.StatusCode))
func (provider *Provider) isReachable() (bool, error) {
echoUrl := fmt.Sprintf("%s/echo", provider.url)
if response, err := provider.client.Get(echoUrl); err != nil {
return false, err
} else if response.StatusCode != 200 {
return false, fmt.Errorf("invalid status code %v", response.StatusCode)
} else {
defer response.Body.Close()
return true, nil
}
}
healthResponse := &shared.HealthResponse{}
if err := json.NewDecoder(response.Body).Decode(&healthResponse); err != nil {
return nil, err
func (provider *Provider) ReportTapperStatus(tapperStatus shared.TapperStatus) error {
tapperStatusUrl := fmt.Sprintf("%s/status/tapperStatus", provider.url)
if jsonValue, err := json.Marshal(tapperStatus); err != nil {
return fmt.Errorf("failed Marshal the tapper status %w", err)
} else {
if response, err := provider.client.Post(tapperStatusUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
return fmt.Errorf("failed sending to API server the tapped pods %w", err)
} else if response.StatusCode != 200 {
return fmt.Errorf("failed sending to API server the tapper status, response status code %v", response.StatusCode)
} else {
logger.Log.Debugf("Reported to server API about tapper status: %v", tapperStatus)
return nil
}
return healthResponse, nil
}
}

View File

@@ -1,6 +1,9 @@
package cmd
import "github.com/up9inc/mizu/cli/apiserver"
import (
"github.com/up9inc/mizu/cli/apiserver"
"github.com/up9inc/mizu/cli/config"
)
func performCleanCommand() {
kubernetesProvider, err := getKubernetesProviderForCli()
@@ -8,5 +11,5 @@ func performCleanCommand() {
return
}
finishMizuExecution(kubernetesProvider, apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout))
finishMizuExecution(kubernetesProvider, apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout), config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
}

View File

@@ -2,11 +2,17 @@ package cmd
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"os/signal"
"syscall"
"github.com/up9inc/mizu/cli/apiserver"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/mizu/fsUtils"
"github.com/up9inc/mizu/cli/resources"
"github.com/up9inc/mizu/cli/telemetry"
"github.com/up9inc/mizu/shared"
"path"
"time"
"github.com/up9inc/mizu/cli/config"
"github.com/up9inc/mizu/cli/config/configStructs"
@@ -31,22 +37,6 @@ func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel
logger.Log.Debugf("proxy ended")
}
func waitForFinish(ctx context.Context, cancel context.CancelFunc) {
logger.Log.Debugf("waiting for finish...")
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
// block until ctx cancel is called or termination signal is received
select {
case <-ctx.Done():
logger.Log.Debugf("ctx done")
break
case <-sigChan:
logger.Log.Debugf("Got termination signal, canceling execution...")
cancel()
}
}
func getKubernetesProviderForCli() (*kubernetes.Provider, error) {
kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath())
if err != nil {
@@ -64,3 +54,31 @@ func handleKubernetesProviderError(err error) {
logger.Log.Error(err)
}
}
func finishMizuExecution(kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider, isNsRestrictedMode bool, mizuResourcesNamespace string) {
telemetry.ReportAPICalls(apiProvider)
removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
defer cancel()
dumpLogsIfNeeded(removalCtx, kubernetesProvider)
resources.CleanUpMizuResources(removalCtx, cancel, kubernetesProvider, isNsRestrictedMode, mizuResourcesNamespace)
}
func dumpLogsIfNeeded(ctx context.Context, kubernetesProvider *kubernetes.Provider) {
if !config.Config.DumpLogs {
return
}
mizuDir := mizu.GetMizuFolderPath()
filePath := path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05")))
if err := fsUtils.DumpLogs(ctx, kubernetesProvider, filePath); err != nil {
logger.Log.Errorf("Failed dump logs %v", err)
}
}
func getSerializedMizuAgentConfig(mizuAgentConfig *shared.MizuAgentConfig) (string, error) {
serializedConfig, err := json.Marshal(mizuAgentConfig)
if err != nil {
return "", err
}
return string(serializedConfig), nil
}

21
cli/cmd/install.go Normal file
View File

@@ -0,0 +1,21 @@
package cmd
import (
"github.com/spf13/cobra"
"github.com/up9inc/mizu/cli/telemetry"
)
var installCmd = &cobra.Command{
Use: "install",
Short: "Installs mizu components",
RunE: func(cmd *cobra.Command, args []string) error {
go telemetry.ReportRun("install", nil)
runMizuInstall()
return nil
},
}
func init() {
rootCmd.AddCommand(installCmd)
}

72
cli/cmd/installRunner.go Normal file
View File

@@ -0,0 +1,72 @@
package cmd
import (
"context"
"errors"
"fmt"
"github.com/creasty/defaults"
"github.com/up9inc/mizu/cli/config"
"github.com/up9inc/mizu/cli/errormessage"
"github.com/up9inc/mizu/cli/resources"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func runMizuInstall() {
kubernetesProvider, err := getKubernetesProviderForCli()
if err != nil {
return
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel() // cancel will be called when this function exits
var serializedValidationRules string
var serializedContract string
var defaultMaxEntriesDBSizeBytes int64 = 200 * 1000 * 1000
defaultResources := shared.Resources{}
defaults.Set(&defaultResources)
mizuAgentConfig := getInstallMizuAgentConfig(defaultMaxEntriesDBSizeBytes, defaultResources)
serializedMizuConfig, err := getSerializedMizuAgentConfig(mizuAgentConfig)
if err != nil {
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error serializing mizu config: %v", errormessage.FormatError(err)))
return
}
if err = resources.CreateInstallMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, nil, defaultMaxEntriesDBSizeBytes, defaultResources, config.Config.ImagePullPolicy(), config.Config.LogLevel(), false); err != nil {
var statusError *k8serrors.StatusError
if errors.As(err, &statusError) {
if statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists {
logger.Log.Info("Mizu is already running in this namespace, change the `mizu-resources-namespace` configuration or run `mizu clean` to remove the currently running Mizu instance")
}
} else {
defer resources.CleanUpMizuResources(ctx, cancel, kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
}
return
}
logger.Log.Infof(uiUtils.Magenta, "Created Mizu Agent components, run `mizu view` to connect to the mizu daemon instance")
}
func getInstallMizuAgentConfig(maxDBSizeBytes int64, tapperResources shared.Resources) *shared.MizuAgentConfig {
mizuAgentConfig := shared.MizuAgentConfig{
MaxDBSizeBytes: maxDBSizeBytes,
AgentImage: config.Config.AgentImage,
PullPolicy: config.Config.ImagePullPolicyStr,
LogLevel: config.Config.LogLevel(),
TapperResources: tapperResources,
MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
AgentDatabasePath: shared.DataDirPath,
StandaloneMode: true,
}
return &mizuAgentConfig
}

View File

@@ -23,6 +23,7 @@ Further info is available at https://github.com/up9inc/mizu`,
if err := config.InitConfig(cmd); err != nil {
logger.Log.Fatal(err)
}
return nil
},
}

View File

@@ -3,6 +3,7 @@ package cmd
import (
"errors"
"fmt"
"github.com/up9inc/mizu/cli/up9"
"os"
"github.com/creasty/defaults"
@@ -62,6 +63,12 @@ Supported protocols are HTTP and gRPC.`,
logger.Log.Errorf("failed to log in, err: %v", err)
return nil
}
} else if isValidToken := up9.IsTokenValid(config.Config.Auth.Token, config.Config.Auth.EnvName); !isValidToken {
logger.Log.Errorf("Token is not valid, please log in again to continue")
if err := auth.Login(); err != nil {
logger.Log.Errorf("failed to log in, err: %v", err)
return nil
}
}
}
}
@@ -112,5 +119,5 @@ func init() {
tapCmd.Flags().StringP(configStructs.WorkspaceTapName, "w", defaultTapConfig.Workspace, "Uploads traffic to your UP9 workspace for further analysis (requires auth)")
tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file path with policy rules")
tapCmd.Flags().String(configStructs.ContractFile, defaultTapConfig.ContractFile, "OAS/Swagger file to validate to monitor the contracts")
tapCmd.Flags().Bool(configStructs.DaemonModeTapName, defaultTapConfig.DaemonMode, "Run mizu in daemon mode, detached from the cli")
tapCmd.Flags().Bool(configStructs.IstioName, defaultTapConfig.Istio, "Record decrypted traffic if the cluster configured with istio and mtls")
}

View File

@@ -4,27 +4,25 @@ import (
"context"
"errors"
"fmt"
"github.com/up9inc/mizu/cli/cmd/goUtils"
"github.com/up9inc/mizu/cli/resources"
"github.com/up9inc/mizu/cli/utils"
"io/ioutil"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"path"
"regexp"
"strings"
"time"
"github.com/getkin/kin-openapi/openapi3"
"gopkg.in/yaml.v3"
core "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/up9inc/mizu/cli/apiserver"
"github.com/up9inc/mizu/cli/cmd/goUtils"
"github.com/up9inc/mizu/cli/config"
"github.com/up9inc/mizu/cli/config/configStructs"
"github.com/up9inc/mizu/cli/errormessage"
"gopkg.in/yaml.v3"
core "k8s.io/api/core/v1"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/mizu/fsUtils"
"github.com/up9inc/mizu/cli/telemetry"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/kubernetes"
@@ -35,8 +33,8 @@ import (
const cleanupTimeout = time.Minute
type tapState struct {
apiServerService *core.Service
tapperSyncer *kubernetes.MizuTapperSyncer
startTime time.Time
targetNamespaces []string
mizuServiceAccountExists bool
}
@@ -44,13 +42,11 @@ var state tapState
var apiProvider *apiserver.Provider
func RunMizuTap() {
mizuApiFilteringOptions, err := getMizuApiFilteringOptions()
apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout)
if err != nil {
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error parsing regex-masking: %v", errormessage.FormatError(err)))
return
}
state.startTime = time.Now()
apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout)
var err error
var serializedValidationRules string
if config.Config.Tap.EnforcePolicyFile != "" {
serializedValidationRules, err = readValidationRules(config.Config.Tap.EnforcePolicyFile)
@@ -92,16 +88,17 @@ func RunMizuTap() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel() // cancel will be called when this function exits
targetNamespaces := getNamespaces(kubernetesProvider)
state.targetNamespaces = getNamespaces(kubernetesProvider)
serializedMizuConfig, err := config.GetSerializedMizuAgentConfig(targetNamespaces, mizuApiFilteringOptions)
mizuAgentConfig := getTapMizuAgentConfig()
serializedMizuConfig, err := getSerializedMizuAgentConfig(mizuAgentConfig)
if err != nil {
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error composing mizu config: %v", errormessage.FormatError(err)))
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error serializing mizu config: %v", errormessage.FormatError(err)))
return
}
if config.Config.IsNsRestrictedMode() {
if len(targetNamespaces) != 1 || !shared.Contains(targetNamespaces, config.Config.MizuResourcesNamespace) {
if len(state.targetNamespaces) != 1 || !shared.Contains(state.targetNamespaces, config.Config.MizuResourcesNamespace) {
logger.Log.Errorf("Not supported mode. Mizu can't resolve IPs in other namespaces when running in namespace restricted mode.\n"+
"You can use the same namespace for --%s and --%s", configStructs.NamespacesTapName, config.MizuResourcesNamespaceConfigName)
return
@@ -109,89 +106,80 @@ func RunMizuTap() {
}
var namespacesStr string
if !shared.Contains(targetNamespaces, kubernetes.K8sAllNamespaces) {
namespacesStr = fmt.Sprintf("namespaces \"%s\"", strings.Join(targetNamespaces, "\", \""))
if !shared.Contains(state.targetNamespaces, kubernetes.K8sAllNamespaces) {
namespacesStr = fmt.Sprintf("namespaces \"%s\"", strings.Join(state.targetNamespaces, "\", \""))
} else {
namespacesStr = "all namespaces"
}
logger.Log.Infof("Tapping pods in %s", namespacesStr)
if err := printTappedPodsPreview(ctx, kubernetesProvider, state.targetNamespaces); err != nil {
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error listing pods: %v", errormessage.FormatError(err)))
}
if config.Config.Tap.DryRun {
return
}
if err := createMizuResources(ctx, cancel, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig); err != nil {
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
logger.Log.Infof("Waiting for Mizu Agent to start...")
if state.mizuServiceAccountExists, err = resources.CreateTapMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, getSyncEntriesConfig(), config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel()); err != nil {
var statusError *k8serrors.StatusError
if errors.As(err, &statusError) {
if statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists {
logger.Log.Info("Mizu is already running in this namespace, change the `mizu-resources-namespace` configuration or run `mizu clean` to remove the currently running Mizu instance")
}
} else {
defer resources.CleanUpMizuResources(ctx, cancel, kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err)))
}
return
}
if config.Config.Tap.DaemonMode {
if err := handleDaemonModePostCreation(cancel, kubernetesProvider); err != nil {
defer finishMizuExecution(kubernetesProvider, apiProvider)
cancel()
} else {
logger.Log.Infof(uiUtils.Magenta, "Mizu is now running in daemon mode, run `mizu view` to connect to the mizu daemon instance")
}
} else {
defer finishMizuExecution(kubernetesProvider, apiProvider)
if err = startTapperSyncer(ctx, cancel, kubernetesProvider, targetNamespaces, *mizuApiFilteringOptions); err != nil {
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error starting mizu tapper syncer: %v", err))
cancel()
}
defer finishMizuExecution(kubernetesProvider, apiProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace)
go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel)
go goUtils.HandleExcWrapper(watchTapperPod, ctx, kubernetesProvider, cancel)
go goUtils.HandleExcWrapper(watchApiServerEvents, ctx, kubernetesProvider, cancel)
go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel)
// block until exit signal or error
waitForFinish(ctx, cancel)
}
// block until exit signal or error
utils.WaitForFinish(ctx, cancel)
}
func handleDaemonModePostCreation(cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) error {
apiProvider := apiserver.NewProvider(GetApiServerUrl(), 90, 1*time.Second)
if err := waitForDaemonModeToBeReady(cancel, kubernetesProvider, apiProvider); err != nil {
return err
}
if err := printDaemonModeTappedPods(apiProvider); err != nil {
return err
func getTapMizuAgentConfig() *shared.MizuAgentConfig {
mizuAgentConfig := shared.MizuAgentConfig{
MaxDBSizeBytes: config.Config.Tap.MaxEntriesDBSizeBytes(),
AgentImage: config.Config.AgentImage,
PullPolicy: config.Config.ImagePullPolicyStr,
LogLevel: config.Config.LogLevel(),
TapperResources: config.Config.Tap.TapperResources,
MizuResourcesNamespace: config.Config.MizuResourcesNamespace,
AgentDatabasePath: shared.DataDirPath,
}
return nil
return &mizuAgentConfig
}
func printDaemonModeTappedPods(apiProvider *apiserver.Provider) error {
if healthStatus, err := apiProvider.GetHealthStatus(); err != nil {
/*
this function is a bit problematic as it might be detached from the actual pods the mizu api server will tap.
The alternative would be to wait for api server to be ready and then query it for the pods it listens to, this has
the arguably worse drawback of taking a relatively very long time before the user sees which pods are targeted, if any.
*/
func printTappedPodsPreview(ctx context.Context, kubernetesProvider *kubernetes.Provider, namespaces []string) error {
if matchingPods, err := kubernetesProvider.ListAllRunningPodsMatchingRegex(ctx, config.Config.Tap.PodRegex(), namespaces); err != nil {
return err
} else {
for _, tappedPod := range healthStatus.TapStatus.Pods {
if len(matchingPods) == 0 {
printNoPodsFoundSuggestion(namespaces)
}
for _, tappedPod := range matchingPods {
logger.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", tappedPod.Name))
}
return nil
}
return nil
}
func waitForDaemonModeToBeReady(cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider) error {
logger.Log.Info("Waiting for mizu to be ready... (may take a few minutes)")
go startProxyReportErrorIfAny(kubernetesProvider, cancel)
// TODO: TRA-3903 add a smarter test to see that tapping/pod watching is functioning properly
if err := apiProvider.TestConnection(); err != nil {
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Mizu was not ready in time, for more info check logs at %s", fsUtils.GetLogFilePath()))
return err
}
return nil
}
func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, mizuApiFilteringOptions api.TrafficFilteringOptions) error {
func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, mizuApiFilteringOptions api.TrafficFilteringOptions, startTime time.Time) error {
tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{
TargetNamespaces: targetNamespaces,
PodFilterRegex: *config.Config.Tap.PodRegex(),
@@ -199,28 +187,17 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider
AgentImage: config.Config.AgentImage,
TapperResources: config.Config.Tap.TapperResources,
ImagePullPolicy: config.Config.ImagePullPolicy(),
DumpLogs: config.Config.DumpLogs,
LogLevel: config.Config.LogLevel(),
IgnoredUserAgents: config.Config.Tap.IgnoredUserAgents,
MizuApiFilteringOptions: mizuApiFilteringOptions,
MizuServiceAccountExists: state.mizuServiceAccountExists,
})
Istio: config.Config.Tap.Istio,
}, startTime)
if err != nil {
return err
}
for _, tappedPod := range tapperSyncer.CurrentlyTappedPods {
logger.Log.Infof(uiUtils.Green, fmt.Sprintf("+%s", tappedPod.Name))
}
if len(tapperSyncer.CurrentlyTappedPods) == 0 {
var suggestionStr string
if !shared.Contains(targetNamespaces, kubernetes.K8sAllNamespaces) {
suggestionStr = ". Select a different namespace with -n or tap all namespaces with -A"
}
logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any pods matching the regex argument%s", suggestionStr))
}
go func() {
for {
select {
@@ -239,6 +216,14 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider
if err := apiProvider.ReportTappedPods(tapperSyncer.CurrentlyTappedPods); err != nil {
logger.Log.Debugf("[Error] failed update tapped pods %v", err)
}
case tapperStatus, ok := <-tapperSyncer.TapperStatusChangedOut:
if !ok {
logger.Log.Debug("mizuTapperSyncer tapper status changed channel closed, ending listener loop")
return
}
if err := apiProvider.ReportTapperStatus(tapperStatus); err != nil {
logger.Log.Debugf("[Error] failed update tapper status %v", err)
}
case <-ctx.Done():
logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done")
return
@@ -246,11 +231,17 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider
}
}()
state.tapperSyncer = tapperSyncer
return nil
}
func printNoPodsFoundSuggestion(targetNamespaces []string) {
var suggestionStr string
if !shared.Contains(targetNamespaces, kubernetes.K8sAllNamespaces) {
suggestionStr = ". You can also try selecting a different namespace with -n or tap all namespaces with -A"
}
logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Did not find any currently running pods that match the regex argument, mizu will automatically tap matching pods if any are created later%s", suggestionStr))
}
func getErrorDisplayTextForK8sTapManagerError(err kubernetes.K8sTapManagerError) string {
switch err.TapManagerReason {
case kubernetes.TapManagerPodListError:
@@ -273,118 +264,6 @@ func readValidationRules(file string) (string, error) {
return string(newContent), nil
}
func createMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string) error {
if !config.Config.IsNsRestrictedMode() {
if err := createMizuNamespace(ctx, kubernetesProvider); err != nil {
return err
}
}
if err := createMizuConfigmap(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig); err != nil {
logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v\n", errormessage.FormatError(err)))
}
var err error
state.mizuServiceAccountExists, err = createRBACIfNecessary(ctx, kubernetesProvider)
if err != nil {
if !config.Config.Tap.DaemonMode {
logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to ensure the resources required for IP resolving. Mizu will not resolve target IPs to names. error: %v", errormessage.FormatError(err)))
}
}
var serviceAccountName string
if state.mizuServiceAccountExists {
serviceAccountName = kubernetes.ServiceAccountName
} else {
serviceAccountName = ""
}
opts := &kubernetes.ApiServerOptions{
Namespace: config.Config.MizuResourcesNamespace,
PodName: kubernetes.ApiServerPodName,
PodImage: config.Config.AgentImage,
ServiceAccountName: serviceAccountName,
IsNamespaceRestricted: config.Config.IsNsRestrictedMode(),
SyncEntriesConfig: getSyncEntriesConfig(),
MaxEntriesDBSizeBytes: config.Config.Tap.MaxEntriesDBSizeBytes(),
Resources: config.Config.Tap.ApiServerResources,
ImagePullPolicy: config.Config.ImagePullPolicy(),
}
if config.Config.Tap.DaemonMode {
if !state.mizuServiceAccountExists {
defer cleanUpMizuResources(ctx, cancel, kubernetesProvider)
logger.Log.Fatalf(uiUtils.Red, fmt.Sprintf("Failed to ensure the resources required for mizu to run in daemon mode. cannot proceed. error: %v", errormessage.FormatError(err)))
}
if err := createMizuApiServerDeployment(ctx, kubernetesProvider, opts); err != nil {
return err
}
} else {
if err := createMizuApiServerPod(ctx, kubernetesProvider, opts); err != nil {
return err
}
}
state.apiServerService, err = kubernetesProvider.CreateService(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName, kubernetes.ApiServerPodName)
if err != nil {
return err
}
logger.Log.Debugf("Successfully created service: %s", kubernetes.ApiServerPodName)
return nil
}
func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string) error {
err := kubernetesProvider.CreateConfigMap(ctx, config.Config.MizuResourcesNamespace, kubernetes.ConfigMapName, serializedValidationRules, serializedContract, serializedMizuConfig)
return err
}
func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Provider) error {
_, err := kubernetesProvider.CreateNamespace(ctx, config.Config.MizuResourcesNamespace)
return err
}
func createMizuApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, opts *kubernetes.ApiServerOptions) error {
pod, err := kubernetesProvider.GetMizuApiServerPodObject(opts, false, "")
if err != nil {
return err
}
if _, err = kubernetesProvider.CreatePod(ctx, config.Config.MizuResourcesNamespace, pod); err != nil {
return err
}
logger.Log.Debugf("Successfully created API server pod: %s", kubernetes.ApiServerPodName)
return nil
}
func createMizuApiServerDeployment(ctx context.Context, kubernetesProvider *kubernetes.Provider, opts *kubernetes.ApiServerOptions) error {
isDefaultStorageClassAvailable, err := kubernetesProvider.IsDefaultStorageProviderAvailable(ctx)
volumeClaimCreated := false
if err != nil {
return err
}
if isDefaultStorageClassAvailable {
if _, err = kubernetesProvider.CreatePersistentVolumeClaim(ctx, config.Config.MizuResourcesNamespace, kubernetes.PersistentVolumeClaimName, config.Config.Tap.MaxEntriesDBSizeBytes()+mizu.DaemonModePersistentVolumeSizeBufferBytes); err != nil {
logger.Log.Warningf(uiUtils.Yellow, "An error has occured while creating a persistent volume claim for mizu, this will mean that mizu's data will be lost on pod restart")
logger.Log.Debugf("error creating persistent volume claim: %v", err)
} else {
volumeClaimCreated = true
}
} else {
logger.Log.Warningf(uiUtils.Yellow, "Could not find default volume provider in this cluster, this will mean that mizu's data will be lost on pod restart")
}
pod, err := kubernetesProvider.GetMizuApiServerPodObject(opts, volumeClaimCreated, kubernetes.PersistentVolumeClaimName)
if err != nil {
return err
}
if _, err = kubernetesProvider.CreateDeployment(ctx, config.Config.MizuResourcesNamespace, opts.PodName, pod); err != nil {
return err
}
logger.Log.Debugf("Successfully created API server deployment: %s", kubernetes.ApiServerPodName)
return nil
}
func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) {
var compiledRegexSlice []*api.SerializableRegexp
@@ -419,216 +298,45 @@ func getSyncEntriesConfig() *shared.SyncEntriesConfig {
}
}
func finishMizuExecution(kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider) {
telemetry.ReportAPICalls(apiProvider)
removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout)
defer cancel()
dumpLogsIfNeeded(removalCtx, kubernetesProvider)
cleanUpMizuResources(removalCtx, cancel, kubernetesProvider)
}
func dumpLogsIfNeeded(ctx context.Context, kubernetesProvider *kubernetes.Provider) {
if !config.Config.DumpLogs {
return
}
mizuDir := mizu.GetMizuFolderPath()
filePath := path.Join(mizuDir, fmt.Sprintf("mizu_logs_%s.zip", time.Now().Format("2006_01_02__15_04_05")))
if err := fsUtils.DumpLogs(ctx, kubernetesProvider, filePath); err != nil {
logger.Log.Errorf("Failed dump logs %v", err)
}
}
func cleanUpMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) {
logger.Log.Infof("\nRemoving mizu resources\n")
var leftoverResources []string
if config.Config.IsNsRestrictedMode() {
leftoverResources = cleanUpRestrictedMode(ctx, kubernetesProvider)
} else {
leftoverResources = cleanUpNonRestrictedMode(ctx, cancel, kubernetesProvider)
}
if len(leftoverResources) > 0 {
errMsg := fmt.Sprintf("Failed to remove the following resources, for more info check logs at %s:", fsUtils.GetLogFilePath())
for _, resource := range leftoverResources {
errMsg += "\n- " + resource
}
logger.Log.Errorf(uiUtils.Error, errMsg)
}
}
func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.Provider) []string {
leftoverResources := make([]string, 0)
if err := kubernetesProvider.RemoveService(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
resourceDesc := fmt.Sprintf("Service %s in namespace %s", kubernetes.ApiServerPodName, config.Config.MizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
if err := kubernetesProvider.RemoveDaemonSet(ctx, config.Config.MizuResourcesNamespace, kubernetes.TapperDaemonSetName); err != nil {
resourceDesc := fmt.Sprintf("DaemonSet %s in namespace %s", kubernetes.TapperDaemonSetName, config.Config.MizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
if err := kubernetesProvider.RemoveConfigMap(ctx, config.Config.MizuResourcesNamespace, kubernetes.ConfigMapName); err != nil {
resourceDesc := fmt.Sprintf("ConfigMap %s in namespace %s", kubernetes.ConfigMapName, config.Config.MizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
if err := kubernetesProvider.RemoveServicAccount(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName); err != nil {
resourceDesc := fmt.Sprintf("Service Account %s in namespace %s", kubernetes.ServiceAccountName, config.Config.MizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
if err := kubernetesProvider.RemoveRole(ctx, config.Config.MizuResourcesNamespace, kubernetes.RoleName); err != nil {
resourceDesc := fmt.Sprintf("Role %s in namespace %s", kubernetes.RoleName, config.Config.MizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
if err := kubernetesProvider.RemovePod(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
resourceDesc := fmt.Sprintf("Pod %s in namespace %s", kubernetes.ApiServerPodName, config.Config.MizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
//daemon mode resources
if err := kubernetesProvider.RemoveRoleBinding(ctx, config.Config.MizuResourcesNamespace, kubernetes.RoleBindingName); err != nil {
resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", kubernetes.RoleBindingName, config.Config.MizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
if err := kubernetesProvider.RemoveDeployment(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
resourceDesc := fmt.Sprintf("Deployment %s in namespace %s", kubernetes.ApiServerPodName, config.Config.MizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
if err := kubernetesProvider.RemovePersistentVolumeClaim(ctx, config.Config.MizuResourcesNamespace, kubernetes.PersistentVolumeClaimName); err != nil {
resourceDesc := fmt.Sprintf("PersistentVolumeClaim %s in namespace %s", kubernetes.PersistentVolumeClaimName, config.Config.MizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
if err := kubernetesProvider.RemoveRole(ctx, config.Config.MizuResourcesNamespace, kubernetes.DaemonRoleName); err != nil {
resourceDesc := fmt.Sprintf("Role %s in namespace %s", kubernetes.DaemonRoleName, config.Config.MizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
if err := kubernetesProvider.RemoveRoleBinding(ctx, config.Config.MizuResourcesNamespace, kubernetes.DaemonRoleBindingName); err != nil {
resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", kubernetes.DaemonRoleBindingName, config.Config.MizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
return leftoverResources
}
func cleanUpNonRestrictedMode(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) []string {
leftoverResources := make([]string, 0)
if err := kubernetesProvider.RemoveNamespace(ctx, config.Config.MizuResourcesNamespace); err != nil {
resourceDesc := fmt.Sprintf("Namespace %s", config.Config.MizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
} else {
defer waitUntilNamespaceDeleted(ctx, cancel, kubernetesProvider)
}
if err := kubernetesProvider.RemoveClusterRole(ctx, kubernetes.ClusterRoleName); err != nil {
resourceDesc := fmt.Sprintf("ClusterRole %s", kubernetes.ClusterRoleName)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
if err := kubernetesProvider.RemoveClusterRoleBinding(ctx, kubernetes.ClusterRoleBindingName); err != nil {
resourceDesc := fmt.Sprintf("ClusterRoleBinding %s", kubernetes.ClusterRoleBindingName)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
return leftoverResources
}
func handleDeletionError(err error, resourceDesc string, leftoverResources *[]string) {
logger.Log.Debugf("Error removing %s: %v", resourceDesc, errormessage.FormatError(err))
*leftoverResources = append(*leftoverResources, resourceDesc)
}
func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) {
// Call cancel if a terminating signal was received. Allows user to skip the wait.
go func() {
waitForFinish(ctx, cancel)
}()
if err := kubernetesProvider.WaitUtilNamespaceDeleted(ctx, config.Config.MizuResourcesNamespace); err != nil {
switch {
case ctx.Err() == context.Canceled:
logger.Log.Debugf("Do nothing. User interrupted the wait")
case err == wait.ErrWaitTimeout:
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Timeout while removing Namespace %s", config.Config.MizuResourcesNamespace))
default:
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error while waiting for Namespace %s to be deleted: %v", config.Config.MizuResourcesNamespace, errormessage.FormatError(err)))
}
}
}
func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", kubernetes.ApiServerPodName))
added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex)
podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex)
eventChan, errorChan := kubernetes.FilteredWatch(ctx, podWatchHelper, []string{config.Config.MizuResourcesNamespace}, podWatchHelper)
isPodReady := false
timeAfter := time.After(25 * time.Second)
for {
select {
case _, ok := <-added:
case wEvent, ok := <-eventChan:
if !ok {
added = nil
eventChan = nil
continue
}
logger.Log.Debugf("Watching API Server pod loop, added")
case _, ok := <-removed:
if !ok {
removed = nil
continue
}
logger.Log.Infof("%s removed", kubernetes.ApiServerPodName)
cancel()
return
case modifiedPod, ok := <-modified:
if !ok {
modified = nil
continue
}
logger.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase)
if modifiedPod.Status.Phase == core.PodPending {
if modifiedPod.Status.Conditions[0].Type == core.PodScheduled && modifiedPod.Status.Conditions[0].Status != core.ConditionTrue {
logger.Log.Debugf("Wasn't able to deploy the API server. Reason: \"%s\"", modifiedPod.Status.Conditions[0].Message)
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Wasn't able to deploy the API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
switch wEvent.Type {
case kubernetes.EventAdded:
logger.Log.Debugf("Watching API Server pod loop, added")
case kubernetes.EventDeleted:
logger.Log.Infof("%s removed", kubernetes.ApiServerPodName)
cancel()
return
case kubernetes.EventModified:
modifiedPod, err := wEvent.ToPod()
if err != nil {
logger.Log.Errorf(uiUtils.Error, err)
cancel()
break
continue
}
if len(modifiedPod.Status.ContainerStatuses) > 0 && modifiedPod.Status.ContainerStatuses[0].State.Waiting != nil && modifiedPod.Status.ContainerStatuses[0].State.Waiting.Reason == "ErrImagePull" {
logger.Log.Debugf("Wasn't able to deploy the API server. (ErrImagePull) Reason: \"%s\"", modifiedPod.Status.ContainerStatuses[0].State.Waiting.Message)
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Wasn't able to deploy the API server: failed to pull the image, for more info check logs at %v", fsUtils.GetLogFilePath()))
cancel()
break
}
}
logger.Log.Debugf("Watching API Server pod loop, modified: %v", modifiedPod.Status.Phase)
if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
isPodReady = true
go startProxyReportErrorIfAny(kubernetesProvider, cancel)
url := GetApiServerUrl()
if err := apiProvider.TestConnection(); err != nil {
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
cancel()
break
}
logger.Log.Infof("Mizu is available at %s\n", url)
uiUtils.OpenBrowser(url)
if err := apiProvider.ReportTappedPods(state.tapperSyncer.CurrentlyTappedPods); err != nil {
logger.Log.Debugf("[Error] failed update tapped pods %v", err)
if modifiedPod.Status.Phase == core.PodRunning && !isPodReady {
isPodReady = true
postApiServerStarted(ctx, kubernetesProvider, cancel, err)
}
case kubernetes.EventBookmark:
break
case kubernetes.EventError:
break
}
case err, ok := <-errorChan:
if !ok {
@@ -651,72 +359,78 @@ func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provi
}
}
func watchTapperPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", kubernetes.TapperDaemonSetName))
added, modified, removed, errorChan := kubernetes.FilteredWatch(ctx, kubernetesProvider, []string{config.Config.MizuResourcesNamespace}, podExactRegex)
var prevPodPhase core.PodPhase
func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) {
podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s", kubernetes.ApiServerPodName))
eventWatchHelper := kubernetes.NewEventWatchHelper(kubernetesProvider, podExactRegex, "pod")
eventChan, errorChan := kubernetes.FilteredWatch(ctx, eventWatchHelper, []string{config.Config.MizuResourcesNamespace}, eventWatchHelper)
for {
select {
case addedPod, ok := <-added:
case wEvent, ok := <-eventChan:
if !ok {
added = nil
eventChan = nil
continue
}
logger.Log.Debugf("Tapper is created [%s]", addedPod.Name)
case removedPod, ok := <-removed:
if !ok {
removed = nil
event, err := wEvent.ToEvent()
if err != nil {
logger.Log.Debugf("[ERROR] parsing Mizu resource event: %+v", err)
continue
}
logger.Log.Debugf("Tapper is removed [%s]", removedPod.Name)
case modifiedPod, ok := <-modified:
if !ok {
modified = nil
if state.startTime.After(event.CreationTimestamp.Time) {
continue
}
if modifiedPod.Status.Phase == core.PodPending && modifiedPod.Status.Conditions[0].Type == core.PodScheduled && modifiedPod.Status.Conditions[0].Status != core.ConditionTrue {
logger.Log.Infof(uiUtils.Red, fmt.Sprintf("Wasn't able to deploy the tapper %s. Reason: \"%s\"", modifiedPod.Name, modifiedPod.Status.Conditions[0].Message))
logger.Log.Debugf(
fmt.Sprintf("Watching API server events loop, event %s, time: %v, resource: %s (%s), reason: %s, note: %s",
event.Name,
event.CreationTimestamp.Time,
event.Regarding.Name,
event.Regarding.Kind,
event.Reason,
event.Note))
switch event.Reason {
case "FailedScheduling", "Failed":
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Mizu API Server status: %s - %s", event.Reason, event.Note))
cancel()
break
}
podStatus := modifiedPod.Status
if podStatus.Phase == core.PodPending && prevPodPhase == podStatus.Phase {
logger.Log.Debugf("Tapper %s is %s", modifiedPod.Name, strings.ToLower(string(podStatus.Phase)))
continue
}
prevPodPhase = podStatus.Phase
if podStatus.Phase == core.PodRunning {
state := podStatus.ContainerStatuses[0].State
if state.Terminated != nil {
switch state.Terminated.Reason {
case "OOMKilled":
logger.Log.Infof(uiUtils.Red, fmt.Sprintf("Tapper %s was terminated (reason: OOMKilled). You should consider increasing machine resources.", modifiedPod.Name))
}
}
}
logger.Log.Debugf("Tapper %s is %s", modifiedPod.Name, strings.ToLower(string(podStatus.Phase)))
case err, ok := <-errorChan:
if !ok {
errorChan = nil
continue
}
logger.Log.Errorf("[Error] Error in mizu tapper watch, err: %v", err)
cancel()
logger.Log.Debugf("[Error] Watching API server events loop, error: %+v", err)
case <-ctx.Done():
logger.Log.Debugf("Watching tapper pod loop, ctx done")
logger.Log.Debugf("Watching API server events loop, ctx done")
return
}
}
}
func postApiServerStarted(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc, err error) {
go startProxyReportErrorIfAny(kubernetesProvider, cancel)
url := GetApiServerUrl()
if err := apiProvider.TestConnection(); err != nil {
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
cancel()
return
}
options, _ := getMizuApiFilteringOptions()
if err = startTapperSyncer(ctx, cancel, kubernetesProvider, state.targetNamespaces, *options, state.startTime); err != nil {
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error starting mizu tapper syncer: %v", err))
cancel()
}
logger.Log.Infof("Mizu is available at %s", url)
if !config.Config.HeadlessMode {
uiUtils.OpenBrowser(url)
}
}
func getNamespaces(kubernetesProvider *kubernetes.Provider) []string {
if config.Config.Tap.AllNamespaces {
return []string{kubernetes.K8sAllNamespaces}
@@ -730,21 +444,3 @@ func getNamespaces(kubernetesProvider *kubernetes.Provider) []string {
return []string{currentNamespace}
}
}
func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) (bool, error) {
if !config.Config.IsNsRestrictedMode() {
if err := kubernetesProvider.CreateMizuRBAC(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.ClusterRoleName, kubernetes.ClusterRoleBindingName, mizu.RBACVersion); err != nil {
return false, err
}
} else {
if err := kubernetesProvider.CreateMizuRBACNamespaceRestricted(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.RoleName, kubernetes.RoleBindingName, mizu.RBACVersion); err != nil {
return false, err
}
}
if config.Config.Tap.DaemonMode {
if err := kubernetesProvider.CreateDaemonsetRBAC(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.DaemonRoleName, kubernetes.DaemonRoleBindingName, mizu.RBACVersion); err != nil {
return false, err
}
}
return true, nil
}

View File

@@ -3,6 +3,7 @@ package cmd
import (
"context"
"fmt"
"github.com/up9inc/mizu/cli/utils"
"net/http"
"github.com/up9inc/mizu/cli/apiserver"
@@ -56,9 +57,11 @@ func runMizuView() {
return
}
logger.Log.Infof("Mizu is available at %s\n", url)
logger.Log.Infof("Mizu is available at %s", url)
uiUtils.OpenBrowser(url)
if !config.Config.HeadlessMode {
uiUtils.OpenBrowser(url)
}
if isCompatible, err := version.CheckVersionCompatibility(apiServerProvider); err != nil {
logger.Log.Errorf("Failed to check versions compatibility %v", err)
@@ -69,5 +72,5 @@ func runMizuView() {
return
}
waitForFinish(ctx, cancel)
utils.WaitForFinish(ctx, cancel)
}

View File

@@ -3,9 +3,7 @@ package config
import (
"errors"
"fmt"
"github.com/up9inc/mizu/tap/api"
"io/ioutil"
"k8s.io/apimachinery/pkg/util/json"
"os"
"reflect"
"strconv"
@@ -51,6 +49,10 @@ func InitConfig(cmd *cobra.Command) error {
cmd.Flags().Visit(initFlag)
if err := Config.validate(); err != nil {
return fmt.Errorf("config validation failed, err: %v", err)
}
finalConfigPrettified, _ := uiUtils.PrettyJson(Config)
logger.Log.Debugf("Init config finished\n Final config: %v", finalConfigPrettified)
@@ -366,37 +368,3 @@ func setZeroForReadonlyFields(currentElem reflect.Value) {
}
}
}
func GetSerializedMizuAgentConfig(targetNamespaces []string, mizuApiFilteringOptions *api.TrafficFilteringOptions) (string, error) {
mizuConfig, err := getMizuAgentConfig(targetNamespaces, mizuApiFilteringOptions)
if err != nil {
return "", err
}
serializedConfig, err := json.Marshal(mizuConfig)
if err != nil {
return "", err
}
return string(serializedConfig), nil
}
func getMizuAgentConfig(targetNamespaces []string, mizuApiFilteringOptions *api.TrafficFilteringOptions) (*shared.MizuAgentConfig, error) {
serializableRegex, err := api.CompileRegexToSerializableRegexp(Config.Tap.PodRegexStr)
if err != nil {
return nil, err
}
config := shared.MizuAgentConfig{
TapTargetRegex: *serializableRegex,
MaxDBSizeBytes: Config.Tap.MaxEntriesDBSizeBytes(),
DaemonMode: Config.Tap.DaemonMode,
TargetNamespaces: targetNamespaces,
AgentImage: Config.AgentImage,
PullPolicy: Config.ImagePullPolicyStr,
DumpLogs: Config.DumpLogs,
IgnoredUserAgents: Config.Tap.IgnoredUserAgents,
TapperResources: Config.Tap.TapperResources,
MizuResourcesNamespace: Config.MizuResourcesNamespace,
MizuApiFilteringOptions: *mizuApiFilteringOptions,
AgentDatabasePath: fmt.Sprintf("%s%s", shared.DataDirPath, "entries.db"),
}
return &config, nil
}

View File

@@ -2,6 +2,7 @@ package config
import (
"fmt"
"github.com/op/go-logging"
"github.com/up9inc/mizu/cli/config/configStructs"
"github.com/up9inc/mizu/cli/mizu"
v1 "k8s.io/api/core/v1"
@@ -31,6 +32,16 @@ type ConfigStruct struct {
DumpLogs bool `yaml:"dump-logs" default:"false"`
KubeConfigPathStr string `yaml:"kube-config-path"`
ConfigFilePath string `yaml:"config-path,omitempty" readonly:""`
HeadlessMode bool `yaml:"headless" default:"false"`
LogLevelStr string `yaml:"log-level,omitempty" default:"INFO" readonly:""`
}
func(config *ConfigStruct) validate() error {
if _, err := logging.LogLevel(config.LogLevelStr); err != nil {
return fmt.Errorf("%s is not a valid log level, err: %v", config.LogLevelStr, err)
}
return nil
}
func (config *ConfigStruct) SetDefaults() {
@@ -59,3 +70,8 @@ func (config *ConfigStruct) KubeConfigPath() string {
home := homedir.HomeDir()
return filepath.Join(home, ".kube", "config")
}
func (config *ConfigStruct) LogLevel() logging.Level {
logLevel, _ := logging.LogLevel(config.LogLevelStr)
return logLevel
}

View File

@@ -3,9 +3,10 @@ package configStructs
import (
"errors"
"fmt"
"github.com/up9inc/mizu/shared"
"regexp"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/units"
)
@@ -21,29 +22,29 @@ const (
WorkspaceTapName = "workspace"
EnforcePolicyFile = "traffic-validation-file"
ContractFile = "contract"
DaemonModeTapName = "daemon"
IstioName = "istio"
)
type TapConfig struct {
UploadIntervalSec int `yaml:"upload-interval" default:"10"`
PodRegexStr string `yaml:"regex" default:".*"`
GuiPort uint16 `yaml:"gui-port" default:"8899"`
ProxyHost string `yaml:"proxy-host" default:"127.0.0.1"`
Namespaces []string `yaml:"namespaces"`
Analysis bool `yaml:"analysis" default:"false"`
AllNamespaces bool `yaml:"all-namespaces" default:"false"`
PlainTextFilterRegexes []string `yaml:"regex-masking"`
IgnoredUserAgents []string `yaml:"ignored-user-agents"`
DisableRedaction bool `yaml:"no-redact" default:"false"`
HumanMaxEntriesDBSize string `yaml:"max-entries-db-size" default:"200MB"`
DryRun bool `yaml:"dry-run" default:"false"`
Workspace string `yaml:"workspace"`
EnforcePolicyFile string `yaml:"traffic-validation-file"`
ContractFile string `yaml:"contract"`
AskUploadConfirmation bool `yaml:"ask-upload-confirmation" default:"true"`
ApiServerResources shared.Resources `yaml:"api-server-resources"`
TapperResources shared.Resources `yaml:"tapper-resources"`
DaemonMode bool `yaml:"daemon" default:"false"`
UploadIntervalSec int `yaml:"upload-interval" default:"10"`
PodRegexStr string `yaml:"regex" default:".*"`
GuiPort uint16 `yaml:"gui-port" default:"8899"`
ProxyHost string `yaml:"proxy-host" default:"127.0.0.1"`
Namespaces []string `yaml:"namespaces"`
Analysis bool `yaml:"analysis" default:"false"`
AllNamespaces bool `yaml:"all-namespaces" default:"false"`
PlainTextFilterRegexes []string `yaml:"regex-masking"`
IgnoredUserAgents []string `yaml:"ignored-user-agents"`
DisableRedaction bool `yaml:"no-redact" default:"false"`
HumanMaxEntriesDBSize string `yaml:"max-entries-db-size" default:"200MB"`
DryRun bool `yaml:"dry-run" default:"false"`
Workspace string `yaml:"workspace"`
EnforcePolicyFile string `yaml:"traffic-validation-file"`
ContractFile string `yaml:"contract"`
AskUploadConfirmation bool `yaml:"ask-upload-confirmation" default:"true"`
ApiServerResources shared.Resources `yaml:"api-server-resources"`
TapperResources shared.Resources `yaml:"tapper-resources"`
Istio bool `yaml:"istio" default:"false"`
}
func (config *TapConfig) PodRegex() *regexp.Regexp {

View File

@@ -8,6 +8,7 @@ require (
github.com/getkin/kin-openapi v0.79.0
github.com/google/go-github/v37 v37.0.0
github.com/google/uuid v1.1.2
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
github.com/spf13/cobra v1.1.3
github.com/spf13/pflag v1.0.5
github.com/up9inc/mizu/shared v0.0.0

View File

@@ -6,14 +6,17 @@ import (
)
var (
SemVer = "0.0.1"
Branch = "develop"
GitCommitHash = "" // this var is overridden using ldflags in makefile when building
BuildTimestamp = "" // this var is overridden using ldflags in makefile when building
RBACVersion = "v1"
DaemonModePersistentVolumeSizeBufferBytes = int64(500 * 1000 * 1000) //500mb
SemVer = "0.0.1"
Branch = "develop"
GitCommitHash = "" // this var is overridden using ldflags in makefile when building
BuildTimestamp = "" // this var is overridden using ldflags in makefile when building
RBACVersion = "v1"
Platform = ""
InstallModePersistentVolumeSizeBufferBytes = int64(500 * 1000 * 1000) //500mb
)
const DEVENVVAR = "MIZU_DISABLE_TELEMTRY"
func GetMizuFolderPath() string {
home, homeDirErr := os.UserHomeDir()
if homeDirErr != nil {

View File

@@ -78,6 +78,6 @@ func DumpLogs(ctx context.Context, provider *kubernetes.Provider, filePath strin
logger.Log.Debugf("Successfully added file %s", GetLogFilePath())
}
logger.Log.Infof("You can find the zip file with all logs in %s\n", filePath)
logger.Log.Infof("You can find the zip file with all logs in %s", filePath)
return nil
}

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"io/ioutil"
"net/http"
"os"
"runtime"
"strings"
"time"
@@ -24,6 +25,11 @@ func CheckVersionCompatibility(apiServerProvider *apiserver.Provider) (bool, err
return false, err
}
if !semver.SemVersion(apiSemVer).IsValid() {
logger.Log.Errorf(uiUtils.Red, fmt.Sprintf("api version (%s) is not a valid SemVer", apiSemVer))
return false, nil
}
if semver.SemVersion(apiSemVer).Major() == semver.SemVersion(mizu.SemVer).Major() &&
semver.SemVersion(apiSemVer).Minor() == semver.SemVersion(mizu.SemVer).Minor() {
return true, nil
@@ -34,6 +40,10 @@ func CheckVersionCompatibility(apiServerProvider *apiserver.Provider) (bool, err
}
func CheckNewerVersion(versionChan chan string) {
if _, present := os.LookupEnv(mizu.DEVENVVAR); present {
versionChan <- ""
return
}
logger.Log.Debugf("Checking for newer version...")
start := time.Now()
client := github.NewClient(nil)

View File

@@ -0,0 +1,168 @@
package resources
import (
"context"
"fmt"
"github.com/up9inc/mizu/cli/errormessage"
"github.com/up9inc/mizu/cli/mizu/fsUtils"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/cli/utils"
"github.com/up9inc/mizu/shared/kubernetes"
"github.com/up9inc/mizu/shared/logger"
"k8s.io/apimachinery/pkg/util/wait"
)
func CleanUpMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, isNsRestrictedMode bool, mizuResourcesNamespace string) {
logger.Log.Infof("\nRemoving mizu resources")
var leftoverResources []string
if isNsRestrictedMode {
leftoverResources = cleanUpRestrictedMode(ctx, kubernetesProvider, mizuResourcesNamespace)
} else {
leftoverResources = cleanUpNonRestrictedMode(ctx, cancel, kubernetesProvider, mizuResourcesNamespace)
}
if len(leftoverResources) > 0 {
errMsg := fmt.Sprintf("Failed to remove the following resources, for more info check logs at %s:", fsUtils.GetLogFilePath())
for _, resource := range leftoverResources {
errMsg += "\n- " + resource
}
logger.Log.Errorf(uiUtils.Error, errMsg)
}
}
func cleanUpNonRestrictedMode(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, mizuResourcesNamespace string) []string {
leftoverResources := make([]string, 0)
if err := kubernetesProvider.RemoveNamespace(ctx, mizuResourcesNamespace); err != nil {
resourceDesc := fmt.Sprintf("Namespace %s", mizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
} else {
defer waitUntilNamespaceDeleted(ctx, cancel, kubernetesProvider, mizuResourcesNamespace)
}
if resources, err := kubernetesProvider.ListManagedClusterRoles(ctx); err != nil {
resourceDesc := "ClusterRoles"
handleDeletionError(err, resourceDesc, &leftoverResources)
} else {
for _, resource := range resources.Items {
if err := kubernetesProvider.RemoveClusterRole(ctx, resource.Name); err != nil {
resourceDesc := fmt.Sprintf("ClusterRole %s", resource.Name)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
}
}
if resources, err := kubernetesProvider.ListManagedClusterRoleBindings(ctx); err != nil {
resourceDesc := "ClusterRoleBindings"
handleDeletionError(err, resourceDesc, &leftoverResources)
} else {
for _, resource := range resources.Items {
if err := kubernetesProvider.RemoveClusterRoleBinding(ctx, resource.Name); err != nil {
resourceDesc := fmt.Sprintf("ClusterRoleBinding %s", resource.Name)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
}
}
return leftoverResources
}
func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, mizuResourcesNamespace string) {
// Call cancel if a terminating signal was received. Allows user to skip the wait.
go func() {
utils.WaitForFinish(ctx, cancel)
}()
if err := kubernetesProvider.WaitUtilNamespaceDeleted(ctx, mizuResourcesNamespace); err != nil {
switch {
case ctx.Err() == context.Canceled:
logger.Log.Debugf("Do nothing. User interrupted the wait")
case err == wait.ErrWaitTimeout:
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Timeout while removing Namespace %s", mizuResourcesNamespace))
default:
logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error while waiting for Namespace %s to be deleted: %v", mizuResourcesNamespace, errormessage.FormatError(err)))
}
}
}
func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuResourcesNamespace string) []string {
leftoverResources := make([]string, 0)
if err := kubernetesProvider.RemoveService(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
resourceDesc := fmt.Sprintf("Service %s in namespace %s", kubernetes.ApiServerPodName, mizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
if err := kubernetesProvider.RemoveDaemonSet(ctx, mizuResourcesNamespace, kubernetes.TapperDaemonSetName); err != nil {
resourceDesc := fmt.Sprintf("DaemonSet %s in namespace %s", kubernetes.TapperDaemonSetName, mizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
if err := kubernetesProvider.RemoveConfigMap(ctx, mizuResourcesNamespace, kubernetes.ConfigMapName); err != nil {
resourceDesc := fmt.Sprintf("ConfigMap %s in namespace %s", kubernetes.ConfigMapName, mizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
if resources, err := kubernetesProvider.ListManagedServiceAccounts(ctx, mizuResourcesNamespace); err != nil {
resourceDesc := fmt.Sprintf("ServiceAccounts in namespace %s", mizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
} else {
for _, resource := range resources.Items {
if err := kubernetesProvider.RemoveServicAccount(ctx, mizuResourcesNamespace, resource.Name); err != nil {
resourceDesc := fmt.Sprintf("ServiceAccount %s in namespace %s", resource.Name, mizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
}
}
if resources, err := kubernetesProvider.ListManagedRoles(ctx, mizuResourcesNamespace); err != nil {
resourceDesc := fmt.Sprintf("Roles in namespace %s", mizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
} else {
for _, resource := range resources.Items {
if err := kubernetesProvider.RemoveRole(ctx, mizuResourcesNamespace, resource.Name); err != nil {
resourceDesc := fmt.Sprintf("Role %s in namespace %s", resource.Name, mizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
}
}
if resources, err := kubernetesProvider.ListManagedRoleBindings(ctx, mizuResourcesNamespace); err != nil {
resourceDesc := fmt.Sprintf("RoleBindings in namespace %s", mizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
} else {
for _, resource := range resources.Items {
if err := kubernetesProvider.RemoveRoleBinding(ctx, mizuResourcesNamespace, resource.Name); err != nil {
resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", resource.Name, mizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
}
}
if err := kubernetesProvider.RemovePod(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
resourceDesc := fmt.Sprintf("Pod %s in namespace %s", kubernetes.ApiServerPodName, mizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
//install mode resources
if err := kubernetesProvider.RemoveDeployment(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil {
resourceDesc := fmt.Sprintf("Deployment %s in namespace %s", kubernetes.ApiServerPodName, mizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
if err := kubernetesProvider.RemovePersistentVolumeClaim(ctx, mizuResourcesNamespace, kubernetes.PersistentVolumeClaimName); err != nil {
resourceDesc := fmt.Sprintf("PersistentVolumeClaim %s in namespace %s", kubernetes.PersistentVolumeClaimName, mizuResourcesNamespace)
handleDeletionError(err, resourceDesc, &leftoverResources)
}
return leftoverResources
}
func handleDeletionError(err error, resourceDesc string, leftoverResources *[]string) {
logger.Log.Debugf("Error removing %s: %v", resourceDesc, errormessage.FormatError(err))
*leftoverResources = append(*leftoverResources, resourceDesc)
}

View File

@@ -0,0 +1,198 @@
package resources
import (
"context"
"fmt"
"github.com/op/go-logging"
"github.com/up9inc/mizu/cli/errormessage"
"github.com/up9inc/mizu/cli/mizu"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/kubernetes"
"github.com/up9inc/mizu/shared/logger"
core "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)
func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level) (bool, error) {
if !isNsRestrictedMode {
if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil {
return false, err
}
}
if err := createMizuConfigmap(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, mizuResourcesNamespace); err != nil {
logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v", errormessage.FormatError(err)))
}
mizuServiceAccountExists, err := createRBACIfNecessary(ctx, kubernetesProvider, isNsRestrictedMode, mizuResourcesNamespace)
if err != nil {
logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to ensure the resources required for IP resolving. Mizu will not resolve target IPs to names. error: %v", errormessage.FormatError(err)))
}
var serviceAccountName string
if mizuServiceAccountExists {
serviceAccountName = kubernetes.ServiceAccountName
} else {
serviceAccountName = ""
}
opts := &kubernetes.ApiServerOptions{
Namespace: mizuResourcesNamespace,
PodName: kubernetes.ApiServerPodName,
PodImage: agentImage,
ServiceAccountName: serviceAccountName,
IsNamespaceRestricted: isNsRestrictedMode,
SyncEntriesConfig: syncEntriesConfig,
MaxEntriesDBSizeBytes: maxEntriesDBSizeBytes,
Resources: apiServerResources,
ImagePullPolicy: imagePullPolicy,
LogLevel: logLevel,
}
if err := createMizuApiServerPod(ctx, kubernetesProvider, opts); err != nil {
return mizuServiceAccountExists, err
}
_, err = kubernetesProvider.CreateService(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName, kubernetes.ApiServerPodName)
if err != nil {
return mizuServiceAccountExists, err
}
logger.Log.Debugf("Successfully created service: %s", kubernetes.ApiServerPodName)
return mizuServiceAccountExists, nil
}
func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, noPersistentVolumeClaim bool) error {
if !isNsRestrictedMode {
if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil {
return err
}
logger.Log.Infof("Created mizu namespace")
}
if err := createMizuConfigmap(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, mizuResourcesNamespace); err != nil {
return err
}
logger.Log.Infof("Created config map")
_, err := createRBACIfNecessary(ctx, kubernetesProvider, isNsRestrictedMode, mizuResourcesNamespace)
if err != nil {
return err
}
if err := kubernetesProvider.CreateDaemonsetRBAC(ctx, mizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.DaemonRoleName, kubernetes.DaemonRoleBindingName, mizu.RBACVersion); err != nil {
return err
}
logger.Log.Infof("Created RBAC")
serviceAccountName := kubernetes.ServiceAccountName
opts := &kubernetes.ApiServerOptions{
Namespace: mizuResourcesNamespace,
PodName: kubernetes.ApiServerPodName,
PodImage: agentImage,
ServiceAccountName: serviceAccountName,
IsNamespaceRestricted: isNsRestrictedMode,
SyncEntriesConfig: syncEntriesConfig,
MaxEntriesDBSizeBytes: maxEntriesDBSizeBytes,
Resources: apiServerResources,
ImagePullPolicy: imagePullPolicy,
LogLevel: logLevel,
}
if err := createMizuApiServerDeployment(ctx, kubernetesProvider, opts, noPersistentVolumeClaim); err != nil {
return err
}
logger.Log.Infof("Created Api Server deployment")
_, err = kubernetesProvider.CreateService(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName, kubernetes.ApiServerPodName)
if err != nil {
return err
}
logger.Log.Infof("Created Api Server service")
return nil
}
func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuResourcesNamespace string) error {
_, err := kubernetesProvider.CreateNamespace(ctx, mizuResourcesNamespace)
return err
}
func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, mizuResourcesNamespace string) error {
err := kubernetesProvider.CreateConfigMap(ctx, mizuResourcesNamespace, kubernetes.ConfigMapName, serializedValidationRules, serializedContract, serializedMizuConfig)
return err
}
func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider, isNsRestrictedMode bool, mizuResourcesNamespace string) (bool, error) {
if !isNsRestrictedMode {
if err := kubernetesProvider.CreateMizuRBAC(ctx, mizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.ClusterRoleName, kubernetes.ClusterRoleBindingName, mizu.RBACVersion); err != nil {
return false, err
}
} else {
if err := kubernetesProvider.CreateMizuRBACNamespaceRestricted(ctx, mizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.RoleName, kubernetes.RoleBindingName, mizu.RBACVersion); err != nil {
return false, err
}
}
return true, nil
}
func createMizuApiServerDeployment(ctx context.Context, kubernetesProvider *kubernetes.Provider, opts *kubernetes.ApiServerOptions, noPersistentVolumeClaim bool) error {
volumeClaimCreated := false
if !noPersistentVolumeClaim {
volumeClaimCreated = tryToCreatePersistentVolumeClaim(ctx, kubernetesProvider, opts)
}
pod, err := kubernetesProvider.GetMizuApiServerPodObject(opts, volumeClaimCreated, kubernetes.PersistentVolumeClaimName)
if err != nil {
return err
}
pod.Spec.Containers[0].LivenessProbe = &core.Probe{
Handler: core.Handler{
HTTPGet: &core.HTTPGetAction{
Path: "/echo",
Port: intstr.FromInt(shared.DefaultApiServerPort),
},
},
InitialDelaySeconds: 1,
PeriodSeconds: 10,
}
if _, err = kubernetesProvider.CreateDeployment(ctx, opts.Namespace, opts.PodName, pod); err != nil {
return err
}
logger.Log.Debugf("Successfully created API server deployment: %s", kubernetes.ApiServerPodName)
return nil
}
func tryToCreatePersistentVolumeClaim(ctx context.Context, kubernetesProvider *kubernetes.Provider, opts *kubernetes.ApiServerOptions) bool {
isDefaultStorageClassAvailable, err := kubernetesProvider.IsDefaultStorageProviderAvailable(ctx)
if err != nil {
logger.Log.Warningf(uiUtils.Yellow, "An error occured when checking if a default storage provider exists in this cluster, this means mizu data will be lost on mizu-api-server pod restart")
logger.Log.Debugf("error checking if default storage class exists: %v", err)
return false
} else if !isDefaultStorageClassAvailable {
logger.Log.Warningf(uiUtils.Yellow, "Could not find default storage provider in this cluster, this means mizu data will be lost on mizu-api-server pod restart")
return false
}
if _, err = kubernetesProvider.CreatePersistentVolumeClaim(ctx, opts.Namespace, kubernetes.PersistentVolumeClaimName, opts.MaxEntriesDBSizeBytes + mizu.InstallModePersistentVolumeSizeBufferBytes); err != nil {
logger.Log.Warningf(uiUtils.Yellow, "An error has occured while creating a persistent volume claim for mizu, this means mizu data will be lost on mizu-api-server pod restart")
logger.Log.Debugf("error creating persistent volume claim: %v", err)
return false
}
return true
}
func createMizuApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, opts *kubernetes.ApiServerOptions) error {
pod, err := kubernetesProvider.GetMizuApiServerPodObject(opts, false, "")
if err != nil {
return err
}
if _, err = kubernetesProvider.CreatePod(ctx, opts.Namespace, pod); err != nil {
return err
}
logger.Log.Debugf("Successfully created API server pod: %s", kubernetes.ApiServerPodName)
return nil
}

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"net/http"
"os"
"github.com/denisbrodbeck/machineid"
"github.com/up9inc/mizu/cli/apiserver"
@@ -62,6 +63,9 @@ func ReportAPICalls(apiProvider *apiserver.Provider) {
}
func shouldRunTelemetry() bool {
if _, present := os.LookupEnv(mizu.DEVENVVAR); present {
return false
}
if !config.Config.Telemetry {
return false
}
@@ -79,6 +83,7 @@ func sendTelemetry(telemetryType string, argsMap map[string]interface{}) error {
argsMap["buildTimestamp"] = mizu.BuildTimestamp
argsMap["branch"] = mizu.Branch
argsMap["version"] = mizu.SemVer
argsMap["Platform"] = mizu.Platform
if machineId, err := machineid.ProtectedID("mizu"); err == nil {
argsMap["machineId"] = machineId

31
cli/up9/provider.go Normal file
View File

@@ -0,0 +1,31 @@
package up9
import (
"fmt"
"net/http"
"net/url"
)
func IsTokenValid(tokenString string, envName string) bool {
whoAmIUrl, _ := url.Parse(fmt.Sprintf("https://trcc.%s/admin/whoami", envName))
req := &http.Request{
Method: http.MethodGet,
URL: whoAmIUrl,
Header: map[string][]string{
"Authorization": {fmt.Sprintf("bearer %s", tokenString)},
},
}
response, err := http.DefaultClient.Do(req)
if err != nil {
return false
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return false
}
return true
}

25
cli/utils/waitUtils.go Normal file
View File

@@ -0,0 +1,25 @@
package utils
import (
"context"
"github.com/up9inc/mizu/shared/logger"
"os"
"os/signal"
"syscall"
)
func WaitForFinish(ctx context.Context, cancel context.CancelFunc) {
logger.Log.Debugf("waiting for finish...")
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
// block until ctx cancel is called or termination signal is received
select {
case <-ctx.Done():
logger.Log.Debugf("ctx done")
break
case <-sigChan:
logger.Log.Debugf("Got termination signal, canceling execution...")
cancel()
}
}

View File

@@ -12,7 +12,7 @@ FROM golang:1.16-alpine AS builder
# Set necessary environment variables needed for our image.
ENV CGO_ENABLED=1 GOOS=linux GOARCH=amd64
RUN apk add libpcap-dev gcc g++ make bash
RUN apk add libpcap-dev gcc g++ make bash perl-utils
# Move to agent working directory (/agent-build).
WORKDIR /app/agent-build
@@ -23,12 +23,12 @@ COPY tap/go.mod tap/go.mod ../tap/
COPY tap/api/go.* ../tap/api/
RUN go mod download
# cheap trick to make the build faster (As long as go.mod wasn't changes)
RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' -e 'sqlite' | xargs go get
RUN go list -f '{{.Path}}@{{.Version}}' -m all | sed 1d | grep -e 'go-cache' | xargs go get
ARG COMMIT_HASH
ARG GIT_BRANCH
ARG BUILD_TIMESTAMP
ARG SEM_VER
ARG SEM_VER=0.0.0
# Copy and build agent code
COPY shared ../shared
@@ -36,17 +36,25 @@ COPY tap ../tap
COPY agent .
RUN go build -gcflags="all=-N -l" -o mizuagent .
# Download Basenine executable, verify the sha1sum and move it to a directory in $PATH
ADD https://github.com/up9inc/basenine/releases/download/v0.2.19/basenine_linux_amd64 ./basenine_linux_amd64
ADD https://github.com/up9inc/basenine/releases/download/v0.2.19/basenine_linux_amd64.sha256 ./basenine_linux_amd64.sha256
RUN shasum -a 256 -c basenine_linux_amd64.sha256
RUN chmod +x ./basenine_linux_amd64
COPY devops/build_extensions_debug.sh ..
RUN cd .. && /bin/bash build_extensions_debug.sh
FROM golang:1.16-alpine
RUN apk add bash libpcap-dev tcpdump
RUN apk add bash libpcap-dev
WORKDIR /app
# Copy binary and config files from /build to root folder of scratch container.
COPY --from=builder ["/app/agent-build/mizuagent", "."]
COPY --from=builder ["/app/agent-build/basenine_linux_amd64", "/usr/local/bin/basenine"]
COPY --from=builder ["/app/agent/build/extensions", "extensions"]
COPY --from=site-build ["/app/ui-build/build", "site"]

89
docs/CONFIGURATION.md Normal file
View File

@@ -0,0 +1,89 @@
![Mizu: The API Traffic Viewer for Kubernetes](../assets/mizu-logo.svg)
# Configuration options for Mizu
Mizu has many configuration options and flags that affect its behavior. Their values can be modified via command-line interface or via configuration file.
The list below covers most useful configuration options.
### Config file
Mizu behaviour can be modified via YAML configuration file located at `$HOME/.mizu/config.yaml`.
Default values for the file can be viewed via `mizu config` command.
### Applying config options via command line
To apply any configuration option via command line, use `--set` following by config option name and value, like in the following example:
```
mizu tap --set tap.dry-run=true
```
Please make sure to use full option name (`tap.dry-run` as opposed to `dry-run` only), incl. section (`tap`, in this example)
## General section
* `agent-image` - full path to Mizu container image, in format `full.path.to/your/image:tag`. Default value is set at compilation time to `gcr.io/up9-docker-hub/mizu/<branch>:<version>`
* `dump-logs` - if set to `true`, saves log files for all Mizu components (tapper, api-server, CLI) in a zip file under `$HOME/.mizu`. Default value is `false`
* `image-pull-policy` - container image pull policy for Kubernetes, default value `Always`. Other accepted values are `Never` or `IfNotExist`. Please mind the implications when changing this.
* `kube-config-path` - path to alternative kubeconfig file to use for all interactions with Kubernetes cluster. By default - `$HOME/.kubeconfig`
* `mizu-resources-namespace` - Kubernetes namespace where all Mizu-related resources are created. Default value `mizu`
* `telemetry` - report anonymous usage statistics. Default value `true`
## section `tap`
* `namespaces` - list of namespace names, in which pods are tapped. Default value is empty, meaning only pods in the current namespace are tapped. Typically supplied as command line options.
* `all-namespaces` - special flag indicating whether Mizu should search and tap pods, matching the regex, in all namespaces. Default is `false`. Please use with caution, tapping too many pods can affect resource consumption.
* `dry-run` - if true, Mizu will print list of pods matching the supplied (or default) regex and exit without actually tapping the traffic. Default value is `false`. Typically supplied as command-line option `--dry-run`
* `proxy-host` - IP address on which proxy to Mizu API service is launched; should be accessible at `proxy-host:gui-port`. Default value is `127.0.0.1`
* `gui-port` - port on which Mizu GUI is accessible, default value is `8899` (stands for `8899/tcp`)
* `regex` - regular expression used to match pods to tap, when no regex is given in the command line; default value is `.*`, which means `mizu tap` with no additional arguments is runnining as `mizu tap .*` (i.e. tap all pods found in current workspace)
* `no-redact` - instructs Mizu whether to redact certain sensitive fields in the collected traffic. Default value is `false`, i.e. Mizu will replace sentitive data values with *REDACTED* placeholder.
* `ignored-user-agents` - array of strings, describing HTTP *User-Agent* header values to be ignored. Useful to ignore Kubernetes healthcheck and other similar noisy periodic probes. Default value is empty.
* `max-entries-db-size` - maximal size of traffic stored locally in the `mizu-api-server` pod. When this size is reached, older traffic is overwritten with new entries. Default value is `200MB`
### section `tap.api-server-resources`
Kubernetes request and limit values for the `mizu-api-server` pod.
Parameters and their default values are same as used natively in Kubernetes pods:
```
cpu-limit: 750m
memory-limit: 1Gi
cpu-requests: 50m
memory-requests: 50Mi
```
### section `tap.tapper-resources`
Kubernetes request and limit values for the `mizu-tapper` pods (launched via daemonset).
Parameters and their default values are same as used natively in Kubernetes pods:
```
cpu-limit: 750m
memory-limit: 1Gi
cpu-requests: 50m
memory-requests: 50Mi
```
--
* `analsys` - enables advanced analysis of collected traffic in the UP9 coud platform. Default value is `false`
* `upload-interval` - in the *analysis* mode, push traffic to UP9 cloud every `upload-interval` seconds. Default value is `10` seconds
* `ask-upload-confirmation` - request user confirmation when uploading tapped data to UP9 cloud
## section `version`
* `debug`- print additional version and build information when `mizu version` command is invoked. Default value is `false`.

View File

@@ -0,0 +1,74 @@
# Mizu install standalone
Mizu can be run detached from the cli using the install command: `mizu install`. This type of mizu instance will run
indefinitely in the cluster.
Please note that install standalone requires you to have RBAC creation permissions, see the [permissions](PERMISSIONS.md)
doc for more details.
```bash
$ mizu install
```
## Stop mizu install
To stop the detached mizu instance and clean all cluster side resources, run `mizu clean`
```bash
$ mizu clean # mizu will continue running in cluster until clean is executed
Removing mizu resources
```
## Expose mizu web app
Mizu could be exposed at a later stage in any of the following ways:
### Using mizu view command
In a machine that can access both the cluster and a browser, you can run `mizu view` command which creates a proxy.
Besides that, all the regular ways to expose k8s pods are valid.
```bash
$ mizu view
Establishing connection to k8s cluster...
Mizu is available at http://localhost:8899
^C
..
```
### Port Forward
```bash
$ kubectl port-forward -n mizu deployment/mizu-api-server 8899:8899
```
### NodePort
```bash
$ kubectl expose -n mizu deployment mizu-api-server --name mizu-node-port --type NodePort --port 80 --target-port 8899
```
Mizu's IP is the IP of any node (get the IP with `kubectl get nodes -o wide`) and the port is the target port of the new
service (`kubectl get services -n mizu mizu-node-port`). Note that this method will expose Mizu to public access if your
nodes are public.
### LoadBalancer
```bash
$ kubectl expose deployment -n mizu --port 80 --target-port 8899 mizu-api-server --type=LoadBalancer --name=mizu-lb
service/mizu-lb exposed
..
$ kubectl get services -n mizu
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
mizu-api-server ClusterIP 10.107.200.100 <none> 80/TCP 5m5s
mizu-lb LoadBalancer 10.107.200.101 34.77.120.116 80:30141/TCP 76s
```
Note that `LoadBalancer` services only work on supported clusters (usually cloud providers) and might incur extra costs
If you changed the `mizu-resources-namespace` value, make sure the `-n mizu` flag of the `kubectl expose` command is
changed to the value of `mizu-resources-namespace`
mizu will now be available both by running `mizu view` or by accessing the `EXTERNAL-IP` of the `mizu-lb` service
through your browser.

46
docs/ISTIO.md Normal file
View File

@@ -0,0 +1,46 @@
![Mizu: The API Traffic Viewer for Kubernetes](../assets/mizu-logo.svg)
# Istio mutual tls (mtls) with Mizu
This document describe how Mizu tapper handles workloads configured with mtls, making the internal traffic between services in a cluster to be encrypted.
Besides Istio there are other service meshes that implement mtls. However, as of now Istio is the most used one, and this is why we are focusing on it.
In order to create an Istio setup for development, follow those steps:
1. Deploy a sample application to a Kubernetes cluster, the sample application needs to make internal service to service calls
2. SSH to one of the nodes, and run `tcpdump`
3. Make sure you see the internal service to service calls in a plain text
4. Deploy Istio to the cluster - make sure it is attached to all pods of the sample application, and that it is configured with mtls (default)
5. Run `tcpdump` again, make sure you don't see the internal service to service calls in a plain text
## The connection between Istio and Envoy
In order to implement its service mesh capabilities, [Istio](https://istio.io) use an [Envoy](https://www.envoyproxy.io) sidecar in front of every pod in the cluster. The Envoy is responsible for the mtls communication, and that's why we are focusing on Envoy proxy.
In the future we might see more players in that field, then we'll have to either add support for each of them or go with a unified eBPF solution.
## Network namespaces
A [linux network namespace](https://man7.org/linux/man-pages/man7/network_namespaces.7.html) is an isolation that limit the process view of the network. In the container world it used to isolate one container from another. In the Kubernetes world it used to isolate a pod from another. That means that two containers running on the same pod share the same network namespace. A container can reach a container in the same pod by accessing `localhost`.
An Envoy proxy configured with mtls receives the inbound traffic directed to the pod, decrypts it and sends it via `localhost` to the target container.
## Tapping mtls traffic
In order for Mizu to be able to see the decrypted traffic it needs to listen on the same network namespace of the target pod. Multiple threads of the same process can have different network namespaces.
[gopacket](https://github.com/google/gopacket) uses [libpacp](https://github.com/the-tcpdump-group/libpcap) by default for capturing the traffic. Libpacap doesn't support network namespaces and we can't ask it to listen to traffic on a different namespace. However, we can change the network namespace of the calling thread and then start libpcap to see the traffic on a different namespace.
## Finding the network namespace of a running process
The network namespace of a running process can be found in `/proc/PID/ns/net` link. Once we have this link, we can ask Linux to change the network namespace of a thread to this one.
This mean that Mizu needs to have access to the `/proc` (procfs) of the running node.
## Finding the network namespace of a running pod
In order for Mizu to be able to listen to mtls traffic, it needs to get the PIDs of the the running pods, filter them according to the user filters and then start listen to their internal network namespace traffic.
There is no official way in Kubernetes to get from pod to PID. The CRI implementation purposefully doesn't force a pod to be a processes on the host. It can be a Virtual Machine as well like [Kata containers](https://katacontainers.io)
While we can provide a solution for various CRIs (like Docker, Containerd and CRI-O) it's better to have a unified solution. In order to achieve that, Mizu scans all the processes in the host, and finds the Envoy processes using their `/proc/PID/exe` link.
Once Mizu detects an Envoy process, it need to check whether this specific Envoy process is relevant according the user filters. The user filters are a list of `CLUSTER_IPS`. The tapper gets them via the `TapOpts.FilterAuthorities` list.
Istio sends an `INSTANCE_IP` environment variable to every Envoy proxy process. By examining the Envoy process's environment variables we can see whether it's relevant or not. Examining a process environment variables is done by reading the `/proc/PID/envion` file.
## Edge cases
The method we use to find Envoy processes and correlate them to the cluster IPs may be inaccurate in certain situations. If, for example, a user runs an Envoy process manually, and set its `INSTANCE_IP` environment variable to one of the `CLUSTER_IPS` the tapper gets, then Mizu will capture traffic for it.

View File

@@ -57,9 +57,11 @@ Mizu needs following permissions on your Kubernetes cluster to run properly
- get
```
## Permissions required for service / pod name resolving (opt)
## Permissions required running with install command or (optional) for service / pod name resolving
Optionally, for proper resolving of IP addresses to Kubernetes service name, Mizu needs below permissions:
Mandatory permissions for running with install command.
Optional for service/pod name resolving in non install standalone
```yaml
- apiGroups:

View File

@@ -7,15 +7,15 @@ rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch", "delete"]
- apiGroups: [ "" ]
- apiGroups: [ "apps" ]
resources: [ "deployments" ]
verbs: [ "create", "delete" ]
verbs: [ "get", "create", "delete" ]
- apiGroups: [""]
resources: ["services"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: ["apps"]
resources: ["daemonsets"]
verbs: ["create", "patch", "delete"]
verbs: ["get", "create", "patch", "delete", "list"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get", "list", "watch", "create", "delete"]
@@ -49,6 +49,9 @@ rules:
- apiGroups: ["", "apps", "extensions"]
resources: ["endpoints"]
verbs: ["get", "list", "watch"]
- apiGroups: ["events.k8s.io"]
resources: ["events"]
verbs: ["list", "watch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1

View File

@@ -23,6 +23,9 @@ rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "create", "delete"]
- apiGroups: ["events.k8s.io"]
resources: ["events"]
verbs: ["list", "watch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1

View File

@@ -46,6 +46,9 @@ rules:
- apiGroups: ["", "apps", "extensions"]
resources: ["endpoints"]
verbs: ["get", "list", "watch"]
- apiGroups: ["events.k8s.io"]
resources: ["events"]
verbs: ["list", "watch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1

View File

@@ -8,7 +8,7 @@ rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch", "delete"]
- apiGroups: [ "" ]
- apiGroups: [ "apps" ]
resources: [ "deployments" ]
verbs: [ "get", "create", "delete" ]
- apiGroups: [""]
@@ -16,7 +16,7 @@ rules:
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: ["apps"]
resources: ["daemonsets"]
verbs: ["get", "create", "patch", "delete"]
verbs: ["get", "create", "patch", "delete", "list"]
- apiGroups: [""]
resources: ["services/proxy"]
verbs: ["get"]
@@ -32,7 +32,7 @@ rules:
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["rolebindings"]
verbs: ["get", "create", "delete"]
- apiGroups: ["apps", "extensions"]
- apiGroups: ["apps", "extensions", ""]
resources: ["pods"]
verbs: ["get", "list", "watch"]
- apiGroups: ["apps", "extensions"]
@@ -41,6 +41,9 @@ rules:
- apiGroups: ["", "apps", "extensions"]
resources: ["endpoints"]
verbs: ["get", "list", "watch"]
- apiGroups: ["events.k8s.io"]
resources: ["events"]
verbs: ["list", "watch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1

View File

@@ -38,6 +38,9 @@ rules:
- apiGroups: ["", "apps", "extensions"]
resources: ["endpoints"]
verbs: ["get", "list", "watch"]
- apiGroups: ["events.k8s.io"]
resources: ["events"]
verbs: ["list", "watch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1

View File

@@ -20,6 +20,9 @@ rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "create", "delete"]
- apiGroups: ["events.k8s.io"]
resources: ["events"]
verbs: ["list", "watch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1

View File

@@ -38,6 +38,9 @@ rules:
- apiGroups: ["", "apps", "extensions"]
resources: ["endpoints"]
verbs: ["get", "list", "watch"]
- apiGroups: ["events.k8s.io"]
resources: ["events"]
verbs: ["list", "watch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1

View File

@@ -13,5 +13,7 @@ const (
ConfigFileName = "mizu-config.json"
GoGCEnvVar = "GOGC"
DefaultApiServerPort = 8899
DebugModeEnvVar = "MIZU_DEBUG"
LogLevelEnvVar = "LOG_LEVEL"
BasenineHost = "localhost"
BaseninePort = "9099"
)

View File

@@ -17,3 +17,12 @@ const (
PersistentVolumeClaimName = MizuResourcesPrefix + "volume-claim"
MinKubernetesServerVersion = "1.16.0"
)
const (
LabelPrefixApp = "app.kubernetes.io/"
LabelManagedBy = LabelPrefixApp + "managed-by"
LabelCreatedBy = LabelPrefixApp + "created-by"
LabelValueMizu = "mizu"
LabelValueMizuCLI = "mizu-cli"
LabelValueMizuAgent = "mizu-agent"
)

View File

@@ -0,0 +1,52 @@
package kubernetes
import (
"context"
"regexp"
"strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
)
type EventWatchHelper struct {
kubernetesProvider *Provider
NameRegexFilter *regexp.Regexp
Kind string
}
func NewEventWatchHelper(kubernetesProvider *Provider, NameRegexFilter *regexp.Regexp, kind string) *EventWatchHelper {
return &EventWatchHelper{
kubernetesProvider: kubernetesProvider,
NameRegexFilter: NameRegexFilter,
Kind: kind,
}
}
// Implements the EventFilterer Interface
func (wh *EventWatchHelper) Filter(wEvent *WatchEvent) (bool, error) {
event, err := wEvent.ToEvent()
if err != nil {
return false, nil
}
if !wh.NameRegexFilter.MatchString(event.Name) {
return false, nil
}
if strings.ToLower(event.Regarding.Kind) != strings.ToLower(wh.Kind) {
return false, nil
}
return true, nil
}
// Implements the WatchCreator Interface
func (wh *EventWatchHelper) NewWatcher(ctx context.Context, namespace string) (watch.Interface, error) {
watcher, err := wh.kubernetesProvider.clientSet.EventsV1().Events(namespace).Watch(ctx, metav1.ListOptions{Watch: true})
if err != nil {
return nil, err
}
return watcher, nil
}

View File

@@ -3,6 +3,7 @@ package kubernetes
import (
"context"
"fmt"
"github.com/op/go-logging"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/debounce"
"github.com/up9inc/mizu/shared/logger"
@@ -15,18 +16,21 @@ import (
const updateTappersDelay = 5 * time.Second
type TappedPodChangeEvent struct {
Added []core.Pod
Removed []core.Pod
Added []core.Pod
Removed []core.Pod
}
// MizuTapperSyncer uses a k8s pod watch to update tapper daemonsets when targeted pods are removed or created
type MizuTapperSyncer struct {
context context.Context
CurrentlyTappedPods []core.Pod
config TapperSyncerConfig
kubernetesProvider *Provider
TapPodChangesOut chan TappedPodChangeEvent
ErrorOut chan K8sTapManagerError
startTime time.Time
context context.Context
CurrentlyTappedPods []core.Pod
config TapperSyncerConfig
kubernetesProvider *Provider
TapPodChangesOut chan TappedPodChangeEvent
TapperStatusChangedOut chan shared.TapperStatus
ErrorOut chan K8sTapManagerError
nodeToTappedPodMap map[string][]core.Pod
}
type TapperSyncerConfig struct {
@@ -36,20 +40,23 @@ type TapperSyncerConfig struct {
AgentImage string
TapperResources shared.Resources
ImagePullPolicy core.PullPolicy
DumpLogs bool
LogLevel logging.Level
IgnoredUserAgents []string
MizuApiFilteringOptions api.TrafficFilteringOptions
MizuServiceAccountExists bool
Istio bool
}
func CreateAndStartMizuTapperSyncer(ctx context.Context, kubernetesProvider *Provider, config TapperSyncerConfig) (*MizuTapperSyncer, error) {
func CreateAndStartMizuTapperSyncer(ctx context.Context, kubernetesProvider *Provider, config TapperSyncerConfig, startTime time.Time) (*MizuTapperSyncer, error) {
syncer := &MizuTapperSyncer{
context: ctx,
CurrentlyTappedPods: make([]core.Pod, 0),
config: config,
kubernetesProvider: kubernetesProvider,
TapPodChangesOut: make(chan TappedPodChangeEvent, 100),
ErrorOut: make(chan K8sTapManagerError, 100),
startTime: startTime.Truncate(time.Second), // Round down because k8s CreationTimestamp is given in 1 sec resolution.
context: ctx,
CurrentlyTappedPods: make([]core.Pod, 0),
config: config,
kubernetesProvider: kubernetesProvider,
TapPodChangesOut: make(chan TappedPodChangeEvent, 100),
TapperStatusChangedOut: make(chan shared.TapperStatus, 100),
ErrorOut: make(chan K8sTapManagerError, 100),
}
if err, _ := syncer.updateCurrentlyTappedPods(); err != nil {
@@ -61,11 +68,120 @@ func CreateAndStartMizuTapperSyncer(ctx context.Context, kubernetesProvider *Pro
}
go syncer.watchPodsForTapping()
go syncer.watchTapperEvents()
go syncer.watchTapperPods()
return syncer, nil
}
func (tapperSyncer *MizuTapperSyncer) watchTapperPods() {
mizuResourceRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", TapperPodName))
podWatchHelper := NewPodWatchHelper(tapperSyncer.kubernetesProvider, mizuResourceRegex)
eventChan, errorChan := FilteredWatch(tapperSyncer.context, podWatchHelper, []string{tapperSyncer.config.MizuResourcesNamespace}, podWatchHelper)
for {
select {
case wEvent, ok := <-eventChan:
if !ok {
eventChan = nil
continue
}
pod, err := wEvent.ToPod()
if err != nil {
logger.Log.Debugf("[ERROR] parsing Mizu resource pod: %+v", err)
continue
}
if tapperSyncer.startTime.After(pod.CreationTimestamp.Time) {
continue
}
logger.Log.Debugf("Watching tapper pods loop, tapper: %v, node: %v, status: %v", pod.Name, pod.Spec.NodeName, pod.Status.Phase)
if pod.Spec.NodeName != "" {
tapperStatus := shared.TapperStatus{TapperName: pod.Name, NodeName: pod.Spec.NodeName, Status: string(pod.Status.Phase)}
tapperSyncer.TapperStatusChangedOut <- tapperStatus
}
case err, ok := <-errorChan:
if !ok {
errorChan = nil
continue
}
logger.Log.Debugf("[ERROR] Watching tapper pods loop, error: %+v", err)
case <-tapperSyncer.context.Done():
logger.Log.Debugf("Watching tapper pods loop, ctx done")
return
}
}
}
func (tapperSyncer *MizuTapperSyncer) watchTapperEvents() {
mizuResourceRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", TapperPodName))
eventWatchHelper := NewEventWatchHelper(tapperSyncer.kubernetesProvider, mizuResourceRegex, "pod")
eventChan, errorChan := FilteredWatch(tapperSyncer.context, eventWatchHelper, []string{tapperSyncer.config.MizuResourcesNamespace}, eventWatchHelper)
for {
select {
case wEvent, ok := <-eventChan:
if !ok {
eventChan = nil
continue
}
event, err := wEvent.ToEvent()
if err != nil {
logger.Log.Debugf("[ERROR] parsing Mizu resource event: %+v", err)
continue
}
if tapperSyncer.startTime.After(event.CreationTimestamp.Time) {
continue
}
logger.Log.Debugf(
fmt.Sprintf("Watching tapper events loop, event %s, time: %v, resource: %s (%s), reason: %s, note: %s",
event.Name,
event.CreationTimestamp.Time,
event.Regarding.Name,
event.Regarding.Kind,
event.Reason,
event.Note))
pod, err1 := tapperSyncer.kubernetesProvider.GetPod(tapperSyncer.context, tapperSyncer.config.MizuResourcesNamespace, event.Regarding.Name)
if err1 != nil {
logger.Log.Debugf(fmt.Sprintf("Couldn't get tapper pod %s", event.Regarding.Name))
continue
}
nodeName := ""
if event.Reason != "FailedScheduling" {
nodeName = pod.Spec.NodeName
} else {
nodeName = pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchFields[0].Values[0]
}
tapperStatus := shared.TapperStatus{TapperName: pod.Name, NodeName: nodeName, Status: string(pod.Status.Phase)}
tapperSyncer.TapperStatusChangedOut <- tapperStatus
case err, ok := <-errorChan:
if !ok {
errorChan = nil
continue
}
logger.Log.Debugf("[ERROR] Watching tapper events loop, error: %+v", err)
case <-tapperSyncer.context.Done():
logger.Log.Debugf("Watching tapper events loop, ctx done")
return
}
}
}
func (tapperSyncer *MizuTapperSyncer) watchPodsForTapping() {
added, modified, removed, errorChan := FilteredWatch(tapperSyncer.context, tapperSyncer.kubernetesProvider, tapperSyncer.config.TargetNamespaces, &tapperSyncer.config.PodFilterRegex)
podWatchHelper := NewPodWatchHelper(tapperSyncer.kubernetesProvider, &tapperSyncer.config.PodFilterRegex)
eventChan, errorChan := FilteredWatch(tapperSyncer.context, podWatchHelper, tapperSyncer.config.TargetNamespaces, podWatchHelper)
restartTappers := func() {
err, changeFound := tapperSyncer.updateCurrentlyTappedPods()
@@ -91,37 +207,40 @@ func (tapperSyncer *MizuTapperSyncer) watchPodsForTapping() {
for {
select {
case pod, ok := <-added:
case wEvent, ok := <-eventChan:
if !ok {
added = nil
eventChan = nil
continue
}
logger.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace)
restartTappersDebouncer.SetOn()
case pod, ok := <-removed:
if !ok {
removed = nil
pod, err := wEvent.ToPod()
if err != nil {
tapperSyncer.handleErrorInWatchLoop(err, restartTappersDebouncer)
continue
}
logger.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace)
restartTappersDebouncer.SetOn()
case pod, ok := <-modified:
if !ok {
modified = nil
continue
}
logger.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP)
// Act only if the modified pod has already obtained an IP address.
// After filtering for IPs, on a normal pod restart this includes the following events:
// - Pod deletion
// - Pod reaches start state
// - Pod reaches ready state
// Ready/unready transitions might also trigger this event.
if pod.Status.PodIP != "" {
switch wEvent.Type {
case EventAdded:
logger.Log.Debugf("Added matching pod %s, ns: %s", pod.Name, pod.Namespace)
restartTappersDebouncer.SetOn()
case EventDeleted:
logger.Log.Debugf("Removed matching pod %s, ns: %s", pod.Name, pod.Namespace)
restartTappersDebouncer.SetOn()
case EventModified:
logger.Log.Debugf("Modified matching pod %s, ns: %s, phase: %s, ip: %s", pod.Name, pod.Namespace, pod.Status.Phase, pod.Status.PodIP)
// Act only if the modified pod has already obtained an IP address.
// After filtering for IPs, on a normal pod restart this includes the following events:
// - Pod deletion
// - Pod reaches start state
// - Pod reaches ready state
// Ready/unready transitions might also trigger this event.
if pod.Status.PodIP != "" {
restartTappersDebouncer.SetOn()
}
case EventBookmark:
break
case EventError:
break
}
case err, ok := <-errorChan:
if !ok {
@@ -129,12 +248,8 @@ func (tapperSyncer *MizuTapperSyncer) watchPodsForTapping() {
continue
}
logger.Log.Debugf("Watching pods loop, got error %v, stopping `restart tappers debouncer`", err)
restartTappersDebouncer.Cancel()
tapperSyncer.ErrorOut <- K8sTapManagerError{
OriginalError: err,
TapManagerReason: TapManagerPodWatchError,
}
tapperSyncer.handleErrorInWatchLoop(err, restartTappersDebouncer)
continue
case <-tapperSyncer.context.Done():
logger.Log.Debugf("Watching pods loop, context done, stopping `restart tappers debouncer`")
@@ -145,6 +260,15 @@ func (tapperSyncer *MizuTapperSyncer) watchPodsForTapping() {
}
}
func (tapperSyncer *MizuTapperSyncer) handleErrorInWatchLoop(err error, restartTappersDebouncer *debounce.Debouncer) {
logger.Log.Debugf("Watching pods loop, got error %v, stopping `restart tappers debouncer`", err)
restartTappersDebouncer.Cancel()
tapperSyncer.ErrorOut <- K8sTapManagerError{
OriginalError: err,
TapManagerReason: TapManagerPodWatchError,
}
}
func (tapperSyncer *MizuTapperSyncer) updateCurrentlyTappedPods() (err error, changesFound bool) {
if matchingPods, err := tapperSyncer.kubernetesProvider.ListAllRunningPodsMatchingRegex(tapperSyncer.context, &tapperSyncer.config.PodFilterRegex, tapperSyncer.config.TargetNamespaces); err != nil {
return err, false
@@ -159,9 +283,10 @@ func (tapperSyncer *MizuTapperSyncer) updateCurrentlyTappedPods() (err error, ch
}
if len(addedPods) > 0 || len(removedPods) > 0 {
tapperSyncer.CurrentlyTappedPods = podsToTap
tapperSyncer.nodeToTappedPodMap = GetNodeHostToTappedPodsMap(tapperSyncer.CurrentlyTappedPods)
tapperSyncer.TapPodChangesOut <- TappedPodChangeEvent{
Added: addedPods,
Removed: removedPods,
Added: addedPods,
Removed: removedPods,
}
return nil, true
}
@@ -170,9 +295,7 @@ func (tapperSyncer *MizuTapperSyncer) updateCurrentlyTappedPods() (err error, ch
}
func (tapperSyncer *MizuTapperSyncer) updateMizuTappers() error {
nodeToTappedPodIPMap := GetNodeHostToTappedPodIpsMap(tapperSyncer.CurrentlyTappedPods)
if len(nodeToTappedPodIPMap) > 0 {
if len(tapperSyncer.nodeToTappedPodMap) > 0 {
var serviceAccountName string
if tapperSyncer.config.MizuServiceAccountExists {
serviceAccountName = ServiceAccountName
@@ -187,16 +310,17 @@ func (tapperSyncer *MizuTapperSyncer) updateMizuTappers() error {
tapperSyncer.config.AgentImage,
TapperPodName,
fmt.Sprintf("%s.%s.svc.cluster.local", ApiServerPodName, tapperSyncer.config.MizuResourcesNamespace),
nodeToTappedPodIPMap,
tapperSyncer.nodeToTappedPodMap,
serviceAccountName,
tapperSyncer.config.TapperResources,
tapperSyncer.config.ImagePullPolicy,
tapperSyncer.config.MizuApiFilteringOptions,
tapperSyncer.config.DumpLogs,
tapperSyncer.config.LogLevel,
tapperSyncer.config.Istio,
); err != nil {
return err
}
logger.Log.Debugf("Successfully created %v tappers", len(nodeToTappedPodIPMap))
logger.Log.Debugf("Successfully created %v tappers", len(tapperSyncer.nodeToTappedPodMap))
} else {
if err := tapperSyncer.kubernetesProvider.RemoveDaemonSet(tapperSyncer.context, tapperSyncer.config.MizuResourcesNamespace, TapperDaemonSetName); err != nil {
return err

View File

@@ -0,0 +1,45 @@
package kubernetes
import (
"context"
"regexp"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
)
type PodWatchHelper struct {
kubernetesProvider *Provider
NameRegexFilter *regexp.Regexp
}
func NewPodWatchHelper(kubernetesProvider *Provider, NameRegexFilter *regexp.Regexp) *PodWatchHelper {
return &PodWatchHelper{
kubernetesProvider: kubernetesProvider,
NameRegexFilter: NameRegexFilter,
}
}
// Implements the EventFilterer Interface
func (wh *PodWatchHelper) Filter(wEvent *WatchEvent) (bool, error) {
pod, err := wEvent.ToPod()
if err != nil {
return false, nil
}
if !wh.NameRegexFilter.MatchString(pod.Name) {
return false, nil
}
return true, nil
}
// Implements the WatchCreator Interface
func (wh *PodWatchHelper) NewWatcher(ctx context.Context, namespace string) (watch.Interface, error) {
watcher, err := wh.kubernetesProvider.clientSet.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{Watch: true})
if err != nil {
return nil, err
}
return watcher, nil
}

View File

@@ -6,11 +6,16 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/url"
"path/filepath"
"regexp"
"github.com/op/go-logging"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/shared/logger"
"github.com/up9inc/mizu/shared/semver"
"github.com/up9inc/mizu/tap/api"
"io"
v1 "k8s.io/api/apps/v1"
core "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
@@ -31,9 +36,6 @@ import (
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/clientcmd"
watchtools "k8s.io/client-go/tools/watch"
"net/url"
"path/filepath"
"regexp"
)
type Provider struct {
@@ -41,10 +43,14 @@ type Provider struct {
kubernetesConfig clientcmd.ClientConfig
clientConfig restclient.Config
Namespace string
managedBy string
createdBy string
}
const (
fieldManagerName = "mizu-manager"
procfsVolumeName = "proc"
procfsMountPath = "/hostproc"
)
func NewProvider(kubeConfigPath string) (*Provider, error) {
@@ -82,6 +88,8 @@ func NewProvider(kubeConfigPath string) (*Provider, error) {
clientSet: clientSet,
kubernetesConfig: kubernetesConfig,
clientConfig: *restClientConfig,
managedBy: LabelValueMizu,
createdBy: LabelValueMizuCLI,
}, nil
}
@@ -99,6 +107,8 @@ func NewProviderInCluster() (*Provider, error) {
clientSet: clientSet,
kubernetesConfig: nil, // not relevant in cluster
clientConfig: *restClientConfig,
managedBy: LabelValueMizu,
createdBy: LabelValueMizuAgent,
}, nil
}
@@ -150,18 +160,14 @@ func (provider *Provider) WaitUtilNamespaceDeleted(ctx context.Context, name str
return err
}
func (provider *Provider) GetPodWatcher(ctx context.Context, namespace string) watch.Interface {
watcher, err := provider.clientSet.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{Watch: true})
if err != nil {
panic(err.Error())
}
return watcher
}
func (provider *Provider) CreateNamespace(ctx context.Context, name string) (*core.Namespace, error) {
namespaceSpec := &core.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Labels: map[string]string{
LabelManagedBy: provider.managedBy,
LabelCreatedBy: provider.createdBy,
},
},
}
return provider.clientSet.CoreV1().Namespaces().Create(ctx, namespaceSpec, metav1.CreateOptions{})
@@ -177,7 +183,7 @@ type ApiServerOptions struct {
MaxEntriesDBSizeBytes int64
Resources shared.Resources
ImagePullPolicy core.PullPolicy
DumpLogs bool
LogLevel logging.Level
}
func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, mountVolumeClaim bool, volumeClaimName string) (*core.Pod, error) {
@@ -239,22 +245,19 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
},
})
volumeMounts = append(volumeMounts, core.VolumeMount{
Name: volumeClaimName,
MountPath: shared.DataDirPath,
Name: volumeClaimName,
MountPath: shared.DataDirPath,
})
}
port := intstr.FromInt(shared.DefaultApiServerPort)
debugMode := ""
if opts.DumpLogs {
debugMode = "1"
}
pod := &core.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: opts.PodName,
Labels: map[string]string{"app": opts.PodName},
Name: opts.PodName,
Labels: map[string]string{
"app": opts.PodName,
LabelManagedBy: provider.managedBy,
LabelCreatedBy: provider.createdBy,
},
},
Spec: core.PodSpec{
Containers: []core.Container{
@@ -262,16 +265,16 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
Name: opts.PodName,
Image: opts.PodImage,
ImagePullPolicy: opts.ImagePullPolicy,
VolumeMounts: volumeMounts,
Command: command,
VolumeMounts: volumeMounts,
Command: command,
Env: []core.EnvVar{
{
Name: shared.SyncEntriesConfigEnvVar,
Value: string(marshaledSyncEntriesConfig),
},
{
Name: shared.DebugModeEnvVar,
Value: debugMode,
Name: shared.LogLevelEnvVar,
Value: opts.LogLevel.String(),
},
},
Resources: core.ResourceRequirements{
@@ -284,28 +287,9 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
"memory": memRequests,
},
},
ReadinessProbe: &core.Probe{
Handler: core.Handler{
TCPSocket: &core.TCPSocketAction{
Port: port,
},
},
InitialDelaySeconds: 5,
PeriodSeconds: 10,
},
LivenessProbe: &core.Probe{
Handler: core.Handler{
HTTPGet: &core.HTTPGetAction{
Path: "/echo",
Port: port,
},
},
InitialDelaySeconds: 5,
PeriodSeconds: 10,
},
},
},
Volumes: volumes,
Volumes: volumes,
DNSPolicy: core.DNSClusterFirstWithHostNet,
TerminationGracePeriodSeconds: new(int64),
},
@@ -318,7 +302,6 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun
return pod, nil
}
func (provider *Provider) CreatePod(ctx context.Context, namespace string, podSpec *core.Pod) (*core.Pod, error) {
return provider.clientSet.CoreV1().Pods(namespace).Create(ctx, podSpec, metav1.CreateOptions{})
}
@@ -333,14 +316,18 @@ func (provider *Provider) CreateDeployment(ctx context.Context, namespace string
}
deployment := &v1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: deploymentName,
Name: deploymentName,
Labels: map[string]string{
LabelManagedBy: provider.managedBy,
LabelCreatedBy: provider.createdBy,
},
},
Spec: v1.DeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": podSpec.ObjectMeta.Labels["app"]},
},
Template: *podTemplate,
Strategy: v1.DeploymentStrategy{},
Template: *podTemplate,
Strategy: v1.DeploymentStrategy{},
},
}
return provider.clientSet.AppsV1().Deployments(namespace).Create(ctx, deployment, metav1.CreateOptions{})
@@ -349,7 +336,11 @@ func (provider *Provider) CreateDeployment(ctx context.Context, namespace string
func (provider *Provider) CreateService(ctx context.Context, namespace string, serviceName string, appLabelValue string) (*core.Service, error) {
service := core.Service{
ObjectMeta: metav1.ObjectMeta{
Name: serviceName,
Name: serviceName,
Labels: map[string]string{
LabelManagedBy: provider.managedBy,
LabelCreatedBy: provider.createdBy,
},
},
Spec: core.ServiceSpec{
Ports: []core.ServicePort{{TargetPort: intstr.FromInt(shared.DefaultApiServerPort), Port: 80}},
@@ -361,8 +352,8 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s
}
func (provider *Provider) DoesServicesExist(ctx context.Context, namespace string, name string) (bool, error) {
resource, err := provider.clientSet.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{})
return provider.doesResourceExist(resource, err)
serviceResource, err := provider.clientSet.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{})
return provider.doesResourceExist(serviceResource, err)
}
func (provider *Provider) doesResourceExist(resource interface{}, err error) (bool, error) {
@@ -381,14 +372,22 @@ func (provider *Provider) doesResourceExist(resource interface{}, err error) (bo
func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, serviceAccountName string, clusterRoleName string, clusterRoleBindingName string, version string) error {
serviceAccount := &core.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: serviceAccountName,
Labels: map[string]string{"mizu-cli-version": version},
Name: serviceAccountName,
Labels: map[string]string{
"mizu-cli-version": version,
LabelManagedBy: provider.managedBy,
LabelCreatedBy: provider.createdBy,
},
},
}
clusterRole := &rbac.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: clusterRoleName,
Labels: map[string]string{"mizu-cli-version": version},
Labels: map[string]string{
"mizu-cli-version": version,
LabelManagedBy: provider.managedBy,
LabelCreatedBy: provider.createdBy,
},
},
Rules: []rbac.PolicyRule{
{
@@ -401,7 +400,11 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string,
clusterRoleBinding := &rbac.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: clusterRoleBindingName,
Labels: map[string]string{"mizu-cli-version": version},
Labels: map[string]string{
"mizu-cli-version": version,
LabelManagedBy: provider.managedBy,
LabelCreatedBy: provider.createdBy,
},
},
RoleRef: rbac.RoleRef{
Name: clusterRoleName,
@@ -434,14 +437,22 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string,
func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context, namespace string, serviceAccountName string, roleName string, roleBindingName string, version string) error {
serviceAccount := &core.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: serviceAccountName,
Labels: map[string]string{"mizu-cli-version": version},
Name: serviceAccountName,
Labels: map[string]string{
"mizu-cli-version": version,
LabelManagedBy: provider.managedBy,
LabelCreatedBy: provider.createdBy,
},
},
}
role := &rbac.Role{
ObjectMeta: metav1.ObjectMeta{
Name: roleName,
Labels: map[string]string{"mizu-cli-version": version},
Labels: map[string]string{
"mizu-cli-version": version,
LabelManagedBy: provider.managedBy,
LabelCreatedBy: provider.createdBy,
},
},
Rules: []rbac.PolicyRule{
{
@@ -454,7 +465,11 @@ func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context,
roleBinding := &rbac.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: roleBindingName,
Labels: map[string]string{"mizu-cli-version": version},
Labels: map[string]string{
"mizu-cli-version": version,
LabelManagedBy: provider.managedBy,
LabelCreatedBy: provider.createdBy,
},
},
RoleRef: rbac.RoleRef{
Name: roleName,
@@ -488,7 +503,11 @@ func (provider *Provider) CreateDaemonsetRBAC(ctx context.Context, namespace str
role := &rbac.Role{
ObjectMeta: metav1.ObjectMeta{
Name: roleName,
Labels: map[string]string{"mizu-cli-version": version},
Labels: map[string]string{
"mizu-cli-version": version,
LabelManagedBy: provider.managedBy,
LabelCreatedBy: provider.createdBy,
},
},
Rules: []rbac.PolicyRule{
{
@@ -496,12 +515,21 @@ func (provider *Provider) CreateDaemonsetRBAC(ctx context.Context, namespace str
Resources: []string{"daemonsets"},
Verbs: []string{"patch", "get", "list", "create", "delete"},
},
{
APIGroups: []string{"events.k8s.io"},
Resources: []string{"events"},
Verbs: []string{"list", "watch"},
},
},
}
roleBinding := &rbac.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: roleBindingName,
Labels: map[string]string{"mizu-cli-version": version},
Labels: map[string]string{
"mizu-cli-version": version,
LabelManagedBy: provider.managedBy,
LabelCreatedBy: provider.createdBy,
},
},
RoleRef: rbac.RoleRef{
Name: roleName,
@@ -582,6 +610,11 @@ func (provider *Provider) RemoveDaemonSet(ctx context.Context, namespace string,
return provider.handleRemovalError(err)
}
func (provider *Provider) RemovePersistentVolumeClaim(ctx context.Context, namespace string, volumeClaimName string) error {
err := provider.clientSet.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, volumeClaimName, metav1.DeleteOptions{})
return provider.handleRemovalError(err)
}
func (provider *Provider) handleRemovalError(err error) error {
// Ignore NotFound - There is nothing to delete.
// Ignore Forbidden - Assume that a user could not have created the resource in the first place.
@@ -608,7 +641,11 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: configMapName,
Name: configMapName,
Labels: map[string]string{
LabelManagedBy: provider.managedBy,
LabelCreatedBy: provider.createdBy,
},
},
Data: configMapData,
}
@@ -618,14 +655,14 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
return nil
}
func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, dumpLogs bool) error {
logger.Log.Debugf("Applying %d tapper daemon sets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodMap map[string][]core.Pod, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, logLevel logging.Level, istio bool) error {
logger.Log.Debugf("Applying %d tapper daemon sets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodMap), namespace, daemonSetName, podImage, tapperPodName)
if len(nodeToTappedPodIPMap) == 0 {
if len(nodeToTappedPodMap) == 0 {
return fmt.Errorf("daemon set %s must tap at least 1 pod", daemonSetName)
}
nodeToTappedPodIPMapJsonStr, err := json.Marshal(nodeToTappedPodIPMap)
nodeToTappedPodMapJsonStr, err := json.Marshal(nodeToTappedPodMap)
if err != nil {
return err
}
@@ -643,21 +680,30 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
"--nodefrag",
}
debugMode := ""
if dumpLogs {
debugMode = "1"
if istio {
mizuCmd = append(mizuCmd, "--procfs", procfsMountPath, "--istio")
}
agentContainer := applyconfcore.Container()
agentContainer.WithName(tapperPodName)
agentContainer.WithImage(podImage)
agentContainer.WithImagePullPolicy(imagePullPolicy)
agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithPrivileged(true))
caps := applyconfcore.Capabilities().WithDrop("ALL").WithAdd("NET_RAW").WithAdd("NET_ADMIN")
if istio {
caps = caps.WithAdd("SYS_ADMIN") // for reading /proc/PID/net/ns
caps = caps.WithAdd("SYS_PTRACE") // for setting netns to other process
caps = caps.WithAdd("DAC_OVERRIDE") // for reading /proc/PID/environ
}
agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithCapabilities(caps))
agentContainer.WithCommand(mizuCmd...)
agentContainer.WithEnv(
applyconfcore.EnvVar().WithName(shared.DebugModeEnvVar).WithValue(debugMode),
applyconfcore.EnvVar().WithName(shared.LogLevelEnvVar).WithValue(logLevel.String()),
applyconfcore.EnvVar().WithName(shared.HostModeEnvVar).WithValue("1"),
applyconfcore.EnvVar().WithName(shared.TappedAddressesPerNodeDictEnvVar).WithValue(string(nodeToTappedPodIPMapJsonStr)),
applyconfcore.EnvVar().WithName(shared.TappedAddressesPerNodeDictEnvVar).WithValue(string(nodeToTappedPodMapJsonStr)),
applyconfcore.EnvVar().WithName(shared.GoGCEnvVar).WithValue("12800"),
applyconfcore.EnvVar().WithName(shared.MizuFilteringOptionsEnvVar).WithValue(string(mizuApiFilteringOptionsJsonStr)),
)
@@ -695,8 +741,8 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
agentResources := applyconfcore.ResourceRequirements().WithRequests(agentResourceRequests).WithLimits(agentResourceLimits)
agentContainer.WithResources(agentResources)
nodeNames := make([]string, 0, len(nodeToTappedPodIPMap))
for nodeName := range nodeToTappedPodIPMap {
nodeNames := make([]string, 0, len(nodeToTappedPodMap))
for nodeName := range nodeToTappedPodMap {
nodeNames = append(nodeNames, nodeName)
}
nodeSelectorRequirement := applyconfcore.NodeSelectorRequirement()
@@ -719,6 +765,14 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
noScheduleToleration.WithOperator(core.TolerationOpExists)
noScheduleToleration.WithEffect(core.TaintEffectNoSchedule)
// Host procfs is needed inside the container because we need access to
// the network namespaces of processes on the machine.
//
procfsVolume := applyconfcore.Volume()
procfsVolume.WithName(procfsVolumeName).WithHostPath(applyconfcore.HostPathVolumeSource().WithPath("/proc"))
volumeMount := applyconfcore.VolumeMount().WithName(procfsVolumeName).WithMountPath(procfsMountPath).WithReadOnly(true)
agentContainer.WithVolumeMounts(volumeMount)
volumeName := ConfigMapName
configMapVolume := applyconfcore.VolumeApplyConfiguration{
Name: &volumeName,
@@ -747,26 +801,35 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
podSpec.WithContainers(agentContainer)
podSpec.WithAffinity(affinity)
podSpec.WithTolerations(noExecuteToleration, noScheduleToleration)
podSpec.WithVolumes(&configMapVolume)
podSpec.WithVolumes(&configMapVolume, procfsVolume)
podTemplate := applyconfcore.PodTemplateSpec()
podTemplate.WithLabels(map[string]string{"app": tapperPodName})
podTemplate.WithLabels(map[string]string{
"app": tapperPodName,
LabelManagedBy: provider.managedBy,
LabelCreatedBy: provider.createdBy,
})
podTemplate.WithSpec(podSpec)
labelSelector := applyconfmeta.LabelSelector()
labelSelector.WithMatchLabels(map[string]string{"app": tapperPodName})
daemonSet := applyconfapp.DaemonSet(daemonSetName, namespace)
daemonSet.WithSpec(applyconfapp.DaemonSetSpec().WithSelector(labelSelector).WithTemplate(podTemplate))
daemonSet.
WithLabels(map[string]string{
LabelManagedBy: provider.managedBy,
LabelCreatedBy: provider.createdBy,
}).
WithSpec(applyconfapp.DaemonSetSpec().WithSelector(labelSelector).WithTemplate(podTemplate))
_, err = provider.clientSet.AppsV1().DaemonSets(namespace).Apply(ctx, daemonSet, metav1.ApplyOptions{FieldManager: fieldManagerName})
return err
}
func (provider *Provider) ListAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespaces []string) ([]core.Pod, error) {
func (provider *Provider) listPodsImpl(ctx context.Context, regex *regexp.Regexp, namespaces []string, listOptions metav1.ListOptions) ([]core.Pod, error) {
var pods []core.Pod
for _, namespace := range namespaces {
namespacePods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
namespacePods, err := provider.clientSet.CoreV1().Pods(namespace).List(ctx, listOptions)
if err != nil {
return nil, fmt.Errorf("failed to get pods in ns: [%s], %w", namespace, err)
}
@@ -783,6 +846,14 @@ func (provider *Provider) ListAllPodsMatchingRegex(ctx context.Context, regex *r
return matchingPods, nil
}
func (provider *Provider) ListAllPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespaces []string) ([]core.Pod, error) {
return provider.listPodsImpl(ctx, regex, namespaces, metav1.ListOptions{})
}
func (provider *Provider) GetPod(ctx context.Context, namespaces string, podName string) (*core.Pod, error) {
return provider.clientSet.CoreV1().Pods(namespaces).Get(ctx, podName, metav1.GetOptions{})
}
func (provider *Provider) ListAllRunningPodsMatchingRegex(ctx context.Context, regex *regexp.Regexp, namespaces []string) ([]core.Pod, error) {
pods, err := provider.ListAllPodsMatchingRegex(ctx, regex, namespaces)
if err != nil {
@@ -823,6 +894,41 @@ func (provider *Provider) GetNamespaceEvents(ctx context.Context, namespace stri
return eventList.String(), nil
}
func (provider *Provider) ListManagedServiceAccounts(ctx context.Context, namespace string) (*core.ServiceAccountList, error) {
listOptions := metav1.ListOptions{
LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy),
}
return provider.clientSet.CoreV1().ServiceAccounts(namespace).List(ctx, listOptions)
}
func (provider *Provider) ListManagedClusterRoles(ctx context.Context) (*rbac.ClusterRoleList, error) {
listOptions := metav1.ListOptions{
LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy),
}
return provider.clientSet.RbacV1().ClusterRoles().List(ctx, listOptions)
}
func (provider *Provider) ListManagedClusterRoleBindings(ctx context.Context) (*rbac.ClusterRoleBindingList, error) {
listOptions := metav1.ListOptions{
LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy),
}
return provider.clientSet.RbacV1().ClusterRoleBindings().List(ctx, listOptions)
}
func (provider *Provider) ListManagedRoles(ctx context.Context, namespace string) (*rbac.RoleList, error) {
listOptions := metav1.ListOptions{
LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy),
}
return provider.clientSet.RbacV1().Roles(namespace).List(ctx, listOptions)
}
func (provider *Provider) ListManagedRoleBindings(ctx context.Context, namespace string) (*rbac.RoleBindingList, error) {
listOptions := metav1.ListOptions{
LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy),
}
return provider.clientSet.RbacV1().RoleBindings(namespace).List(ctx, listOptions)
}
func (provider *Provider) IsDefaultStorageProviderAvailable(ctx context.Context) (bool, error) {
storageClassList, err := provider.clientSet.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{})
if err != nil {
@@ -841,10 +947,14 @@ func (provider *Provider) CreatePersistentVolumeClaim(ctx context.Context, names
volumeClaim := &core.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: volumeClaimName,
Labels: map[string]string{
LabelManagedBy: provider.managedBy,
LabelCreatedBy: provider.createdBy,
},
},
Spec: core.PersistentVolumeClaimSpec{
AccessModes: []core.PersistentVolumeAccessMode{core.ReadWriteOnce},
Resources: core.ResourceRequirements{
Spec: core.PersistentVolumeClaimSpec{
AccessModes: []core.PersistentVolumeAccessMode{core.ReadWriteOnce},
Resources: core.ResourceRequirements{
Limits: core.ResourceList{
core.ResourceStorage: *sizeLimitQuantity,
},
@@ -858,10 +968,6 @@ func (provider *Provider) CreatePersistentVolumeClaim(ctx context.Context, names
return provider.clientSet.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, volumeClaim, metav1.CreateOptions{})
}
func (provider *Provider) RemovePersistentVolumeClaim(ctx context.Context, namespace string, volumeClaimName string) error {
return provider.clientSet.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, volumeClaimName, metav1.DeleteOptions{})
}
func getClientSet(config *restclient.Config) (*kubernetes.Clientset, error) {
clientSet, err := kubernetes.NewForConfig(config)
if err != nil {

View File

@@ -28,9 +28,9 @@ func StartProxy(kubernetesProvider *Provider, proxyHost string, mizuPort uint16,
return err
}
mux := http.NewServeMux()
mux.Handle(k8sProxyApiPrefix, proxyHandler)
mux.Handle(k8sProxyApiPrefix, getRerouteHttpHandlerMizuAPI(proxyHandler, mizuNamespace, mizuServiceName))
mux.Handle("/static/", getRerouteHttpHandlerMizuStatic(proxyHandler, mizuNamespace, mizuServiceName))
mux.Handle("/mizu/", getRerouteHttpHandlerMizuAPI(proxyHandler, mizuNamespace, mizuServiceName))
l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", proxyHost, int(mizuPort)))
if err != nil {
@@ -45,16 +45,21 @@ func StartProxy(kubernetesProvider *Provider, proxyHost string, mizuPort uint16,
}
func getMizuApiServerProxiedHostAndPath(mizuNamespace string, mizuServiceName string) string {
return fmt.Sprintf("/api/v1/namespaces/%s/services/%s:%d/proxy/", mizuNamespace, mizuServiceName, mizuServicePort)
return fmt.Sprintf("/api/v1/namespaces/%s/services/%s:%d/proxy", mizuNamespace, mizuServiceName, mizuServicePort)
}
func GetMizuApiServerProxiedHostAndPath(mizuPort uint16) string {
return fmt.Sprintf("localhost:%d/mizu", mizuPort)
return fmt.Sprintf("localhost:%d", mizuPort)
}
func getRerouteHttpHandlerMizuAPI(proxyHandler http.Handler, mizuNamespace string, mizuServiceName string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r.URL.Path = strings.Replace(r.URL.Path, "/mizu/", getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName), 1)
proxiedPath := getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName)
//avoid redirecting several times
if !strings.Contains(r.URL.Path, proxiedPath) {
r.URL.Path = fmt.Sprintf("%s%s", getMizuApiServerProxiedHostAndPath(mizuNamespace, mizuServiceName), r.URL.Path)
}
proxyHandler.ServeHTTP(w, r)
})
}

View File

@@ -1,22 +1,38 @@
package kubernetes
import (
"regexp"
"github.com/up9inc/mizu/shared"
core "k8s.io/api/core/v1"
"regexp"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func GetNodeHostToTappedPodIpsMap(tappedPods []core.Pod) map[string][]string {
nodeToTappedPodIPMap := make(map[string][]string, 0)
func GetNodeHostToTappedPodsMap(tappedPods []core.Pod) map[string][]core.Pod {
nodeToTappedPodMap := make(map[string][]core.Pod, 0)
for _, pod := range tappedPods {
existingList := nodeToTappedPodIPMap[pod.Spec.NodeName]
minimizedPod := getMinimizedPod(pod)
existingList := nodeToTappedPodMap[pod.Spec.NodeName]
if existingList == nil {
nodeToTappedPodIPMap[pod.Spec.NodeName] = []string{pod.Status.PodIP}
nodeToTappedPodMap[pod.Spec.NodeName] = []core.Pod{minimizedPod}
} else {
nodeToTappedPodIPMap[pod.Spec.NodeName] = append(nodeToTappedPodIPMap[pod.Spec.NodeName], pod.Status.PodIP)
nodeToTappedPodMap[pod.Spec.NodeName] = append(nodeToTappedPodMap[pod.Spec.NodeName], minimizedPod)
}
}
return nodeToTappedPodIPMap
return nodeToTappedPodMap
}
func getMinimizedPod(fullPod core.Pod) core.Pod {
return core.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: fullPod.Name,
},
Status: v1.PodStatus{
PodIP: fullPod.Status.PodIP,
},
}
}
func excludeMizuPods(pods []core.Pod) []core.Pod {
@@ -57,11 +73,10 @@ func getMissingPods(pods1 []core.Pod, pods2 []core.Pod) []core.Pod {
return missingPods
}
func GetPodInfosForPods(pods []core.Pod) []shared.PodInfo {
podInfos := make([]shared.PodInfo, 0)
for _, pod := range pods {
podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace})
podInfos = append(podInfos, shared.PodInfo{Name: pod.Name, Namespace: pod.Namespace, NodeName: pod.Spec.NodeName})
}
return podInfos
}

View File

@@ -6,19 +6,22 @@ import (
"fmt"
"github.com/up9inc/mizu/shared/debounce"
"github.com/up9inc/mizu/shared/logger"
"regexp"
"sync"
"time"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/watch"
)
func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetNamespaces []string, podFilter *regexp.Regexp) (chan *corev1.Pod, chan *corev1.Pod, chan *corev1.Pod, chan error) {
addedChan := make(chan *corev1.Pod)
modifiedChan := make(chan *corev1.Pod)
removedChan := make(chan *corev1.Pod)
type EventFilterer interface {
Filter(*WatchEvent) (bool, error)
}
type WatchCreator interface {
NewWatcher(ctx context.Context, namespace string) (watch.Interface, error)
}
func FilteredWatch(ctx context.Context, watcherCreator WatchCreator, targetNamespaces []string, filterer EventFilterer) (<-chan *WatchEvent, <-chan error) {
eventChan := make(chan *WatchEvent)
errorChan := make(chan error)
var wg sync.WaitGroup
@@ -31,8 +34,13 @@ func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetName
watchRestartDebouncer := debounce.NewDebouncer(1 * time.Minute, func() {})
for {
watcher := kubernetesProvider.GetPodWatcher(ctx, targetNamespace)
err := startWatchLoop(ctx, watcher, podFilter, addedChan, modifiedChan, removedChan) // blocking
watcher, err := watcherCreator.NewWatcher(ctx, targetNamespace)
if err != nil {
errorChan <- fmt.Errorf("error in k8s watch: %v", err)
break
}
err = startWatchLoop(ctx, watcher, filterer, eventChan) // blocking
watcher.Stop()
select {
@@ -43,7 +51,7 @@ func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetName
}
if err != nil {
errorChan <- fmt.Errorf("error in k8 watch: %v", err)
errorChan <- fmt.Errorf("error in k8s watch: %v", err)
break
} else {
if !watchRestartDebouncer.IsOn() {
@@ -63,16 +71,14 @@ func FilteredWatch(ctx context.Context, kubernetesProvider *Provider, targetName
go func() {
<-ctx.Done()
wg.Wait()
close(addedChan)
close(modifiedChan)
close(removedChan)
close(eventChan)
close(errorChan)
}()
return addedChan, modifiedChan, removedChan, errorChan
return eventChan, errorChan
}
func startWatchLoop(ctx context.Context, watcher watch.Interface, podFilter *regexp.Regexp, addedChan chan *corev1.Pod, modifiedChan chan *corev1.Pod, removedChan chan *corev1.Pod) error {
func startWatchLoop(ctx context.Context, watcher watch.Interface, filterer EventFilterer, eventChan chan<- *WatchEvent) error {
resultChan := watcher.ResultChan()
for {
select {
@@ -81,27 +87,19 @@ func startWatchLoop(ctx context.Context, watcher watch.Interface, podFilter *reg
return nil
}
if e.Type == watch.Error {
return apierrors.FromObject(e.Object)
wEvent := WatchEvent(e)
if wEvent.Type == watch.Error {
return wEvent.ToError()
}
pod, ok := e.Object.(*corev1.Pod)
if !ok {
if pass, err := filterer.Filter(&wEvent); err != nil {
return err
} else if !pass {
continue
}
if !podFilter.MatchString(pod.Name) {
continue
}
switch e.Type {
case watch.Added:
addedChan <- pod
case watch.Modified:
modifiedChan <- pod
case watch.Deleted:
removedChan <- pod
}
eventChan <- &wEvent
case <-ctx.Done():
return nil
}

View File

@@ -0,0 +1,52 @@
package kubernetes
import (
"fmt"
"reflect"
corev1 "k8s.io/api/core/v1"
eventsv1 "k8s.io/api/events/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/watch"
)
const (
EventAdded watch.EventType = watch.Added
EventModified watch.EventType = watch.Modified
EventDeleted watch.EventType = watch.Deleted
EventBookmark watch.EventType = watch.Bookmark
EventError watch.EventType = watch.Error
)
type InvalidObjectType struct {
RequestedType reflect.Type
}
// Implements the error interface
func (iot *InvalidObjectType) Error() string {
return fmt.Sprintf("Cannot convert event to type %s", iot.RequestedType)
}
type WatchEvent watch.Event
func (we *WatchEvent) ToPod() (*corev1.Pod, error) {
pod, ok := we.Object.(*corev1.Pod)
if !ok {
return nil, &InvalidObjectType{RequestedType: reflect.TypeOf(pod)}
}
return pod, nil
}
func (we *WatchEvent) ToEvent() (*eventsv1.Event, error) {
event, ok := we.Object.(*eventsv1.Event)
if !ok {
return nil, &InvalidObjectType{RequestedType: reflect.TypeOf(event)}
}
return event, nil
}
func (we *WatchEvent) ToError() error {
return apierrors.FromObject(we.Object)
}

View File

@@ -9,7 +9,7 @@ import (
var Log = logging.MustGetLogger("mizu")
var format = logging.MustStringFormatter(
`%{time} %{level:.5s} ▶ %{pid} %{shortfile} %{shortfunc} ▶ %{message}`,
`[%{time:2006-01-02T15:04:05.000-0700}] %{level:-5s} ▶ %{message} ▶ [%{pid} %{shortfile} %{shortfunc}]`,
)
func InitLogger(logPath string) {

View File

@@ -1,11 +1,13 @@
package shared
import (
"github.com/up9inc/mizu/tap/api"
"io/ioutil"
"log"
"strings"
"github.com/op/go-logging"
"github.com/up9inc/mizu/shared/logger"
v1 "k8s.io/api/core/v1"
"gopkg.in/yaml.v3"
)
@@ -17,6 +19,10 @@ const (
WebSocketMessageTypeUpdateStatus WebSocketMessageType = "status"
WebSocketMessageTypeAnalyzeStatus WebSocketMessageType = "analyzeStatus"
WebsocketMessageTypeOutboundLink WebSocketMessageType = "outboundLink"
WebSocketMessageTypeToast WebSocketMessageType = "toast"
WebSocketMessageTypeQueryMetadata WebSocketMessageType = "queryMetadata"
WebSocketMessageTypeStartTime WebSocketMessageType = "startTime"
WebSocketMessageTypeTapConfig WebSocketMessageType = "tapConfig"
)
type Resources struct {
@@ -27,18 +33,14 @@ type Resources struct {
}
type MizuAgentConfig struct {
TapTargetRegex api.SerializableRegexp `json:"tapTargetRegex"`
MaxDBSizeBytes int64 `json:"maxDBSizeBytes"`
DaemonMode bool `json:"daemonMode"`
TargetNamespaces []string `json:"targetNamespaces"`
AgentImage string `json:"agentImage"`
PullPolicy string `json:"pullPolicy"`
DumpLogs bool `json:"dumpLogs"`
IgnoredUserAgents []string `json:"ignoredUserAgents"`
TapperResources Resources `json:"tapperResources"`
MizuResourcesNamespace string `json:"mizuResourceNamespace"`
MizuApiFilteringOptions api.TrafficFilteringOptions `json:"mizuApiFilteringOptions"`
AgentDatabasePath string `json:"agentDatabasePath"`
MaxDBSizeBytes int64 `json:"maxDBSizeBytes"`
AgentImage string `json:"agentImage"`
PullPolicy string `json:"pullPolicy"`
LogLevel logging.Level `json:"logLevel"`
TapperResources Resources `json:"tapperResources"`
MizuResourcesNamespace string `json:"mizuResourceNamespace"`
AgentDatabasePath string `json:"agentDatabasePath"`
StandaloneMode bool `json:"standaloneMode"`
}
type WebSocketMessageMetadata struct {
@@ -59,17 +61,34 @@ type AnalyzeStatus struct {
type WebSocketStatusMessage struct {
*WebSocketMessageMetadata
TappingStatus TapStatus `json:"tappingStatus"`
TappingStatus []TappedPodStatus `json:"tappingStatus"`
}
type WebSocketTapConfigMessage struct {
*WebSocketMessageMetadata
TapTargets []v1.Pod `json:"pods"`
}
type TapperStatus struct {
TapperName string `json:"tapperName"`
NodeName string `json:"nodeName"`
Status string `json:"status"`
}
type TappedPodStatus struct {
Name string `json:"name"`
Namespace string `json:"namespace"`
IsTapped bool `json:"isTapped"`
}
type TapStatus struct {
Pods []PodInfo `json:"pods"`
TLSLinks []TLSLinkInfo `json:"tlsLinks"`
Pods []PodInfo `json:"pods"`
}
type PodInfo struct {
Namespace string `json:"namespace"`
Name string `json:"name"`
NodeName string `json:"nodeName"`
}
type TLSLinkInfo struct {
@@ -86,12 +105,12 @@ type SyncEntriesConfig struct {
UploadIntervalSec int `json:"interval"`
}
func CreateWebSocketStatusMessage(tappingStatus TapStatus) WebSocketStatusMessage {
func CreateWebSocketStatusMessage(tappedPodsStatus []TappedPodStatus) WebSocketStatusMessage {
return WebSocketStatusMessage{
WebSocketMessageMetadata: &WebSocketMessageMetadata{
MessageType: WebSocketMessageTypeUpdateStatus,
},
TappingStatus: tappingStatus,
TappingStatus: tappedPodsStatus,
}
}
@@ -105,8 +124,9 @@ func CreateWebSocketMessageTypeAnalyzeStatus(analyzeStatus AnalyzeStatus) WebSoc
}
type HealthResponse struct {
TapStatus TapStatus `json:"tapStatus"`
TappersCount int `json:"tappersCount"`
TapStatus TapStatus `json:"tapStatus"`
TappersCount int `json:"tappersCount"`
TappersStatus []TapperStatus `json:"tappersStatus"`
}
type VersionResponse struct {
@@ -137,14 +157,12 @@ func (r *RulePolicy) validateType() bool {
permitedTypes := []string{"json", "header", "slo"}
_, found := Find(permitedTypes, r.Type)
if !found {
log.Printf("Error: %s. ", r.Name)
log.Printf("Only json, header and slo types are supported on rule definition. This rule will be ignored\n")
logger.Log.Errorf("Only json, header and slo types are supported on rule definition. This rule will be ignored. rule name: %s", r.Name)
found = false
}
if strings.ToLower(r.Type) == "slo" {
if r.ResponseTime <= 0 {
log.Printf("Error: %s. ", r.Name)
log.Printf("When type=slo, the field response-time should be specified and have a value >= 1\n\n")
logger.Log.Errorf("When rule type is slo, the field response-time should be specified and have a value >= 1. rule name: %s", r.Name)
found = false
}
}

View File

@@ -18,7 +18,8 @@ import (
type Protocol struct {
Name string `json:"name"`
LongName string `json:"longName"`
Abbreviation string `json:"abbreviation"`
Abbreviation string `json:"abbr"`
Macro string `json:"macro"`
Version string `json:"version"`
BackgroundColor string `json:"backgroundColor"`
ForegroundColor string `json:"foregroundColor"`
@@ -28,6 +29,12 @@ type Protocol struct {
Priority uint8 `json:"priority"`
}
type TCP struct {
IP string `json:"ip"`
Port string `json:"port"`
Name string `json:"name"`
}
type Extension struct {
Protocol *Protocol
Path string
@@ -74,6 +81,7 @@ type OutputChannelItem struct {
Timestamp int64
ConnectionInfo *ConnectionInfo
Pair *RequestResponsePair
Summary *BaseEntryDetails
}
type SuperTimer struct {
@@ -89,9 +97,10 @@ type Dissector interface {
Register(*Extension)
Ping()
Dissect(b *bufio.Reader, isClient bool, tcpID *TcpID, counterPair *CounterPair, superTimer *SuperTimer, superIdentifier *SuperIdentifier, emitter Emitter, options *TrafficFilteringOptions) error
Analyze(item *OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *MizuEntry
Analyze(item *OutputChannelItem, resolvedSource string, resolvedDestination string) *MizuEntry
Summarize(entry *MizuEntry) *BaseEntryDetails
Represent(entry *MizuEntry) (protocol Protocol, object []byte, bodySize int64, err error)
Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error)
Macros() map[string]string
}
type Emitting struct {
@@ -109,39 +118,27 @@ func (e *Emitting) Emit(item *OutputChannelItem) {
}
type MizuEntry struct {
ID uint `gorm:"primarykey"`
CreatedAt time.Time
UpdatedAt time.Time
ProtocolName string `json:"protocolName" gorm:"column:protocolName"`
ProtocolLongName string `json:"protocolLongName" gorm:"column:protocolLongName"`
ProtocolAbbreviation string `json:"protocolAbbreviation" gorm:"column:protocolAbbreviation"`
ProtocolVersion string `json:"protocolVersion" gorm:"column:protocolVersion"`
ProtocolBackgroundColor string `json:"protocolBackgroundColor" gorm:"column:protocolBackgroundColor"`
ProtocolForegroundColor string `json:"protocolForegroundColor" gorm:"column:protocolForegroundColor"`
ProtocolFontSize int8 `json:"protocolFontSize" gorm:"column:protocolFontSize"`
ProtocolReferenceLink string `json:"protocolReferenceLink" gorm:"column:protocolReferenceLink"`
Entry string `json:"entry,omitempty" gorm:"column:entry"`
EntryId string `json:"entryId" gorm:"column:entryId"`
Url string `json:"url" gorm:"column:url"`
Method string `json:"method" gorm:"column:method"`
Status int `json:"status" gorm:"column:status"`
RequestSenderIp string `json:"requestSenderIp" gorm:"column:requestSenderIp"`
Service string `json:"service" gorm:"column:service"`
Timestamp int64 `json:"timestamp" gorm:"column:timestamp"`
ElapsedTime int64 `json:"elapsedTime" gorm:"column:elapsedTime"`
Path string `json:"path" gorm:"column:path"`
ResolvedSource string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"`
ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
SourceIp string `json:"sourceIp,omitempty" gorm:"column:sourceIp"`
DestinationIp string `json:"destinationIp,omitempty" gorm:"column:destinationIp"`
SourcePort string `json:"sourcePort,omitempty" gorm:"column:sourcePort"`
DestinationPort string `json:"destinationPort,omitempty" gorm:"column:destinationPort"`
IsOutgoing bool `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"`
ContractStatus ContractStatus `json:"contractStatus,omitempty" gorm:"column:contractStatus"`
ContractRequestReason string `json:"contractRequestReason,omitempty" gorm:"column:contractRequestReason"`
ContractResponseReason string `json:"contractResponseReason,omitempty" gorm:"column:contractResponseReason"`
ContractContent string `json:"contractContent,omitempty" gorm:"column:contractContent"`
EstimatedSizeBytes int `json:"-" gorm:"column:estimatedSizeBytes"`
Id uint `json:"id"`
Protocol Protocol `json:"proto"`
Source *TCP `json:"src"`
Destination *TCP `json:"dst"`
Outgoing bool `json:"outgoing"`
Timestamp int64 `json:"timestamp"`
StartTime time.Time `json:"startTime"`
Request map[string]interface{} `json:"request"`
Response map[string]interface{} `json:"response"`
Base *BaseEntryDetails `json:"base"`
Summary string `json:"summary"`
Method string `json:"method"`
Status int `json:"status"`
ElapsedTime int64 `json:"elapsedTime"`
Path string `json:"path"`
IsOutgoing bool `json:"isOutgoing,omitempty"`
ContractStatus ContractStatus `json:"contractStatus,omitempty"`
ContractRequestReason string `json:"contractRequestReason,omitempty"`
ContractResponseReason string `json:"contractResponseReason,omitempty"`
ContractContent string `json:"contractContent,omitempty"`
HTTPPair string `json:"httpPair,omitempty"`
}
type MizuEntryWrapper struct {
@@ -154,24 +151,20 @@ type MizuEntryWrapper struct {
}
type BaseEntryDetails struct {
Id string `json:"id,omitempty"`
Protocol Protocol `json:"protocol,omitempty"`
Url string `json:"url,omitempty"`
RequestSenderIp string `json:"requestSenderIp,omitempty"`
Service string `json:"service,omitempty"`
Path string `json:"path,omitempty"`
Summary string `json:"summary,omitempty"`
StatusCode int `json:"statusCode"`
Method string `json:"method,omitempty"`
Timestamp int64 `json:"timestamp,omitempty"`
SourceIp string `json:"sourceIp,omitempty"`
DestinationIp string `json:"destinationIp,omitempty"`
SourcePort string `json:"sourcePort,omitempty"`
DestinationPort string `json:"destinationPort,omitempty"`
IsOutgoing bool `json:"isOutgoing,omitempty"`
Latency int64 `json:"latency"`
Rules ApplicableRules `json:"rules,omitempty"`
ContractStatus ContractStatus `json:"contractStatus"`
Id uint `json:"id"`
Protocol Protocol `json:"protocol,omitempty"`
Url string `json:"url,omitempty"`
Path string `json:"path,omitempty"`
Summary string `json:"summary,omitempty"`
StatusCode int `json:"statusCode"`
Method string `json:"method,omitempty"`
Timestamp int64 `json:"timestamp,omitempty"`
Source *TCP `json:"src"`
Destination *TCP `json:"dst"`
IsOutgoing bool `json:"isOutgoing,omitempty"`
Latency int64 `json:"latency"`
Rules ApplicableRules `json:"rules,omitempty"`
ContractStatus ContractStatus `json:"contractStatus"`
}
type ApplicableRules struct {
@@ -194,29 +187,15 @@ type DataUnmarshaler interface {
}
func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error {
bed.Protocol = Protocol{
Name: entry.ProtocolName,
LongName: entry.ProtocolLongName,
Abbreviation: entry.ProtocolAbbreviation,
Version: entry.ProtocolVersion,
BackgroundColor: entry.ProtocolBackgroundColor,
ForegroundColor: entry.ProtocolForegroundColor,
FontSize: entry.ProtocolFontSize,
ReferenceLink: entry.ProtocolReferenceLink,
}
bed.Id = entry.EntryId
bed.Url = entry.Url
bed.RequestSenderIp = entry.RequestSenderIp
bed.Service = entry.Service
bed.Protocol = entry.Protocol
bed.Id = entry.Id
bed.Path = entry.Path
bed.Summary = entry.Path
bed.Summary = entry.Summary
bed.StatusCode = entry.Status
bed.Method = entry.Method
bed.Timestamp = entry.Timestamp
bed.SourceIp = entry.SourceIp
bed.DestinationIp = entry.DestinationIp
bed.SourcePort = entry.SourcePort
bed.DestinationPort = entry.DestinationPort
bed.Source = entry.Source
bed.Destination = entry.Destination
bed.IsOutgoing = entry.IsOutgoing
bed.Latency = entry.ElapsedTime
bed.ContractStatus = entry.ContractStatus
@@ -228,6 +207,21 @@ const (
BODY string = "body"
)
type SectionData struct {
Type string `json:"type"`
Title string `json:"title"`
Data string `json:"data"`
Encoding string `json:"encoding,omitempty"`
MimeType string `json:"mimeType,omitempty"`
Selector string `json:"selector,omitempty"`
}
type TableData struct {
Name string `json:"name"`
Value interface{} `json:"value"`
Selector string `json:"selector"`
}
const (
TypeHttpRequest = iota
TypeHttpResponse
@@ -259,7 +253,6 @@ func (h HTTPPayload) MarshalJSON() ([]byte, error) {
}
return json.Marshal(&HTTPWrapper{
Method: harRequest.Method,
Url: "",
Details: harRequest,
RawRequest: &HTTPRequestWrapper{Request: h.Data.(*http.Request)},
})
@@ -275,7 +268,7 @@ func (h HTTPPayload) MarshalJSON() ([]byte, error) {
RawResponse: &HTTPResponseWrapper{Response: h.Data.(*http.Response)},
})
default:
panic(fmt.Sprintf("HTTP payload cannot be marshaled: %s\n", h.Type))
panic(fmt.Sprintf("HTTP payload cannot be marshaled: %s", h.Type))
}
}

View File

@@ -39,7 +39,7 @@ func StartMemoryProfiler(envDumpPath string, envTimeInterval string) {
filename := fmt.Sprintf("%s/%s__mem.prof", dumpPath, t.Format("15_04_05"))
logger.Log.Infof("Writing memory profile to %s\n", filename)
logger.Log.Infof("Writing memory profile to %s", filename)
f, err := os.Create(filename)
if err != nil {

View File

@@ -131,97 +131,109 @@ func representProperties(properties map[string]interface{}, rep []interface{}) (
userId := ""
appId := ""
if properties["ContentType"] != nil {
contentType = properties["ContentType"].(string)
if properties["contentType"] != nil {
contentType = properties["contentType"].(string)
}
if properties["ContentEncoding"] != nil {
contentEncoding = properties["ContentEncoding"].(string)
if properties["contentEncoding"] != nil {
contentEncoding = properties["contentEncoding"].(string)
}
if properties["Delivery Mode"] != nil {
deliveryMode = fmt.Sprintf("%g", properties["DeliveryMode"].(float64))
if properties["deliveryMode"] != nil {
deliveryMode = fmt.Sprintf("%g", properties["deliveryMode"].(float64))
}
if properties["Priority"] != nil {
priority = fmt.Sprintf("%g", properties["Priority"].(float64))
if properties["priority"] != nil {
priority = fmt.Sprintf("%g", properties["priority"].(float64))
}
if properties["CorrelationId"] != nil {
correlationId = properties["CorrelationId"].(string)
if properties["correlationId"] != nil {
correlationId = properties["correlationId"].(string)
}
if properties["ReplyTo"] != nil {
replyTo = properties["ReplyTo"].(string)
if properties["replyTo"] != nil {
replyTo = properties["replyTo"].(string)
}
if properties["Expiration"] != nil {
expiration = properties["Expiration"].(string)
if properties["expiration"] != nil {
expiration = properties["expiration"].(string)
}
if properties["MessageId"] != nil {
messageId = properties["MessageId"].(string)
if properties["messageId"] != nil {
messageId = properties["messageId"].(string)
}
if properties["Timestamp"] != nil {
timestamp = properties["Timestamp"].(string)
if properties["timestamp"] != nil {
timestamp = properties["timestamp"].(string)
}
if properties["Type"] != nil {
_type = properties["Type"].(string)
if properties["type"] != nil {
_type = properties["type"].(string)
}
if properties["UserId"] != nil {
userId = properties["UserId"].(string)
if properties["userId"] != nil {
userId = properties["userId"].(string)
}
if properties["AppId"] != nil {
appId = properties["AppId"].(string)
if properties["appId"] != nil {
appId = properties["appId"].(string)
}
props, _ := json.Marshal([]map[string]string{
props, _ := json.Marshal([]api.TableData{
{
"name": "Content Type",
"value": contentType,
Name: "Content Type",
Value: contentType,
Selector: `request.properties.contentType`,
},
{
"name": "Content Encoding",
"value": contentEncoding,
Name: "Content Encoding",
Value: contentEncoding,
Selector: `request.properties.contentEncoding`,
},
{
"name": "Delivery Mode",
"value": deliveryMode,
Name: "Delivery Mode",
Value: deliveryMode,
Selector: `request.properties.deliveryMode`,
},
{
"name": "Priority",
"value": priority,
Name: "Priority",
Value: priority,
Selector: `request.properties.priority`,
},
{
"name": "Correlation ID",
"value": correlationId,
Name: "Correlation ID",
Value: correlationId,
Selector: `request.properties.correlationId`,
},
{
"name": "Reply To",
"value": replyTo,
Name: "Reply To",
Value: replyTo,
Selector: `request.properties.replyTo`,
},
{
"name": "Expiration",
"value": expiration,
Name: "Expiration",
Value: expiration,
Selector: `request.properties.expiration`,
},
{
"name": "Message ID",
"value": messageId,
Name: "Message ID",
Value: messageId,
Selector: `request.properties.messageId`,
},
{
"name": "Timestamp",
"value": timestamp,
Name: "Timestamp",
Value: timestamp,
Selector: `request.properties.timestamp`,
},
{
"name": "Type",
"value": _type,
Name: "Type",
Value: _type,
Selector: `request.properties.type`,
},
{
"name": "User ID",
"value": userId,
Name: "User ID",
Value: userId,
Selector: `request.properties.userId`,
},
{
"name": "App ID",
"value": appId,
Name: "App ID",
Value: appId,
Selector: `request.properties.appId`,
},
})
rep = append(rep, map[string]string{
"type": api.TABLE,
"title": "Properties",
"data": string(props),
rep = append(rep, api.SectionData{
Type: api.TABLE,
Title: "Properties",
Data: string(props),
})
return rep, contentType, contentEncoding
@@ -230,56 +242,62 @@ func representProperties(properties map[string]interface{}, rep []interface{}) (
func representBasicPublish(event map[string]interface{}) []interface{} {
rep := make([]interface{}, 0)
details, _ := json.Marshal([]map[string]string{
details, _ := json.Marshal([]api.TableData{
{
"name": "Exchange",
"value": event["Exchange"].(string),
Name: "Exchange",
Value: event["exchange"].(string),
Selector: `request.exchange`,
},
{
"name": "Routing Key",
"value": event["RoutingKey"].(string),
Name: "Routing Key",
Value: event["routingKey"].(string),
Selector: `request.routingKey`,
},
{
"name": "Mandatory",
"value": strconv.FormatBool(event["Mandatory"].(bool)),
Name: "Mandatory",
Value: strconv.FormatBool(event["mandatory"].(bool)),
Selector: `request.mandatory`,
},
{
"name": "Immediate",
"value": strconv.FormatBool(event["Immediate"].(bool)),
Name: "Immediate",
Value: strconv.FormatBool(event["immediate"].(bool)),
Selector: `request.immediate`,
},
})
rep = append(rep, map[string]string{
"type": api.TABLE,
"title": "Details",
"data": string(details),
rep = append(rep, api.SectionData{
Type: api.TABLE,
Title: "Details",
Data: string(details),
})
properties := event["Properties"].(map[string]interface{})
properties := event["properties"].(map[string]interface{})
rep, contentType, _ := representProperties(properties, rep)
if properties["Headers"] != nil {
headers := make([]map[string]string, 0)
for name, value := range properties["Headers"].(map[string]interface{}) {
headers = append(headers, map[string]string{
"name": name,
"value": value.(string),
if properties["headers"] != nil {
headers := make([]api.TableData, 0)
for name, value := range properties["headers"].(map[string]interface{}) {
headers = append(headers, api.TableData{
Name: name,
Value: value.(string),
Selector: fmt.Sprintf(`request.properties.headers["%s"]`, name),
})
}
headersMarshaled, _ := json.Marshal(headers)
rep = append(rep, map[string]string{
"type": api.TABLE,
"title": "Headers",
"data": string(headersMarshaled),
rep = append(rep, api.SectionData{
Type: api.TABLE,
Title: "Headers",
Data: string(headersMarshaled),
})
}
if event["Body"] != nil {
rep = append(rep, map[string]string{
"type": api.BODY,
"title": "Body",
"encoding": "base64",
"mime_type": contentType,
"data": event["Body"].(string),
if event["body"] != nil {
rep = append(rep, api.SectionData{
Type: api.BODY,
Title: "Body",
Encoding: "base64",
MimeType: contentType,
Data: event["body"].(string),
Selector: `request.body`,
})
}
@@ -293,70 +311,77 @@ func representBasicDeliver(event map[string]interface{}) []interface{} {
deliveryTag := ""
redelivered := ""
if event["ConsumerTag"] != nil {
consumerTag = event["ConsumerTag"].(string)
if event["consumerTag"] != nil {
consumerTag = event["consumerTag"].(string)
}
if event["DeliveryTag"] != nil {
deliveryTag = fmt.Sprintf("%g", event["DeliveryTag"].(float64))
if event["deliveryTag"] != nil {
deliveryTag = fmt.Sprintf("%g", event["deliveryTag"].(float64))
}
if event["Redelivered"] != nil {
redelivered = strconv.FormatBool(event["Redelivered"].(bool))
if event["redelivered"] != nil {
redelivered = strconv.FormatBool(event["redelivered"].(bool))
}
details, _ := json.Marshal([]map[string]string{
details, _ := json.Marshal([]api.TableData{
{
"name": "Consumer Tag",
"value": consumerTag,
Name: "Consumer Tag",
Value: consumerTag,
Selector: `request.consumerTag`,
},
{
"name": "Delivery Tag",
"value": deliveryTag,
Name: "Delivery Tag",
Value: deliveryTag,
Selector: `request.deliveryTag`,
},
{
"name": "Redelivered",
"value": redelivered,
Name: "Redelivered",
Value: redelivered,
Selector: `request.redelivered`,
},
{
"name": "Exchange",
"value": event["Exchange"].(string),
Name: "Exchange",
Value: event["exchange"].(string),
Selector: `request.exchange`,
},
{
"name": "Routing Key",
"value": event["RoutingKey"].(string),
Name: "Routing Key",
Value: event["routingKey"].(string),
Selector: `request.routingKey`,
},
})
rep = append(rep, map[string]string{
"type": api.TABLE,
"title": "Details",
"data": string(details),
rep = append(rep, api.SectionData{
Type: api.TABLE,
Title: "Details",
Data: string(details),
})
properties := event["Properties"].(map[string]interface{})
properties := event["properties"].(map[string]interface{})
rep, contentType, _ := representProperties(properties, rep)
if properties["Headers"] != nil {
headers := make([]map[string]string, 0)
for name, value := range properties["Headers"].(map[string]interface{}) {
headers = append(headers, map[string]string{
"name": name,
"value": value.(string),
if properties["headers"] != nil {
headers := make([]api.TableData, 0)
for name, value := range properties["headers"].(map[string]interface{}) {
headers = append(headers, api.TableData{
Name: name,
Value: value.(string),
Selector: fmt.Sprintf(`request.properties.headers["%s"]`, name),
})
}
headersMarshaled, _ := json.Marshal(headers)
rep = append(rep, map[string]string{
"type": api.TABLE,
"title": "Headers",
"data": string(headersMarshaled),
rep = append(rep, api.SectionData{
Type: api.TABLE,
Title: "Headers",
Data: string(headersMarshaled),
})
}
if event["Body"] != nil {
rep = append(rep, map[string]string{
"type": api.BODY,
"title": "Body",
"encoding": "base64",
"mime_type": contentType,
"data": event["Body"].(string),
if event["body"] != nil {
rep = append(rep, api.SectionData{
Type: api.BODY,
Title: "Body",
Encoding: "base64",
MimeType: contentType,
Data: event["body"].(string),
Selector: `request.body`,
})
}
@@ -366,51 +391,58 @@ func representBasicDeliver(event map[string]interface{}) []interface{} {
func representQueueDeclare(event map[string]interface{}) []interface{} {
rep := make([]interface{}, 0)
details, _ := json.Marshal([]map[string]string{
details, _ := json.Marshal([]api.TableData{
{
"name": "Queue",
"value": event["Queue"].(string),
Name: "Queue",
Value: event["queue"].(string),
Selector: `request.queue`,
},
{
"name": "Passive",
"value": strconv.FormatBool(event["Passive"].(bool)),
Name: "Passive",
Value: strconv.FormatBool(event["passive"].(bool)),
Selector: `request.queue`,
},
{
"name": "Durable",
"value": strconv.FormatBool(event["Durable"].(bool)),
Name: "Durable",
Value: strconv.FormatBool(event["durable"].(bool)),
Selector: `request.durable`,
},
{
"name": "Exclusive",
"value": strconv.FormatBool(event["Exclusive"].(bool)),
Name: "Exclusive",
Value: strconv.FormatBool(event["exclusive"].(bool)),
Selector: `request.exclusive`,
},
{
"name": "Auto Delete",
"value": strconv.FormatBool(event["AutoDelete"].(bool)),
Name: "Auto Delete",
Value: strconv.FormatBool(event["autoDelete"].(bool)),
Selector: `request.autoDelete`,
},
{
"name": "NoWait",
"value": strconv.FormatBool(event["NoWait"].(bool)),
Name: "NoWait",
Value: strconv.FormatBool(event["noWait"].(bool)),
Selector: `request.noWait`,
},
})
rep = append(rep, map[string]string{
"type": api.TABLE,
"title": "Details",
"data": string(details),
rep = append(rep, api.SectionData{
Type: api.TABLE,
Title: "Details",
Data: string(details),
})
if event["Arguments"] != nil {
headers := make([]map[string]string, 0)
for name, value := range event["Arguments"].(map[string]interface{}) {
headers = append(headers, map[string]string{
"name": name,
"value": value.(string),
if event["arguments"] != nil {
headers := make([]api.TableData, 0)
for name, value := range event["arguments"].(map[string]interface{}) {
headers = append(headers, api.TableData{
Name: name,
Value: value.(string),
Selector: fmt.Sprintf(`request.arguments["%s"]`, name),
})
}
headersMarshaled, _ := json.Marshal(headers)
rep = append(rep, map[string]string{
"type": api.TABLE,
"title": "Arguments",
"data": string(headersMarshaled),
rep = append(rep, api.SectionData{
Type: api.TABLE,
Title: "Arguments",
Data: string(headersMarshaled),
})
}
@@ -420,55 +452,63 @@ func representQueueDeclare(event map[string]interface{}) []interface{} {
func representExchangeDeclare(event map[string]interface{}) []interface{} {
rep := make([]interface{}, 0)
details, _ := json.Marshal([]map[string]string{
details, _ := json.Marshal([]api.TableData{
{
"name": "Exchange",
"value": event["Exchange"].(string),
Name: "Exchange",
Value: event["exchange"].(string),
Selector: `request.exchange`,
},
{
"name": "Type",
"value": event["Type"].(string),
Name: "Type",
Value: event["type"].(string),
Selector: `request.type`,
},
{
"name": "Passive",
"value": strconv.FormatBool(event["Passive"].(bool)),
Name: "Passive",
Value: strconv.FormatBool(event["passive"].(bool)),
Selector: `request.passive`,
},
{
"name": "Durable",
"value": strconv.FormatBool(event["Durable"].(bool)),
Name: "Durable",
Value: strconv.FormatBool(event["durable"].(bool)),
Selector: `request.durable`,
},
{
"name": "Auto Delete",
"value": strconv.FormatBool(event["AutoDelete"].(bool)),
Name: "Auto Delete",
Value: strconv.FormatBool(event["autoDelete"].(bool)),
Selector: `request.autoDelete`,
},
{
"name": "Internal",
"value": strconv.FormatBool(event["Internal"].(bool)),
Name: "Internal",
Value: strconv.FormatBool(event["internal"].(bool)),
Selector: `request.internal`,
},
{
"name": "NoWait",
"value": strconv.FormatBool(event["NoWait"].(bool)),
Name: "NoWait",
Value: strconv.FormatBool(event["noWait"].(bool)),
Selector: `request.noWait`,
},
})
rep = append(rep, map[string]string{
"type": api.TABLE,
"title": "Details",
"data": string(details),
rep = append(rep, api.SectionData{
Type: api.TABLE,
Title: "Details",
Data: string(details),
})
if event["Arguments"] != nil {
headers := make([]map[string]string, 0)
for name, value := range event["Arguments"].(map[string]interface{}) {
headers = append(headers, map[string]string{
"name": name,
"value": value.(string),
if event["arguments"] != nil {
headers := make([]api.TableData, 0)
for name, value := range event["arguments"].(map[string]interface{}) {
headers = append(headers, api.TableData{
Name: name,
Value: value.(string),
Selector: fmt.Sprintf(`request.arguments["%s"]`, name),
})
}
headersMarshaled, _ := json.Marshal(headers)
rep = append(rep, map[string]string{
"type": api.TABLE,
"title": "Arguments",
"data": string(headersMarshaled),
rep = append(rep, api.SectionData{
Type: api.TABLE,
Title: "Arguments",
Data: string(headersMarshaled),
})
}
@@ -478,33 +518,37 @@ func representExchangeDeclare(event map[string]interface{}) []interface{} {
func representConnectionStart(event map[string]interface{}) []interface{} {
rep := make([]interface{}, 0)
details, _ := json.Marshal([]map[string]string{
details, _ := json.Marshal([]api.TableData{
{
"name": "Version Major",
"value": fmt.Sprintf("%g", event["VersionMajor"].(float64)),
Name: "Version Major",
Value: fmt.Sprintf("%g", event["versionMajor"].(float64)),
Selector: `request.versionMajor`,
},
{
"name": "Version Minor",
"value": fmt.Sprintf("%g", event["VersionMinor"].(float64)),
Name: "Version Minor",
Value: fmt.Sprintf("%g", event["versionMinor"].(float64)),
Selector: `request.versionMinor`,
},
{
"name": "Mechanisms",
"value": event["Mechanisms"].(string),
Name: "Mechanisms",
Value: event["mechanisms"].(string),
Selector: `request.mechanisms`,
},
{
"name": "Locales",
"value": event["Locales"].(string),
Name: "Locales",
Value: event["locales"].(string),
Selector: `request.locales`,
},
})
rep = append(rep, map[string]string{
"type": api.TABLE,
"title": "Details",
"data": string(details),
rep = append(rep, api.SectionData{
Type: api.TABLE,
Title: "Details",
Data: string(details),
})
if event["ServerProperties"] != nil {
headers := make([]map[string]string, 0)
for name, value := range event["ServerProperties"].(map[string]interface{}) {
if event["serverProperties"] != nil {
headers := make([]api.TableData, 0)
for name, value := range event["serverProperties"].(map[string]interface{}) {
var outcome string
switch value.(type) {
case string:
@@ -517,16 +561,17 @@ func representConnectionStart(event map[string]interface{}) []interface{} {
default:
panic("Unknown data type for the server property!")
}
headers = append(headers, map[string]string{
"name": name,
"value": outcome,
headers = append(headers, api.TableData{
Name: name,
Value: outcome,
Selector: fmt.Sprintf(`request.serverProperties["%s"]`, name),
})
}
headersMarshaled, _ := json.Marshal(headers)
rep = append(rep, map[string]string{
"type": api.TABLE,
"title": "Server Properties",
"data": string(headersMarshaled),
rep = append(rep, api.SectionData{
Type: api.TABLE,
Title: "Server Properties",
Data: string(headersMarshaled),
})
}
@@ -534,30 +579,40 @@ func representConnectionStart(event map[string]interface{}) []interface{} {
}
func representConnectionClose(event map[string]interface{}) []interface{} {
replyCode := ""
if event["replyCode"] != nil {
replyCode = fmt.Sprintf("%g", event["replyCode"].(float64))
}
rep := make([]interface{}, 0)
details, _ := json.Marshal([]map[string]string{
details, _ := json.Marshal([]api.TableData{
{
"name": "Reply Code",
"value": fmt.Sprintf("%g", event["ReplyCode"].(float64)),
Name: "Reply Code",
Value: replyCode,
Selector: `request.replyCode`,
},
{
"name": "Reply Text",
"value": event["ReplyText"].(string),
Name: "Reply Text",
Value: event["replyText"].(string),
Selector: `request.replyText`,
},
{
"name": "Class ID",
"value": fmt.Sprintf("%g", event["ClassId"].(float64)),
Name: "Class ID",
Value: fmt.Sprintf("%g", event["classId"].(float64)),
Selector: `request.classId`,
},
{
"name": "Method ID",
"value": fmt.Sprintf("%g", event["MethodId"].(float64)),
Name: "Method ID",
Value: fmt.Sprintf("%g", event["methodId"].(float64)),
Selector: `request.methodId`,
},
})
rep = append(rep, map[string]string{
"type": api.TABLE,
"title": "Details",
"data": string(details),
rep = append(rep, api.SectionData{
Type: api.TABLE,
Title: "Details",
Data: string(details),
})
return rep
@@ -566,43 +621,48 @@ func representConnectionClose(event map[string]interface{}) []interface{} {
func representQueueBind(event map[string]interface{}) []interface{} {
rep := make([]interface{}, 0)
details, _ := json.Marshal([]map[string]string{
details, _ := json.Marshal([]api.TableData{
{
"name": "Queue",
"value": event["Queue"].(string),
Name: "Queue",
Value: event["queue"].(string),
Selector: `request.queue`,
},
{
"name": "Exchange",
"value": event["Exchange"].(string),
Name: "Exchange",
Value: event["exchange"].(string),
Selector: `request.exchange`,
},
{
"name": "RoutingKey",
"value": event["RoutingKey"].(string),
Name: "RoutingKey",
Value: event["routingKey"].(string),
Selector: `request.routingKey`,
},
{
"name": "NoWait",
"value": strconv.FormatBool(event["NoWait"].(bool)),
Name: "NoWait",
Value: strconv.FormatBool(event["noWait"].(bool)),
Selector: `request.noWait`,
},
})
rep = append(rep, map[string]string{
"type": api.TABLE,
"title": "Details",
"data": string(details),
rep = append(rep, api.SectionData{
Type: api.TABLE,
Title: "Details",
Data: string(details),
})
if event["Arguments"] != nil {
headers := make([]map[string]string, 0)
for name, value := range event["Arguments"].(map[string]interface{}) {
headers = append(headers, map[string]string{
"name": name,
"value": value.(string),
if event["arguments"] != nil {
headers := make([]api.TableData, 0)
for name, value := range event["arguments"].(map[string]interface{}) {
headers = append(headers, api.TableData{
Name: name,
Value: value.(string),
Selector: fmt.Sprintf(`request.arguments["%s"]`, name),
})
}
headersMarshaled, _ := json.Marshal(headers)
rep = append(rep, map[string]string{
"type": api.TABLE,
"title": "Arguments",
"data": string(headersMarshaled),
rep = append(rep, api.SectionData{
Type: api.TABLE,
Title: "Arguments",
Data: string(headersMarshaled),
})
}
@@ -612,51 +672,58 @@ func representQueueBind(event map[string]interface{}) []interface{} {
func representBasicConsume(event map[string]interface{}) []interface{} {
rep := make([]interface{}, 0)
details, _ := json.Marshal([]map[string]string{
details, _ := json.Marshal([]api.TableData{
{
"name": "Queue",
"value": event["Queue"].(string),
Name: "Queue",
Value: event["queue"].(string),
Selector: `request.queue`,
},
{
"name": "Consumer Tag",
"value": event["ConsumerTag"].(string),
Name: "Consumer Tag",
Value: event["consumerTag"].(string),
Selector: `request.consumerTag`,
},
{
"name": "No Local",
"value": strconv.FormatBool(event["NoLocal"].(bool)),
Name: "No Local",
Value: strconv.FormatBool(event["noLocal"].(bool)),
Selector: `request.noLocal`,
},
{
"name": "No Ack",
"value": strconv.FormatBool(event["NoAck"].(bool)),
Name: "No Ack",
Value: strconv.FormatBool(event["noAck"].(bool)),
Selector: `request.noAck`,
},
{
"name": "Exclusive",
"value": strconv.FormatBool(event["Exclusive"].(bool)),
Name: "Exclusive",
Value: strconv.FormatBool(event["exclusive"].(bool)),
Selector: `request.exclusive`,
},
{
"name": "NoWait",
"value": strconv.FormatBool(event["NoWait"].(bool)),
Name: "NoWait",
Value: strconv.FormatBool(event["noWait"].(bool)),
Selector: `request.noWait`,
},
})
rep = append(rep, map[string]string{
"type": api.TABLE,
"title": "Details",
"data": string(details),
rep = append(rep, api.SectionData{
Type: api.TABLE,
Title: "Details",
Data: string(details),
})
if event["Arguments"] != nil {
headers := make([]map[string]string, 0)
for name, value := range event["Arguments"].(map[string]interface{}) {
headers = append(headers, map[string]string{
"name": name,
"value": value.(string),
if event["arguments"] != nil {
headers := make([]api.TableData, 0)
for name, value := range event["arguments"].(map[string]interface{}) {
headers = append(headers, api.TableData{
Name: name,
Value: value.(string),
Selector: fmt.Sprintf(`request.arguments["%s"]`, name),
})
}
headersMarshaled, _ := json.Marshal(headers)
rep = append(rep, map[string]string{
"type": api.TABLE,
"title": "Arguments",
"data": string(headersMarshaled),
rep = append(rep, api.SectionData{
Type: api.TABLE,
Title: "Arguments",
Data: string(headersMarshaled),
})
}

View File

@@ -16,6 +16,7 @@ var protocol api.Protocol = api.Protocol{
Name: "amqp",
LongName: "Advanced Message Queuing Protocol 0-9-1",
Abbreviation: "AMQP",
Macro: "amqp",
Version: "0-9-1",
BackgroundColor: "#ff6600",
ForegroundColor: "#ffffff",
@@ -36,7 +37,7 @@ func (d dissecting) Register(extension *api.Extension) {
}
func (d dissecting) Ping() {
log.Printf("pong %s\n", protocol.Name)
log.Printf("pong %s", protocol.Name)
}
const amqpRequest string = "amqp_request"
@@ -217,102 +218,86 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
}
default:
// log.Printf("unexpected frame: %+v\n", f)
// log.Printf("unexpected frame: %+v", f)
}
}
}
func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry {
func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
request := item.Pair.Request.Payload.(map[string]interface{})
reqDetails := request["details"].(map[string]interface{})
service := "amqp"
if resolvedDestination != "" {
service = resolvedDestination
} else if resolvedSource != "" {
service = resolvedSource
}
summary := ""
switch request["method"] {
case basicMethodMap[40]:
summary = reqDetails["Exchange"].(string)
summary = reqDetails["exchange"].(string)
break
case basicMethodMap[60]:
summary = reqDetails["Exchange"].(string)
summary = reqDetails["exchange"].(string)
break
case exchangeMethodMap[10]:
summary = reqDetails["Exchange"].(string)
summary = reqDetails["exchange"].(string)
break
case queueMethodMap[10]:
summary = reqDetails["Queue"].(string)
summary = reqDetails["queue"].(string)
break
case connectionMethodMap[10]:
summary = fmt.Sprintf(
"%s.%s",
strconv.Itoa(int(reqDetails["VersionMajor"].(float64))),
strconv.Itoa(int(reqDetails["VersionMinor"].(float64))),
strconv.Itoa(int(reqDetails["versionMajor"].(float64))),
strconv.Itoa(int(reqDetails["versionMinor"].(float64))),
)
break
case connectionMethodMap[50]:
summary = reqDetails["ReplyText"].(string)
summary = reqDetails["replyText"].(string)
break
case queueMethodMap[20]:
summary = reqDetails["Queue"].(string)
summary = reqDetails["queue"].(string)
break
case basicMethodMap[20]:
summary = reqDetails["Queue"].(string)
summary = reqDetails["queue"].(string)
break
}
request["url"] = summary
entryBytes, _ := json.Marshal(item.Pair)
reqDetails["method"] = request["method"]
return &api.MizuEntry{
ProtocolName: protocol.Name,
ProtocolLongName: protocol.LongName,
ProtocolAbbreviation: protocol.Abbreviation,
ProtocolVersion: protocol.Version,
ProtocolBackgroundColor: protocol.BackgroundColor,
ProtocolForegroundColor: protocol.ForegroundColor,
ProtocolFontSize: protocol.FontSize,
ProtocolReferenceLink: protocol.ReferenceLink,
EntryId: entryId,
Entry: string(entryBytes),
Url: fmt.Sprintf("%s%s", service, summary),
Method: request["method"].(string),
Status: 0,
RequestSenderIp: item.ConnectionInfo.ClientIP,
Service: service,
Timestamp: item.Timestamp,
ElapsedTime: 0,
Path: summary,
ResolvedSource: resolvedSource,
ResolvedDestination: resolvedDestination,
SourceIp: item.ConnectionInfo.ClientIP,
DestinationIp: item.ConnectionInfo.ServerIP,
SourcePort: item.ConnectionInfo.ClientPort,
DestinationPort: item.ConnectionInfo.ServerPort,
IsOutgoing: item.ConnectionInfo.IsOutgoing,
Protocol: protocol,
Source: &api.TCP{
Name: resolvedSource,
IP: item.ConnectionInfo.ClientIP,
Port: item.ConnectionInfo.ClientPort,
},
Destination: &api.TCP{
Name: resolvedDestination,
IP: item.ConnectionInfo.ServerIP,
Port: item.ConnectionInfo.ServerPort,
},
Outgoing: item.ConnectionInfo.IsOutgoing,
Request: reqDetails,
Method: request["method"].(string),
Status: 0,
Timestamp: item.Timestamp,
StartTime: item.Pair.Request.CaptureTime,
ElapsedTime: 0,
Summary: summary,
IsOutgoing: item.ConnectionInfo.IsOutgoing,
}
}
func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
return &api.BaseEntryDetails{
Id: entry.EntryId,
Protocol: protocol,
Url: entry.Url,
RequestSenderIp: entry.RequestSenderIp,
Service: entry.Service,
Summary: entry.Path,
StatusCode: entry.Status,
Method: entry.Method,
Timestamp: entry.Timestamp,
SourceIp: entry.SourceIp,
DestinationIp: entry.DestinationIp,
SourcePort: entry.SourcePort,
DestinationPort: entry.DestinationPort,
IsOutgoing: entry.IsOutgoing,
Latency: entry.ElapsedTime,
Id: entry.Id,
Protocol: protocol,
Summary: entry.Summary,
StatusCode: entry.Status,
Method: entry.Method,
Timestamp: entry.Timestamp,
Source: entry.Source,
Destination: entry.Destination,
IsOutgoing: entry.IsOutgoing,
Latency: entry.ElapsedTime,
Rules: api.ApplicableRules{
Latency: 0,
Status: false,
@@ -320,39 +305,34 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
}
}
func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) {
p = protocol
func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) {
bodySize = 0
var root map[string]interface{}
json.Unmarshal([]byte(entry.Entry), &root)
representation := make(map[string]interface{}, 0)
request := root["request"].(map[string]interface{})["payload"].(map[string]interface{})
var repRequest []interface{}
details := request["details"].(map[string]interface{})
switch request["method"].(string) {
case basicMethodMap[40]:
repRequest = representBasicPublish(details)
repRequest = representBasicPublish(request)
break
case basicMethodMap[60]:
repRequest = representBasicDeliver(details)
repRequest = representBasicDeliver(request)
break
case queueMethodMap[10]:
repRequest = representQueueDeclare(details)
repRequest = representQueueDeclare(request)
break
case exchangeMethodMap[10]:
repRequest = representExchangeDeclare(details)
repRequest = representExchangeDeclare(request)
break
case connectionMethodMap[10]:
repRequest = representConnectionStart(details)
repRequest = representConnectionStart(request)
break
case connectionMethodMap[50]:
repRequest = representConnectionClose(details)
repRequest = representConnectionClose(request)
break
case queueMethodMap[20]:
repRequest = representQueueBind(details)
repRequest = representQueueBind(request)
break
case basicMethodMap[20]:
repRequest = representBasicConsume(details)
repRequest = representBasicConsume(request)
break
}
representation["request"] = repRequest
@@ -360,4 +340,10 @@ func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []by
return
}
func (d dissecting) Macros() map[string]string {
return map[string]string{
`amqp`: fmt.Sprintf(`proto.name == "%s"`, protocol.Name),
}
}
var Dissector dissecting

View File

@@ -71,11 +71,11 @@ func isSoftExceptionCode(code int) bool {
}
type ConnectionStart struct {
VersionMajor byte
VersionMinor byte
ServerProperties Table
Mechanisms string
Locales string
VersionMajor byte `json:"versionMajor"`
VersionMinor byte `json:"versionMinor"`
ServerProperties Table `json:"serverProperties"`
Mechanisms string `json:"mechanisms"`
Locales string `json:"locales"`
}
func (msg *ConnectionStart) id() (uint16, uint16) {
@@ -429,10 +429,10 @@ func (msg *connectionOpenOk) read(r io.Reader) (err error) {
}
type ConnectionClose struct {
ReplyCode uint16
ReplyText string
ClassId uint16
MethodId uint16
ReplyCode uint16 `json:"relyCode"`
ReplyText string `json:"replyText"`
ClassId uint16 `json:"classId"`
MethodId uint16 `json:"methodId"`
}
func (msg *ConnectionClose) id() (uint16, uint16) {
@@ -767,14 +767,14 @@ func (msg *channelCloseOk) read(r io.Reader) (err error) {
type ExchangeDeclare struct {
reserved1 uint16
Exchange string
Type string
Passive bool
Durable bool
AutoDelete bool
Internal bool
NoWait bool
Arguments Table
Exchange string `json:"exchange"`
Type string `json:"type"`
Passive bool `json:"passive"`
Durable bool `json:"durable"`
AutoDelete bool `json:"autoDelete"`
Internal bool `json:"internal"`
NoWait bool `json:"noWait"`
Arguments Table `json:"arguments"`
}
func (msg *ExchangeDeclare) id() (uint16, uint16) {
@@ -1163,13 +1163,13 @@ func (msg *exchangeUnbindOk) read(r io.Reader) (err error) {
type QueueDeclare struct {
reserved1 uint16
Queue string
Passive bool
Durable bool
Exclusive bool
AutoDelete bool
NoWait bool
Arguments Table
Queue string `json:"queue"`
Passive bool `json:"passive"`
Durable bool `json:"durable"`
Exclusive bool `json:"exclusive"`
AutoDelete bool `json:"autoDelete"`
NoWait bool `json:"noWait"`
Arguments Table `json:"arguments"`
}
func (msg *QueueDeclare) id() (uint16, uint16) {
@@ -1297,11 +1297,11 @@ func (msg *QueueDeclareOk) read(r io.Reader) (err error) {
type QueueBind struct {
reserved1 uint16
Queue string
Exchange string
RoutingKey string
NoWait bool
Arguments Table
Queue string `json:"queue"`
Exchange string `json:"exchange"`
RoutingKey string `json:"routingKey"`
NoWait bool `json:"noWait"`
Arguments Table `json:"arguments"`
}
func (msg *QueueBind) id() (uint16, uint16) {
@@ -1737,13 +1737,13 @@ func (msg *basicQosOk) read(r io.Reader) (err error) {
type BasicConsume struct {
reserved1 uint16
Queue string
ConsumerTag string
NoLocal bool
NoAck bool
Exclusive bool
NoWait bool
Arguments Table
Queue string `json:"queue"`
ConsumerTag string `json:"consumerTag"`
NoLocal bool `json:"noLocal"`
NoAck bool `json:"noAck"`
Exclusive bool `json:"exclusive"`
NoWait bool `json:"noWait"`
Arguments Table `json:"arguments"`
}
func (msg *BasicConsume) id() (uint16, uint16) {
@@ -1932,12 +1932,12 @@ func (msg *basicCancelOk) read(r io.Reader) (err error) {
type BasicPublish struct {
reserved1 uint16
Exchange string
RoutingKey string
Mandatory bool
Immediate bool
Properties Properties
Body []byte
Exchange string `json:"exchange"`
RoutingKey string `json:"routingKey"`
Mandatory bool `json:"mandatory"`
Immediate bool `json:"immediate"`
Properties Properties `json:"properties"`
Body []byte `json:"body"`
}
func (msg *BasicPublish) id() (uint16, uint16) {
@@ -2072,13 +2072,13 @@ func (msg *basicReturn) read(r io.Reader) (err error) {
}
type BasicDeliver struct {
ConsumerTag string
DeliveryTag uint64
Redelivered bool
Exchange string
RoutingKey string
Properties Properties
Body []byte
ConsumerTag string `json:"consumerTag"`
DeliveryTag uint64 `json:"deliveryTag"`
Redelivered bool `json:"redelivered"`
Exchange string `json:"exchange"`
RoutingKey string `json:"routingKey"`
Properties Properties `json:"properties"`
Body []byte `json:"body"`
}
func (msg *BasicDeliver) id() (uint16, uint16) {

View File

@@ -93,19 +93,19 @@ func (e Error) Error() string {
// Used by header frames to capture routing and header information
type Properties struct {
ContentType string // MIME content type
ContentEncoding string // MIME content encoding
Headers Table // Application or header exchange table
DeliveryMode uint8 // queue implementation use - Transient (1) or Persistent (2)
Priority uint8 // queue implementation use - 0 to 9
CorrelationId string // application use - correlation identifier
ReplyTo string // application use - address to to reply to (ex: RPC)
Expiration string // implementation use - message expiration spec
MessageId string // application use - message identifier
Timestamp time.Time // application use - message timestamp
Type string // application use - message type name
UserId string // application use - creating user id
AppId string // application use - creating application
ContentType string `json:"contentType"` // MIME content type
ContentEncoding string `json:"contentEncoding"` // MIME content encoding
Headers Table `json:"headers"` // Application or header exchange table
DeliveryMode uint8 `json:"deliveryMode"` // queue implementation use - Transient (1) or Persistent (2)
Priority uint8 `json:"priority"` // queue implementation use - 0 to 9
CorrelationId string `json:"correlationId"` // application use - correlation identifier
ReplyTo string `json:"replyTo"` // application use - address to to reply to (ex: RPC)
Expiration string `json:"expiration"` // implementation use - message expiration spec
MessageId string `json:"messageId"` // application use - message identifier
Timestamp time.Time `json:"timestamp"` // application use - message timestamp
Type string `json:"type"` // application use - message type name
UserId string `json:"userId"` // application use - creating user id
AppId string `json:"appId"` // application use - creating application
reserved1 string // was cluster-id - process for buffer consumption
}

View File

@@ -7,6 +7,7 @@ import (
"io"
"io/ioutil"
"net/http"
"strings"
"github.com/up9inc/mizu/tap/api"
)
@@ -23,8 +24,8 @@ func filterAndEmit(item *api.OutputChannelItem, emitter api.Emitter, options *ap
emitter.Emit(item)
}
func handleHTTP2Stream(grpcAssembler *GrpcAssembler, tcpID *api.TcpID, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
streamID, messageHTTP1, err := grpcAssembler.readMessage()
func handleHTTP2Stream(http2Assembler *Http2Assembler, tcpID *api.TcpID, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
streamID, messageHTTP1, isGrpc, err := http2Assembler.readMessage()
if err != nil {
return err
}
@@ -34,12 +35,13 @@ func handleHTTP2Stream(grpcAssembler *GrpcAssembler, tcpID *api.TcpID, superTime
switch messageHTTP1 := messageHTTP1.(type) {
case http.Request:
ident := fmt.Sprintf(
"%s->%s %s->%s %d",
"%s->%s %s->%s %d %s",
tcpID.SrcIP,
tcpID.DstIP,
tcpID.SrcPort,
tcpID.DstPort,
streamID,
"HTTP2",
)
item = reqResMatcher.registerRequest(ident, &messageHTTP1, superTimer.CaptureTime)
if item != nil {
@@ -53,12 +55,13 @@ func handleHTTP2Stream(grpcAssembler *GrpcAssembler, tcpID *api.TcpID, superTime
}
case http.Response:
ident := fmt.Sprintf(
"%s->%s %s->%s %d",
"%s->%s %s->%s %d %s",
tcpID.DstIP,
tcpID.SrcIP,
tcpID.DstPort,
tcpID.SrcPort,
streamID,
"HTTP2",
)
item = reqResMatcher.registerResponse(ident, &messageHTTP1, superTimer.CaptureTime)
if item != nil {
@@ -73,30 +76,41 @@ func handleHTTP2Stream(grpcAssembler *GrpcAssembler, tcpID *api.TcpID, superTime
}
if item != nil {
item.Protocol = http2Protocol
if isGrpc {
item.Protocol = grpcProtocol
} else {
item.Protocol = http2Protocol
}
filterAndEmit(item, emitter, options)
}
return nil
}
func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
req, err := http.ReadRequest(b)
func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) (switchingProtocolsHTTP2 bool, req *http.Request, err error) {
req, err = http.ReadRequest(b)
if err != nil {
return err
return
}
counterPair.Request++
body, err := ioutil.ReadAll(req.Body)
// Check HTTP2 upgrade - HTTP2 Over Cleartext (H2C)
if strings.Contains(strings.ToLower(req.Header.Get("Connection")), "upgrade") && strings.ToLower(req.Header.Get("Upgrade")) == "h2c" {
switchingProtocolsHTTP2 = true
}
var body []byte
body, err = ioutil.ReadAll(req.Body)
req.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind
ident := fmt.Sprintf(
"%s->%s %s->%s %d",
"%s->%s %s->%s %d %s",
tcpID.SrcIP,
tcpID.DstIP,
tcpID.SrcPort,
tcpID.DstPort,
counterPair.Request,
"HTTP1",
)
item := reqResMatcher.registerRequest(ident, req, superTimer.CaptureTime)
if item != nil {
@@ -109,26 +123,34 @@ func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api
}
filterAndEmit(item, emitter, options)
}
return nil
return
}
func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
res, err := http.ReadResponse(b, nil)
func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, emitter api.Emitter, options *api.TrafficFilteringOptions) (switchingProtocolsHTTP2 bool, err error) {
var res *http.Response
res, err = http.ReadResponse(b, nil)
if err != nil {
return err
return
}
counterPair.Response++
body, err := ioutil.ReadAll(res.Body)
// Check HTTP2 upgrade - HTTP2 Over Cleartext (H2C)
if res.StatusCode == 101 && strings.Contains(strings.ToLower(res.Header.Get("Connection")), "upgrade") && strings.ToLower(res.Header.Get("Upgrade")) == "h2c" {
switchingProtocolsHTTP2 = true
}
var body []byte
body, err = ioutil.ReadAll(res.Body)
res.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind
ident := fmt.Sprintf(
"%s->%s %s->%s %d",
"%s->%s %s->%s %d %s",
tcpID.DstIP,
tcpID.SrcIP,
tcpID.DstPort,
tcpID.SrcPort,
counterPair.Response,
"HTTP1",
)
item := reqResMatcher.registerResponse(ident, res, superTimer.CaptureTime)
if item != nil {
@@ -141,5 +163,5 @@ func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api
}
filterAndEmit(item, emitter, options)
}
return nil
return
}

View File

@@ -0,0 +1,35 @@
package main
import (
"encoding/json"
"fmt"
"github.com/up9inc/mizu/tap/api"
)
func mapSliceRebuildAsMap(mapSlice []interface{}) (newMap map[string]interface{}) {
newMap = make(map[string]interface{})
for _, header := range mapSlice {
h := header.(map[string]interface{})
newMap[h["name"].(string)] = h["value"]
}
return
}
func representMapSliceAsTable(mapSlice []interface{}, selectorPrefix string) (representation string) {
var table []api.TableData
for _, header := range mapSlice {
h := header.(map[string]interface{})
selector := fmt.Sprintf("%s[\"%s\"]", selectorPrefix, h["name"].(string))
table = append(table, api.TableData{
Name: h["name"].(string),
Value: h["value"],
Selector: selector,
})
}
obj, _ := json.Marshal(table)
representation = string(obj)
return
}

View File

@@ -10,6 +10,7 @@ import (
"math"
"net/http"
"net/url"
"strconv"
"strings"
"golang.org/x/net/http2"
@@ -27,6 +28,26 @@ const protoMinorHTTP2 = 0
var maxHTTP2DataLen = 1 * 1024 * 1024 // 1MB
var grpcStatusCodes = []string{
"OK",
"CANCELLED",
"UNKNOWN",
"INVALID_ARGUMENT",
"DEADLINE_EXCEEDED",
"NOT_FOUND",
"ALREADY_EXISTS",
"PERMISSION_DENIED",
"RESOURCE_EXHAUSTED",
"FAILED_PRECONDITION",
"ABORTED",
"OUT_OF_RANGE",
"UNIMPLEMENTED",
"INTERNAL",
"UNAVAILABLE",
"DATA_LOSS",
"UNAUTHENTICATED",
}
type messageFragment struct {
headers []hpack.HeaderField
data []byte
@@ -71,37 +92,38 @@ func (fbs *fragmentsByStream) pop(streamID uint32) ([]hpack.HeaderField, []byte)
return headers, data
}
func createGrpcAssembler(b *bufio.Reader) *GrpcAssembler {
func createHTTP2Assembler(b *bufio.Reader) *Http2Assembler {
var framerOutput bytes.Buffer
framer := http2.NewFramer(&framerOutput, b)
framer.ReadMetaHeaders = hpack.NewDecoder(initialHeaderTableSize, nil)
return &GrpcAssembler{
return &Http2Assembler{
fragmentsByStream: make(fragmentsByStream),
framer: framer,
}
}
type GrpcAssembler struct {
type Http2Assembler struct {
fragmentsByStream fragmentsByStream
framer *http2.Framer
}
func (ga *GrpcAssembler) readMessage() (uint32, interface{}, error) {
func (ga *Http2Assembler) readMessage() (streamID uint32, messageHTTP1 interface{}, isGrpc bool, err error) {
// Exactly one Framer is used for each half connection.
// (Instead of creating a new Framer for each ReadFrame operation)
// This is needed in order to decompress the headers,
// because the compression context is updated with each requests/response.
frame, err := ga.framer.ReadFrame()
if err != nil {
return 0, nil, err
return
}
streamID := frame.Header().StreamID
streamID = frame.Header().StreamID
ga.fragmentsByStream.appendFrame(streamID, frame)
if !(ga.isStreamEnd(frame)) {
return 0, nil, nil
streamID = 0
return
}
headers, data := ga.fragmentsByStream.pop(streamID)
@@ -115,13 +137,29 @@ func (ga *GrpcAssembler) readMessage() (uint32, interface{}, error) {
dataString := base64.StdEncoding.EncodeToString(data)
// Use http1 types only because they are expected in http_matcher.
// TODO: Create an interface that will be used by http_matcher:registerRequest and http_matcher:registerRequest
// to accept both HTTP/1.x and HTTP/2 requests and responses
var messageHTTP1 interface{}
if _, ok := headersHTTP1[":method"]; ok {
method := headersHTTP1.Get(":method")
status := headersHTTP1.Get(":status")
// gRPC detection
grpcStatus := headersHTTP1.Get("Grpc-Status")
if grpcStatus != "" {
isGrpc = true
status = grpcStatus
}
if strings.Contains(headersHTTP1.Get("Content-Type"), "application/grpc") {
isGrpc = true
grpcPath := headersHTTP1.Get(":path")
pathSegments := strings.Split(grpcPath, "/")
if len(pathSegments) > 0 {
method = pathSegments[len(pathSegments)-1]
}
}
if method != "" {
messageHTTP1 = http.Request{
URL: &url.URL{},
Method: "POST",
Method: method,
Header: headersHTTP1,
Proto: protoHTTP2,
ProtoMajor: protoMajorHTTP2,
@@ -129,8 +167,16 @@ func (ga *GrpcAssembler) readMessage() (uint32, interface{}, error) {
Body: io.NopCloser(strings.NewReader(dataString)),
ContentLength: int64(len(dataString)),
}
} else if _, ok := headersHTTP1[":status"]; ok {
} else if status != "" {
var statusCode int
statusCode, err = strconv.Atoi(status)
if err != nil {
return
}
messageHTTP1 = http.Response{
StatusCode: statusCode,
Header: headersHTTP1,
Proto: protoHTTP2,
ProtoMajor: protoMajorHTTP2,
@@ -139,13 +185,14 @@ func (ga *GrpcAssembler) readMessage() (uint32, interface{}, error) {
ContentLength: int64(len(dataString)),
}
} else {
return 0, nil, errors.New("failed to assemble stream: neither a request nor a message")
err = errors.New("failed to assemble stream: neither a request nor a message")
return
}
return streamID, messageHTTP1, nil
return
}
func (ga *GrpcAssembler) isStreamEnd(frame http2.Frame) bool {
func (ga *Http2Assembler) isStreamEnd(frame http2.Frame) bool {
switch frame := frame.(type) {
case *http2.MetaHeadersFrame:
if frame.StreamEnded() {

View File

@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"log"
"net/http"
"net/url"
"time"
@@ -17,25 +18,41 @@ var protocol api.Protocol = api.Protocol{
Name: "http",
LongName: "Hypertext Transfer Protocol -- HTTP/1.1",
Abbreviation: "HTTP",
Macro: "http",
Version: "1.1",
BackgroundColor: "#205cf5",
ForegroundColor: "#ffffff",
FontSize: 12,
ReferenceLink: "https://datatracker.ietf.org/doc/html/rfc2616",
Ports: []string{"80", "8080", "50051"},
Ports: []string{"80", "443", "8080"},
Priority: 0,
}
var http2Protocol api.Protocol = api.Protocol{
Name: "http",
LongName: "Hypertext Transfer Protocol Version 2 (HTTP/2) (gRPC)",
LongName: "Hypertext Transfer Protocol Version 2 (HTTP/2)",
Abbreviation: "HTTP/2",
Macro: "http2",
Version: "2.0",
BackgroundColor: "#244c5a",
ForegroundColor: "#ffffff",
FontSize: 11,
ReferenceLink: "https://datatracker.ietf.org/doc/html/rfc7540",
Ports: []string{"80", "8080"},
Ports: []string{"80", "443", "8080"},
Priority: 0,
}
var grpcProtocol api.Protocol = api.Protocol{
Name: "http",
LongName: "Hypertext Transfer Protocol Version 2 (HTTP/2) [ gRPC over HTTP/2 ]",
Abbreviation: "gRPC",
Macro: "grpc",
Version: "2.0",
BackgroundColor: "#244c5a",
ForegroundColor: "#ffffff",
FontSize: 11,
ReferenceLink: "https://grpc.github.io/grpc/core/md_doc_statuscodes.html",
Ports: []string{"80", "443", "8080", "50051"},
Priority: 0,
}
@@ -56,26 +73,34 @@ func (d dissecting) Register(extension *api.Extension) {
}
func (d dissecting) Ping() {
log.Printf("pong %s\n", protocol.Name)
log.Printf("pong %s", protocol.Name)
}
func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, superTimer *api.SuperTimer, superIdentifier *api.SuperIdentifier, emitter api.Emitter, options *api.TrafficFilteringOptions) error {
isHTTP2, err := checkIsHTTP2Connection(b, isClient)
var grpcAssembler *GrpcAssembler
var http2Assembler *Http2Assembler
if isHTTP2 {
prepareHTTP2Connection(b, isClient)
grpcAssembler = createGrpcAssembler(b)
http2Assembler = createHTTP2Assembler(b)
}
dissected := false
switchingProtocolsHTTP2 := false
for {
if switchingProtocolsHTTP2 {
switchingProtocolsHTTP2 = false
isHTTP2, err = checkIsHTTP2Connection(b, isClient)
prepareHTTP2Connection(b, isClient)
http2Assembler = createHTTP2Assembler(b)
}
if superIdentifier.Protocol != nil && superIdentifier.Protocol != &protocol {
return errors.New("Identified by another protocol")
}
if isHTTP2 {
err = handleHTTP2Stream(grpcAssembler, tcpID, superTimer, emitter, options)
err = handleHTTP2Stream(http2Assembler, tcpID, superTimer, emitter, options)
if err == io.EOF || err == io.ErrUnexpectedEOF {
break
} else if err != nil {
@@ -83,15 +108,39 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
}
dissected = true
} else if isClient {
err = handleHTTP1ClientStream(b, tcpID, counterPair, superTimer, emitter, options)
var req *http.Request
switchingProtocolsHTTP2, req, err = handleHTTP1ClientStream(b, tcpID, counterPair, superTimer, emitter, options)
if err == io.EOF || err == io.ErrUnexpectedEOF {
break
} else if err != nil {
continue
}
dissected = true
// In case of an HTTP2 upgrade, duplicate the HTTP1 request into HTTP2 with stream ID 1
if switchingProtocolsHTTP2 {
ident := fmt.Sprintf(
"%s->%s %s->%s 1 %s",
tcpID.SrcIP,
tcpID.DstIP,
tcpID.SrcPort,
tcpID.DstPort,
"HTTP2",
)
item := reqResMatcher.registerRequest(ident, req, superTimer.CaptureTime)
if item != nil {
item.ConnectionInfo = &api.ConnectionInfo{
ClientIP: tcpID.SrcIP,
ClientPort: tcpID.SrcPort,
ServerIP: tcpID.DstIP,
ServerPort: tcpID.DstPort,
IsOutgoing: true,
}
filterAndEmit(item, emitter, options)
}
}
} else {
err = handleHTTP1ServerStream(b, tcpID, counterPair, superTimer, emitter, options)
switchingProtocolsHTTP2, err = handleHTTP1ServerStream(b, tcpID, counterPair, superTimer, emitter, options)
if err == io.EOF || err == io.ErrUnexpectedEOF {
break
} else if err != nil {
@@ -108,23 +157,16 @@ func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, co
return nil
}
func SetHostname(address, newHostname string) string {
replacedUrl, err := url.Parse(address)
if err != nil {
return address
}
replacedUrl.Host = newHostname
return replacedUrl.String()
}
func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry {
var host, scheme, authority, path, service string
func (d dissecting) Analyze(item *api.OutputChannelItem, resolvedSource string, resolvedDestination string) *api.MizuEntry {
var host, authority, path string
request := item.Pair.Request.Payload.(map[string]interface{})
response := item.Pair.Response.Payload.(map[string]interface{})
reqDetails := request["details"].(map[string]interface{})
resDetails := response["details"].(map[string]interface{})
isRequestUpgradedH2C := false
for _, header := range reqDetails["headers"].([]interface{}) {
h := header.(map[string]interface{})
if h["name"] == "Host" {
@@ -133,83 +175,111 @@ func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolve
if h["name"] == ":authority" {
authority = h["value"].(string)
}
if h["name"] == ":scheme" {
scheme = h["value"].(string)
}
if h["name"] == ":path" {
path = h["value"].(string)
}
if h["name"] == "Upgrade" {
if h["value"].(string) == "h2c" {
isRequestUpgradedH2C = true
}
}
}
if item.Protocol.Version == "2.0" {
service = fmt.Sprintf("%s://%s", scheme, authority)
if resDetails["bodySize"].(float64) < 0 {
resDetails["bodySize"] = 0
}
if item.Protocol.Version == "2.0" && !isRequestUpgradedH2C {
if resolvedDestination == "" {
resolvedDestination = authority
}
if resolvedDestination == "" {
resolvedDestination = host
}
} else {
service = fmt.Sprintf("http://%s", host)
path = reqDetails["url"].(string)
u, err := url.Parse(reqDetails["url"].(string))
if err != nil {
path = reqDetails["url"].(string)
} else {
path = u.Path
}
}
request["url"] = path
if resolvedDestination != "" {
service = SetHostname(service, resolvedDestination)
} else if resolvedSource != "" {
service = SetHostname(service, resolvedSource)
request["url"] = reqDetails["url"].(string)
reqDetails["targetUri"] = reqDetails["url"]
reqDetails["path"] = path
reqDetails["summary"] = path
// Rearrange the maps for the querying
reqDetails["_headers"] = reqDetails["headers"]
reqDetails["headers"] = mapSliceRebuildAsMap(reqDetails["_headers"].([]interface{}))
resDetails["_headers"] = resDetails["headers"]
resDetails["headers"] = mapSliceRebuildAsMap(resDetails["_headers"].([]interface{}))
reqDetails["_cookies"] = reqDetails["cookies"]
reqDetails["cookies"] = mapSliceRebuildAsMap(reqDetails["_cookies"].([]interface{}))
resDetails["_cookies"] = resDetails["cookies"]
resDetails["cookies"] = mapSliceRebuildAsMap(resDetails["_cookies"].([]interface{}))
reqDetails["_queryString"] = reqDetails["queryString"]
reqDetails["queryString"] = mapSliceRebuildAsMap(reqDetails["_queryString"].([]interface{}))
method := reqDetails["method"].(string)
statusCode := int(resDetails["status"].(float64))
if item.Protocol.Abbreviation == "gRPC" {
resDetails["statusText"] = grpcStatusCodes[statusCode]
}
if item.Protocol.Version == "2.0" && !isRequestUpgradedH2C {
reqDetails["url"] = path
request["url"] = path
}
elapsedTime := item.Pair.Response.CaptureTime.Sub(item.Pair.Request.CaptureTime).Round(time.Millisecond).Milliseconds()
entryBytes, _ := json.Marshal(item.Pair)
if elapsedTime < 0 {
elapsedTime = 0
}
httpPair, _ := json.Marshal(item.Pair)
return &api.MizuEntry{
ProtocolName: protocol.Name,
ProtocolLongName: protocol.LongName,
ProtocolAbbreviation: protocol.Abbreviation,
ProtocolVersion: item.Protocol.Version,
ProtocolBackgroundColor: protocol.BackgroundColor,
ProtocolForegroundColor: protocol.ForegroundColor,
ProtocolFontSize: protocol.FontSize,
ProtocolReferenceLink: protocol.ReferenceLink,
EntryId: entryId,
Entry: string(entryBytes),
Url: fmt.Sprintf("%s%s", service, path),
Method: reqDetails["method"].(string),
Status: int(resDetails["status"].(float64)),
RequestSenderIp: item.ConnectionInfo.ClientIP,
Service: service,
Timestamp: item.Timestamp,
ElapsedTime: elapsedTime,
Path: path,
ResolvedSource: resolvedSource,
ResolvedDestination: resolvedDestination,
SourceIp: item.ConnectionInfo.ClientIP,
DestinationIp: item.ConnectionInfo.ServerIP,
SourcePort: item.ConnectionInfo.ClientPort,
DestinationPort: item.ConnectionInfo.ServerPort,
IsOutgoing: item.ConnectionInfo.IsOutgoing,
Protocol: item.Protocol,
Source: &api.TCP{
Name: resolvedSource,
IP: item.ConnectionInfo.ClientIP,
Port: item.ConnectionInfo.ClientPort,
},
Destination: &api.TCP{
Name: resolvedDestination,
IP: item.ConnectionInfo.ServerIP,
Port: item.ConnectionInfo.ServerPort,
},
Outgoing: item.ConnectionInfo.IsOutgoing,
Request: reqDetails,
Response: resDetails,
Method: method,
Status: statusCode,
Timestamp: item.Timestamp,
StartTime: item.Pair.Request.CaptureTime,
ElapsedTime: elapsedTime,
Summary: path,
IsOutgoing: item.ConnectionInfo.IsOutgoing,
HTTPPair: string(httpPair),
}
}
func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
var p api.Protocol
if entry.ProtocolVersion == "2.0" {
p = http2Protocol
} else {
p = protocol
}
return &api.BaseEntryDetails{
Id: entry.EntryId,
Protocol: p,
Url: entry.Url,
RequestSenderIp: entry.RequestSenderIp,
Service: entry.Service,
Path: entry.Path,
Summary: entry.Path,
StatusCode: entry.Status,
Method: entry.Method,
Timestamp: entry.Timestamp,
SourceIp: entry.SourceIp,
DestinationIp: entry.DestinationIp,
SourcePort: entry.SourcePort,
DestinationPort: entry.DestinationPort,
IsOutgoing: entry.IsOutgoing,
Latency: entry.ElapsedTime,
Id: entry.Id,
Protocol: entry.Protocol,
Path: entry.Path,
Summary: entry.Summary,
StatusCode: entry.Status,
Method: entry.Method,
Timestamp: entry.Timestamp,
Source: entry.Source,
Destination: entry.Destination,
IsOutgoing: entry.IsOutgoing,
Latency: entry.ElapsedTime,
Rules: api.ApplicableRules{
Latency: 0,
Status: false,
@@ -218,45 +288,50 @@ func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
}
func representRequest(request map[string]interface{}) (repRequest []interface{}) {
details, _ := json.Marshal([]map[string]string{
details, _ := json.Marshal([]api.TableData{
{
"name": "Method",
"value": request["method"].(string),
Name: "Method",
Value: request["method"].(string),
Selector: `request.method`,
},
{
"name": "URL",
"value": request["url"].(string),
Name: "Target URI",
Value: request["targetUri"].(string),
Selector: `request.targetUri`,
},
{
"name": "Body Size",
"value": fmt.Sprintf("%g bytes", request["bodySize"].(float64)),
Name: "Path",
Value: request["path"].(string),
Selector: `request.path`,
},
{
Name: "Body Size (bytes)",
Value: int64(request["bodySize"].(float64)),
Selector: `request.bodySize`,
},
})
repRequest = append(repRequest, map[string]string{
"type": api.TABLE,
"title": "Details",
"data": string(details),
repRequest = append(repRequest, api.SectionData{
Type: api.TABLE,
Title: "Details",
Data: string(details),
})
headers, _ := json.Marshal(request["headers"].([]interface{}))
repRequest = append(repRequest, map[string]string{
"type": api.TABLE,
"title": "Headers",
"data": string(headers),
repRequest = append(repRequest, api.SectionData{
Type: api.TABLE,
Title: "Headers",
Data: representMapSliceAsTable(request["_headers"].([]interface{}), `request.headers`),
})
cookies, _ := json.Marshal(request["cookies"].([]interface{}))
repRequest = append(repRequest, map[string]string{
"type": api.TABLE,
"title": "Cookies",
"data": string(cookies),
repRequest = append(repRequest, api.SectionData{
Type: api.TABLE,
Title: "Cookies",
Data: representMapSliceAsTable(request["_cookies"].([]interface{}), `request.cookies`),
})
queryString, _ := json.Marshal(request["queryString"].([]interface{}))
repRequest = append(repRequest, map[string]string{
"type": api.TABLE,
"title": "Query String",
"data": string(queryString),
repRequest = append(repRequest, api.SectionData{
Type: api.TABLE,
Title: "Query String",
Data: representMapSliceAsTable(request["_queryString"].([]interface{}), `request.queryString`),
})
postData, _ := request["postData"].(map[string]interface{})
@@ -266,12 +341,12 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
}
text, _ := postData["text"]
if text != nil {
repRequest = append(repRequest, map[string]string{
"type": api.BODY,
"title": "POST Data (text/plain)",
"encoding": "",
"mime_type": mimeType.(string),
"data": text.(string),
repRequest = append(repRequest, api.SectionData{
Type: api.BODY,
Title: "POST Data (text/plain)",
MimeType: mimeType.(string),
Data: text.(string),
Selector: `request.postData.text`,
})
}
@@ -285,16 +360,16 @@ func representRequest(request map[string]interface{}) (repRequest []interface{})
"value": string(params),
},
})
repRequest = append(repRequest, map[string]string{
"type": api.TABLE,
"title": "POST Data (multipart/form-data)",
"data": string(multipart),
repRequest = append(repRequest, api.SectionData{
Type: api.TABLE,
Title: "POST Data (multipart/form-data)",
Data: string(multipart),
})
} else {
repRequest = append(repRequest, map[string]string{
"type": api.TABLE,
"title": "POST Data (application/x-www-form-urlencoded)",
"data": string(params),
repRequest = append(repRequest, api.SectionData{
Type: api.TABLE,
Title: "POST Data (application/x-www-form-urlencoded)",
Data: representMapSliceAsTable(postData["params"].([]interface{}), `request.postData.params`),
})
}
}
@@ -308,38 +383,39 @@ func representResponse(response map[string]interface{}) (repResponse []interface
bodySize = int64(response["bodySize"].(float64))
details, _ := json.Marshal([]map[string]string{
details, _ := json.Marshal([]api.TableData{
{
"name": "Status",
"value": fmt.Sprintf("%g", response["status"].(float64)),
Name: "Status",
Value: int64(response["status"].(float64)),
Selector: `response.status`,
},
{
"name": "Status Text",
"value": response["statusText"].(string),
Name: "Status Text",
Value: response["statusText"].(string),
Selector: `response.statusText`,
},
{
"name": "Body Size",
"value": fmt.Sprintf("%d bytes", bodySize),
Name: "Body Size (bytes)",
Value: bodySize,
Selector: `response.bodySize`,
},
})
repResponse = append(repResponse, map[string]string{
"type": api.TABLE,
"title": "Details",
"data": string(details),
repResponse = append(repResponse, api.SectionData{
Type: api.TABLE,
Title: "Details",
Data: string(details),
})
headers, _ := json.Marshal(response["headers"].([]interface{}))
repResponse = append(repResponse, map[string]string{
"type": api.TABLE,
"title": "Headers",
"data": string(headers),
repResponse = append(repResponse, api.SectionData{
Type: api.TABLE,
Title: "Headers",
Data: representMapSliceAsTable(response["_headers"].([]interface{}), `response.headers`),
})
cookies, _ := json.Marshal(response["cookies"].([]interface{}))
repResponse = append(repResponse, map[string]string{
"type": api.TABLE,
"title": "Cookies",
"data": string(cookies),
repResponse = append(repResponse, api.SectionData{
Type: api.TABLE,
Title: "Cookies",
Data: representMapSliceAsTable(response["_cookies"].([]interface{}), `response.cookies`),
})
content, _ := response["content"].(map[string]interface{})
@@ -350,37 +426,35 @@ func representResponse(response map[string]interface{}) (repResponse []interface
encoding, _ := content["encoding"]
text, _ := content["text"]
if text != nil {
repResponse = append(repResponse, map[string]string{
"type": api.BODY,
"title": "Body",
"encoding": encoding.(string),
"mime_type": mimeType.(string),
"data": text.(string),
repResponse = append(repResponse, api.SectionData{
Type: api.BODY,
Title: "Body",
Encoding: encoding.(string),
MimeType: mimeType.(string),
Data: text.(string),
Selector: `response.content.text`,
})
}
return
}
func (d dissecting) Represent(entry *api.MizuEntry) (p api.Protocol, object []byte, bodySize int64, err error) {
if entry.ProtocolVersion == "2.0" {
p = http2Protocol
} else {
p = protocol
}
var root map[string]interface{}
json.Unmarshal([]byte(entry.Entry), &root)
func (d dissecting) Represent(request map[string]interface{}, response map[string]interface{}) (object []byte, bodySize int64, err error) {
representation := make(map[string]interface{}, 0)
request := root["request"].(map[string]interface{})["payload"].(map[string]interface{})
response := root["response"].(map[string]interface{})["payload"].(map[string]interface{})
reqDetails := request["details"].(map[string]interface{})
resDetails := response["details"].(map[string]interface{})
repRequest := representRequest(reqDetails)
repResponse, bodySize := representResponse(resDetails)
repRequest := representRequest(request)
repResponse, bodySize := representResponse(response)
representation["request"] = repRequest
representation["response"] = repResponse
object, err = json.Marshal(representation)
return
}
func (d dissecting) Macros() map[string]string {
return map[string]string{
`http`: fmt.Sprintf(`proto.name == "%s" and proto.version == "%s"`, protocol.Name, protocol.Version),
`http2`: fmt.Sprintf(`proto.name == "%s" and proto.version == "%s"`, protocol.Name, http2Protocol.Version),
`grpc`: fmt.Sprintf(`proto.name == "%s" and proto.version == "%s" and proto.macro == "%s"`, protocol.Name, grpcProtocol.Version, grpcProtocol.Macro),
}
}
var Dissector dissecting

View File

@@ -92,6 +92,6 @@ func splitIdent(ident string) []string {
}
func genKey(split []string) string {
key := fmt.Sprintf("%s:%s->%s:%s,%s", split[0], split[2], split[1], split[3], split[4])
key := fmt.Sprintf("%s:%s->%s:%s,%s%s", split[0], split[2], split[1], split[3], split[4], split[5])
return key
}

View File

@@ -16,6 +16,7 @@ import (
)
const maskedFieldPlaceholderValue = "[REDACTED]"
const userAgent = "user-agent"
//these values MUST be all lower case and contain no `-` or `_` characters
var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password",
@@ -32,7 +33,7 @@ func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteri
request := item.Pair.Request.Payload.(api.HTTPPayload).Data.(*http.Request)
for headerKey, headerValues := range request.Header {
if strings.ToLower(headerKey) == "user-agent" {
if strings.ToLower(headerKey) == userAgent {
for _, userAgent := range options.IgnoredUserAgents {
for _, headerValue := range headerValues {
if strings.Contains(strings.ToLower(headerValue), strings.ToLower(userAgent)) {
@@ -89,6 +90,10 @@ func filterResponseBody(response *http.Response, options *api.TrafficFilteringOp
func filterHeaders(headers *http.Header) {
for key, _ := range *headers {
if strings.ToLower(key) == userAgent {
continue
}
if strings.ToLower(key) == "cookie" {
headers.Del(key)
} else if isFieldNameSensitive(key) {

Some files were not shown because too many files have changed in this diff Show More