From a75bac181d95cf68df03126f1dbf4cc4f3a76f06 Mon Sep 17 00:00:00 2001 From: David Levanon Date: Mon, 20 Dec 2021 13:57:58 +0200 Subject: [PATCH 01/72] support linkerd (#547) * support linkerd - initial commit * renaming readEnvironmentVariable --- tap/source/discoverer_util.go | 38 +++++++++++++ tap/source/envoy_discoverer.go | 36 ++----------- tap/source/linkerd_discoverer.go | 83 +++++++++++++++++++++++++++++ tap/source/packet_source_manager.go | 30 +++++++++-- 4 files changed, 149 insertions(+), 38 deletions(-) create mode 100644 tap/source/discoverer_util.go create mode 100644 tap/source/linkerd_discoverer.go diff --git a/tap/source/discoverer_util.go b/tap/source/discoverer_util.go new file mode 100644 index 000000000..7c9640d4f --- /dev/null +++ b/tap/source/discoverer_util.go @@ -0,0 +1,38 @@ +package source + +import ( + "io/ioutil" + "regexp" + "strings" + + "github.com/up9inc/mizu/shared/logger" +) + +var numberRegex = regexp.MustCompile("[0-9]+") + +func getSingleValueFromEnvironmentVariableFile(filePath string, variableName string) (string, error) { + bytes, err := ioutil.ReadFile(filePath) + + if err != nil { + logger.Log.Warningf("Error reading environment file %v - %v", filePath, err) + return "", err + } + + envs := strings.Split(string(bytes), string([]byte{0})) + + for _, env := range envs { + if !strings.Contains(env, "=") { + continue + } + + parts := strings.Split(env, "=") + varName := parts[0] + value := parts[1] + + if variableName == varName { + return value, nil + } + } + + return "", nil +} diff --git a/tap/source/envoy_discoverer.go b/tap/source/envoy_discoverer.go index 0230b1273..d90f96943 100644 --- a/tap/source/envoy_discoverer.go +++ b/tap/source/envoy_discoverer.go @@ -4,7 +4,6 @@ import ( "fmt" "io/ioutil" "os" - "regexp" "strings" "github.com/up9inc/mizu/shared/logger" @@ -13,8 +12,6 @@ import ( const envoyBinary = "/envoy" -var numberRegex = regexp.MustCompile("[0-9]+") - func discoverRelevantEnvoyPids(procfs string, pods []v1.Pod) ([]string, error) { result := make([]string, 0) @@ -36,7 +33,7 @@ func discoverRelevantEnvoyPids(procfs string, pods []v1.Pod) ([]string, error) { continue } - if checkPid(procfs, pid.Name(), pods) { + if checkEnvoyPid(procfs, pid.Name(), pods) { result = append(result, pid.Name()) } } @@ -46,7 +43,7 @@ func discoverRelevantEnvoyPids(procfs string, pods []v1.Pod) ([]string, error) { return result, nil } -func checkPid(procfs string, pid string, pods []v1.Pod) bool { +func checkEnvoyPid(procfs string, pid string, pods []v1.Pod) bool { execLink := fmt.Sprintf("%v/%v/exe", procfs, pid) exec, err := os.Readlink(execLink) @@ -63,7 +60,7 @@ func checkPid(procfs string, pid string, pods []v1.Pod) bool { } environmentFile := fmt.Sprintf("%v/%v/environ", procfs, pid) - podIp, err := readEnvironmentVariable(environmentFile, "INSTANCE_IP") + podIp, err := getSingleValueFromEnvironmentVariableFile(environmentFile, "INSTANCE_IP") if err != nil { return false @@ -84,30 +81,3 @@ func checkPid(procfs string, pid string, pods []v1.Pod) bool { return false } - -func readEnvironmentVariable(file string, name string) (string, error) { - bytes, err := ioutil.ReadFile(file) - - if err != nil { - logger.Log.Warningf("Error reading environment file %v - %v", file, err) - return "", err - } - - envs := strings.Split(string(bytes), string([]byte{0})) - - for _, env := range envs { - if !strings.Contains(env, "=") { - continue - } - - parts := strings.Split(env, "=") - varName := parts[0] - value := parts[1] - - if name == varName { - return value, nil - } - } - - return "", nil -} diff --git a/tap/source/linkerd_discoverer.go b/tap/source/linkerd_discoverer.go new file mode 100644 index 000000000..808fc8036 --- /dev/null +++ b/tap/source/linkerd_discoverer.go @@ -0,0 +1,83 @@ +package source + +import ( + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/up9inc/mizu/shared/logger" + v1 "k8s.io/api/core/v1" +) + +const linkerdBinary = "/linkerd2-proxy" + +func discoverRelevantLinkerdPids(procfs string, pods []v1.Pod) ([]string, error) { + result := make([]string, 0) + + pids, err := ioutil.ReadDir(procfs) + + if err != nil { + return result, err + } + + logger.Log.Infof("Starting linkerd auto discoverer %v %v - scanning %v potential pids", + procfs, pods, len(pids)) + + for _, pid := range pids { + if !pid.IsDir() { + continue + } + + if !numberRegex.MatchString(pid.Name()) { + continue + } + + if checkLinkerdPid(procfs, pid.Name(), pods) { + result = append(result, pid.Name()) + } + } + + logger.Log.Infof("Found %v relevant linkerd processes - %v", len(result), result) + + return result, nil +} + +func checkLinkerdPid(procfs string, pid string, pods []v1.Pod) bool { + execLink := fmt.Sprintf("%v/%v/exe", procfs, pid) + exec, err := os.Readlink(execLink) + + if err != nil { + // Debug on purpose - it may happen due to many reasons and we only care + // for it during troubleshooting + // + logger.Log.Debugf("Unable to read link %v - %v\n", execLink, err) + return false + } + + if !strings.HasSuffix(exec, linkerdBinary) { + return false + } + + environmentFile := fmt.Sprintf("%v/%v/environ", procfs, pid) + podName, err := getSingleValueFromEnvironmentVariableFile(environmentFile, "_pod_name") + + if err != nil { + return false + } + + if podName == "" { + logger.Log.Debugf("Found a linkerd process without _pod_name variable %v\n", pid) + return false + } + + logger.Log.Infof("Found linkerd pid %v with pod name %v", pid, podName) + + for _, pod := range pods { + if pod.Name == podName { + return true + } + } + + return false +} diff --git a/tap/source/packet_source_manager.go b/tap/source/packet_source_manager.go index af88554ea..912d03c9a 100644 --- a/tap/source/packet_source_manager.go +++ b/tap/source/packet_source_manager.go @@ -16,7 +16,7 @@ type PacketSourceManager struct { } func NewPacketSourceManager(procfs string, pids string, filename string, interfaceName string, - istio bool, pods []v1.Pod, behaviour TcpPacketSourceBehaviour) (*PacketSourceManager, error) { + mtls bool, pods []v1.Pod, behaviour TcpPacketSourceBehaviour) (*PacketSourceManager, error) { sources := make([]*tcpPacketSource, 0) sources, err := createHostSource(sources, filename, interfaceName, behaviour) @@ -25,7 +25,8 @@ func NewPacketSourceManager(procfs string, pids string, filename string, interfa } sources = createSourcesFromPids(sources, procfs, pids, interfaceName, behaviour) - sources = createSourcesFromEnvoy(sources, istio, procfs, pods, interfaceName, behaviour) + sources = createSourcesFromEnvoy(sources, mtls, procfs, pods, interfaceName, behaviour) + sources = createSourcesFromLinkerd(sources, mtls, procfs, pods, interfaceName, behaviour) return &PacketSourceManager{ sources: sources, @@ -54,13 +55,13 @@ func createSourcesFromPids(sources []*tcpPacketSource, procfs string, pids strin return sources } -func createSourcesFromEnvoy(sources []*tcpPacketSource, istio bool, procfs string, clusterIps []v1.Pod, +func createSourcesFromEnvoy(sources []*tcpPacketSource, mtls bool, procfs string, pods []v1.Pod, interfaceName string, behaviour TcpPacketSourceBehaviour) []*tcpPacketSource { - if !istio { + if !mtls { return sources } - envoyPids, err := discoverRelevantEnvoyPids(procfs, clusterIps) + envoyPids, err := discoverRelevantEnvoyPids(procfs, pods) if err != nil { logger.Log.Warningf("Unable to discover envoy pids - %v", err) @@ -73,6 +74,25 @@ func createSourcesFromEnvoy(sources []*tcpPacketSource, istio bool, procfs strin return sources } +func createSourcesFromLinkerd(sources []*tcpPacketSource, mtls bool, procfs string, pods []v1.Pod, + interfaceName string, behaviour TcpPacketSourceBehaviour) []*tcpPacketSource { + if !mtls { + return sources + } + + linkerdPids, err := discoverRelevantLinkerdPids(procfs, pods) + + if err != nil { + logger.Log.Warningf("Unable to discover linkerd pids - %v", err) + return sources + } + + netnsSources := newNetnsPacketSources(procfs, linkerdPids, interfaceName, behaviour) + sources = append(sources, netnsSources...) + + return sources +} + func newHostPacketSource(filename string, interfaceName string, behaviour TcpPacketSourceBehaviour) (*tcpPacketSource, error) { var name string From 12aaa762f69e4f4e7c4c67652fc73ca0148890fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Mert=20Y=C4=B1ld=C4=B1ran?= Date: Wed, 22 Dec 2021 20:23:21 +0300 Subject: [PATCH 02/72] Fix `React Hook useEffect has a missing dependency: 'handleQueryChange'` warning (#551) --- ui/src/components/TrafficPage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/components/TrafficPage.tsx b/ui/src/components/TrafficPage.tsx index 0faf7a816..334003555 100644 --- a/ui/src/components/TrafficPage.tsx +++ b/ui/src/components/TrafficPage.tsx @@ -91,7 +91,7 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS useEffect(() => { handleQueryChange(query); - }, [query]); + }, [query, handleQueryChange]); useEffect(() => { if (query) { From 32dfe40e189f414806b5cd3b6117e52d5253d2aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Mert=20Y=C4=B1ld=C4=B1ran?= Date: Thu, 23 Dec 2021 10:25:39 +0300 Subject: [PATCH 03/72] Make `EntryItem` more responsive (#552) --- .../components/EntryListItem/EntryListItem.module.sass | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ui/src/components/EntryListItem/EntryListItem.module.sass b/ui/src/components/EntryListItem/EntryListItem.module.sass index 7e396d136..f9622229f 100644 --- a/ui/src/components/EntryListItem/EntryListItem.module.sass +++ b/ui/src/components/EntryListItem/EntryListItem.module.sass @@ -92,3 +92,13 @@ .ip margin-left: 5px + +@media (max-width: 1760px) + .timestamp + display: none + .separatorRight + border-right: 0px + +@media (max-width: 1340px) + .separatorRight + display: none From 03b1313a9f029e570d2e36446d5951cc157ef556 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Mert=20Y=C4=B1ld=C4=B1ran?= Date: Thu, 23 Dec 2021 17:45:51 +0300 Subject: [PATCH 04/72] Don't use `Queryable` for the `Mime type` and `Encoding` fields but use it directly in `CollapsibleTitle` suffixed components and only enable it for `EntryBodySection` (#550) Co-authored-by: gadotroee <55343099+gadotroee@users.noreply.github.com> --- .../EntryDetailed/EntrySections.tsx | 77 +++++++++++++------ ui/src/components/UI/CollapsibleContainer.tsx | 25 ++---- 2 files changed, 60 insertions(+), 42 deletions(-) diff --git a/ui/src/components/EntryDetailed/EntrySections.tsx b/ui/src/components/EntryDetailed/EntrySections.tsx index 7201b5206..17bc33357 100644 --- a/ui/src/components/EntryDetailed/EntrySections.tsx +++ b/ui/src/components/EntryDetailed/EntrySections.tsx @@ -10,12 +10,14 @@ import ProtobufDecoder from "protobuf-decoder"; interface EntryViewLineProps { label: string; value: number | string; - updateQuery: any; - selector: string; + updateQuery?: any; + selector?: string; overrideQueryValue?: string; + displayIconOnMouseOver?: boolean; + useTooltip?: boolean; } -const EntryViewLine: React.FC = ({label, value, updateQuery, selector, overrideQueryValue}) => { +const EntryViewLine: React.FC = ({label, value, updateQuery = null, selector = "", overrideQueryValue = "", displayIconOnMouseOver = true, useTooltip = true}) => { let query: string; if (!selector) { query = ""; @@ -34,7 +36,8 @@ const EntryViewLine: React.FC = ({label, value, updateQuery, style={{float: "right", height: "18px"}} iconStyle={{marginRight: "20px"}} flipped={true} - displayIconOnMouseOver={true} + useTooltip={useTooltip} + displayIconOnMouseOver={displayIconOnMouseOver} > {label} @@ -55,30 +58,47 @@ const EntryViewLine: React.FC = ({label, value, updateQuery, interface EntrySectionCollapsibleTitleProps { title: string, color: string, - isExpanded: boolean, + expanded: boolean, + setExpanded: any, + query?: string, + updateQuery?: any, } -const EntrySectionCollapsibleTitle: React.FC = ({title, color, isExpanded}) => { +const EntrySectionCollapsibleTitle: React.FC = ({title, color, expanded, setExpanded, query = "", updateQuery = null}) => { return
-
- {isExpanded ? '-' : '+'} +
{ + setExpanded(!expanded) + }} + > + {expanded ? '-' : '+'}
- {title} + + {title} +
} interface EntrySectionContainerProps { title: string, color: string, + query?: string, + updateQuery?: any, } -export const EntrySectionContainer: React.FC = ({title, color, children}) => { +export const EntrySectionContainer: React.FC = ({title, color, children, query = "", updateQuery = null}) => { const [expanded, setExpanded] = useState(true); return setExpanded(!expanded)} - title={} + expanded={expanded} + title={} > {children} @@ -133,11 +153,16 @@ export const EntryBodySection: React.FC = ({ } return - {content && content?.length > 0 && + {content && content?.length > 0 && - - {encoding && } + + {encoding && }
@@ -195,13 +220,20 @@ interface EntryPolicySectionProps { interface EntryPolicySectionCollapsibleTitleProps { label: string; matched: string; - isExpanded: boolean; + expanded: boolean; + setExpanded: any; } -const EntryPolicySectionCollapsibleTitle: React.FC = ({label, matched, isExpanded}) => { +const EntryPolicySectionCollapsibleTitle: React.FC = ({label, matched, expanded, setExpanded}) => { return
- - {isExpanded ? '-' : '+'} + { + setExpanded(!expanded) + }} + > + {expanded ? '-' : '+'} @@ -222,9 +254,8 @@ export const EntryPolicySectionContainer: React.FC setExpanded(!expanded)} - title={} + expanded={expanded} + title={} > {children} diff --git a/ui/src/components/UI/CollapsibleContainer.tsx b/ui/src/components/UI/CollapsibleContainer.tsx index 4c0452623..c59fcb6be 100644 --- a/ui/src/components/UI/CollapsibleContainer.tsx +++ b/ui/src/components/UI/CollapsibleContainer.tsx @@ -1,38 +1,25 @@ -import React, {useState} from "react"; +import React from "react"; import collapsedImg from "../assets/collapsed.svg"; import expandedImg from "../assets/expanded.svg"; import "./style/CollapsibleContainer.sass"; interface Props { title: string | React.ReactNode, - onClick?: (e: React.SyntheticEvent) => void, - isExpanded?: boolean, + expanded: boolean, titleClassName?: string, - stickyHeader?: boolean, className?: string, - initialExpanded?: boolean; - passiveOnClick?: boolean; //whether specifying onClick overrides internal _isExpanded state handling + stickyHeader?: boolean, } -const CollapsibleContainer: React.FC = ({title, children, isExpanded, onClick, titleClassName, stickyHeader = false, className, initialExpanded = true, passiveOnClick}) => { - const [_isExpanded, _setExpanded] = useState(initialExpanded); - let expanded = isExpanded !== undefined ? isExpanded : _isExpanded; +const CollapsibleContainer: React.FC = ({title, children, expanded, titleClassName, className, stickyHeader = false}) => { const classNames = `CollapsibleContainer ${expanded ? "CollapsibleContainer-Expanded" : "CollapsibleContainer-Collapsed"} ${className ? className : ''}`; - // This is needed to achieve the sticky header feature. + // This is needed to achieve the sticky header feature. // It is needed an un-contained component for the css to work properly. const content =
{ - if (onClick) { - onClick(e) - if (passiveOnClick !== true) - return; - } - _setExpanded(!_isExpanded) - }} > { React.isValidElement(title)? From e358aa4c8f39183b35e89f33f8bd0b6011f125df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Mert=20Y=C4=B1ld=C4=B1ran?= Date: Sun, 26 Dec 2021 15:10:02 +0300 Subject: [PATCH 05/72] Remove TCP related logging to reduce the amount of logs on `DEBUG` level (#553) --- tap/tcp_assembler.go | 7 +++---- tap/tcp_stream.go | 11 +---------- tap/tcp_stream_factory.go | 6 ------ 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/tap/tcp_assembler.go b/tap/tcp_assembler.go index 150834967..8bba8f81d 100644 --- a/tap/tcp_assembler.go +++ b/tap/tcp_assembler.go @@ -65,11 +65,11 @@ func (a *tcpAssembler) processPackets(dumpPacket bool, packets <-chan source.Tcp for packetInfo := range packets { packetsCount := diagnose.AppStats.IncPacketsCount() - - if packetsCount % PACKETS_SEEN_LOG_THRESHOLD == 0 { + + if packetsCount%PACKETS_SEEN_LOG_THRESHOLD == 0 { logger.Log.Debugf("Packets seen: #%d", packetsCount) } - + packet := packetInfo.Packet data := packet.Data() diagnose.AppStats.UpdateProcessedBytes(uint64(len(data))) @@ -91,7 +91,6 @@ func (a *tcpAssembler) processPackets(dumpPacket bool, packets <-chan source.Tcp CaptureInfo: packet.Metadata().CaptureInfo, } diagnose.InternalStats.Totalsz += len(tcp.Payload) - logger.Log.Debugf("%s:%v -> %s:%v", packet.NetworkLayer().NetworkFlow().Src(), tcp.SrcPort, packet.NetworkLayer().NetworkFlow().Dst(), tcp.DstPort) a.assemblerMutex.Lock() a.AssembleWithContext(packet.NetworkLayer().NetworkFlow(), tcp, &c) a.assemblerMutex.Unlock() diff --git a/tap/tcp_stream.go b/tap/tcp_stream.go index 7a9b53e30..c979c2991 100644 --- a/tap/tcp_stream.go +++ b/tap/tcp_stream.go @@ -2,7 +2,6 @@ package tap import ( "encoding/binary" - "fmt" "sync" "github.com/google/gopacket" @@ -75,7 +74,7 @@ func (t *tcpStream) Accept(tcp *layers.TCP, ci gopacket.CaptureInfo, dir reassem } func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.AssemblerContext) { - dir, start, end, skip := sg.Info() + dir, _, _, skip := sg.Info() length, saved := sg.Lengths() // update stats sgStats := sg.Stats() @@ -103,13 +102,6 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass diagnose.InternalStats.OverlapBytes += sgStats.OverlapBytes diagnose.InternalStats.OverlapPackets += sgStats.OverlapPackets - var ident string - if dir == reassembly.TCPDirClientToServer { - ident = fmt.Sprintf("%v %v(%s): ", t.net, t.transport, dir) - } else { - ident = fmt.Sprintf("%v %v(%s): ", t.net.Reverse(), t.transport.Reverse(), dir) - } - diagnose.TapErrors.Debug("%s: SG reassembled packet with %d bytes (start:%v,end:%v,skip:%d,saved:%d,nb:%d,%d,overlap:%d,%d)", ident, length, start, end, skip, saved, sgStats.Packets, sgStats.Chunks, sgStats.OverlapBytes, sgStats.OverlapPackets) if skip == -1 && *allowmissinginit { // this is allowed } else if skip != 0 { @@ -174,7 +166,6 @@ func (t *tcpStream) ReassembledSG(sg reassembly.ScatterGather, ac reassembly.Ass } func (t *tcpStream) ReassemblyComplete(ac reassembly.AssemblerContext) bool { - diagnose.TapErrors.Debug("%s: Connection closed", t.ident) if t.isTapTarget && !t.isClosed { t.Close() } diff --git a/tap/tcp_stream_factory.go b/tap/tcp_stream_factory.go index 847ca10d1..c063f7bfe 100644 --- a/tap/tcp_stream_factory.go +++ b/tap/tcp_stream_factory.go @@ -54,7 +54,6 @@ func NewTcpStreamFactory(emitter api.Emitter, streamsMap *tcpStreamMap, opts *Ta } func (factory *tcpStreamFactory) New(net, transport gopacket.Flow, tcp *layers.TCP, ac reassembly.AssemblerContext) reassembly.Stream { - logger.Log.Debugf("* NEW: %s %s", net, transport) fsmOptions := reassembly.TCPSimpleFSMOptions{ SupportMissingEstablishment: *allowmissinginit, } @@ -153,21 +152,16 @@ func inArrayPod(pods []v1.Pod, address string) bool { func (factory *tcpStreamFactory) getStreamProps(srcIP string, srcPort string, dstIP string, dstPort string) *streamProps { if factory.opts.HostMode { if inArrayPod(factory.opts.FilterAuthorities, fmt.Sprintf("%s:%s", dstIP, dstPort)) { - logger.Log.Debugf("getStreamProps %s", fmt.Sprintf("+ host1 %s:%s", dstIP, dstPort)) return &streamProps{isTapTarget: true, isOutgoing: false} } else if inArrayPod(factory.opts.FilterAuthorities, dstIP) { - logger.Log.Debugf("getStreamProps %s", fmt.Sprintf("+ host2 %s", dstIP)) return &streamProps{isTapTarget: true, isOutgoing: false} } else if inArrayPod(factory.opts.FilterAuthorities, fmt.Sprintf("%s:%s", srcIP, srcPort)) { - logger.Log.Debugf("getStreamProps %s", fmt.Sprintf("+ host3 %s:%s", srcIP, srcPort)) return &streamProps{isTapTarget: true, isOutgoing: true} } else if inArrayPod(factory.opts.FilterAuthorities, srcIP) { - logger.Log.Debugf("getStreamProps %s", fmt.Sprintf("+ host4 %s", srcIP)) return &streamProps{isTapTarget: true, isOutgoing: true} } return &streamProps{isTapTarget: false, isOutgoing: false} } else { - logger.Log.Debugf("getStreamProps %s", fmt.Sprintf("+ notHost3 %s:%s -> %s:%s", srcIP, srcPort, dstIP, dstPort)) return &streamProps{isTapTarget: true} } } From 3a8353159011d6a2544fce18508a5e7db08f1663 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Mert=20Y=C4=B1ld=C4=B1ran?= Date: Sun, 26 Dec 2021 16:12:17 +0300 Subject: [PATCH 06/72] Use `react-lowlight` to highlight and `json-beautify`, `xml-formatter` to prettify the `EntryBodySection` (#554) * Use `react-lowlight` to highlight and `json-beautify` to prettify the `EntryBodySection` * Bring back the line numbers * Make the Base64 decoding optional but make it `true` by default * Align line numbers to right and don't have a dot character * Make line numbers semi transparent * Make `markers` code more elegant * Prettify XML as well --- ui/package-lock.json | 66 +++++++- ui/package.json | 6 +- .../EntryDetailed/EntrySections.tsx | 72 +++++---- ui/src/components/Filters.tsx | 14 -- .../UI/SyntaxHighlighter/highlighterStyle.ts | 152 ------------------ .../UI/SyntaxHighlighter/index.scss | 30 ++-- .../components/UI/SyntaxHighlighter/index.tsx | 51 ++++-- 7 files changed, 166 insertions(+), 225 deletions(-) delete mode 100644 ui/src/components/UI/SyntaxHighlighter/highlighterStyle.ts diff --git a/ui/package-lock.json b/ui/package-lock.json index a0fa2a952..97b475cc3 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -7747,9 +7747,9 @@ "integrity": "sha512-l9sfDFsuqtOqKDsQdqrMRk0U85RZc0RtOR9yPI7mRVOa4FsR/BVnZ0shmQRM96Ji99kYZP/7hn1cedc1+ApsTQ==" }, "highlight.js": { - "version": "10.7.2", - "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.2.tgz", - "integrity": "sha512-oFLl873u4usRM9K63j4ME9u3etNF0PLiJhSQ8rdfuL51Wn3zkD6drf9ZW0dOzjnZI22YYG24z30JcmfCZjMgYg==" + "version": "11.3.1", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.3.1.tgz", + "integrity": "sha512-PUhCRnPjLtiLHZAQ5A/Dt5F8cWZeMyj9KRsACsWT+OD6OP0x6dp5OmT5jdx0JgEyPxPZZIPQpRN2TciUT7occw==" }, "hmac-drbg": { "version": "1.0.1", @@ -10234,6 +10234,11 @@ "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==" }, + "json-beautify": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/json-beautify/-/json-beautify-1.1.1.tgz", + "integrity": "sha512-17j+Hk2lado0xqKtUcyAjK0AtoHnPSIgktWRsEXgdFQFG9UnaGw6CHa0J7xsvulxRpFl6CrkDFHght1p5ZJc4A==" + }, "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", @@ -10612,6 +10617,13 @@ "requires": { "fault": "^1.0.0", "highlight.js": "~10.7.0" + }, + "dependencies": { + "highlight.js": { + "version": "10.7.3", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", + "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==" + } } }, "lru-cache": { @@ -13577,6 +13589,34 @@ "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==" }, + "react-lowlight": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/react-lowlight/-/react-lowlight-3.0.0.tgz", + "integrity": "sha512-s0+T81PsCbUZYd/0XrplGc6kQEUdiwLKI0G6umJP1ViqRoZRCvSuHvXOy20Usd2ywDKWLuVETQgBDPeNQhPNZg==", + "requires": { + "lowlight": "^2.4.1" + }, + "dependencies": { + "fault": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fault/-/fault-2.0.1.tgz", + "integrity": "sha512-WtySTkS4OKev5JtpHXnib4Gxiurzh5NCGvWrFaZ34m6JehfTUhKZvn9njTfw48t6JumVQOmrKqpmGcdwxnhqBQ==", + "requires": { + "format": "^0.2.0" + } + }, + "lowlight": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/lowlight/-/lowlight-2.4.1.tgz", + "integrity": "sha512-mQkAG0zGQ9lcYecEft+hl9uV1fD6HpURA83/TYrsxKvb8xX2mfyB+aaV/A/aWmhhEcWVzr9Cc+l/fvUYfEUumw==", + "requires": { + "@types/hast": "^2.0.0", + "fault": "^2.0.0", + "highlight.js": "~11.3.0" + } + } + } + }, "react-refresh": { "version": "0.8.3", "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.8.3.tgz", @@ -13663,6 +13703,13 @@ "lowlight": "^1.17.0", "prismjs": "^1.22.0", "refractor": "^3.2.0" + }, + "dependencies": { + "highlight.js": { + "version": "10.7.3", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", + "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==" + } } }, "react-toastify": { @@ -18149,11 +18196,24 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.5.tgz", "integrity": "sha512-xzyu3hFvomRfXKH8vOFMU3OguG6oOvhXMo3xsGy3xWExqaM2dxBbVxuD99O7m3ZUFMvvscsZDqxfgMaRr/Nr1g==" }, + "xml-formatter": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/xml-formatter/-/xml-formatter-2.6.0.tgz", + "integrity": "sha512-+bQeoiE5W3CJdDCHTlveYSWFfQWnYB3uHGeRJ6LlEsL5kT++mWy9iN1cMeEDfBbgOnXO2DNUbmQ6elkR/mCcjg==", + "requires": { + "xml-parser-xo": "^3.2.0" + } + }, "xml-name-validator": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" }, + "xml-parser-xo": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/xml-parser-xo/-/xml-parser-xo-3.2.0.tgz", + "integrity": "sha512-8LRU6cq+d7mVsoDaMhnkkt3CTtAs4153p49fRo+HIB3I1FD1o5CeXRjRH29sQevIfVJIcPjKSsPU/+Ujhq09Rg==" + }, "xmlchars": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", diff --git a/ui/package.json b/ui/package.json index f8fcebd2a..71b1ed470 100644 --- a/ui/package.json +++ b/ui/package.json @@ -15,6 +15,8 @@ "@types/react-dom": "^17.0.3", "@uiw/react-textarea-code-editor": "^1.4.12", "axios": "^0.21.1", + "highlight.js": "^11.3.1", + "json-beautify": "^1.1.1", "jsonpath": "^1.1.1", "moment": "^2.29.1", "node-sass": "^5.0.0", @@ -23,12 +25,14 @@ "react": "^17.0.2", "react-copy-to-clipboard": "^5.0.3", "react-dom": "^17.0.2", + "react-lowlight": "^3.0.0", "react-scripts": "4.0.3", "react-scrollable-feed-virtualized": "^1.4.9", "react-syntax-highlighter": "^15.4.3", "react-toastify": "^8.0.3", "typescript": "^4.2.4", - "web-vitals": "^1.1.1" + "web-vitals": "^1.1.1", + "xml-formatter": "^2.6.0" }, "scripts": { "start": "react-scripts start", diff --git a/ui/src/components/EntryDetailed/EntrySections.tsx b/ui/src/components/EntryDetailed/EntrySections.tsx index 17bc33357..9a777705f 100644 --- a/ui/src/components/EntryDetailed/EntrySections.tsx +++ b/ui/src/components/EntryDetailed/EntrySections.tsx @@ -6,6 +6,8 @@ import FancyTextDisplay from "../UI/FancyTextDisplay"; import Queryable from "../UI/Queryable"; import Checkbox from "../UI/Checkbox"; import ProtobufDecoder from "protobuf-decoder"; +import {default as jsonBeautify} from "json-beautify"; +import {default as xmlBeautify} from "xml-formatter"; interface EntryViewLineProps { label: string; @@ -121,23 +123,41 @@ export const EntryBodySection: React.FC = ({ contentType, selector, }) => { - const MAXIMUM_BYTES_TO_HIGHLIGHT = 10000; // The maximum of chars to highlight in body, in case the response can be megabytes - const supportedLanguages = [['html', 'html'], ['json', 'json'], ['application/grpc', 'json']]; // [[indicator, languageToUse],...] - const jsonLikeFormats = ['json']; + const MAXIMUM_BYTES_TO_FORMAT = 1000000; // The maximum of chars to highlight in body, in case the response can be megabytes + const jsonLikeFormats = ['json', 'yaml', 'yml']; + const xmlLikeFormats = ['xml', 'html']; const protobufFormats = ['application/grpc']; - const [isWrapped, setIsWrapped] = useState(false); + const supportedFormats = jsonLikeFormats.concat(xmlLikeFormats, protobufFormats); - const formatTextBody = (body): string => { - const chunk = body.slice(0, MAXIMUM_BYTES_TO_HIGHLIGHT); - const bodyBuf = encoding === 'base64' ? atob(chunk) : chunk; + const [isPretty, setIsPretty] = useState(true); + const [showLineNumbers, setShowLineNumbers] = useState(true); + const [decodeBase64, setDecodeBase64] = useState(true); + + const isBase64Encoding = encoding === 'base64'; + const supportsPrettying = supportedFormats.some(format => contentType?.indexOf(format) > -1); + + const formatTextBody = (body: any): string => { + if (!decodeBase64) return body; + + const chunk = body.slice(0, MAXIMUM_BYTES_TO_FORMAT); + const bodyBuf = isBase64Encoding ? atob(chunk) : chunk; + + if (!isPretty) return bodyBuf; try { if (jsonLikeFormats.some(format => contentType?.indexOf(format) > -1)) { - return JSON.stringify(JSON.parse(bodyBuf), null, 2); + return jsonBeautify(JSON.parse(bodyBuf), null, 2, 80); + } else if (xmlLikeFormats.some(format => contentType?.indexOf(format) > -1)) { + return xmlBeautify(bodyBuf, { + indentation: ' ', + filter: (node) => node.type !== 'Comment', + collapseContent: true, + lineSeparator: '\n' + }); } else if (protobufFormats.some(format => contentType?.indexOf(format) > -1)) { // Replace all non printable characters (ASCII) const protobufDecoder = new ProtobufDecoder(bodyBuf, true); - return JSON.stringify(protobufDecoder.decode().toSimple(), null, 2); + return jsonBeautify(protobufDecoder.decode().toSimple(), null, 2, 80); } } catch (error) { console.error(error); @@ -145,13 +165,6 @@ export const EntryBodySection: React.FC = ({ return bodyBuf; } - const getLanguage = (mimetype) => { - const chunk = content?.slice(0, 100); - if (chunk.indexOf('html') > 0 || chunk.indexOf('HTML') > 0) return supportedLanguages[0][1]; - const language = supportedLanguages.find(el => (mimetype + contentType).indexOf(el[0]) > -1); - return language ? language[1] : 'default'; - } - return {content && content?.length > 0 && = ({ query={`${selector} == r".*"`} updateQuery={updateQuery} > - - - - {encoding && } - -
+
+ {supportsPrettying &&
+ {setIsPretty(!isPretty)}}/> +
} + {supportsPrettying && Pretty} -
setIsWrapped(!isWrapped)}> -
- {}}/> +
+ {setShowLineNumbers(!showLineNumbers)}}/>
- Wrap text + Line numbers + + {isBase64Encoding &&
+ {setDecodeBase64(!decodeBase64)}}/> +
} + {isBase64Encoding && Decode Base64}
} @@ -334,7 +349,6 @@ export const EntryContractSection: React.FC = ({color } {contractContent && diff --git a/ui/src/components/Filters.tsx b/ui/src/components/Filters.tsx index 0afc7325d..9e281c9b6 100644 --- a/ui/src/components/Filters.tsx +++ b/ui/src/components/Filters.tsx @@ -167,7 +167,6 @@ export const QueryForm: React.FC = ({query, setQuery, background This is a simple query that matches to HTTP packets with request path "/catalogue": = ({query, setQuery, background The same query can be negated for HTTP path and written like this: = ({query, setQuery, background The syntax supports regular expressions. Here is a query that matches the HTTP requests that send JSON to a server: = ({query, setQuery, background Here is another query that matches HTTP responses with status code 4xx: = ({query, setQuery, background The same exact query can be as integer comparison: = 400`} language="python" @@ -212,7 +207,6 @@ export const QueryForm: React.FC = ({query, setQuery, background The results can be queried based on their timestamps: = ({query, setQuery, background Since Mizu supports various protocols like gRPC, AMQP, Kafka and Redis. It's possible to write complex queries that match multiple protocols like this: = ({query, setQuery, background Such that; clicking this icon in left-pane, would append the query below: = ({query, setQuery, background A query that compares one selector to another is also a valid query: = ({query, setQuery, background true if the given selector's value starts with the string: = ({query, setQuery, background true if the given selector's value ends with the string: = ({query, setQuery, background true if the given selector's value contains the string: = ({query, setQuery, background returns the UNIX timestamp which is the equivalent of the time that's provided by the string. Invalid input evaluates to false: = datetime("10/19/2021, 6:29:02.593 PM")`} language="python" @@ -312,7 +299,6 @@ export const QueryForm: React.FC = ({query, setQuery, background limits the number of records that are streamed back as a result of a query. Always evaluates to true: code[class*=\"language-\"]": { - "background": "#F7F9FC", - "padding": ".1em", - "borderRadius": ".3em" - }, - "comment": { - "color": "#5d6aa0" - }, - "prolog": { - "color": "#494677" - }, - "doctype": { - "color": "#494677" - }, - "cdata": { - "color": "#494677" - }, - "punctuation": { - "color": "#494677" - }, - ".namespace": { - "Opacity": ".7" - }, - "property": { - "color": "#627ef7" - }, - "keyword": { - "color": "#627ef7" - }, - "tag": { - "color": "#627ef7" - }, - "class-name": { - "color": "#3eb545", - "textDecoration": "underline" - }, - "boolean": { - "color": "#3eb545" - }, - "constant": { - "color": "#3eb545" - }, - "symbol": { - "color": "#ff3a30" - }, - "deleted": { - "color": "#ff3a30" - }, - "number": { - "color": "#ff16f7" - }, - "selector": { - "color": "rgb(9,224,19)" - }, - "attr-name": { - "color": "rgb(9,224,19)" - }, - "string": { - "color": "rgb(9,224,19)" - }, - "char": { - "color": "rgb(9,224,19)" - }, - "builtin": { - "color": "rgb(9,224,19)" - }, - "inserted": { - "color": "rgb(9,224,19)" - }, - "variable": { - "color": "#C6C5FE" - }, - "operator": { - "color": "#A1A1A1" - }, - "entity": { - "color": "#fdab2b", - "cursor": "help" - }, - "url": { - "color": "#96CBFE" - }, - ".language-css .token.string": { - "color": "#87C38A" - }, - ".style .token.string": { - "color": "#87C38A" - }, - "atrule": { - "color": "#fdab2b" - }, - "attr-value": { - "color": "#f8c575" - }, - "function": { - "color": "#fdab2b" - }, - "regex": { - "color": "#fab248" - }, - "important": { - "color": "#fd971f", - "fontWeight": "bold" - }, - "bold": { - "fontWeight": "bold" - }, - "italic": { - "fontStyle": "italic" - } -}; diff --git a/ui/src/components/UI/SyntaxHighlighter/index.scss b/ui/src/components/UI/SyntaxHighlighter/index.scss index 623c71509..d735e6fe6 100644 --- a/ui/src/components/UI/SyntaxHighlighter/index.scss +++ b/ui/src/components/UI/SyntaxHighlighter/index.scss @@ -26,12 +26,24 @@ } } -.wrapped{ - pre { - code { - &:last-child { - white-space: pre-wrap!important - } - } - } -} \ No newline at end of file +code.hljs { + white-space: pre-wrap; +} + +code.hljs:before { + counter-reset: listing; +} + +code.hljs .hljs-marker-line { + counter-increment: listing; +} + +code.hljs .hljs-marker-line:before { + content: counter(listing) " "; + display: inline-block; + width: 3rem; + padding-left: auto; + margin-left: auto; + text-align: right; + opacity: .5; +} diff --git a/ui/src/components/UI/SyntaxHighlighter/index.tsx b/ui/src/components/UI/SyntaxHighlighter/index.tsx index 3b2efe644..313971bbf 100644 --- a/ui/src/components/UI/SyntaxHighlighter/index.tsx +++ b/ui/src/components/UI/SyntaxHighlighter/index.tsx @@ -1,30 +1,47 @@ import React from 'react'; -import {Prism as SyntaxHighlighterContainer} from 'react-syntax-highlighter'; -import {highlighterStyle} from './highlighterStyle' +import Lowlight from 'react-lowlight' +import 'highlight.js/styles/atom-one-light.css' import './index.scss'; +import xml from 'highlight.js/lib/languages/xml' +import json from 'highlight.js/lib/languages/json' +import protobuf from 'highlight.js/lib/languages/protobuf' +import javascript from 'highlight.js/lib/languages/javascript' +import actionscript from 'highlight.js/lib/languages/actionscript' +import wasm from 'highlight.js/lib/languages/wasm' +import handlebars from 'highlight.js/lib/languages/handlebars' +import yaml from 'highlight.js/lib/languages/yaml' +import python from 'highlight.js/lib/languages/python' + +Lowlight.registerLanguage('python', python); +Lowlight.registerLanguage('xml', xml); +Lowlight.registerLanguage('json', json); +Lowlight.registerLanguage('yaml', yaml); +Lowlight.registerLanguage('protobuf', protobuf); +Lowlight.registerLanguage('javascript', javascript); +Lowlight.registerLanguage('actionscript', actionscript); +Lowlight.registerLanguage('wasm', wasm); +Lowlight.registerLanguage('handlebars', handlebars); + interface Props { code: string; - style?: any; showLineNumbers?: boolean; - className?: string; language?: string; - isWrapped?: boolean; } export const SyntaxHighlighter: React.FC = ({ - code, - style = highlighterStyle, - showLineNumbers = true, - className, - language = 'python', - isWrapped = false, - }) => { - return
- - {code ?? ""} - -
; + code, + showLineNumbers = false, + language = null + }) => { + const markers = showLineNumbers ? code.split("\n").map((item, i) => { + return { + line: i + 1, + className: 'hljs-marker-line' + } + }) : []; + + return
; }; export default SyntaxHighlighter; From 52ce6044ead93d7d2d1bfc9a95ce0dda453cf8cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Mert=20Y=C4=B1ld=C4=B1ran?= Date: Sun, 26 Dec 2021 16:20:06 +0300 Subject: [PATCH 07/72] Add `4px` padding to the top of `endpointServiceContainer` (#555) --- ui/src/components/EntryListItem/EntryListItem.module.sass | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/src/components/EntryListItem/EntryListItem.module.sass b/ui/src/components/EntryListItem/EntryListItem.module.sass index f9622229f..bb0fc9424 100644 --- a/ui/src/components/EntryListItem/EntryListItem.module.sass +++ b/ui/src/components/EntryListItem/EntryListItem.module.sass @@ -68,6 +68,7 @@ flex-direction: column overflow: hidden padding-right: 10px + padding-top: 4px flex-grow: 1 .separatorRight From fd97a096240b01ed0492ad8ec9ad9dc9cdf871d5 Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Mon, 27 Dec 2021 11:32:48 +0200 Subject: [PATCH 08/72] extracted create and clean resources from tap runner (#557) --- cli/cmd/cleanRunner.go | 7 +- cli/cmd/common.go | 44 +---- cli/cmd/tapRunner.go | 280 +------------------------------ cli/cmd/viewRunner.go | 3 +- cli/resources/cleanResources.go | 141 ++++++++++++++++ cli/resources/createResources.go | 163 ++++++++++++++++++ cli/utils/waitUtils.go | 25 +++ 7 files changed, 347 insertions(+), 316 deletions(-) create mode 100644 cli/resources/cleanResources.go create mode 100644 cli/resources/createResources.go create mode 100644 cli/utils/waitUtils.go diff --git a/cli/cmd/cleanRunner.go b/cli/cmd/cleanRunner.go index 5cc3ca6b1..14c594668 100644 --- a/cli/cmd/cleanRunner.go +++ b/cli/cmd/cleanRunner.go @@ -1,6 +1,9 @@ package cmd -import "github.com/up9inc/mizu/cli/apiserver" +import ( + "github.com/up9inc/mizu/cli/apiserver" + "github.com/up9inc/mizu/cli/config" +) func performCleanCommand() { kubernetesProvider, err := getKubernetesProviderForCli() @@ -8,5 +11,5 @@ func performCleanCommand() { return } - finishMizuExecution(kubernetesProvider, apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout)) + finishMizuExecution(kubernetesProvider, apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout), config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace) } diff --git a/cli/cmd/common.go b/cli/cmd/common.go index 822335c8e..9f1df8a11 100644 --- a/cli/cmd/common.go +++ b/cli/cmd/common.go @@ -7,11 +7,9 @@ import ( "github.com/up9inc/mizu/cli/apiserver" "github.com/up9inc/mizu/cli/mizu" "github.com/up9inc/mizu/cli/mizu/fsUtils" + "github.com/up9inc/mizu/cli/resources" "github.com/up9inc/mizu/cli/telemetry" - "os" - "os/signal" "path" - "syscall" "time" "github.com/up9inc/mizu/cli/config" @@ -37,22 +35,6 @@ func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, cancel logger.Log.Debugf("proxy ended") } -func waitForFinish(ctx context.Context, cancel context.CancelFunc) { - logger.Log.Debugf("waiting for finish...") - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) - - // block until ctx cancel is called or termination signal is received - select { - case <-ctx.Done(): - logger.Log.Debugf("ctx done") - break - case <-sigChan: - logger.Log.Debugf("Got termination signal, canceling execution...") - cancel() - } -} - func getKubernetesProviderForCli() (*kubernetes.Provider, error) { kubernetesProvider, err := kubernetes.NewProvider(config.Config.KubeConfigPath()) if err != nil { @@ -71,12 +53,12 @@ func handleKubernetesProviderError(err error) { } } -func finishMizuExecution(kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider) { +func finishMizuExecution(kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider, isNsRestrictedMode bool, mizuResourcesNamespace string) { telemetry.ReportAPICalls(apiProvider) removalCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) defer cancel() dumpLogsIfNeeded(removalCtx, kubernetesProvider) - cleanUpMizuResources(removalCtx, cancel, kubernetesProvider) + resources.CleanUpMizuResources(removalCtx, cancel, kubernetesProvider, isNsRestrictedMode, mizuResourcesNamespace) } func dumpLogsIfNeeded(ctx context.Context, kubernetesProvider *kubernetes.Provider) { @@ -89,23 +71,3 @@ func dumpLogsIfNeeded(ctx context.Context, kubernetesProvider *kubernetes.Provid logger.Log.Errorf("Failed dump logs %v", err) } } - -func cleanUpMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) { - logger.Log.Infof("\nRemoving mizu resources") - - var leftoverResources []string - - if config.Config.IsNsRestrictedMode() { - leftoverResources = cleanUpRestrictedMode(ctx, kubernetesProvider) - } else { - leftoverResources = cleanUpNonRestrictedMode(ctx, cancel, kubernetesProvider) - } - - if len(leftoverResources) > 0 { - errMsg := fmt.Sprintf("Failed to remove the following resources, for more info check logs at %s:", fsUtils.GetLogFilePath()) - for _, resource := range leftoverResources { - errMsg += "\n- " + resource - } - logger.Log.Errorf(uiUtils.Error, errMsg) - } -} diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 90b7eb071..804122985 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -4,26 +4,24 @@ import ( "context" "errors" "fmt" + "github.com/up9inc/mizu/cli/resources" + "github.com/up9inc/mizu/cli/utils" "io/ioutil" "regexp" "strings" "time" - "k8s.io/apimachinery/pkg/util/intstr" - "github.com/getkin/kin-openapi/openapi3" "gopkg.in/yaml.v3" core "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" "github.com/up9inc/mizu/cli/apiserver" "github.com/up9inc/mizu/cli/cmd/goUtils" "github.com/up9inc/mizu/cli/config" "github.com/up9inc/mizu/cli/config/configStructs" "github.com/up9inc/mizu/cli/errormessage" - "github.com/up9inc/mizu/cli/mizu" "github.com/up9inc/mizu/cli/mizu/fsUtils" "github.com/up9inc/mizu/cli/uiUtils" "github.com/up9inc/mizu/shared" @@ -37,9 +35,6 @@ const cleanupTimeout = time.Minute type tapState struct { startTime time.Time targetNamespaces []string - - apiServerService *core.Service - tapperSyncer *kubernetes.MizuTapperSyncer mizuServiceAccountExists bool } @@ -131,7 +126,7 @@ func RunMizuTap() { } logger.Log.Infof("Waiting for Mizu Agent to start...") - if err := createMizuResources(ctx, cancel, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig); err != nil { + if state.mizuServiceAccountExists, err = resources.CreateMizuResources(ctx, cancel, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.Tap.DaemonMode, config.Config.AgentImage, getSyncEntriesConfig(), config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel(), config.Config.Tap.NoPersistentVolumeClaim); err != nil { logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err))) var statusError *k8serrors.StatusError @@ -143,24 +138,24 @@ func RunMizuTap() { return } if config.Config.Tap.DaemonMode { - if err := handleDaemonModePostCreation(ctx, cancel, kubernetesProvider, state.targetNamespaces); err != nil { - defer finishMizuExecution(kubernetesProvider, apiProvider) + if err := handleDaemonModePostCreation(cancel, kubernetesProvider); err != nil { + defer finishMizuExecution(kubernetesProvider, apiProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace) cancel() } else { logger.Log.Infof(uiUtils.Magenta, "Mizu is now running in daemon mode, run `mizu view` to connect to the mizu daemon instance") } } else { - defer finishMizuExecution(kubernetesProvider, apiProvider) + defer finishMizuExecution(kubernetesProvider, apiProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace) go goUtils.HandleExcWrapper(watchApiServerEvents, ctx, kubernetesProvider, cancel) go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel) // block until exit signal or error - waitForFinish(ctx, cancel) + utils.WaitForFinish(ctx, cancel) } } -func handleDaemonModePostCreation(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, namespaces []string) error { +func handleDaemonModePostCreation(cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) error { apiProvider := apiserver.NewProvider(GetApiServerUrl(), 90, 1*time.Second) if err := waitForDaemonModeToBeReady(cancel, kubernetesProvider, apiProvider); err != nil { @@ -252,8 +247,6 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider } }() - state.tapperSyncer = tapperSyncer - return nil } @@ -287,137 +280,6 @@ func readValidationRules(file string) (string, error) { return string(newContent), nil } -func createMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string) error { - if !config.Config.IsNsRestrictedMode() { - if err := createMizuNamespace(ctx, kubernetesProvider); err != nil { - return err - } - } - - if err := createMizuConfigmap(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig); err != nil { - logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v", errormessage.FormatError(err))) - } - - var err error - state.mizuServiceAccountExists, err = createRBACIfNecessary(ctx, kubernetesProvider) - if err != nil { - if !config.Config.Tap.DaemonMode { - logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to ensure the resources required for IP resolving. Mizu will not resolve target IPs to names. error: %v", errormessage.FormatError(err))) - } - } - - var serviceAccountName string - if state.mizuServiceAccountExists { - serviceAccountName = kubernetes.ServiceAccountName - } else { - serviceAccountName = "" - } - - opts := &kubernetes.ApiServerOptions{ - Namespace: config.Config.MizuResourcesNamespace, - PodName: kubernetes.ApiServerPodName, - PodImage: config.Config.AgentImage, - ServiceAccountName: serviceAccountName, - IsNamespaceRestricted: config.Config.IsNsRestrictedMode(), - SyncEntriesConfig: getSyncEntriesConfig(), - MaxEntriesDBSizeBytes: config.Config.Tap.MaxEntriesDBSizeBytes(), - Resources: config.Config.Tap.ApiServerResources, - ImagePullPolicy: config.Config.ImagePullPolicy(), - LogLevel: config.Config.LogLevel(), - } - - if config.Config.Tap.DaemonMode { - if !state.mizuServiceAccountExists { - defer cleanUpMizuResources(ctx, cancel, kubernetesProvider) - logger.Log.Fatalf(uiUtils.Red, fmt.Sprintf("Failed to ensure the resources required for mizu to run in daemon mode. cannot proceed. error: %v", errormessage.FormatError(err))) - } - if err := createMizuApiServerDeployment(ctx, kubernetesProvider, opts); err != nil { - return err - } - } else { - if err := createMizuApiServerPod(ctx, kubernetesProvider, opts); err != nil { - return err - } - } - - state.apiServerService, err = kubernetesProvider.CreateService(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName, kubernetes.ApiServerPodName) - if err != nil { - return err - } - logger.Log.Debugf("Successfully created service: %s", kubernetes.ApiServerPodName) - - return nil -} - -func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string) error { - err := kubernetesProvider.CreateConfigMap(ctx, config.Config.MizuResourcesNamespace, kubernetes.ConfigMapName, serializedValidationRules, serializedContract, serializedMizuConfig) - return err -} - -func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Provider) error { - _, err := kubernetesProvider.CreateNamespace(ctx, config.Config.MizuResourcesNamespace) - return err -} - -func createMizuApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, opts *kubernetes.ApiServerOptions) error { - pod, err := kubernetesProvider.GetMizuApiServerPodObject(opts, false, "") - if err != nil { - return err - } - if _, err = kubernetesProvider.CreatePod(ctx, config.Config.MizuResourcesNamespace, pod); err != nil { - return err - } - logger.Log.Debugf("Successfully created API server pod: %s", kubernetes.ApiServerPodName) - return nil -} - -func createMizuApiServerDeployment(ctx context.Context, kubernetesProvider *kubernetes.Provider, opts *kubernetes.ApiServerOptions) error { - volumeClaimCreated := false - if !config.Config.Tap.NoPersistentVolumeClaim { - volumeClaimCreated = TryToCreatePersistentVolumeClaim(ctx, kubernetesProvider) - } - - pod, err := kubernetesProvider.GetMizuApiServerPodObject(opts, volumeClaimCreated, kubernetes.PersistentVolumeClaimName) - if err != nil { - return err - } - pod.Spec.Containers[0].LivenessProbe = &core.Probe{ - Handler: core.Handler{ - HTTPGet: &core.HTTPGetAction{ - Path: "/echo", - Port: intstr.FromInt(shared.DefaultApiServerPort), - }, - }, - InitialDelaySeconds: 1, - PeriodSeconds: 10, - } - if _, err = kubernetesProvider.CreateDeployment(ctx, config.Config.MizuResourcesNamespace, opts.PodName, pod); err != nil { - return err - } - logger.Log.Debugf("Successfully created API server deployment: %s", kubernetes.ApiServerPodName) - return nil -} - -func TryToCreatePersistentVolumeClaim(ctx context.Context, kubernetesProvider *kubernetes.Provider) bool { - isDefaultStorageClassAvailable, err := kubernetesProvider.IsDefaultStorageProviderAvailable(ctx) - if err != nil { - logger.Log.Warningf(uiUtils.Yellow, "An error occured when checking if a default storage provider exists in this cluster, this means mizu data will be lost on mizu-api-server pod restart") - logger.Log.Debugf("error checking if default storage class exists: %v", err) - return false - } else if !isDefaultStorageClassAvailable { - logger.Log.Warningf(uiUtils.Yellow, "Could not find default storage provider in this cluster, this means mizu data will be lost on mizu-api-server pod restart") - return false - } - - if _, err = kubernetesProvider.CreatePersistentVolumeClaim(ctx, config.Config.MizuResourcesNamespace, kubernetes.PersistentVolumeClaimName, config.Config.Tap.MaxEntriesDBSizeBytes()+mizu.DaemonModePersistentVolumeSizeBufferBytes); err != nil { - logger.Log.Warningf(uiUtils.Yellow, "An error has occured while creating a persistent volume claim for mizu, this means mizu data will be lost on mizu-api-server pod restart") - logger.Log.Debugf("error creating persistent volume claim: %v", err) - return false - } - - return true -} - func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) { var compiledRegexSlice []*api.SerializableRegexp @@ -452,114 +314,6 @@ func getSyncEntriesConfig() *shared.SyncEntriesConfig { } } -func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.Provider) []string { - leftoverResources := make([]string, 0) - - if err := kubernetesProvider.RemoveService(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil { - resourceDesc := fmt.Sprintf("Service %s in namespace %s", kubernetes.ApiServerPodName, config.Config.MizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - - if err := kubernetesProvider.RemoveDaemonSet(ctx, config.Config.MizuResourcesNamespace, kubernetes.TapperDaemonSetName); err != nil { - resourceDesc := fmt.Sprintf("DaemonSet %s in namespace %s", kubernetes.TapperDaemonSetName, config.Config.MizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - - if err := kubernetesProvider.RemoveConfigMap(ctx, config.Config.MizuResourcesNamespace, kubernetes.ConfigMapName); err != nil { - resourceDesc := fmt.Sprintf("ConfigMap %s in namespace %s", kubernetes.ConfigMapName, config.Config.MizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - - if err := kubernetesProvider.RemoveServicAccount(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName); err != nil { - resourceDesc := fmt.Sprintf("Service Account %s in namespace %s", kubernetes.ServiceAccountName, config.Config.MizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - - if err := kubernetesProvider.RemoveRole(ctx, config.Config.MizuResourcesNamespace, kubernetes.RoleName); err != nil { - resourceDesc := fmt.Sprintf("Role %s in namespace %s", kubernetes.RoleName, config.Config.MizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - - if err := kubernetesProvider.RemovePod(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil { - resourceDesc := fmt.Sprintf("Pod %s in namespace %s", kubernetes.ApiServerPodName, config.Config.MizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - - //daemon mode resources - if err := kubernetesProvider.RemoveRoleBinding(ctx, config.Config.MizuResourcesNamespace, kubernetes.RoleBindingName); err != nil { - resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", kubernetes.RoleBindingName, config.Config.MizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - - if err := kubernetesProvider.RemoveDeployment(ctx, config.Config.MizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil { - resourceDesc := fmt.Sprintf("Deployment %s in namespace %s", kubernetes.ApiServerPodName, config.Config.MizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - - if err := kubernetesProvider.RemovePersistentVolumeClaim(ctx, config.Config.MizuResourcesNamespace, kubernetes.PersistentVolumeClaimName); err != nil { - resourceDesc := fmt.Sprintf("PersistentVolumeClaim %s in namespace %s", kubernetes.PersistentVolumeClaimName, config.Config.MizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - - if err := kubernetesProvider.RemoveRole(ctx, config.Config.MizuResourcesNamespace, kubernetes.DaemonRoleName); err != nil { - resourceDesc := fmt.Sprintf("Role %s in namespace %s", kubernetes.DaemonRoleName, config.Config.MizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - - if err := kubernetesProvider.RemoveRoleBinding(ctx, config.Config.MizuResourcesNamespace, kubernetes.DaemonRoleBindingName); err != nil { - resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", kubernetes.DaemonRoleBindingName, config.Config.MizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - - return leftoverResources -} - -func cleanUpNonRestrictedMode(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) []string { - leftoverResources := make([]string, 0) - - if err := kubernetesProvider.RemoveNamespace(ctx, config.Config.MizuResourcesNamespace); err != nil { - resourceDesc := fmt.Sprintf("Namespace %s", config.Config.MizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } else { - defer waitUntilNamespaceDeleted(ctx, cancel, kubernetesProvider) - } - - if err := kubernetesProvider.RemoveClusterRole(ctx, kubernetes.ClusterRoleName); err != nil { - resourceDesc := fmt.Sprintf("ClusterRole %s", kubernetes.ClusterRoleName) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - - if err := kubernetesProvider.RemoveClusterRoleBinding(ctx, kubernetes.ClusterRoleBindingName); err != nil { - resourceDesc := fmt.Sprintf("ClusterRoleBinding %s", kubernetes.ClusterRoleBindingName) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - - return leftoverResources -} - -func handleDeletionError(err error, resourceDesc string, leftoverResources *[]string) { - logger.Log.Debugf("Error removing %s: %v", resourceDesc, errormessage.FormatError(err)) - *leftoverResources = append(*leftoverResources, resourceDesc) -} - -func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) { - // Call cancel if a terminating signal was received. Allows user to skip the wait. - go func() { - waitForFinish(ctx, cancel) - }() - - if err := kubernetesProvider.WaitUtilNamespaceDeleted(ctx, config.Config.MizuResourcesNamespace); err != nil { - switch { - case ctx.Err() == context.Canceled: - logger.Log.Debugf("Do nothing. User interrupted the wait") - case err == wait.ErrWaitTimeout: - logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Timeout while removing Namespace %s", config.Config.MizuResourcesNamespace)) - default: - logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error while waiting for Namespace %s to be deleted: %v", config.Config.MizuResourcesNamespace, errormessage.FormatError(err))) - } - } -} - func watchApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, cancel context.CancelFunc) { podExactRegex := regexp.MustCompile(fmt.Sprintf("^%s$", kubernetes.ApiServerPodName)) podWatchHelper := kubernetes.NewPodWatchHelper(kubernetesProvider, podExactRegex) @@ -705,21 +459,3 @@ func getNamespaces(kubernetesProvider *kubernetes.Provider) []string { return []string{currentNamespace} } } - -func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider) (bool, error) { - if !config.Config.IsNsRestrictedMode() { - if err := kubernetesProvider.CreateMizuRBAC(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.ClusterRoleName, kubernetes.ClusterRoleBindingName, mizu.RBACVersion); err != nil { - return false, err - } - } else { - if err := kubernetesProvider.CreateMizuRBACNamespaceRestricted(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.RoleName, kubernetes.RoleBindingName, mizu.RBACVersion); err != nil { - return false, err - } - } - if config.Config.Tap.DaemonMode { - if err := kubernetesProvider.CreateDaemonsetRBAC(ctx, config.Config.MizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.DaemonRoleName, kubernetes.DaemonRoleBindingName, mizu.RBACVersion); err != nil { - return false, err - } - } - return true, nil -} diff --git a/cli/cmd/viewRunner.go b/cli/cmd/viewRunner.go index c9c056be1..2cba89126 100644 --- a/cli/cmd/viewRunner.go +++ b/cli/cmd/viewRunner.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "github.com/up9inc/mizu/cli/utils" "net/http" "github.com/up9inc/mizu/cli/apiserver" @@ -71,5 +72,5 @@ func runMizuView() { return } - waitForFinish(ctx, cancel) + utils.WaitForFinish(ctx, cancel) } diff --git a/cli/resources/cleanResources.go b/cli/resources/cleanResources.go new file mode 100644 index 000000000..8fc3b9936 --- /dev/null +++ b/cli/resources/cleanResources.go @@ -0,0 +1,141 @@ +package resources + +import ( + "context" + "fmt" + "github.com/up9inc/mizu/cli/errormessage" + "github.com/up9inc/mizu/cli/mizu/fsUtils" + "github.com/up9inc/mizu/cli/uiUtils" + "github.com/up9inc/mizu/cli/utils" + "github.com/up9inc/mizu/shared/kubernetes" + "github.com/up9inc/mizu/shared/logger" + "k8s.io/apimachinery/pkg/util/wait" +) + +func CleanUpMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, isNsRestrictedMode bool, mizuResourcesNamespace string) { + logger.Log.Infof("\nRemoving mizu resources") + + var leftoverResources []string + + if isNsRestrictedMode { + leftoverResources = cleanUpRestrictedMode(ctx, kubernetesProvider, mizuResourcesNamespace) + } else { + leftoverResources = cleanUpNonRestrictedMode(ctx, cancel, kubernetesProvider, mizuResourcesNamespace) + } + + if len(leftoverResources) > 0 { + errMsg := fmt.Sprintf("Failed to remove the following resources, for more info check logs at %s:", fsUtils.GetLogFilePath()) + for _, resource := range leftoverResources { + errMsg += "\n- " + resource + } + logger.Log.Errorf(uiUtils.Error, errMsg) + } +} + +func cleanUpNonRestrictedMode(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, mizuResourcesNamespace string) []string { + leftoverResources := make([]string, 0) + + if err := kubernetesProvider.RemoveNamespace(ctx, mizuResourcesNamespace); err != nil { + resourceDesc := fmt.Sprintf("Namespace %s", mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } else { + defer waitUntilNamespaceDeleted(ctx, cancel, kubernetesProvider, mizuResourcesNamespace) + } + + if err := kubernetesProvider.RemoveClusterRole(ctx, kubernetes.ClusterRoleName); err != nil { + resourceDesc := fmt.Sprintf("ClusterRole %s", kubernetes.ClusterRoleName) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + + if err := kubernetesProvider.RemoveClusterRoleBinding(ctx, kubernetes.ClusterRoleBindingName); err != nil { + resourceDesc := fmt.Sprintf("ClusterRoleBinding %s", kubernetes.ClusterRoleBindingName) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + + return leftoverResources +} + +func waitUntilNamespaceDeleted(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, mizuResourcesNamespace string) { + // Call cancel if a terminating signal was received. Allows user to skip the wait. + go func() { + utils.WaitForFinish(ctx, cancel) + }() + + if err := kubernetesProvider.WaitUtilNamespaceDeleted(ctx, mizuResourcesNamespace); err != nil { + switch { + case ctx.Err() == context.Canceled: + logger.Log.Debugf("Do nothing. User interrupted the wait") + case err == wait.ErrWaitTimeout: + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Timeout while removing Namespace %s", mizuResourcesNamespace)) + default: + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error while waiting for Namespace %s to be deleted: %v", mizuResourcesNamespace, errormessage.FormatError(err))) + } + } +} + +func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuResourcesNamespace string) []string { + leftoverResources := make([]string, 0) + + if err := kubernetesProvider.RemoveService(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil { + resourceDesc := fmt.Sprintf("Service %s in namespace %s", kubernetes.ApiServerPodName, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + + if err := kubernetesProvider.RemoveDaemonSet(ctx, mizuResourcesNamespace, kubernetes.TapperDaemonSetName); err != nil { + resourceDesc := fmt.Sprintf("DaemonSet %s in namespace %s", kubernetes.TapperDaemonSetName, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + + if err := kubernetesProvider.RemoveConfigMap(ctx, mizuResourcesNamespace, kubernetes.ConfigMapName); err != nil { + resourceDesc := fmt.Sprintf("ConfigMap %s in namespace %s", kubernetes.ConfigMapName, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + + if err := kubernetesProvider.RemoveServicAccount(ctx, mizuResourcesNamespace, kubernetes.ServiceAccountName); err != nil { + resourceDesc := fmt.Sprintf("Service Account %s in namespace %s", kubernetes.ServiceAccountName, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + + if err := kubernetesProvider.RemoveRole(ctx, mizuResourcesNamespace, kubernetes.RoleName); err != nil { + resourceDesc := fmt.Sprintf("Role %s in namespace %s", kubernetes.RoleName, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + + if err := kubernetesProvider.RemovePod(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil { + resourceDesc := fmt.Sprintf("Pod %s in namespace %s", kubernetes.ApiServerPodName, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + + //install mode resources + if err := kubernetesProvider.RemoveRoleBinding(ctx, mizuResourcesNamespace, kubernetes.RoleBindingName); err != nil { + resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", kubernetes.RoleBindingName, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + + if err := kubernetesProvider.RemoveDeployment(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil { + resourceDesc := fmt.Sprintf("Deployment %s in namespace %s", kubernetes.ApiServerPodName, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + + if err := kubernetesProvider.RemovePersistentVolumeClaim(ctx, mizuResourcesNamespace, kubernetes.PersistentVolumeClaimName); err != nil { + resourceDesc := fmt.Sprintf("PersistentVolumeClaim %s in namespace %s", kubernetes.PersistentVolumeClaimName, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + + if err := kubernetesProvider.RemoveRole(ctx, mizuResourcesNamespace, kubernetes.DaemonRoleName); err != nil { + resourceDesc := fmt.Sprintf("Role %s in namespace %s", kubernetes.DaemonRoleName, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + + if err := kubernetesProvider.RemoveRoleBinding(ctx, mizuResourcesNamespace, kubernetes.DaemonRoleBindingName); err != nil { + resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", kubernetes.DaemonRoleBindingName, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + + return leftoverResources +} + +func handleDeletionError(err error, resourceDesc string, leftoverResources *[]string) { + logger.Log.Debugf("Error removing %s: %v", resourceDesc, errormessage.FormatError(err)) + *leftoverResources = append(*leftoverResources, resourceDesc) +} diff --git a/cli/resources/createResources.go b/cli/resources/createResources.go new file mode 100644 index 000000000..74d3186c2 --- /dev/null +++ b/cli/resources/createResources.go @@ -0,0 +1,163 @@ +package resources + +import ( + "context" + "fmt" + "github.com/op/go-logging" + "github.com/up9inc/mizu/cli/errormessage" + "github.com/up9inc/mizu/cli/mizu" + "github.com/up9inc/mizu/cli/uiUtils" + "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/shared/kubernetes" + "github.com/up9inc/mizu/shared/logger" + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func CreateMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, isInstallMode bool, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, noPersistentVolumeClaim bool) (bool, error) { + if !isNsRestrictedMode { + if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil { + return false, err + } + } + + if err := createMizuConfigmap(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, mizuResourcesNamespace); err != nil { + logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v", errormessage.FormatError(err))) + } + + mizuServiceAccountExists, err := createRBACIfNecessary(ctx, kubernetesProvider, isNsRestrictedMode, mizuResourcesNamespace, isInstallMode) + if err != nil { + if !isInstallMode { + logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to ensure the resources required for IP resolving. Mizu will not resolve target IPs to names. error: %v", errormessage.FormatError(err))) + } + } + + var serviceAccountName string + if mizuServiceAccountExists { + serviceAccountName = kubernetes.ServiceAccountName + } else { + serviceAccountName = "" + } + + opts := &kubernetes.ApiServerOptions{ + Namespace: mizuResourcesNamespace, + PodName: kubernetes.ApiServerPodName, + PodImage: agentImage, + ServiceAccountName: serviceAccountName, + IsNamespaceRestricted: isNsRestrictedMode, + SyncEntriesConfig: syncEntriesConfig, + MaxEntriesDBSizeBytes: maxEntriesDBSizeBytes, + Resources: apiServerResources, + ImagePullPolicy: imagePullPolicy, + LogLevel: logLevel, + } + + if isInstallMode { + if !mizuServiceAccountExists { + defer CleanUpMizuResources(ctx, cancel, kubernetesProvider, isNsRestrictedMode, mizuResourcesNamespace) + logger.Log.Fatalf(uiUtils.Red, fmt.Sprintf("Failed to ensure the resources required for mizu to run in daemon mode. cannot proceed. error: %v", errormessage.FormatError(err))) + } + if err := createMizuApiServerDeployment(ctx, kubernetesProvider, opts, noPersistentVolumeClaim); err != nil { + return mizuServiceAccountExists, err + } + } else { + if err := createMizuApiServerPod(ctx, kubernetesProvider, opts); err != nil { + return mizuServiceAccountExists, err + } + } + + _, err = kubernetesProvider.CreateService(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName, kubernetes.ApiServerPodName) + if err != nil { + return mizuServiceAccountExists, err + } + logger.Log.Debugf("Successfully created service: %s", kubernetes.ApiServerPodName) + + return mizuServiceAccountExists, nil +} + +func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuResourcesNamespace string) error { + _, err := kubernetesProvider.CreateNamespace(ctx, mizuResourcesNamespace) + return err +} + +func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, mizuResourcesNamespace string) error { + err := kubernetesProvider.CreateConfigMap(ctx, mizuResourcesNamespace, kubernetes.ConfigMapName, serializedValidationRules, serializedContract, serializedMizuConfig) + return err +} + +func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider, isNsRestrictedMode bool, mizuResourcesNamespace string, isInstallMode bool) (bool, error) { + if !isNsRestrictedMode { + if err := kubernetesProvider.CreateMizuRBAC(ctx, mizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.ClusterRoleName, kubernetes.ClusterRoleBindingName, mizu.RBACVersion); err != nil { + return false, err + } + } else { + if err := kubernetesProvider.CreateMizuRBACNamespaceRestricted(ctx, mizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.RoleName, kubernetes.RoleBindingName, mizu.RBACVersion); err != nil { + return false, err + } + } + if isInstallMode { + if err := kubernetesProvider.CreateDaemonsetRBAC(ctx, mizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.DaemonRoleName, kubernetes.DaemonRoleBindingName, mizu.RBACVersion); err != nil { + return false, err + } + } + return true, nil +} + +func createMizuApiServerDeployment(ctx context.Context, kubernetesProvider *kubernetes.Provider, opts *kubernetes.ApiServerOptions, noPersistentVolumeClaim bool) error { + volumeClaimCreated := false + if !noPersistentVolumeClaim { + volumeClaimCreated = tryToCreatePersistentVolumeClaim(ctx, kubernetesProvider, opts) + } + + pod, err := kubernetesProvider.GetMizuApiServerPodObject(opts, volumeClaimCreated, kubernetes.PersistentVolumeClaimName) + if err != nil { + return err + } + pod.Spec.Containers[0].LivenessProbe = &core.Probe{ + Handler: core.Handler{ + HTTPGet: &core.HTTPGetAction{ + Path: "/echo", + Port: intstr.FromInt(shared.DefaultApiServerPort), + }, + }, + InitialDelaySeconds: 1, + PeriodSeconds: 10, + } + if _, err = kubernetesProvider.CreateDeployment(ctx, opts.Namespace, opts.PodName, pod); err != nil { + return err + } + logger.Log.Debugf("Successfully created API server deployment: %s", kubernetes.ApiServerPodName) + return nil +} + +func tryToCreatePersistentVolumeClaim(ctx context.Context, kubernetesProvider *kubernetes.Provider, opts *kubernetes.ApiServerOptions) bool { + isDefaultStorageClassAvailable, err := kubernetesProvider.IsDefaultStorageProviderAvailable(ctx) + if err != nil { + logger.Log.Warningf(uiUtils.Yellow, "An error occured when checking if a default storage provider exists in this cluster, this means mizu data will be lost on mizu-api-server pod restart") + logger.Log.Debugf("error checking if default storage class exists: %v", err) + return false + } else if !isDefaultStorageClassAvailable { + logger.Log.Warningf(uiUtils.Yellow, "Could not find default storage provider in this cluster, this means mizu data will be lost on mizu-api-server pod restart") + return false + } + + if _, err = kubernetesProvider.CreatePersistentVolumeClaim(ctx, opts.Namespace, kubernetes.PersistentVolumeClaimName, opts.MaxEntriesDBSizeBytes + mizu.DaemonModePersistentVolumeSizeBufferBytes); err != nil { + logger.Log.Warningf(uiUtils.Yellow, "An error has occured while creating a persistent volume claim for mizu, this means mizu data will be lost on mizu-api-server pod restart") + logger.Log.Debugf("error creating persistent volume claim: %v", err) + return false + } + + return true +} + +func createMizuApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, opts *kubernetes.ApiServerOptions) error { + pod, err := kubernetesProvider.GetMizuApiServerPodObject(opts, false, "") + if err != nil { + return err + } + if _, err = kubernetesProvider.CreatePod(ctx, opts.Namespace, pod); err != nil { + return err + } + logger.Log.Debugf("Successfully created API server pod: %s", kubernetes.ApiServerPodName) + return nil +} diff --git a/cli/utils/waitUtils.go b/cli/utils/waitUtils.go new file mode 100644 index 000000000..75a5e83ff --- /dev/null +++ b/cli/utils/waitUtils.go @@ -0,0 +1,25 @@ +package utils + +import ( + "context" + "github.com/up9inc/mizu/shared/logger" + "os" + "os/signal" + "syscall" +) + +func WaitForFinish(ctx context.Context, cancel context.CancelFunc) { + logger.Log.Debugf("waiting for finish...") + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + // block until ctx cancel is called or termination signal is received + select { + case <-ctx.Done(): + logger.Log.Debugf("ctx done") + break + case <-sigChan: + logger.Log.Debugf("Got termination signal, canceling execution...") + cancel() + } +} From cfe9e863b7318f66460ec477b0b40f3f580e3c43 Mon Sep 17 00:00:00 2001 From: RamiBerm <54766858+RamiBerm@users.noreply.github.com> Date: Mon, 27 Dec 2021 11:50:34 +0200 Subject: [PATCH 09/72] TRA-4065 support inflight tap target update (#556) * WIP * WIP * Update main.go * Update main.go and passive_tapper.go * Update passive_tapper.go * Update passive_tapper.go * Update passive_tapper.go * Update passive_tapper.go --- agent/main.go | 22 ++++++++++++++- shared/models.go | 12 ++++++-- tap/passive_tapper.go | 59 +++++++++++++++++++++++++++------------ tap/tcp_stream_factory.go | 8 +++--- 4 files changed, 76 insertions(+), 25 deletions(-) diff --git a/agent/main.go b/agent/main.go index f90dc0f96..6f1df5b3c 100644 --- a/agent/main.go +++ b/agent/main.go @@ -55,7 +55,7 @@ var extensionsMap map[string]*tapApi.Extension // global var startTime int64 const ( - socketConnectionRetries = 10 + socketConnectionRetries = 30 socketConnectionRetryDelay = time.Second * 2 socketHandshakeTimeout = time.Second * 2 ) @@ -425,12 +425,32 @@ func dialSocketWithRetry(socketAddress string, retryAmount int, retryDelay time. time.Sleep(retryDelay) } } else { + go handleIncomingMessageAsTapper(socketConnection) return socketConnection, nil } } return nil, lastErr } +func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) { + for { + if _, message, err := socketConnection.ReadMessage(); err != nil { + logger.Log.Errorf("error reading message from socket connection, err: %s, (%v,%+v)", err, err, err) + if errors.Is(err, syscall.EPIPE) { + // socket has disconnected, we can safely stop this goroutine + return + } + } else { + var tapConfigMessage *shared.WebSocketTapConfigMessage + if err := json.Unmarshal(message, &tapConfigMessage); err != nil { + logger.Log.Errorf("received unknown message from socket connection: %s, err: %s, (%v,%+v)", string(message), err, err, err) + } else { + tap.UpdateTapTargets(tapConfigMessage.TapTargets) + } + } + } +} + func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider) (*kubernetes.MizuTapperSyncer, error) { tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{ TargetNamespaces: config.Config.TargetNamespaces, diff --git a/shared/models.go b/shared/models.go index ef0cb2949..f18b833e8 100644 --- a/shared/models.go +++ b/shared/models.go @@ -1,11 +1,13 @@ package shared import ( + "io/ioutil" + "strings" + "github.com/op/go-logging" "github.com/up9inc/mizu/shared/logger" "github.com/up9inc/mizu/tap/api" - "io/ioutil" - "strings" + v1 "k8s.io/api/core/v1" "gopkg.in/yaml.v3" ) @@ -21,6 +23,7 @@ const ( WebSocketMessageTypeToast WebSocketMessageType = "toast" WebSocketMessageTypeQueryMetadata WebSocketMessageType = "queryMetadata" WebSocketMessageTypeStartTime WebSocketMessageType = "startTime" + WebSocketMessageTypeTapConfig WebSocketMessageType = "tapConfig" ) type Resources struct { @@ -67,6 +70,11 @@ type WebSocketStatusMessage struct { TappingStatus []TappedPodStatus `json:"tappingStatus"` } +type WebSocketTapConfigMessage struct { + *WebSocketMessageMetadata + TapTargets []v1.Pod `json:"pods"` +} + type TapperStatus struct { TapperName string `json:"tapperName"` NodeName string `json:"nodeName"` diff --git a/tap/passive_tapper.go b/tap/passive_tapper.go index 275f0288d..95f4e1c4d 100644 --- a/tap/passive_tapper.go +++ b/tap/passive_tapper.go @@ -11,6 +11,7 @@ package tap import ( "encoding/json" "flag" + "fmt" "os" "runtime" "strings" @@ -60,8 +61,11 @@ type TapOpts struct { FilterAuthorities []v1.Pod } -var extensions []*api.Extension // global -var filteringOptions *api.TrafficFilteringOptions // global +var extensions []*api.Extension // global +var filteringOptions *api.TrafficFilteringOptions // global +var tapTargets []v1.Pod // global +var packetSourceManager *source.PacketSourceManager // global +var mainPacketInputChan chan source.TcpPacketInfo // global func inArrayInt(arr []int, valueToCheck int) bool { for _, value := range arr { @@ -86,7 +90,9 @@ func StartPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem, filteringOptions = options if opts.FilterAuthorities == nil { - opts.FilterAuthorities = []v1.Pod{} + tapTargets = []v1.Pod{} + } else { + tapTargets = opts.FilterAuthorities } if GetMemoryProfilingEnabled() { @@ -96,6 +102,23 @@ func StartPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem, go startPassiveTapper(opts, outputItems) } +func UpdateTapTargets(newTapTargets []v1.Pod) { + tapTargets = newTapTargets + if err := initializePacketSources(); err != nil { + logger.Log.Fatal(err) + } + printNewTapTargets() +} + +func printNewTapTargets() { + printStr := "" + for _, tapTarget := range tapTargets { + printStr += fmt.Sprintf("%s (%s), ", tapTarget.Status.PodIP, tapTarget.Name) + } + printStr = strings.TrimRight(printStr, ", ") + logger.Log.Infof("Now tapping: %s", printStr) +} + func printPeriodicStats(cleaner *Cleaner) { statsPeriod := time.Second * time.Duration(*statsevery) ticker := time.NewTicker(statsPeriod) @@ -136,7 +159,11 @@ func printPeriodicStats(cleaner *Cleaner) { } } -func initializePacketSources(opts *TapOpts) (*source.PacketSourceManager, error) { +func initializePacketSources() error { + if packetSourceManager != nil { + packetSourceManager.Close() + } + var bpffilter string if len(flag.Args()) > 0 { bpffilter = strings.Join(flag.Args(), " ") @@ -151,7 +178,13 @@ func initializePacketSources(opts *TapOpts) (*source.PacketSourceManager, error) BpfFilter: bpffilter, } - return source.NewPacketSourceManager(*procfs, *pids, *fname, *iface, *istio, opts.FilterAuthorities, behaviour) + var err error + if packetSourceManager, err = source.NewPacketSourceManager(*procfs, *pids, *fname, *iface, *istio, tapTargets, behaviour); err != nil { + return err + } else { + packetSourceManager.ReadPackets(!*nodefrag, mainPacketInputChan) + return nil + } } func startPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem) { @@ -161,25 +194,15 @@ func startPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem) diagnose.InitializeErrorsMap(*debug, *verbose, *quiet) diagnose.InitializeTapperInternalStats() - sources, err := initializePacketSources(opts) - - if err != nil { + if err := initializePacketSources(); err != nil { logger.Log.Fatal(err) } - defer sources.Close() - - if err != nil { - logger.Log.Fatal(err) - } - - packets := make(chan source.TcpPacketInfo) + mainPacketInputChan = make(chan source.TcpPacketInfo) assembler := NewTcpAssembler(outputItems, streamsMap, opts) diagnose.AppStats.SetStartTime(time.Now()) - sources.ReadPackets(!*nodefrag, packets) - staleConnectionTimeout := time.Second * time.Duration(*staleTimeoutSeconds) cleaner := Cleaner{ assembler: assembler.Assembler, @@ -191,7 +214,7 @@ func startPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem) go printPeriodicStats(&cleaner) - assembler.processPackets(*hexdumppkt, packets) + assembler.processPackets(*hexdumppkt, mainPacketInputChan) if diagnose.TapErrors.OutputLevel >= 2 { assembler.dumpStreamPool() diff --git a/tap/tcp_stream_factory.go b/tap/tcp_stream_factory.go index c063f7bfe..90f09b5d8 100644 --- a/tap/tcp_stream_factory.go +++ b/tap/tcp_stream_factory.go @@ -151,13 +151,13 @@ func inArrayPod(pods []v1.Pod, address string) bool { func (factory *tcpStreamFactory) getStreamProps(srcIP string, srcPort string, dstIP string, dstPort string) *streamProps { if factory.opts.HostMode { - if inArrayPod(factory.opts.FilterAuthorities, fmt.Sprintf("%s:%s", dstIP, dstPort)) { + if inArrayPod(tapTargets, fmt.Sprintf("%s:%s", dstIP, dstPort)) { return &streamProps{isTapTarget: true, isOutgoing: false} - } else if inArrayPod(factory.opts.FilterAuthorities, dstIP) { + } else if inArrayPod(tapTargets, dstIP) { return &streamProps{isTapTarget: true, isOutgoing: false} - } else if inArrayPod(factory.opts.FilterAuthorities, fmt.Sprintf("%s:%s", srcIP, srcPort)) { + } else if inArrayPod(tapTargets, fmt.Sprintf("%s:%s", srcIP, srcPort)) { return &streamProps{isTapTarget: true, isOutgoing: true} - } else if inArrayPod(factory.opts.FilterAuthorities, srcIP) { + } else if inArrayPod(tapTargets, srcIP) { return &streamProps{isTapTarget: true, isOutgoing: true} } return &streamProps{isTapTarget: false, isOutgoing: false} From 06785ec877cea641da382b761c19ade90e572e75 Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Mon, 27 Dec 2021 14:45:42 +0200 Subject: [PATCH 10/72] fixed acceptance test go.sum (#560) --- acceptanceTests/go.sum | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/acceptanceTests/go.sum b/acceptanceTests/go.sum index 808c5bc66..02e707750 100644 --- a/acceptanceTests/go.sum +++ b/acceptanceTests/go.sum @@ -110,6 +110,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -156,6 +157,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0= github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= @@ -191,8 +193,10 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -243,6 +247,7 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -288,8 +293,10 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -462,6 +469,7 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -521,6 +529,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -630,6 +639,7 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8X gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -641,6 +651,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= @@ -653,7 +664,9 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.21.2 h1:vz7DqmRsXTCSa6pNxXwQ1IYeAZgdIsua+DZU+o+SX3Y= k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU= +k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc= k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM= k8s.io/cli-runtime v0.21.2/go.mod h1:8u/jFcM0QpoI28f6sfrAAIslLCXUYKD5SsPPMWiHYrI= k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA= @@ -664,6 +677,7 @@ k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= k8s.io/kubectl v0.21.2/go.mod h1:PgeUclpG8VVmmQIl8zpLar3IQEpFc9mrmvlwY3CK1xo= @@ -677,5 +691,6 @@ sigs.k8s.io/kustomize/cmd/config v0.9.10/go.mod h1:Mrby0WnRH7hA6OwOYnYpfpiY0WJIM sigs.k8s.io/kustomize/kustomize/v4 v4.1.2/go.mod h1:PxBvo4WGYlCLeRPL+ziT64wBXqbgfcalOS/SXa/tcyo= sigs.k8s.io/kustomize/kyaml v0.10.17/go.mod h1:mlQFagmkm1P+W4lZJbJ/yaxMd8PqMRSC4cPcfUVt5Hg= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From 2da7c0f0ed245a26e5f284828dd1b840329bb927 Mon Sep 17 00:00:00 2001 From: RamiBerm <54766858+RamiBerm@users.noreply.github.com> Date: Mon, 27 Dec 2021 15:11:36 +0200 Subject: [PATCH 11/72] TRA-4065 fix mizu tapping (#561) * Update main.go and passive_tapper.go * Update main.go --- agent/main.go | 18 ++++++++++++++---- tap/passive_tapper.go | 3 ++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/agent/main.go b/agent/main.go index 6f1df5b3c..06812d4e0 100644 --- a/agent/main.go +++ b/agent/main.go @@ -441,11 +441,21 @@ func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) { return } } else { - var tapConfigMessage *shared.WebSocketTapConfigMessage - if err := json.Unmarshal(message, &tapConfigMessage); err != nil { - logger.Log.Errorf("received unknown message from socket connection: %s, err: %s, (%v,%+v)", string(message), err, err, err) + var socketMessageBase shared.WebSocketMessageMetadata + if err := json.Unmarshal(message, &socketMessageBase); err != nil { + logger.Log.Errorf("Could not unmarshal websocket message %v", err) } else { - tap.UpdateTapTargets(tapConfigMessage.TapTargets) + switch socketMessageBase.MessageType { + case shared.WebSocketMessageTypeTapConfig: + var tapConfigMessage *shared.WebSocketTapConfigMessage + if err := json.Unmarshal(message, &tapConfigMessage); err != nil { + logger.Log.Errorf("received unknown message from socket connection: %s, err: %s, (%v,%+v)", string(message), err, err, err) + } else { + tap.UpdateTapTargets(tapConfigMessage.TapTargets) + } + default: + logger.Log.Warningf("Received socket message of type %s for which no handlers are defined", socketMessageBase.MessageType) + } } } } diff --git a/tap/passive_tapper.go b/tap/passive_tapper.go index 95f4e1c4d..fcf0a9eda 100644 --- a/tap/passive_tapper.go +++ b/tap/passive_tapper.go @@ -194,11 +194,12 @@ func startPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem) diagnose.InitializeErrorsMap(*debug, *verbose, *quiet) diagnose.InitializeTapperInternalStats() + mainPacketInputChan = make(chan source.TcpPacketInfo) + if err := initializePacketSources(); err != nil { logger.Log.Fatal(err) } - mainPacketInputChan = make(chan source.TcpPacketInfo) assembler := NewTcpAssembler(outputItems, streamsMap, opts) diagnose.AppStats.SetStartTime(time.Now()) From 15895d5947d0ba74c575cfab66932ed34d8543eb Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Mon, 27 Dec 2021 17:19:00 +0200 Subject: [PATCH 12/72] Changed sync tappers to start only when sync tappers config is true (#562) --- agent/main.go | 32 ++++++++++++---------- agent/pkg/config/config.go | 2 +- agent/pkg/controllers/status_controller.go | 2 +- cli/config/config.go | 2 +- shared/models.go | 2 +- 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/agent/main.go b/agent/main.go index 06812d4e0..9ea0369ac 100644 --- a/agent/main.go +++ b/agent/main.go @@ -131,6 +131,10 @@ func main() { } } + if config.Config.SyncTappers { + startSyncingTappers() + } + hostApi(outputItemsChannel) } else if *harsReaderMode { outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000) @@ -260,20 +264,6 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) { routes.StatusRoutes(app) routes.NotFoundRoute(app) - if config.Config.DaemonMode { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - kubernetesProvider, err := kubernetes.NewProviderInCluster() - if err != nil { - logger.Log.Fatalf("error creating k8s provider: %+v", err) - } - - if _, err := startMizuTapperSyncer(ctx, kubernetesProvider); err != nil { - logger.Log.Fatalf("error initializing tapper syncer: %+v", err) - } - } - utils.StartServer(app) } @@ -461,6 +451,20 @@ func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) { } } +func startSyncingTappers() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + kubernetesProvider, err := kubernetes.NewProviderInCluster() + if err != nil { + logger.Log.Fatalf("error creating k8s provider: %+v", err) + } + + if _, err := startMizuTapperSyncer(ctx, kubernetesProvider); err != nil { + logger.Log.Fatalf("error initializing tapper syncer: %+v", err) + } +} + func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider) (*kubernetes.MizuTapperSyncer, error) { tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{ TargetNamespaces: config.Config.TargetNamespaces, diff --git a/agent/pkg/config/config.go b/agent/pkg/config/config.go index ac002e0c0..a25ead8aa 100644 --- a/agent/pkg/config/config.go +++ b/agent/pkg/config/config.go @@ -56,6 +56,6 @@ func getDefaultConfig() (*shared.MizuAgentConfig, error) { TapTargetRegex: *regex, MaxDBSizeBytes: defaultMaxDatabaseSizeBytes, AgentDatabasePath: DefaultDatabasePath, - DaemonMode: false, + SyncTappers: false, }, nil } diff --git a/agent/pkg/controllers/status_controller.go b/agent/pkg/controllers/status_controller.go index 0987ddc16..512cb4ddb 100644 --- a/agent/pkg/controllers/status_controller.go +++ b/agent/pkg/controllers/status_controller.go @@ -17,7 +17,7 @@ import ( ) func HealthCheck(c *gin.Context) { - if config.Config.DaemonMode { + if config.Config.SyncTappers { if providers.ExpectedTapperAmount != providers.TappersCount { c.JSON(http.StatusInternalServerError, fmt.Sprintf("expecting more tappers than are actually connected (%d expected, %d connected)", providers.ExpectedTapperAmount, providers.TappersCount)) return diff --git a/cli/config/config.go b/cli/config/config.go index dc9501fab..982f4d2b0 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -392,7 +392,6 @@ func getMizuAgentConfig(targetNamespaces []string, mizuApiFilteringOptions *api. config := shared.MizuAgentConfig{ TapTargetRegex: *serializableRegex, MaxDBSizeBytes: Config.Tap.MaxEntriesDBSizeBytes(), - DaemonMode: Config.Tap.DaemonMode, TargetNamespaces: targetNamespaces, AgentImage: Config.AgentImage, PullPolicy: Config.ImagePullPolicyStr, @@ -403,6 +402,7 @@ func getMizuAgentConfig(targetNamespaces []string, mizuApiFilteringOptions *api. MizuApiFilteringOptions: *mizuApiFilteringOptions, AgentDatabasePath: shared.DataDirPath, Istio: Config.Tap.Istio, + SyncTappers: Config.Tap.DaemonMode, } return &config, nil } diff --git a/shared/models.go b/shared/models.go index f18b833e8..43fbf6af2 100644 --- a/shared/models.go +++ b/shared/models.go @@ -36,7 +36,6 @@ type Resources struct { type MizuAgentConfig struct { TapTargetRegex api.SerializableRegexp `json:"tapTargetRegex"` MaxDBSizeBytes int64 `json:"maxDBSizeBytes"` - DaemonMode bool `json:"daemonMode"` TargetNamespaces []string `json:"targetNamespaces"` AgentImage string `json:"agentImage"` PullPolicy string `json:"pullPolicy"` @@ -47,6 +46,7 @@ type MizuAgentConfig struct { MizuApiFilteringOptions api.TrafficFilteringOptions `json:"mizuApiFilteringOptions"` AgentDatabasePath string `json:"agentDatabasePath"` Istio bool `json:"istio"` + SyncTappers bool `json:"syncTappers"` } type WebSocketMessageMetadata struct { From 1e1b5f0c0fac521f84a2f2ee121883d69e5452b8 Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Tue, 28 Dec 2021 09:59:21 +0200 Subject: [PATCH 13/72] move mizu agent config from config (#563) --- cli/cmd/common.go | 11 +++++++++++ cli/cmd/tapRunner.go | 35 +++++++++++++++++++++++++++++++++-- cli/config/config.go | 38 -------------------------------------- 3 files changed, 44 insertions(+), 40 deletions(-) diff --git a/cli/cmd/common.go b/cli/cmd/common.go index 9f1df8a11..b99b1011d 100644 --- a/cli/cmd/common.go +++ b/cli/cmd/common.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "encoding/json" "errors" "fmt" "github.com/up9inc/mizu/cli/apiserver" @@ -9,6 +10,7 @@ import ( "github.com/up9inc/mizu/cli/mizu/fsUtils" "github.com/up9inc/mizu/cli/resources" "github.com/up9inc/mizu/cli/telemetry" + "github.com/up9inc/mizu/shared" "path" "time" @@ -71,3 +73,12 @@ func dumpLogsIfNeeded(ctx context.Context, kubernetesProvider *kubernetes.Provid logger.Log.Errorf("Failed dump logs %v", err) } } + +func getSerializedMizuAgentConfig(mizuAgentConfig *shared.MizuAgentConfig) (string, error) { + serializedConfig, err := json.Marshal(mizuAgentConfig) + if err != nil { + return "", err + } + + return string(serializedConfig), nil +} diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 804122985..28c0bd67e 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -94,9 +94,15 @@ func RunMizuTap() { state.targetNamespaces = getNamespaces(kubernetesProvider) - serializedMizuConfig, err := config.GetSerializedMizuAgentConfig(state.targetNamespaces, mizuApiFilteringOptions) + mizuAgentConfig, err := getMizuAgentConfig(state.targetNamespaces, mizuApiFilteringOptions) if err != nil { - logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error composing mizu config: %v", errormessage.FormatError(err))) + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error getting mizu config: %v", errormessage.FormatError(err))) + return + } + + serializedMizuConfig, err := getSerializedMizuAgentConfig(mizuAgentConfig) + if err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error serializing mizu config: %v", errormessage.FormatError(err))) return } @@ -165,6 +171,31 @@ func handleDaemonModePostCreation(cancel context.CancelFunc, kubernetesProvider return nil } +func getMizuAgentConfig(targetNamespaces []string, mizuApiFilteringOptions *api.TrafficFilteringOptions) (*shared.MizuAgentConfig, error) { + serializableRegex, err := api.CompileRegexToSerializableRegexp(config.Config.Tap.PodRegexStr) + if err != nil { + return nil, err + } + + mizuAgentConfig := shared.MizuAgentConfig{ + TapTargetRegex: *serializableRegex, + MaxDBSizeBytes: config.Config.Tap.MaxEntriesDBSizeBytes(), + TargetNamespaces: targetNamespaces, + AgentImage: config.Config.AgentImage, + PullPolicy: config.Config.ImagePullPolicyStr, + LogLevel: config.Config.LogLevel(), + IgnoredUserAgents: config.Config.Tap.IgnoredUserAgents, + TapperResources: config.Config.Tap.TapperResources, + MizuResourcesNamespace: config.Config.MizuResourcesNamespace, + MizuApiFilteringOptions: *mizuApiFilteringOptions, + AgentDatabasePath: shared.DataDirPath, + Istio: config.Config.Tap.Istio, + SyncTappers: config.Config.Tap.DaemonMode, + } + + return &mizuAgentConfig, nil +} + /* this function is a bit problematic as it might be detached from the actual pods the mizu api server will tap. The alternative would be to wait for api server to be ready and then query it for the pods it listens to, this has diff --git a/cli/config/config.go b/cli/config/config.go index 982f4d2b0..f00cc6f09 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -9,9 +9,6 @@ import ( "strconv" "strings" - "github.com/up9inc/mizu/tap/api" - "k8s.io/apimachinery/pkg/util/json" - "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/logger" @@ -371,38 +368,3 @@ func setZeroForReadonlyFields(currentElem reflect.Value) { } } } - -func GetSerializedMizuAgentConfig(targetNamespaces []string, mizuApiFilteringOptions *api.TrafficFilteringOptions) (string, error) { - mizuConfig, err := getMizuAgentConfig(targetNamespaces, mizuApiFilteringOptions) - if err != nil { - return "", err - } - serializedConfig, err := json.Marshal(mizuConfig) - if err != nil { - return "", err - } - return string(serializedConfig), nil -} - -func getMizuAgentConfig(targetNamespaces []string, mizuApiFilteringOptions *api.TrafficFilteringOptions) (*shared.MizuAgentConfig, error) { - serializableRegex, err := api.CompileRegexToSerializableRegexp(Config.Tap.PodRegexStr) - if err != nil { - return nil, err - } - config := shared.MizuAgentConfig{ - TapTargetRegex: *serializableRegex, - MaxDBSizeBytes: Config.Tap.MaxEntriesDBSizeBytes(), - TargetNamespaces: targetNamespaces, - AgentImage: Config.AgentImage, - PullPolicy: Config.ImagePullPolicyStr, - LogLevel: Config.LogLevel(), - IgnoredUserAgents: Config.Tap.IgnoredUserAgents, - TapperResources: Config.Tap.TapperResources, - MizuResourcesNamespace: Config.MizuResourcesNamespace, - MizuApiFilteringOptions: *mizuApiFilteringOptions, - AgentDatabasePath: shared.DataDirPath, - Istio: Config.Tap.Istio, - SyncTappers: Config.Tap.DaemonMode, - } - return &config, nil -} From b039c2abadc885199ecddc5f03a6c853b714de65 Mon Sep 17 00:00:00 2001 From: Igor Gov Date: Tue, 28 Dec 2021 13:24:51 +0200 Subject: [PATCH 14/72] Fix: [EKS] server could not find the requested resourceevents.events.k8s.io (#565) --- agent/pkg/utils/utils.go | 3 +- cli/cmd/tapRunner.go | 9 ++--- shared/kubernetes/mizuTapperSyncer.go | 49 ++++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/agent/pkg/utils/utils.go b/agent/pkg/utils/utils.go index 95b92e288..a783d128a 100644 --- a/agent/pkg/utils/utils.go +++ b/agent/pkg/utils/utils.go @@ -48,7 +48,8 @@ func StartServer(app *gin.Engine) { func GetTappedPodsStatus() []shared.TappedPodStatus { tappedPodsStatus := make([]shared.TappedPodStatus, 0) for _, pod := range providers.TapStatus.Pods { - isTapped := strings.ToLower(providers.TappersStatus[pod.NodeName].Status) == "started" + status := strings.ToLower(providers.TappersStatus[pod.NodeName].Status) + isTapped := status == "running" tappedPodsStatus = append(tappedPodsStatus, shared.TappedPodStatus{Name: pod.Name, Namespace: pod.Namespace, IsTapped: isTapped}) } return tappedPodsStatus diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 28c0bd67e..9b53f7f17 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -33,8 +33,8 @@ import ( const cleanupTimeout = time.Minute type tapState struct { - startTime time.Time - targetNamespaces []string + startTime time.Time + targetNamespaces []string mizuServiceAccountExists bool } @@ -420,7 +420,8 @@ func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Pr event, err := wEvent.ToEvent() if err != nil { - logger.Log.Errorf(fmt.Sprintf("Error parsing Mizu resource event: %+v", err)) + logger.Log.Debugf("[ERROR] parsing Mizu resource event: %+v", err) + continue } if state.startTime.After(event.CreationTimestamp.Time) { @@ -448,7 +449,7 @@ func watchApiServerEvents(ctx context.Context, kubernetesProvider *kubernetes.Pr continue } - logger.Log.Errorf("Watching API server events loop, error: %+v", err) + logger.Log.Debugf("[Error] Watching API server events loop, error: %+v", err) case <-ctx.Done(): logger.Log.Debugf("Watching API server events loop, ctx done") return diff --git a/shared/kubernetes/mizuTapperSyncer.go b/shared/kubernetes/mizuTapperSyncer.go index 31dba52d1..2f9748a57 100644 --- a/shared/kubernetes/mizuTapperSyncer.go +++ b/shared/kubernetes/mizuTapperSyncer.go @@ -70,9 +70,49 @@ func CreateAndStartMizuTapperSyncer(ctx context.Context, kubernetesProvider *Pro go syncer.watchPodsForTapping() go syncer.watchTapperEvents() + go syncer.watchTapperPods() return syncer, nil } +func (tapperSyncer *MizuTapperSyncer) watchTapperPods() { + mizuResourceRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", TapperPodName)) + podWatchHelper := NewPodWatchHelper(tapperSyncer.kubernetesProvider, mizuResourceRegex) + eventChan, errorChan := FilteredWatch(tapperSyncer.context, podWatchHelper, []string{tapperSyncer.config.MizuResourcesNamespace}, podWatchHelper) + + for { + select { + case wEvent, ok := <-eventChan: + if !ok { + eventChan = nil + continue + } + + pod, err := wEvent.ToPod() + if err != nil { + logger.Log.Debugf("[ERROR] parsing Mizu resource pod: %+v", err) + continue + } + + logger.Log.Debugf("Watching tapper pods loop, tapper: %v, node: %v, status: %v", pod.Name, pod.Spec.NodeName, pod.Status.Phase) + if pod.Spec.NodeName != "" { + tapperStatus := shared.TapperStatus{TapperName: pod.Name, NodeName: pod.Spec.NodeName, Status: string(pod.Status.Phase)} + tapperSyncer.TapperStatusChangedOut <- tapperStatus + } + + case err, ok := <-errorChan: + if !ok { + errorChan = nil + continue + } + logger.Log.Debugf("[ERROR] Watching tapper pods loop, error: %+v", err) + + case <-tapperSyncer.context.Done(): + logger.Log.Debugf("Watching tapper pods loop, ctx done") + return + } + } +} + func (tapperSyncer *MizuTapperSyncer) watchTapperEvents() { mizuResourceRegex := regexp.MustCompile(fmt.Sprintf("^%s.*", TapperPodName)) eventWatchHelper := NewEventWatchHelper(tapperSyncer.kubernetesProvider, mizuResourceRegex, "pod") @@ -88,7 +128,8 @@ func (tapperSyncer *MizuTapperSyncer) watchTapperEvents() { event, err := wEvent.ToEvent() if err != nil { - logger.Log.Errorf(fmt.Sprintf("Error parsing Mizu resource event: %+v", err)) + logger.Log.Debugf("[ERROR] parsing Mizu resource event: %+v", err) + continue } if tapperSyncer.startTime.After(event.CreationTimestamp.Time) { @@ -117,8 +158,8 @@ func (tapperSyncer *MizuTapperSyncer) watchTapperEvents() { nodeName = pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchFields[0].Values[0] } - taperStatus := shared.TapperStatus{TapperName: pod.Name, NodeName: nodeName, Status: event.Reason} - tapperSyncer.TapperStatusChangedOut <- taperStatus + tapperStatus := shared.TapperStatus{TapperName: pod.Name, NodeName: nodeName, Status: string(pod.Status.Phase)} + tapperSyncer.TapperStatusChangedOut <- tapperStatus case err, ok := <-errorChan: if !ok { @@ -126,7 +167,7 @@ func (tapperSyncer *MizuTapperSyncer) watchTapperEvents() { continue } - logger.Log.Errorf("Watching tapper events loop, error: %+v", err) + logger.Log.Debugf("[ERROR] Watching tapper events loop, error: %+v", err) case <-tapperSyncer.context.Done(): logger.Log.Debugf("Watching tapper events loop, ctx done") From de046c65b037e668523bb2502ba10d5952b196d2 Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Tue, 28 Dec 2021 15:44:37 +0200 Subject: [PATCH 15/72] Mizu install mode (#566) --- README.md | 6 +- acceptanceTests/tap_test.go | 248 ------------------ acceptanceTests/testsUtils.go | 15 -- agent/main.go | 33 +-- agent/pkg/config/config.go | 8 - agent/pkg/controllers/status_controller.go | 9 - agent/pkg/providers/status_provider.go | 2 +- cli/cmd/install.go | 21 ++ cli/cmd/installRunner.go | 71 +++++ cli/cmd/tap.go | 1 - cli/cmd/tapRunner.go | 91 ++----- cli/config/configStructs/tapConfig.go | 9 - cli/mizu/consts.go | 14 +- cli/resources/createResources.go | 83 ++++-- docs/CONFIGURATION.md | 2 - .../{DAEMON_MODE.md => INSTALL_STANDALONE.md} | 16 +- docs/PERMISSIONS.md | 6 +- shared/models.go | 21 +- 18 files changed, 207 insertions(+), 449 deletions(-) create mode 100644 cli/cmd/install.go create mode 100644 cli/cmd/installRunner.go rename docs/{DAEMON_MODE.md => INSTALL_STANDALONE.md} (79%) diff --git a/README.md b/README.md index 56d537899..625ee0887 100644 --- a/README.md +++ b/README.md @@ -179,9 +179,9 @@ tap proxy-host: 0.0.0.0 and when changed it will support accessing by IP -### Run in daemon mode +### Install Mizu standalone -Mizu can be run detached from the cli using the daemon flag: `mizu tap --daemon`. This type of mizu instance will run +Mizu can be run detached from the cli using the install command: `mizu install`. This type of mizu instance will run indefinitely in the cluster. -For more information please refer to [DAEMON MODE](docs/DAEMON_MODE.md) +For more information please refer to [INSTALL STANDALONE](docs/INSTALL_STANDALONE.md) diff --git a/acceptanceTests/tap_test.go b/acceptanceTests/tap_test.go index 51b57a4ea..965b77c87 100644 --- a/acceptanceTests/tap_test.go +++ b/acceptanceTests/tap_test.go @@ -880,251 +880,3 @@ func TestTapDumpLogs(t *testing.T) { return } } - -func TestDaemonSeeTraffic(t *testing.T) { - if testing.Short() { - t.Skip("ignored acceptance test") - } - - tests := []int{50} - - for _, entriesCount := range tests { - t.Run(fmt.Sprintf("%d", entriesCount), func(t *testing.T) { - cliPath, cliPathErr := getCliPath() - if cliPathErr != nil { - t.Errorf("failed to get cli path, err: %v", cliPathErr) - return - } - - tapDaemonCmdArgs := getDefaultTapCommandArgsWithDaemonMode() - - tapNamespace := getDefaultTapNamespace() - tapDaemonCmdArgs = append(tapDaemonCmdArgs, tapNamespace...) - - tapCmd := exec.Command(cliPath, tapDaemonCmdArgs...) - - viewCmd := exec.Command(cliPath, getDefaultViewCommandArgs()...) - - t.Cleanup(func() { - daemonCleanup(t, viewCmd) - }) - - t.Logf("running command: %v", tapCmd.String()) - if err := tapCmd.Run(); err != nil { - t.Errorf("error occured while running the tap command, err: %v", err) - return - } - - t.Logf("running command: %v", viewCmd.String()) - if err := viewCmd.Start(); err != nil { - t.Errorf("error occured while running the view command, err: %v", err) - return - } - - apiServerUrl := getApiServerUrl(defaultApiServerPort) - if err := waitTapPodsReady(apiServerUrl); err != nil { - t.Errorf("failed to start tap pods on time, err: %v", err) - return - } - - proxyUrl := getProxyUrl(defaultNamespaceName, defaultServiceName) - for i := 0; i < entriesCount; i++ { - if _, requestErr := executeHttpGetRequest(fmt.Sprintf("%v/get", proxyUrl)); requestErr != nil { - t.Errorf("failed to send proxy request, err: %v", requestErr) - return - } - } - - entriesCheckFunc := func() error { - timestamp := time.Now().UnixNano() / int64(time.Millisecond) - - entries, err := getDBEntries(timestamp, entriesCount, 1*time.Second) - if err != nil { - return err - } - err = checkEntriesAtLeast(entries, 1) - if err != nil { - return err - } - entry := entries[0] - - entryUrl := fmt.Sprintf("%v/entries/%v", apiServerUrl, entry["id"]) - requestResult, requestErr := executeHttpGetRequest(entryUrl) - if requestErr != nil { - return fmt.Errorf("failed to get entry, err: %v", requestErr) - } - - if requestResult == nil { - return fmt.Errorf("unexpected nil entry result") - } - - return nil - } - if err := retriesExecute(shortRetriesCount, entriesCheckFunc); err != nil { - t.Errorf("%v", err) - return - } - }) - } -} - -func TestDaemonMultipleNamespacesSeePods(t *testing.T) { - if testing.Short() { - t.Skip("ignored acceptance test") - } - - expectedPods := []PodDescriptor{ - {Name: "httpbin", Namespace: "mizu-tests"}, - {Name: "httpbin2", Namespace: "mizu-tests"}, - {Name: "httpbin", Namespace: "mizu-tests2"}, - } - - cliPath, cliPathErr := getCliPath() - if cliPathErr != nil { - t.Errorf("failed to get cli path, err: %v", cliPathErr) - return - } - - tapCmdArgs := getDefaultTapCommandArgsWithDaemonMode() - var namespacesCmd []string - for _, expectedPod := range expectedPods { - namespacesCmd = append(namespacesCmd, "-n", expectedPod.Namespace) - } - tapCmdArgs = append(tapCmdArgs, namespacesCmd...) - - tapCmd := exec.Command(cliPath, tapCmdArgs...) - - viewCmd := exec.Command(cliPath, getDefaultViewCommandArgs()...) - - t.Cleanup(func() { - daemonCleanup(t, viewCmd) - }) - - t.Logf("running command: %v", tapCmd.String()) - if err := tapCmd.Run(); err != nil { - t.Errorf("failed to start tap command, err: %v", err) - return - } - - t.Logf("running command: %v", viewCmd.String()) - if err := viewCmd.Start(); err != nil { - t.Errorf("error occured while running the view command, err: %v", err) - return - } - - apiServerUrl := getApiServerUrl(defaultApiServerPort) - if err := waitTapPodsReady(apiServerUrl); err != nil { - t.Errorf("failed to start tap pods on time, err: %v", err) - return - } - - podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl) - requestResult, requestErr := executeHttpGetRequest(podsUrl) - if requestErr != nil { - t.Errorf("failed to get tap status, err: %v", requestErr) - return - } - - pods, err := getPods(requestResult) - if err != nil { - t.Errorf("failed to get pods, err: %v", err) - return - } - - if len(expectedPods) != len(pods) { - t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods)) - return - } - - for _, expectedPod := range expectedPods { - if !isPodDescriptorInPodArray(pods, expectedPod) { - t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name) - return - } - } -} - -func TestDaemonSingleNamespaceSeePods(t *testing.T) { - if testing.Short() { - t.Skip("ignored acceptance test") - } - - expectedPods := []PodDescriptor{ - {Name: "httpbin", Namespace: "mizu-tests"}, - {Name: "httpbin2", Namespace: "mizu-tests"}, - } - unexpectedPods := []PodDescriptor{ - {Name: "httpbin", Namespace: "mizu-tests2"}, - } - - cliPath, cliPathErr := getCliPath() - if cliPathErr != nil { - t.Errorf("failed to get cli path, err: %v", cliPathErr) - return - } - - tapCmdArgs := getDefaultTapCommandArgsWithDaemonMode() - var namespacesCmd []string - for _, expectedPod := range expectedPods { - namespacesCmd = append(namespacesCmd, "-n", expectedPod.Namespace) - } - tapCmdArgs = append(tapCmdArgs, namespacesCmd...) - - tapCmd := exec.Command(cliPath, tapCmdArgs...) - - viewCmd := exec.Command(cliPath, getDefaultViewCommandArgs()...) - - t.Cleanup(func() { - daemonCleanup(t, viewCmd) - }) - - t.Logf("running command: %v", tapCmd.String()) - if err := tapCmd.Run(); err != nil { - t.Errorf("failed to start tap command, err: %v", err) - return - } - - t.Logf("running command: %v", viewCmd.String()) - if err := viewCmd.Start(); err != nil { - t.Errorf("error occured while running the view command, err: %v", err) - return - } - - apiServerUrl := getApiServerUrl(defaultApiServerPort) - if err := waitTapPodsReady(apiServerUrl); err != nil { - t.Errorf("failed to start tap pods on time, err: %v", err) - return - } - - podsUrl := fmt.Sprintf("%v/status/tap", apiServerUrl) - requestResult, requestErr := executeHttpGetRequest(podsUrl) - if requestErr != nil { - t.Errorf("failed to get tap status, err: %v", requestErr) - return - } - - pods, err := getPods(requestResult) - if err != nil { - t.Errorf("failed to get pods, err: %v", err) - return - } - - for _, unexpectedPod := range unexpectedPods { - if isPodDescriptorInPodArray(pods, unexpectedPod) { - t.Errorf("unexpected result - unexpected pod found, pod namespace: %v, pod name: %v", unexpectedPod.Namespace, unexpectedPod.Name) - return - } - } - - if len(expectedPods) != len(pods) { - t.Errorf("unexpected result - expected pods length: %v, actual pods length: %v", len(expectedPods), len(pods)) - return - } - - for _, expectedPod := range expectedPods { - if !isPodDescriptorInPodArray(pods, expectedPod) { - t.Errorf("unexpected result - expected pod not found, pod namespace: %v, pod name: %v", expectedPod.Namespace, expectedPod.Name) - return - } - } -} diff --git a/acceptanceTests/testsUtils.go b/acceptanceTests/testsUtils.go index 715df51d7..e1aee5806 100644 --- a/acceptanceTests/testsUtils.go +++ b/acceptanceTests/testsUtils.go @@ -13,7 +13,6 @@ import ( "strings" "sync" "syscall" - "testing" "time" "github.com/gorilla/websocket" @@ -105,10 +104,6 @@ func getDefaultTapCommandArgs() []string { return append([]string{tapCommand}, defaultCmdArgs...) } -func getDefaultTapCommandArgsWithDaemonMode() []string { - return append(getDefaultTapCommandArgs(), "--daemon") -} - func getDefaultTapCommandArgsWithRegex(regex string) []string { tapCommand := "tap" defaultCmdArgs := getDefaultCommandArgs() @@ -324,16 +319,6 @@ func getLogsPath() (string, error) { return logsPath, nil } -func daemonCleanup(t *testing.T, viewCmd *exec.Cmd) { - if err := runMizuClean(); err != nil { - t.Logf("error running mizu clean: %v", err) - } - - if err := cleanupCommand(viewCmd); err != nil { - t.Logf("failed to cleanup view command, err: %v", err) - } -} - // waitTimeout waits for the waitgroup for the specified max timeout. // Returns true if waiting timed out. func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool { diff --git a/agent/main.go b/agent/main.go index 9ea0369ac..38c7e569d 100644 --- a/agent/main.go +++ b/agent/main.go @@ -22,6 +22,7 @@ import ( "path" "path/filepath" "plugin" + "regexp" "sort" "syscall" "time" @@ -131,10 +132,6 @@ func main() { } } - if config.Config.SyncTappers { - startSyncingTappers() - } - hostApi(outputItemsChannel) } else if *harsReaderMode { outputItemsChannel := make(chan *tapApi.OutputChannelItem, 1000) @@ -451,33 +448,19 @@ func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) { } } -func startSyncingTappers() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - kubernetesProvider, err := kubernetes.NewProviderInCluster() - if err != nil { - logger.Log.Fatalf("error creating k8s provider: %+v", err) - } - - if _, err := startMizuTapperSyncer(ctx, kubernetesProvider); err != nil { - logger.Log.Fatalf("error initializing tapper syncer: %+v", err) - } -} - -func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider) (*kubernetes.MizuTapperSyncer, error) { +func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, istio bool) (*kubernetes.MizuTapperSyncer, error) { tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{ - TargetNamespaces: config.Config.TargetNamespaces, - PodFilterRegex: config.Config.TapTargetRegex.Regexp, + TargetNamespaces: targetNamespaces, + PodFilterRegex: podFilterRegex, MizuResourcesNamespace: config.Config.MizuResourcesNamespace, AgentImage: config.Config.AgentImage, TapperResources: config.Config.TapperResources, ImagePullPolicy: v1.PullPolicy(config.Config.PullPolicy), LogLevel: config.Config.LogLevel, - IgnoredUserAgents: config.Config.IgnoredUserAgents, - MizuApiFilteringOptions: config.Config.MizuApiFilteringOptions, - MizuServiceAccountExists: true, //assume service account exists since daemon mode will not function without it anyway - Istio: config.Config.Istio, + IgnoredUserAgents: ignoredUserAgents, + MizuApiFilteringOptions: mizuApiFilteringOptions, + MizuServiceAccountExists: true, //assume service account exists since install mode will not function without it anyway + Istio: istio, }, time.Now()) if err != nil { diff --git a/agent/pkg/config/config.go b/agent/pkg/config/config.go index a25ead8aa..6a38e61e9 100644 --- a/agent/pkg/config/config.go +++ b/agent/pkg/config/config.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "github.com/up9inc/mizu/shared" - "github.com/up9inc/mizu/tap/api" "io/ioutil" "os" ) @@ -12,7 +11,6 @@ import ( // these values are used when the config.json file is not present const ( defaultMaxDatabaseSizeBytes int64 = 200 * 1000 * 1000 - defaultRegexTarget string = ".*" DefaultDatabasePath string = "./entries" ) @@ -48,14 +46,8 @@ func applyDefaultConfig() error { } func getDefaultConfig() (*shared.MizuAgentConfig, error) { - regex, err := api.CompileRegexToSerializableRegexp(defaultRegexTarget) - if err != nil { - return nil, err - } return &shared.MizuAgentConfig{ - TapTargetRegex: *regex, MaxDBSizeBytes: defaultMaxDatabaseSizeBytes, AgentDatabasePath: DefaultDatabasePath, - SyncTappers: false, }, nil } diff --git a/agent/pkg/controllers/status_controller.go b/agent/pkg/controllers/status_controller.go index 512cb4ddb..8819238db 100644 --- a/agent/pkg/controllers/status_controller.go +++ b/agent/pkg/controllers/status_controller.go @@ -2,12 +2,10 @@ package controllers import ( "encoding/json" - "fmt" "github.com/gin-gonic/gin" "github.com/up9inc/mizu/shared" "github.com/up9inc/mizu/shared/logger" "mizuserver/pkg/api" - "mizuserver/pkg/config" "mizuserver/pkg/holder" "mizuserver/pkg/providers" "mizuserver/pkg/up9" @@ -17,13 +15,6 @@ import ( ) func HealthCheck(c *gin.Context) { - if config.Config.SyncTappers { - if providers.ExpectedTapperAmount != providers.TappersCount { - c.JSON(http.StatusInternalServerError, fmt.Sprintf("expecting more tappers than are actually connected (%d expected, %d connected)", providers.ExpectedTapperAmount, providers.TappersCount)) - return - } - } - tappers := make([]shared.TapperStatus, 0) for _, value := range providers.TappersStatus { tappers = append(tappers, value) diff --git a/agent/pkg/providers/status_provider.go b/agent/pkg/providers/status_provider.go index 85fe8793b..9ed1a2a46 100644 --- a/agent/pkg/providers/status_provider.go +++ b/agent/pkg/providers/status_provider.go @@ -20,7 +20,7 @@ var ( TappersStatus map[string]shared.TapperStatus authStatus *models.AuthStatus RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime) - ExpectedTapperAmount = -1 //only relevant in daemon mode as cli manages tappers otherwise + ExpectedTapperAmount = -1 //only relevant in install mode as cli manages tappers otherwise tappersCountLock = sync.Mutex{} ) diff --git a/cli/cmd/install.go b/cli/cmd/install.go new file mode 100644 index 000000000..dfca9bf1a --- /dev/null +++ b/cli/cmd/install.go @@ -0,0 +1,21 @@ +package cmd + +import ( + "github.com/spf13/cobra" + "github.com/up9inc/mizu/cli/telemetry" +) + +var installCmd = &cobra.Command{ + Use: "install", + Short: "Installs mizu components", + RunE: func(cmd *cobra.Command, args []string) error { + go telemetry.ReportRun("install", nil) + runMizuInstall() + return nil + }, +} + +func init() { + rootCmd.AddCommand(installCmd) +} + diff --git a/cli/cmd/installRunner.go b/cli/cmd/installRunner.go new file mode 100644 index 000000000..e73b26be6 --- /dev/null +++ b/cli/cmd/installRunner.go @@ -0,0 +1,71 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "github.com/creasty/defaults" + "github.com/up9inc/mizu/cli/config" + "github.com/up9inc/mizu/cli/errormessage" + "github.com/up9inc/mizu/cli/resources" + "github.com/up9inc/mizu/cli/uiUtils" + "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/shared/logger" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func runMizuInstall() { + kubernetesProvider, err := getKubernetesProviderForCli() + if err != nil { + return + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() // cancel will be called when this function exits + + var serializedValidationRules string + var serializedContract string + + var defaultMaxEntriesDBSizeBytes int64 = 200 * 1000 * 1000 + + defaultResources := shared.Resources{} + defaults.Set(&defaultResources) + + mizuAgentConfig := getInstallMizuAgentConfig(defaultMaxEntriesDBSizeBytes, defaultResources) + serializedMizuConfig, err := getSerializedMizuAgentConfig(mizuAgentConfig) + if err != nil { + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error serializing mizu config: %v", errormessage.FormatError(err))) + return + } + + if err = resources.CreateInstallMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, nil, defaultMaxEntriesDBSizeBytes, defaultResources, config.Config.ImagePullPolicy(), config.Config.LogLevel(), false); err != nil { + var statusError *k8serrors.StatusError + if errors.As(err, &statusError) { + if statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists { + logger.Log.Info("Mizu is already running in this namespace, change the `mizu-resources-namespace` configuration or run `mizu clean` to remove the currently running Mizu instance") + } + } else { + defer resources.CleanUpMizuResources(ctx, cancel, kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace) + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err))) + } + + return + } + + logger.Log.Infof(uiUtils.Magenta, "Created Mizu Agent components, run `mizu view` to connect to the mizu daemon instance") +} + +func getInstallMizuAgentConfig(maxDBSizeBytes int64, tapperResources shared.Resources) *shared.MizuAgentConfig { + mizuAgentConfig := shared.MizuAgentConfig{ + MaxDBSizeBytes: maxDBSizeBytes, + AgentImage: config.Config.AgentImage, + PullPolicy: config.Config.ImagePullPolicyStr, + LogLevel: config.Config.LogLevel(), + TapperResources: tapperResources, + MizuResourcesNamespace: config.Config.MizuResourcesNamespace, + AgentDatabasePath: shared.DataDirPath, + } + + return &mizuAgentConfig +} diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index f93a8f65a..ebaa2befd 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -119,6 +119,5 @@ func init() { tapCmd.Flags().StringP(configStructs.WorkspaceTapName, "w", defaultTapConfig.Workspace, "Uploads traffic to your UP9 workspace for further analysis (requires auth)") tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file path with policy rules") tapCmd.Flags().String(configStructs.ContractFile, defaultTapConfig.ContractFile, "OAS/Swagger file to validate to monitor the contracts") - tapCmd.Flags().Bool(configStructs.DaemonModeTapName, defaultTapConfig.DaemonMode, "Run mizu in daemon mode, detached from the cli") tapCmd.Flags().Bool(configStructs.IstioName, defaultTapConfig.Istio, "Record decrypted traffic if the cluster configured with istio and mtls") } diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 9b53f7f17..44ac2532f 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -44,13 +44,9 @@ var apiProvider *apiserver.Provider func RunMizuTap() { state.startTime = time.Now() - mizuApiFilteringOptions, err := getMizuApiFilteringOptions() apiProvider = apiserver.NewProvider(GetApiServerUrl(), apiserver.DefaultRetries, apiserver.DefaultTimeout) - if err != nil { - logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error parsing regex-masking: %v", errormessage.FormatError(err))) - return - } + var err error var serializedValidationRules string if config.Config.Tap.EnforcePolicyFile != "" { serializedValidationRules, err = readValidationRules(config.Config.Tap.EnforcePolicyFile) @@ -94,12 +90,7 @@ func RunMizuTap() { state.targetNamespaces = getNamespaces(kubernetesProvider) - mizuAgentConfig, err := getMizuAgentConfig(state.targetNamespaces, mizuApiFilteringOptions) - if err != nil { - logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error getting mizu config: %v", errormessage.FormatError(err))) - return - } - + mizuAgentConfig := getTapMizuAgentConfig() serializedMizuConfig, err := getSerializedMizuAgentConfig(mizuAgentConfig) if err != nil { logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error serializing mizu config: %v", errormessage.FormatError(err))) @@ -132,68 +123,41 @@ func RunMizuTap() { } logger.Log.Infof("Waiting for Mizu Agent to start...") - if state.mizuServiceAccountExists, err = resources.CreateMizuResources(ctx, cancel, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.Tap.DaemonMode, config.Config.AgentImage, getSyncEntriesConfig(), config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel(), config.Config.Tap.NoPersistentVolumeClaim); err != nil { - logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err))) - + if state.mizuServiceAccountExists, err = resources.CreateTapMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, getSyncEntriesConfig(), config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel()); err != nil { var statusError *k8serrors.StatusError if errors.As(err, &statusError) { if statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists { logger.Log.Info("Mizu is already running in this namespace, change the `mizu-resources-namespace` configuration or run `mizu clean` to remove the currently running Mizu instance") } + } else { + defer resources.CleanUpMizuResources(ctx, cancel, kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace) + logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Error creating resources: %v", errormessage.FormatError(err))) } + return } - if config.Config.Tap.DaemonMode { - if err := handleDaemonModePostCreation(cancel, kubernetesProvider); err != nil { - defer finishMizuExecution(kubernetesProvider, apiProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace) - cancel() - } else { - logger.Log.Infof(uiUtils.Magenta, "Mizu is now running in daemon mode, run `mizu view` to connect to the mizu daemon instance") - } - } else { - defer finishMizuExecution(kubernetesProvider, apiProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace) - go goUtils.HandleExcWrapper(watchApiServerEvents, ctx, kubernetesProvider, cancel) - go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel) + defer finishMizuExecution(kubernetesProvider, apiProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace) - // block until exit signal or error - utils.WaitForFinish(ctx, cancel) - } + go goUtils.HandleExcWrapper(watchApiServerEvents, ctx, kubernetesProvider, cancel) + go goUtils.HandleExcWrapper(watchApiServerPod, ctx, kubernetesProvider, cancel) + + // block until exit signal or error + utils.WaitForFinish(ctx, cancel) } -func handleDaemonModePostCreation(cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider) error { - apiProvider := apiserver.NewProvider(GetApiServerUrl(), 90, 1*time.Second) - - if err := waitForDaemonModeToBeReady(cancel, kubernetesProvider, apiProvider); err != nil { - return err - } - - return nil -} - -func getMizuAgentConfig(targetNamespaces []string, mizuApiFilteringOptions *api.TrafficFilteringOptions) (*shared.MizuAgentConfig, error) { - serializableRegex, err := api.CompileRegexToSerializableRegexp(config.Config.Tap.PodRegexStr) - if err != nil { - return nil, err - } - +func getTapMizuAgentConfig() *shared.MizuAgentConfig { mizuAgentConfig := shared.MizuAgentConfig{ - TapTargetRegex: *serializableRegex, - MaxDBSizeBytes: config.Config.Tap.MaxEntriesDBSizeBytes(), - TargetNamespaces: targetNamespaces, - AgentImage: config.Config.AgentImage, - PullPolicy: config.Config.ImagePullPolicyStr, - LogLevel: config.Config.LogLevel(), - IgnoredUserAgents: config.Config.Tap.IgnoredUserAgents, - TapperResources: config.Config.Tap.TapperResources, - MizuResourcesNamespace: config.Config.MizuResourcesNamespace, - MizuApiFilteringOptions: *mizuApiFilteringOptions, - AgentDatabasePath: shared.DataDirPath, - Istio: config.Config.Tap.Istio, - SyncTappers: config.Config.Tap.DaemonMode, + MaxDBSizeBytes: config.Config.Tap.MaxEntriesDBSizeBytes(), + AgentImage: config.Config.AgentImage, + PullPolicy: config.Config.ImagePullPolicyStr, + LogLevel: config.Config.LogLevel(), + TapperResources: config.Config.Tap.TapperResources, + MizuResourcesNamespace: config.Config.MizuResourcesNamespace, + AgentDatabasePath: shared.DataDirPath, } - return &mizuAgentConfig, nil + return &mizuAgentConfig } /* @@ -215,17 +179,6 @@ func printTappedPodsPreview(ctx context.Context, kubernetesProvider *kubernetes. } } -func waitForDaemonModeToBeReady(cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, apiProvider *apiserver.Provider) error { - go startProxyReportErrorIfAny(kubernetesProvider, cancel) - - // TODO: TRA-3903 add a smarter test to see that tapping/pod watching is functioning properly - if err := apiProvider.TestConnection(); err != nil { - logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Mizu was not ready in time, for more info check logs at %s", fsUtils.GetLogFilePath())) - return err - } - return nil -} - func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider *kubernetes.Provider, targetNamespaces []string, mizuApiFilteringOptions api.TrafficFilteringOptions, startTime time.Time) error { tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{ TargetNamespaces: targetNamespaces, diff --git a/cli/config/configStructs/tapConfig.go b/cli/config/configStructs/tapConfig.go index 43c29523d..bd90ca262 100644 --- a/cli/config/configStructs/tapConfig.go +++ b/cli/config/configStructs/tapConfig.go @@ -3,8 +3,6 @@ package configStructs import ( "errors" "fmt" - "github.com/up9inc/mizu/cli/uiUtils" - "github.com/up9inc/mizu/shared/logger" "regexp" "github.com/up9inc/mizu/shared" @@ -24,7 +22,6 @@ const ( WorkspaceTapName = "workspace" EnforcePolicyFile = "traffic-validation-file" ContractFile = "contract" - DaemonModeTapName = "daemon" IstioName = "istio" ) @@ -47,8 +44,6 @@ type TapConfig struct { AskUploadConfirmation bool `yaml:"ask-upload-confirmation" default:"true"` ApiServerResources shared.Resources `yaml:"api-server-resources"` TapperResources shared.Resources `yaml:"tapper-resources"` - DaemonMode bool `yaml:"daemon" default:"false"` - NoPersistentVolumeClaim bool `yaml:"no-persistent-volume-claim" default:"false"` Istio bool `yaml:"istio" default:"false"` } @@ -84,9 +79,5 @@ func (config *TapConfig) Validate() error { return errors.New(fmt.Sprintf("Can't run with both --%s and --%s flags", AnalysisTapName, WorkspaceTapName)) } - if config.NoPersistentVolumeClaim && !config.DaemonMode { - logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("the --set tap.no-persistent-volume-claim=true flag has no effect without the --%s flag, the claim will not be created anyway.", DaemonModeTapName)) - } - return nil } diff --git a/cli/mizu/consts.go b/cli/mizu/consts.go index 73621a2ff..cf67a5ab4 100644 --- a/cli/mizu/consts.go +++ b/cli/mizu/consts.go @@ -6,13 +6,13 @@ import ( ) var ( - SemVer = "0.0.1" - Branch = "develop" - GitCommitHash = "" // this var is overridden using ldflags in makefile when building - BuildTimestamp = "" // this var is overridden using ldflags in makefile when building - RBACVersion = "v1" - Platform = "" - DaemonModePersistentVolumeSizeBufferBytes = int64(500 * 1000 * 1000) //500mb + SemVer = "0.0.1" + Branch = "develop" + GitCommitHash = "" // this var is overridden using ldflags in makefile when building + BuildTimestamp = "" // this var is overridden using ldflags in makefile when building + RBACVersion = "v1" + Platform = "" + InstallModePersistentVolumeSizeBufferBytes = int64(500 * 1000 * 1000) //500mb ) const DEVENVVAR = "MIZU_DISABLE_TELEMTRY" diff --git a/cli/resources/createResources.go b/cli/resources/createResources.go index 74d3186c2..80af4a513 100644 --- a/cli/resources/createResources.go +++ b/cli/resources/createResources.go @@ -14,7 +14,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) -func CreateMizuResources(ctx context.Context, cancel context.CancelFunc, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, isInstallMode bool, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, noPersistentVolumeClaim bool) (bool, error) { +func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level) (bool, error) { if !isNsRestrictedMode { if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil { return false, err @@ -25,11 +25,9 @@ func CreateMizuResources(ctx context.Context, cancel context.CancelFunc, kuberne logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v", errormessage.FormatError(err))) } - mizuServiceAccountExists, err := createRBACIfNecessary(ctx, kubernetesProvider, isNsRestrictedMode, mizuResourcesNamespace, isInstallMode) + mizuServiceAccountExists, err := createRBACIfNecessary(ctx, kubernetesProvider, isNsRestrictedMode, mizuResourcesNamespace) if err != nil { - if !isInstallMode { - logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to ensure the resources required for IP resolving. Mizu will not resolve target IPs to names. error: %v", errormessage.FormatError(err))) - } + logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to ensure the resources required for IP resolving. Mizu will not resolve target IPs to names. error: %v", errormessage.FormatError(err))) } var serviceAccountName string @@ -52,29 +50,70 @@ func CreateMizuResources(ctx context.Context, cancel context.CancelFunc, kuberne LogLevel: logLevel, } - if isInstallMode { - if !mizuServiceAccountExists { - defer CleanUpMizuResources(ctx, cancel, kubernetesProvider, isNsRestrictedMode, mizuResourcesNamespace) - logger.Log.Fatalf(uiUtils.Red, fmt.Sprintf("Failed to ensure the resources required for mizu to run in daemon mode. cannot proceed. error: %v", errormessage.FormatError(err))) - } - if err := createMizuApiServerDeployment(ctx, kubernetesProvider, opts, noPersistentVolumeClaim); err != nil { - return mizuServiceAccountExists, err - } - } else { - if err := createMizuApiServerPod(ctx, kubernetesProvider, opts); err != nil { - return mizuServiceAccountExists, err - } + if err := createMizuApiServerPod(ctx, kubernetesProvider, opts); err != nil { + return mizuServiceAccountExists, err } _, err = kubernetesProvider.CreateService(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName, kubernetes.ApiServerPodName) if err != nil { return mizuServiceAccountExists, err } + logger.Log.Debugf("Successfully created service: %s", kubernetes.ApiServerPodName) return mizuServiceAccountExists, nil } +func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, noPersistentVolumeClaim bool) error { + if !isNsRestrictedMode { + if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil { + return err + } + logger.Log.Infof("Created mizu namespace") + } + + if err := createMizuConfigmap(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, mizuResourcesNamespace); err != nil { + return err + } + logger.Log.Infof("Created config map") + + _, err := createRBACIfNecessary(ctx, kubernetesProvider, isNsRestrictedMode, mizuResourcesNamespace) + if err != nil { + return err + } + if err := kubernetesProvider.CreateDaemonsetRBAC(ctx, mizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.DaemonRoleName, kubernetes.DaemonRoleBindingName, mizu.RBACVersion); err != nil { + return err + } + logger.Log.Infof("Created RBAC") + + serviceAccountName := kubernetes.ServiceAccountName + opts := &kubernetes.ApiServerOptions{ + Namespace: mizuResourcesNamespace, + PodName: kubernetes.ApiServerPodName, + PodImage: agentImage, + ServiceAccountName: serviceAccountName, + IsNamespaceRestricted: isNsRestrictedMode, + SyncEntriesConfig: syncEntriesConfig, + MaxEntriesDBSizeBytes: maxEntriesDBSizeBytes, + Resources: apiServerResources, + ImagePullPolicy: imagePullPolicy, + LogLevel: logLevel, + } + + if err := createMizuApiServerDeployment(ctx, kubernetesProvider, opts, noPersistentVolumeClaim); err != nil { + return err + } + logger.Log.Infof("Created Api Server deployment") + + _, err = kubernetesProvider.CreateService(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName, kubernetes.ApiServerPodName) + if err != nil { + return err + } + logger.Log.Infof("Created Api Server service") + + return nil +} + func createMizuNamespace(ctx context.Context, kubernetesProvider *kubernetes.Provider, mizuResourcesNamespace string) error { _, err := kubernetesProvider.CreateNamespace(ctx, mizuResourcesNamespace) return err @@ -85,7 +124,7 @@ func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Pro return err } -func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider, isNsRestrictedMode bool, mizuResourcesNamespace string, isInstallMode bool) (bool, error) { +func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider, isNsRestrictedMode bool, mizuResourcesNamespace string) (bool, error) { if !isNsRestrictedMode { if err := kubernetesProvider.CreateMizuRBAC(ctx, mizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.ClusterRoleName, kubernetes.ClusterRoleBindingName, mizu.RBACVersion); err != nil { return false, err @@ -95,11 +134,7 @@ func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.P return false, err } } - if isInstallMode { - if err := kubernetesProvider.CreateDaemonsetRBAC(ctx, mizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.DaemonRoleName, kubernetes.DaemonRoleBindingName, mizu.RBACVersion); err != nil { - return false, err - } - } + return true, nil } @@ -141,7 +176,7 @@ func tryToCreatePersistentVolumeClaim(ctx context.Context, kubernetesProvider *k return false } - if _, err = kubernetesProvider.CreatePersistentVolumeClaim(ctx, opts.Namespace, kubernetes.PersistentVolumeClaimName, opts.MaxEntriesDBSizeBytes + mizu.DaemonModePersistentVolumeSizeBufferBytes); err != nil { + if _, err = kubernetesProvider.CreatePersistentVolumeClaim(ctx, opts.Namespace, kubernetes.PersistentVolumeClaimName, opts.MaxEntriesDBSizeBytes + mizu.InstallModePersistentVolumeSizeBufferBytes); err != nil { logger.Log.Warningf(uiUtils.Yellow, "An error has occured while creating a persistent volume claim for mizu, this means mizu data will be lost on mizu-api-server pod restart") logger.Log.Debugf("error creating persistent volume claim: %v", err) return false diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index ad9c64889..50609f4ab 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -38,8 +38,6 @@ Please make sure to use full option name (`tap.dry-run` as opposed to `dry-run` * `all-namespaces` - special flag indicating whether Mizu should search and tap pods, matching the regex, in all namespaces. Default is `false`. Please use with caution, tapping too many pods can affect resource consumption. -* `daemon` - instructs Mizu whether to run daemon mode (where CLI command exits after launch, and tapper & api-server pods in Kubernetes continue to run without controlling CLI). Typically supplied as command-line option `--daemon`. Default valie is `false` - * `dry-run` - if true, Mizu will print list of pods matching the supplied (or default) regex and exit without actually tapping the traffic. Default value is `false`. Typically supplied as command-line option `--dry-run` * `proxy-host` - IP address on which proxy to Mizu API service is launched; should be accessible at `proxy-host:gui-port`. Default value is `127.0.0.1` diff --git a/docs/DAEMON_MODE.md b/docs/INSTALL_STANDALONE.md similarity index 79% rename from docs/DAEMON_MODE.md rename to docs/INSTALL_STANDALONE.md index ab28d7d12..28b5b8620 100644 --- a/docs/DAEMON_MODE.md +++ b/docs/INSTALL_STANDALONE.md @@ -1,22 +1,16 @@ -# Mizu daemon mode +# Mizu install standalone -Mizu can be run detached from the cli using the daemon flag: `mizu tap --daemon`. This type of mizu instance will run +Mizu can be run detached from the cli using the install command: `mizu install`. This type of mizu instance will run indefinitely in the cluster. -Please note that daemon mode requires you to have RBAC creation permissions, see the [permissions](PERMISSIONS.md) +Please note that install standalone requires you to have RBAC creation permissions, see the [permissions](PERMISSIONS.md) doc for more details. ```bash -$ mizu tap "^ca.*" --daemon - Mizu will store up to 200MB of traffic, old traffic will be cleared once the limit is reached. - Tapping pods in namespaces "sock-shop" - Waiting for mizu to be ready... (may take a few minutes) - +carts-66c77f5fbb-fq65r - +catalogue-5f4cb7cf5-7zrmn - .. +$ mizu install ``` -## Stop mizu daemon +## Stop mizu install To stop the detached mizu instance and clean all cluster side resources, run `mizu clean` diff --git a/docs/PERMISSIONS.md b/docs/PERMISSIONS.md index 12eb69b8e..b200b7da6 100644 --- a/docs/PERMISSIONS.md +++ b/docs/PERMISSIONS.md @@ -57,11 +57,11 @@ Mizu needs following permissions on your Kubernetes cluster to run properly - get ``` -## Permissions required running with --daemon flag or (optional) for service / pod name resolving +## Permissions required running with install command or (optional) for service / pod name resolving -Mandatory permissions for running with `--daemon` flag. +Mandatory permissions for running with install command. -Optional for service/pod name resolving in non daemon mode +Optional for service/pod name resolving in non install standalone ```yaml - apiGroups: diff --git a/shared/models.go b/shared/models.go index 43fbf6af2..17efc230a 100644 --- a/shared/models.go +++ b/shared/models.go @@ -6,7 +6,6 @@ import ( "github.com/op/go-logging" "github.com/up9inc/mizu/shared/logger" - "github.com/up9inc/mizu/tap/api" v1 "k8s.io/api/core/v1" "gopkg.in/yaml.v3" @@ -34,19 +33,13 @@ type Resources struct { } type MizuAgentConfig struct { - TapTargetRegex api.SerializableRegexp `json:"tapTargetRegex"` - MaxDBSizeBytes int64 `json:"maxDBSizeBytes"` - TargetNamespaces []string `json:"targetNamespaces"` - AgentImage string `json:"agentImage"` - PullPolicy string `json:"pullPolicy"` - LogLevel logging.Level `json:"logLevel"` - IgnoredUserAgents []string `json:"ignoredUserAgents"` - TapperResources Resources `json:"tapperResources"` - MizuResourcesNamespace string `json:"mizuResourceNamespace"` - MizuApiFilteringOptions api.TrafficFilteringOptions `json:"mizuApiFilteringOptions"` - AgentDatabasePath string `json:"agentDatabasePath"` - Istio bool `json:"istio"` - SyncTappers bool `json:"syncTappers"` + MaxDBSizeBytes int64 `json:"maxDBSizeBytes"` + AgentImage string `json:"agentImage"` + PullPolicy string `json:"pullPolicy"` + LogLevel logging.Level `json:"logLevel"` + TapperResources Resources `json:"tapperResources"` + MizuResourcesNamespace string `json:"mizuResourceNamespace"` + AgentDatabasePath string `json:"agentDatabasePath"` } type WebSocketMessageMetadata struct { From 84f7107a333af7f22affec64017a25550930d796 Mon Sep 17 00:00:00 2001 From: Igor Gov Date: Wed, 29 Dec 2021 10:50:41 +0200 Subject: [PATCH 16/72] API server provider readiness check with echo (#569) --- cli/apiserver/provider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/apiserver/provider.go b/cli/apiserver/provider.go index dc5dff5e9..565d7566a 100644 --- a/cli/apiserver/provider.go +++ b/cli/apiserver/provider.go @@ -58,7 +58,7 @@ func (provider *Provider) TestConnection() error { } func (provider *Provider) GetHealthStatus() (*shared.HealthResponse, error) { - healthUrl := fmt.Sprintf("%s/status/health", provider.url) + healthUrl := fmt.Sprintf("%s/echo", provider.url) if response, err := provider.client.Get(healthUrl); err != nil { return nil, err } else if response.StatusCode > 299 { From 9ada330fcf501ec2b76a4c11ba21367aa827632e Mon Sep 17 00:00:00 2001 From: lirazyehezkel <61656597+lirazyehezkel@users.noreply.github.com> Date: Wed, 29 Dec 2021 11:49:44 +0200 Subject: [PATCH 17/72] Mizu enterprise preparing (#567) --- ui/src/App.sass | 13 -- ui/src/App.tsx | 138 ++---------------- .../AnalyzeButton/AnalyzeButton.tsx | 86 +++++++++++ .../AuthPresentation/AuthPresentation.sass | 13 ++ .../AuthPresentation/AuthPresentation.tsx | 30 ++++ ui/src/components/Header/Header.tsx | 22 +++ ui/src/components/TLSWarning/TLSWarning.sass | 12 ++ ui/src/components/TLSWarning/TLSWarning.tsx | 42 ++++++ 8 files changed, 216 insertions(+), 140 deletions(-) create mode 100644 ui/src/components/AnalyzeButton/AnalyzeButton.tsx create mode 100644 ui/src/components/AuthPresentation/AuthPresentation.sass create mode 100644 ui/src/components/AuthPresentation/AuthPresentation.tsx create mode 100644 ui/src/components/Header/Header.tsx create mode 100644 ui/src/components/TLSWarning/TLSWarning.sass create mode 100644 ui/src/components/TLSWarning/TLSWarning.tsx diff --git a/ui/src/App.sass b/ui/src/App.sass index 0bc2e9505..c9c6ae28a 100644 --- a/ui/src/App.sass +++ b/ui/src/App.sass @@ -23,16 +23,3 @@ font-size: 11px font-weight: bold color: $light-blue-color - - .httpsDomains - display: none - margin: 0 - padding: 0 - list-style: none - - .customWarningStyle - &:hover - overflow-y: scroll - height: 85px - .httpsDomains - display: block diff --git a/ui/src/App.tsx b/ui/src/App.tsx index 8f654a325..8e6ab26e1 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -1,51 +1,15 @@ -import React, {useEffect, useState} from 'react'; +import React, {useState} from 'react'; import './App.sass'; -import logo from './components/assets/Mizu-logo.svg'; -import logo_up9 from './components/assets/logo_up9.svg'; -import {Button, Snackbar} from "@material-ui/core"; import {TrafficPage} from "./components/TrafficPage"; -import Tooltip from "./components/UI/Tooltip"; -import {makeStyles} from "@material-ui/core/styles"; -import MuiAlert from '@material-ui/lab/Alert'; -import Api from "./helpers/api"; - - -const useStyles = makeStyles(() => ({ - tooltip: { - backgroundColor: "#3868dc", - color: "white", - fontSize: 13, - }, -})); - -const api = new Api(); +import {TLSWarning} from "./components/TLSWarning/TLSWarning"; +import {Header} from "./components/Header/Header"; const App = () => { - const classes = useStyles(); - const [analyzeStatus, setAnalyzeStatus] = useState(null); const [showTLSWarning, setShowTLSWarning] = useState(false); const [userDismissedTLSWarning, setUserDismissedTLSWarning] = useState(false); - const [addressesWithTLS, setAddressesWithTLS] = useState(new Set()); - const [statusAuth, setStatusAuth] = useState(null); - - useEffect(() => { - (async () => { - try { - const recentTLSLinks = await api.getRecentTLSLinks(); - if (recentTLSLinks?.length > 0) { - setAddressesWithTLS(new Set(recentTLSLinks)); - setShowTLSWarning(true); - } - const auth = await api.getAuthStatus(); - setStatusAuth(auth); - } catch (e) { - console.error(e); - } - - })(); - }, []); + const [addressesWithTLS, setAddressesWithTLS] = useState(new Set()); const onTLSDetected = (destAddress: string) => { addressesWithTLS.add(destAddress); @@ -56,96 +20,16 @@ const App = () => { } }; - const analysisMessage = analyzeStatus?.isRemoteReady ? - - - - - - - - - - -
StatusAvailable
Messages{analyzeStatus?.sentCount}
-
: - analyzeStatus?.sentCount > 0 ? - - - - - - - - - - - - - -
StatusProcessing
Messages{analyzeStatus?.sentCount}
Please allow a few minutes for the analysis to complete
-
: - - - - - - - - - - -
StatusWaiting for traffic
Messages{analyzeStatus?.sentCount}
- -
- return (
-
-
-
logo
-
Traffic viewer for Kubernetes
-
-
- - {analyzeStatus?.isAnalyzing && -
- -
- -
-
-
- } - {statusAuth?.email &&
-
-
{statusAuth.email}
-
{statusAuth.model}
-
-
} -
-
+
- - setUserDismissedTLSWarning(true)} severity="warning"> - Mizu is detecting TLS traffic, this type of traffic will not be displayed. - {addressesWithTLS.size > 0 &&
    {Array.from(addressesWithTLS, address =>
  • {address}
  • )}
} -
-
+
); } diff --git a/ui/src/components/AnalyzeButton/AnalyzeButton.tsx b/ui/src/components/AnalyzeButton/AnalyzeButton.tsx new file mode 100644 index 000000000..4f6e0f45b --- /dev/null +++ b/ui/src/components/AnalyzeButton/AnalyzeButton.tsx @@ -0,0 +1,86 @@ +import {Button} from "@material-ui/core"; +import React from "react"; +import Tooltip from "../UI/Tooltip"; +import logo_up9 from "../assets/logo_up9.svg"; +import {makeStyles} from "@material-ui/core/styles"; + +const useStyles = makeStyles(() => ({ + tooltip: { + backgroundColor: "#3868dc", + color: "white", + fontSize: 13, + }, +})); + +interface AnalyseButtonProps { + analyzeStatus: any +} + +export const AnalyzeButton: React.FC = ({analyzeStatus}) => { + + const classes = useStyles(); + + const analysisMessage = analyzeStatus?.isRemoteReady ? + + + + + + + + + + +
StatusAvailable
Messages{analyzeStatus?.sentCount}
+
: + analyzeStatus?.sentCount > 0 ? + + + + + + + + + + + + + +
StatusProcessing
Messages{analyzeStatus?.sentCount}
Please allow a few minutes for the analysis to complete
+
: + + + + + + + + + + +
StatusWaiting for traffic
Messages{analyzeStatus?.sentCount}
+
+ + return (
+ +
+ +
+
+
); +} diff --git a/ui/src/components/AuthPresentation/AuthPresentation.sass b/ui/src/components/AuthPresentation/AuthPresentation.sass new file mode 100644 index 000000000..2d973ec0c --- /dev/null +++ b/ui/src/components/AuthPresentation/AuthPresentation.sass @@ -0,0 +1,13 @@ +.authPresentationContainer + display: flex + border-left: 2px #87878759 solid + padding-left: 10px + margin-left: 10px + color: rgba(0,0,0,0.75) + + .authEmail + font-weight: 600 + font-size: 13px + + .authModel + font-size: 11px diff --git a/ui/src/components/AuthPresentation/AuthPresentation.tsx b/ui/src/components/AuthPresentation/AuthPresentation.tsx new file mode 100644 index 000000000..8efa739ea --- /dev/null +++ b/ui/src/components/AuthPresentation/AuthPresentation.tsx @@ -0,0 +1,30 @@ +import React, {useEffect, useState} from "react"; +import Api from "../../helpers/api"; +import './AuthPresentation.sass'; + +const api = new Api(); + +export const AuthPresentation = () => { + + const [statusAuth, setStatusAuth] = useState(null); + + useEffect(() => { + (async () => { + try { + const auth = await api.getAuthStatus(); + setStatusAuth(auth); + } catch (e) { + console.error(e); + } + })(); + }, []); + + return <> + {statusAuth?.email &&
+
+
{statusAuth.email}
+
{statusAuth.model}
+
+
} + ; +} diff --git a/ui/src/components/Header/Header.tsx b/ui/src/components/Header/Header.tsx new file mode 100644 index 000000000..7fa7d4a9d --- /dev/null +++ b/ui/src/components/Header/Header.tsx @@ -0,0 +1,22 @@ +import React from "react"; +import {AuthPresentation} from "../AuthPresentation/AuthPresentation"; +import {AnalyzeButton} from "../AnalyzeButton/AnalyzeButton"; +import logo from '../assets/Mizu-logo.svg'; + +interface HeaderProps { + analyzeStatus: any +} + +export const Header: React.FC = ({analyzeStatus}) => { + + return
+
+
logo
+
Traffic viewer for Kubernetes
+
+
+ {analyzeStatus?.isAnalyzing && } + +
+
; +} diff --git a/ui/src/components/TLSWarning/TLSWarning.sass b/ui/src/components/TLSWarning/TLSWarning.sass new file mode 100644 index 000000000..76f77b401 --- /dev/null +++ b/ui/src/components/TLSWarning/TLSWarning.sass @@ -0,0 +1,12 @@ +.httpsDomains + display: none + margin: 0 + padding: 0 + list-style: none + +.customWarningStyle + &:hover + overflow-y: scroll + height: 85px + .httpsDomains + display: block diff --git a/ui/src/components/TLSWarning/TLSWarning.tsx b/ui/src/components/TLSWarning/TLSWarning.tsx new file mode 100644 index 000000000..e96002c57 --- /dev/null +++ b/ui/src/components/TLSWarning/TLSWarning.tsx @@ -0,0 +1,42 @@ +import {Snackbar} from "@material-ui/core"; +import MuiAlert from "@material-ui/lab/Alert"; +import React, {useEffect} from "react"; +import Api from "../../helpers/api"; +import './TLSWarning.sass'; + +const api = new Api(); + +interface TLSWarningProps { + showTLSWarning: boolean + setShowTLSWarning: (show: boolean) => void + addressesWithTLS: Set + setAddressesWithTLS: (addresses: Set) => void + userDismissedTLSWarning: boolean + setUserDismissedTLSWarning: (flag: boolean) => void +} + +export const TLSWarning: React.FC = ({showTLSWarning, setShowTLSWarning, addressesWithTLS, setAddressesWithTLS, userDismissedTLSWarning, setUserDismissedTLSWarning}) => { + + useEffect(() => { + (async () => { + try { + const recentTLSLinks = await api.getRecentTLSLinks(); + if (recentTLSLinks?.length > 0) { + setAddressesWithTLS(new Set(recentTLSLinks)); + setShowTLSWarning(true); + } + } catch (e) { + console.error(e); + } + })(); + }, []); + + return ( + setUserDismissedTLSWarning(true)} severity="warning"> + Mizu is detecting TLS traffic, this type of traffic will not be displayed. + {addressesWithTLS.size > 0 && +
    {Array.from(addressesWithTLS, address =>
  • {address}
  • )}
} +
+
); +} From da2aaa9bd895edd660ac61af728a7efed0686815 Mon Sep 17 00:00:00 2001 From: Igor Gov Date: Wed, 29 Dec 2021 12:21:12 +0200 Subject: [PATCH 18/72] API server provider readiness check with echo (#570) --- cli/apiserver/provider.go | 31 ++++++++----------------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/cli/apiserver/provider.go b/cli/apiserver/provider.go index 565d7566a..67c6eb7a1 100644 --- a/cli/apiserver/provider.go +++ b/cli/apiserver/provider.go @@ -4,11 +4,9 @@ import ( "bytes" "encoding/json" "fmt" - "io" "io/ioutil" "net/http" "net/url" - "strings" "time" "github.com/up9inc/mizu/shared/kubernetes" @@ -41,7 +39,7 @@ func NewProvider(url string, retries int, timeout time.Duration) *Provider { func (provider *Provider) TestConnection() error { retriesLeft := provider.retries for retriesLeft > 0 { - if _, err := provider.GetHealthStatus(); err != nil { + if isReachable, err := provider.isReachable(); err != nil || !isReachable { logger.Log.Debugf("api server not ready yet %v", err) } else { logger.Log.Debugf("connection test to api server passed successfully") @@ -57,27 +55,14 @@ func (provider *Provider) TestConnection() error { return nil } -func (provider *Provider) GetHealthStatus() (*shared.HealthResponse, error) { - healthUrl := fmt.Sprintf("%s/echo", provider.url) - if response, err := provider.client.Get(healthUrl); err != nil { - return nil, err - } else if response.StatusCode > 299 { - responseBody := new(strings.Builder) - - if _, err := io.Copy(responseBody, response.Body); err != nil { - return nil, fmt.Errorf("status code: %d - (bad response - %v)", response.StatusCode, err) - } else { - singleLineResponse := strings.ReplaceAll(responseBody.String(), "\n", "") - return nil, fmt.Errorf("status code: %d - (response - %v)", response.StatusCode, singleLineResponse) - } +func (provider *Provider) isReachable() (bool, error) { + echoUrl := fmt.Sprintf("%s/echo", provider.url) + if response, err := provider.client.Get(echoUrl); err != nil { + return false, err + } else if response.StatusCode != 200 { + return false, fmt.Errorf("invalid status code %v", response.StatusCode) } else { - defer response.Body.Close() - - healthResponse := &shared.HealthResponse{} - if err := json.NewDecoder(response.Body).Decode(&healthResponse); err != nil { - return nil, err - } - return healthResponse, nil + return true, nil } } From f535719ddde3d23b484f01a7e69428920e8e3e5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Mert=20Y=C4=B1ld=C4=B1ran?= Date: Thu, 30 Dec 2021 07:59:05 +0300 Subject: [PATCH 19/72] Use `wss://` instead of `ws://` in case of HTTPS (#573) --- ui/src/components/TrafficPage.tsx | 2 +- ui/src/helpers/api.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ui/src/components/TrafficPage.tsx b/ui/src/components/TrafficPage.tsx index 334003555..0983ad27c 100644 --- a/ui/src/components/TrafficPage.tsx +++ b/ui/src/components/TrafficPage.tsx @@ -211,7 +211,7 @@ export const TrafficPage: React.FC = ({setAnalyzeStatus, onTLS const entryData = await api.getEntry(focusedEntryId); setSelectedEntryData(entryData); } catch (error) { - if (error.response) { + if (error.response?.data?.type) { toast[error.response.data.type](`Entry[${focusedEntryId}]: ${error.response.data.msg}`, { position: "bottom-right", theme: "colored", diff --git a/ui/src/helpers/api.js b/ui/src/helpers/api.js index ef7404996..524b58334 100644 --- a/ui/src/helpers/api.js +++ b/ui/src/helpers/api.js @@ -1,7 +1,8 @@ import * as axios from "axios"; // When working locally cp `cp .env.example .env` -export const MizuWebsocketURL = process.env.REACT_APP_OVERRIDE_WS_URL ? process.env.REACT_APP_OVERRIDE_WS_URL : `ws://${window.location.host}/ws`; +export const MizuWebsocketURL = process.env.REACT_APP_OVERRIDE_WS_URL ? process.env.REACT_APP_OVERRIDE_WS_URL : + window.location.protocol === 'https:' ? `wss://${window.location.host}/ws` : `ws://${window.location.host}/ws`; const CancelToken = axios.CancelToken; From 482036182bc59ba39e3e0a52a34bcdac64c1f025 Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Thu, 30 Dec 2021 14:58:53 +0200 Subject: [PATCH 20/72] Added setting ui mode - enterprise or normal (#575) --- agent/main.go | 24 ++++++++++++++++++++++++ cli/cmd/installRunner.go | 1 + shared/models.go | 1 + 3 files changed, 26 insertions(+) diff --git a/agent/main.go b/agent/main.go index 38c7e569d..e9609c586 100644 --- a/agent/main.go +++ b/agent/main.go @@ -24,6 +24,8 @@ import ( "plugin" "regexp" "sort" + "strconv" + "strings" "syscall" "time" @@ -59,6 +61,7 @@ const ( socketConnectionRetries = 30 socketConnectionRetryDelay = time.Second * 2 socketHandshakeTimeout = time.Second * 2 + uiIndexPath = "./site/index.html" ) func main() { @@ -251,7 +254,12 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) { } app.Use(DisableRootStaticCache()) + + if err := setUIMode(); err != nil { + logger.Log.Panicf("Error setting ui mode, err: %v", err) + } app.Use(static.ServeRoot("/", "./site")) + app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before api.WebSocketRoutes(app, &eventHandlers, startTime) @@ -291,6 +299,22 @@ func CORSMiddleware() gin.HandlerFunc { } } +func setUIMode() error { + read, err := ioutil.ReadFile(uiIndexPath) + if err != nil { + return err + } + + replacedContent := strings.Replace(string(read), "__IS_STANDALONE__", strconv.FormatBool(config.Config.StandaloneMode), 1) + + err = ioutil.WriteFile(uiIndexPath, []byte(replacedContent), 0) + if err != nil { + return err + } + + return nil +} + func parseEnvVar(env string) map[string][]v1.Pod { var mapOfList map[string][]v1.Pod diff --git a/cli/cmd/installRunner.go b/cli/cmd/installRunner.go index e73b26be6..962d32098 100644 --- a/cli/cmd/installRunner.go +++ b/cli/cmd/installRunner.go @@ -65,6 +65,7 @@ func getInstallMizuAgentConfig(maxDBSizeBytes int64, tapperResources shared.Reso TapperResources: tapperResources, MizuResourcesNamespace: config.Config.MizuResourcesNamespace, AgentDatabasePath: shared.DataDirPath, + StandaloneMode: true, } return &mizuAgentConfig diff --git a/shared/models.go b/shared/models.go index 17efc230a..abf0c8b67 100644 --- a/shared/models.go +++ b/shared/models.go @@ -40,6 +40,7 @@ type MizuAgentConfig struct { TapperResources Resources `json:"tapperResources"` MizuResourcesNamespace string `json:"mizuResourceNamespace"` AgentDatabasePath string `json:"agentDatabasePath"` + StandaloneMode bool `json:"standaloneMode"` } type WebSocketMessageMetadata struct { From 9c45166075a657bbc641cd1767a4e7d7165cbefe Mon Sep 17 00:00:00 2001 From: Igor Gov Date: Sun, 2 Jan 2022 10:42:10 +0200 Subject: [PATCH 21/72] Fixing Mizu url in readme (#580) --- README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 625ee0887..aa50a5bbb 100644 --- a/README.md +++ b/README.md @@ -52,8 +52,8 @@ Pick one from the [Releases](https://github.com/up9inc/mizu/releases) page ## How to Run 1. Find pods you'd like to tap to in your Kubernetes cluster -2. Run `mizu tap` or `mizu tap PODNAME` -3. Open browser on `http://localhost:8899/mizu` **or** as instructed in the CLI +2. Run `mizu tap` or `mizu tap PODNAME` +3. Open browser on `http://localhost:8899` **or** as instructed in the CLI 4. Watch the API traffic flowing 5. Type ^C to stop @@ -172,12 +172,10 @@ Please see [CONTRACT MONITORING](docs/CONTRACT_MONITORING.md) page for more deta ### Configure proxy host -By default, mizu will be accessible via local host: 'http://localhost:8899/mizu/', it is possible to change the host, -for instance, to '0.0.0.0' which can grant access via machine IP address. -This setting can be changed via command line flag `--set tap.proxy-host=` or via config file: -tap - proxy-host: 0.0.0.0 -and when changed it will support accessing by IP +By default, mizu will be accessible via local host: 'http://localhost:8899', it is possible to change the host, for +instance, to '0.0.0.0' which can grant access via machine IP address. This setting can be changed via command line +flag `--set tap.proxy-host=` or via config file: +tap proxy-host: 0.0.0.0 and when changed it will support accessing by IP ### Install Mizu standalone From 2834ae1e85e6eed3eab4788eaf625cdf00f70453 Mon Sep 17 00:00:00 2001 From: Nimrod Gilboa Markevich <59927337+nimrod-up9@users.noreply.github.com> Date: Tue, 4 Jan 2022 09:48:22 +0200 Subject: [PATCH 22/72] Support custom RBAC resources (#572) Motivation: Allow users to change the default RBAC resources (ServiceAccount, ClusterRole, ClusterRoleBinding, Role and RoleBinding) without having Mizu delete them every run. Adds app.kubernetes.io/created-by and app.kubernetes.io/managed-by labels to all resources. The value of app.kubernetes.io/created-by is either mizu-cli or mizu-agent. The value of app.kubernetes.io/managed-by is mizu. When Mizu cleans resources (ctrl-c in tap cmd or mizu clean cmd) it removes all RBAC resources that have managed-by=mizu, and only those. A user may have a ClusterRole named mizu-clusterrole. If it doesn't have the label app.kubernetes.io/managed-by=mizu, then Mizu won't overwrite it and won't delete it. Other resources (deployments, services etc.) are always removed, regardless of their labels. --- cli/resources/cleanResources.go | 71 ++++++++++++------ shared/kubernetes/consts.go | 9 +++ shared/kubernetes/provider.go | 128 +++++++++++++++++++++++++++++--- 3 files changed, 175 insertions(+), 33 deletions(-) diff --git a/cli/resources/cleanResources.go b/cli/resources/cleanResources.go index 8fc3b9936..e92e1471d 100644 --- a/cli/resources/cleanResources.go +++ b/cli/resources/cleanResources.go @@ -42,14 +42,29 @@ func cleanUpNonRestrictedMode(ctx context.Context, cancel context.CancelFunc, ku defer waitUntilNamespaceDeleted(ctx, cancel, kubernetesProvider, mizuResourcesNamespace) } - if err := kubernetesProvider.RemoveClusterRole(ctx, kubernetes.ClusterRoleName); err != nil { - resourceDesc := fmt.Sprintf("ClusterRole %s", kubernetes.ClusterRoleName) + if resources, err := kubernetesProvider.ListManagedClusterRoles(ctx); err != nil { + resourceDesc := "ClusterRoles" handleDeletionError(err, resourceDesc, &leftoverResources) + } else { + for _, resource := range resources.Items { + if err := kubernetesProvider.RemoveClusterRole(ctx, resource.Name); err != nil { + resourceDesc := fmt.Sprintf("ClusterRole %s", resource.Name) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + } } - if err := kubernetesProvider.RemoveClusterRoleBinding(ctx, kubernetes.ClusterRoleBindingName); err != nil { - resourceDesc := fmt.Sprintf("ClusterRoleBinding %s", kubernetes.ClusterRoleBindingName) + + if resources, err := kubernetesProvider.ListManagedClusterRoleBindings(ctx); err != nil { + resourceDesc := "ClusterRoleBindings" handleDeletionError(err, resourceDesc, &leftoverResources) + } else { + for _, resource := range resources.Items { + if err := kubernetesProvider.RemoveClusterRoleBinding(ctx, resource.Name); err != nil { + resourceDesc := fmt.Sprintf("ClusterRoleBinding %s", resource.Name) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + } } return leftoverResources @@ -91,14 +106,40 @@ func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.P handleDeletionError(err, resourceDesc, &leftoverResources) } - if err := kubernetesProvider.RemoveServicAccount(ctx, mizuResourcesNamespace, kubernetes.ServiceAccountName); err != nil { - resourceDesc := fmt.Sprintf("Service Account %s in namespace %s", kubernetes.ServiceAccountName, mizuResourcesNamespace) + if resources, err := kubernetesProvider.ListManagedServiceAccounts(ctx, mizuResourcesNamespace); err != nil { + resourceDesc := fmt.Sprintf("ServiceAccounts in namespace %s", mizuResourcesNamespace) handleDeletionError(err, resourceDesc, &leftoverResources) + } else { + for _, resource := range resources.Items { + if err := kubernetesProvider.RemoveServicAccount(ctx, mizuResourcesNamespace, resource.Name); err != nil { + resourceDesc := fmt.Sprintf("ServiceAccount %s in namespace %s", resource.Name, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + } } - if err := kubernetesProvider.RemoveRole(ctx, mizuResourcesNamespace, kubernetes.RoleName); err != nil { - resourceDesc := fmt.Sprintf("Role %s in namespace %s", kubernetes.RoleName, mizuResourcesNamespace) + if resources, err := kubernetesProvider.ListManagedRoles(ctx, mizuResourcesNamespace); err != nil { + resourceDesc := fmt.Sprintf("Roles in namespace %s", mizuResourcesNamespace) handleDeletionError(err, resourceDesc, &leftoverResources) + } else { + for _, resource := range resources.Items { + if err := kubernetesProvider.RemoveRole(ctx, mizuResourcesNamespace, resource.Name); err != nil { + resourceDesc := fmt.Sprintf("Role %s in namespace %s", resource.Name, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + } + } + + if resources, err := kubernetesProvider.ListManagedRoleBindings(ctx, mizuResourcesNamespace); err != nil { + resourceDesc := fmt.Sprintf("RoleBindings in namespace %s", mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } else { + for _, resource := range resources.Items { + if err := kubernetesProvider.RemoveRoleBinding(ctx, mizuResourcesNamespace, resource.Name); err != nil { + resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", resource.Name, mizuResourcesNamespace) + handleDeletionError(err, resourceDesc, &leftoverResources) + } + } } if err := kubernetesProvider.RemovePod(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil { @@ -107,10 +148,6 @@ func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.P } //install mode resources - if err := kubernetesProvider.RemoveRoleBinding(ctx, mizuResourcesNamespace, kubernetes.RoleBindingName); err != nil { - resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", kubernetes.RoleBindingName, mizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } if err := kubernetesProvider.RemoveDeployment(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName); err != nil { resourceDesc := fmt.Sprintf("Deployment %s in namespace %s", kubernetes.ApiServerPodName, mizuResourcesNamespace) @@ -122,16 +159,6 @@ func cleanUpRestrictedMode(ctx context.Context, kubernetesProvider *kubernetes.P handleDeletionError(err, resourceDesc, &leftoverResources) } - if err := kubernetesProvider.RemoveRole(ctx, mizuResourcesNamespace, kubernetes.DaemonRoleName); err != nil { - resourceDesc := fmt.Sprintf("Role %s in namespace %s", kubernetes.DaemonRoleName, mizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - - if err := kubernetesProvider.RemoveRoleBinding(ctx, mizuResourcesNamespace, kubernetes.DaemonRoleBindingName); err != nil { - resourceDesc := fmt.Sprintf("RoleBinding %s in namespace %s", kubernetes.DaemonRoleBindingName, mizuResourcesNamespace) - handleDeletionError(err, resourceDesc, &leftoverResources) - } - return leftoverResources } diff --git a/shared/kubernetes/consts.go b/shared/kubernetes/consts.go index 68acdd1ec..de84d5714 100644 --- a/shared/kubernetes/consts.go +++ b/shared/kubernetes/consts.go @@ -17,3 +17,12 @@ const ( PersistentVolumeClaimName = MizuResourcesPrefix + "volume-claim" MinKubernetesServerVersion = "1.16.0" ) + +const ( + LabelPrefixApp = "app.kubernetes.io/" + LabelManagedBy = LabelPrefixApp + "managed-by" + LabelCreatedBy = LabelPrefixApp + "created-by" + LabelValueMizu = "mizu" + LabelValueMizuCLI = "mizu-cli" + LabelValueMizuAgent = "mizu-agent" +) diff --git a/shared/kubernetes/provider.go b/shared/kubernetes/provider.go index a1a0195d1..d0a28cc48 100644 --- a/shared/kubernetes/provider.go +++ b/shared/kubernetes/provider.go @@ -43,6 +43,8 @@ type Provider struct { kubernetesConfig clientcmd.ClientConfig clientConfig restclient.Config Namespace string + managedBy string + createdBy string } const ( @@ -86,6 +88,8 @@ func NewProvider(kubeConfigPath string) (*Provider, error) { clientSet: clientSet, kubernetesConfig: kubernetesConfig, clientConfig: *restClientConfig, + managedBy: LabelValueMizu, + createdBy: LabelValueMizuCLI, }, nil } @@ -103,6 +107,8 @@ func NewProviderInCluster() (*Provider, error) { clientSet: clientSet, kubernetesConfig: nil, // not relevant in cluster clientConfig: *restClientConfig, + managedBy: LabelValueMizu, + createdBy: LabelValueMizuAgent, }, nil } @@ -158,6 +164,10 @@ func (provider *Provider) CreateNamespace(ctx context.Context, name string) (*co namespaceSpec := &core.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: name, + Labels: map[string]string{ + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, } return provider.clientSet.CoreV1().Namespaces().Create(ctx, namespaceSpec, metav1.CreateOptions{}) @@ -243,7 +253,11 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun pod := &core.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: opts.PodName, - Labels: map[string]string{"app": opts.PodName}, + Labels: map[string]string{ + "app": opts.PodName, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, Spec: core.PodSpec{ Containers: []core.Container{ @@ -303,6 +317,10 @@ func (provider *Provider) CreateDeployment(ctx context.Context, namespace string deployment := &v1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: deploymentName, + Labels: map[string]string{ + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, Spec: v1.DeploymentSpec{ Selector: &metav1.LabelSelector{ @@ -319,6 +337,10 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s service := core.Service{ ObjectMeta: metav1.ObjectMeta{ Name: serviceName, + Labels: map[string]string{ + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, Spec: core.ServiceSpec{ Ports: []core.ServicePort{{TargetPort: intstr.FromInt(shared.DefaultApiServerPort), Port: 80}}, @@ -351,13 +373,21 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, serviceAccount := &core.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: serviceAccountName, - Labels: map[string]string{"mizu-cli-version": version}, + Labels: map[string]string{ + "mizu-cli-version": version, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, } clusterRole := &rbac.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ Name: clusterRoleName, - Labels: map[string]string{"mizu-cli-version": version}, + Labels: map[string]string{ + "mizu-cli-version": version, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, Rules: []rbac.PolicyRule{ { @@ -370,7 +400,11 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, clusterRoleBinding := &rbac.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: clusterRoleBindingName, - Labels: map[string]string{"mizu-cli-version": version}, + Labels: map[string]string{ + "mizu-cli-version": version, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, RoleRef: rbac.RoleRef{ Name: clusterRoleName, @@ -404,13 +438,21 @@ func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context, serviceAccount := &core.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: serviceAccountName, - Labels: map[string]string{"mizu-cli-version": version}, + Labels: map[string]string{ + "mizu-cli-version": version, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, } role := &rbac.Role{ ObjectMeta: metav1.ObjectMeta{ Name: roleName, - Labels: map[string]string{"mizu-cli-version": version}, + Labels: map[string]string{ + "mizu-cli-version": version, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, Rules: []rbac.PolicyRule{ { @@ -423,7 +465,11 @@ func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context, roleBinding := &rbac.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: roleBindingName, - Labels: map[string]string{"mizu-cli-version": version}, + Labels: map[string]string{ + "mizu-cli-version": version, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, RoleRef: rbac.RoleRef{ Name: roleName, @@ -457,7 +503,11 @@ func (provider *Provider) CreateDaemonsetRBAC(ctx context.Context, namespace str role := &rbac.Role{ ObjectMeta: metav1.ObjectMeta{ Name: roleName, - Labels: map[string]string{"mizu-cli-version": version}, + Labels: map[string]string{ + "mizu-cli-version": version, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, Rules: []rbac.PolicyRule{ { @@ -475,7 +525,11 @@ func (provider *Provider) CreateDaemonsetRBAC(ctx context.Context, namespace str roleBinding := &rbac.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: roleBindingName, - Labels: map[string]string{"mizu-cli-version": version}, + Labels: map[string]string{ + "mizu-cli-version": version, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, RoleRef: rbac.RoleRef{ Name: roleName, @@ -588,6 +642,10 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, }, ObjectMeta: metav1.ObjectMeta{ Name: configMapName, + Labels: map[string]string{ + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, Data: configMapData, } @@ -746,14 +804,23 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac podSpec.WithVolumes(&configMapVolume, procfsVolume) podTemplate := applyconfcore.PodTemplateSpec() - podTemplate.WithLabels(map[string]string{"app": tapperPodName}) + podTemplate.WithLabels(map[string]string{ + "app": tapperPodName, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }) podTemplate.WithSpec(podSpec) labelSelector := applyconfmeta.LabelSelector() labelSelector.WithMatchLabels(map[string]string{"app": tapperPodName}) daemonSet := applyconfapp.DaemonSet(daemonSetName, namespace) - daemonSet.WithSpec(applyconfapp.DaemonSetSpec().WithSelector(labelSelector).WithTemplate(podTemplate)) + daemonSet. + WithLabels(map[string]string{ + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }). + WithSpec(applyconfapp.DaemonSetSpec().WithSelector(labelSelector).WithTemplate(podTemplate)) _, err = provider.clientSet.AppsV1().DaemonSets(namespace).Apply(ctx, daemonSet, metav1.ApplyOptions{FieldManager: fieldManagerName}) return err @@ -827,6 +894,41 @@ func (provider *Provider) GetNamespaceEvents(ctx context.Context, namespace stri return eventList.String(), nil } +func (provider *Provider) ListManagedServiceAccounts(ctx context.Context, namespace string) (*core.ServiceAccountList, error) { + listOptions := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy), + } + return provider.clientSet.CoreV1().ServiceAccounts(namespace).List(ctx, listOptions) +} + +func (provider *Provider) ListManagedClusterRoles(ctx context.Context) (*rbac.ClusterRoleList, error) { + listOptions := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy), + } + return provider.clientSet.RbacV1().ClusterRoles().List(ctx, listOptions) +} + +func (provider *Provider) ListManagedClusterRoleBindings(ctx context.Context) (*rbac.ClusterRoleBindingList, error) { + listOptions := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy), + } + return provider.clientSet.RbacV1().ClusterRoleBindings().List(ctx, listOptions) +} + +func (provider *Provider) ListManagedRoles(ctx context.Context, namespace string) (*rbac.RoleList, error) { + listOptions := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy), + } + return provider.clientSet.RbacV1().Roles(namespace).List(ctx, listOptions) +} + +func (provider *Provider) ListManagedRoleBindings(ctx context.Context, namespace string) (*rbac.RoleBindingList, error) { + listOptions := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", LabelManagedBy, provider.managedBy), + } + return provider.clientSet.RbacV1().RoleBindings(namespace).List(ctx, listOptions) +} + func (provider *Provider) IsDefaultStorageProviderAvailable(ctx context.Context) (bool, error) { storageClassList, err := provider.clientSet.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{}) if err != nil { @@ -845,6 +947,10 @@ func (provider *Provider) CreatePersistentVolumeClaim(ctx context.Context, names volumeClaim := &core.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: volumeClaimName, + Labels: map[string]string{ + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, + }, }, Spec: core.PersistentVolumeClaimSpec{ AccessModes: []core.PersistentVolumeAccessMode{core.ReadWriteOnce}, From 02b2cbaa03a7ce581315fa0fd4def336816d04f5 Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Tue, 4 Jan 2022 17:27:14 +0200 Subject: [PATCH 23/72] Added update config route for install mode (#581) --- agent/main.go | 75 +--------- agent/pkg/api/socket_routes.go | 9 +- .../pkg/controllers/standalone_controller.go | 133 ++++++++++++++++++ agent/pkg/controllers/status_controller.go | 15 +- agent/pkg/models/models.go | 4 + agent/pkg/providers/status_provider.go | 1 - agent/pkg/routes/standalone_routes.go | 13 ++ shared/kubernetes/mizuTapperSyncer.go | 8 +- 8 files changed, 179 insertions(+), 79 deletions(-) create mode 100644 agent/pkg/controllers/standalone_controller.go create mode 100644 agent/pkg/routes/standalone_routes.go diff --git a/agent/main.go b/agent/main.go index e9609c586..40b92fd54 100644 --- a/agent/main.go +++ b/agent/main.go @@ -1,7 +1,6 @@ package main import ( - "context" "encoding/json" "errors" "flag" @@ -11,7 +10,6 @@ import ( "mizuserver/pkg/config" "mizuserver/pkg/controllers" "mizuserver/pkg/models" - "mizuserver/pkg/providers" "mizuserver/pkg/routes" "mizuserver/pkg/up9" "mizuserver/pkg/utils" @@ -22,14 +20,12 @@ import ( "path" "path/filepath" "plugin" - "regexp" "sort" "strconv" "strings" "syscall" "time" - "github.com/up9inc/mizu/shared/kubernetes" v1 "k8s.io/api/core/v1" "github.com/antelman107/net-wait-go/wait" @@ -256,13 +252,18 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) { app.Use(DisableRootStaticCache()) if err := setUIMode(); err != nil { - logger.Log.Panicf("Error setting ui mode, err: %v", err) + logger.Log.Errorf("Error setting ui mode, err: %v", err) } app.Use(static.ServeRoot("/", "./site")) app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before api.WebSocketRoutes(app, &eventHandlers, startTime) + + if config.Config.StandaloneMode { + routes.StandaloneRoutes(app) + } + routes.QueryRoutes(app) routes.EntriesRoutes(app) routes.MetadataRoutes(app) @@ -471,67 +472,3 @@ func handleIncomingMessageAsTapper(socketConnection *websocket.Conn) { } } } - -func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, istio bool) (*kubernetes.MizuTapperSyncer, error) { - tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{ - TargetNamespaces: targetNamespaces, - PodFilterRegex: podFilterRegex, - MizuResourcesNamespace: config.Config.MizuResourcesNamespace, - AgentImage: config.Config.AgentImage, - TapperResources: config.Config.TapperResources, - ImagePullPolicy: v1.PullPolicy(config.Config.PullPolicy), - LogLevel: config.Config.LogLevel, - IgnoredUserAgents: ignoredUserAgents, - MizuApiFilteringOptions: mizuApiFilteringOptions, - MizuServiceAccountExists: true, //assume service account exists since install mode will not function without it anyway - Istio: istio, - }, time.Now()) - - if err != nil { - return nil, err - } - - // handle tapperSyncer events (pod changes and errors) - go func() { - for { - select { - case syncerErr, ok := <-tapperSyncer.ErrorOut: - if !ok { - logger.Log.Debug("mizuTapperSyncer err channel closed, ending listener loop") - return - } - logger.Log.Fatalf("fatal tap syncer error: %v", syncerErr) - case tapPodChangeEvent, ok := <-tapperSyncer.TapPodChangesOut: - if !ok { - logger.Log.Debug("mizuTapperSyncer pod changes channel closed, ending listener loop") - return - } - providers.TapStatus = shared.TapStatus{Pods: kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods)} - - tappedPodsStatus := utils.GetTappedPodsStatus() - - serializedTapStatus, err := json.Marshal(shared.CreateWebSocketStatusMessage(tappedPodsStatus)) - if err != nil { - logger.Log.Fatalf("error serializing tap status: %v", err) - } - api.BroadcastToBrowserClients(serializedTapStatus) - providers.ExpectedTapperAmount = tapPodChangeEvent.ExpectedTapperAmount - case tapperStatus, ok := <-tapperSyncer.TapperStatusChangedOut: - if !ok { - logger.Log.Debug("mizuTapperSyncer tapper status changed channel closed, ending listener loop") - return - } - if providers.TappersStatus == nil { - providers.TappersStatus = make(map[string]shared.TapperStatus) - } - providers.TappersStatus[tapperStatus.NodeName] = tapperStatus - - case <-ctx.Done(): - logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done") - return - } - } - }() - - return tapperSyncer, nil -} diff --git a/agent/pkg/api/socket_routes.go b/agent/pkg/api/socket_routes.go index a96bb11ff..89e8f45f0 100644 --- a/agent/pkg/api/socket_routes.go +++ b/agent/pkg/api/socket_routes.go @@ -83,10 +83,10 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even meta := make(chan []byte) defer func() { + socketCleanup(socketId, connectedWebsockets[socketId]) data <- []byte(basenine.CloseChannel) meta <- []byte(basenine.CloseChannel) connection.Close() - socketCleanup(socketId, connectedWebsockets[socketId]) }() eventHandlers.WebSocketConnect(socketId, isTapper) @@ -97,7 +97,12 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even for { _, msg, err := ws.ReadMessage() if err != nil { - logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err) + if _, ok := err.(*websocket.CloseError); ok { + logger.Log.Debugf("Received websocket close message, socket id: %d", socketId) + } else { + logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err) + } + break } diff --git a/agent/pkg/controllers/standalone_controller.go b/agent/pkg/controllers/standalone_controller.go new file mode 100644 index 000000000..ea9468772 --- /dev/null +++ b/agent/pkg/controllers/standalone_controller.go @@ -0,0 +1,133 @@ +package controllers + +import ( + "context" + "github.com/gin-gonic/gin" + "github.com/up9inc/mizu/shared" + "github.com/up9inc/mizu/shared/kubernetes" + "github.com/up9inc/mizu/shared/logger" + tapApi "github.com/up9inc/mizu/tap/api" + v1 "k8s.io/api/core/v1" + "mizuserver/pkg/config" + "mizuserver/pkg/models" + "mizuserver/pkg/providers" + "net/http" + "regexp" + "time" +) + +var globalTapConfig *models.StandaloneTapConfig +var cancelTapperSyncer context.CancelFunc +var kubernetesProvider *kubernetes.Provider + +func PostTapConfig(c *gin.Context) { + tapConfig := &models.StandaloneTapConfig{} + + if err := c.Bind(tapConfig); err != nil { + c.JSON(http.StatusBadRequest, err) + return + } + + if cancelTapperSyncer != nil { + cancelTapperSyncer() + + providers.TapStatus = shared.TapStatus{} + providers.TappersStatus = make(map[string]shared.TapperStatus) + + broadcastTappedPodsStatus() + } + + if kubernetesProvider == nil { + var err error + kubernetesProvider, err = kubernetes.NewProviderInCluster() + if err != nil { + c.JSON(http.StatusBadRequest, err) + return + } + } + + ctx, cancel := context.WithCancel(context.Background()) + + var tappedNamespaces []string + for namespace, tapped := range tapConfig.TappedNamespaces { + if tapped { + tappedNamespaces = append(tappedNamespaces, namespace) + } + } + + podRegex, _ := regexp.Compile(".*") + + if _, err := startMizuTapperSyncer(ctx, kubernetesProvider, tappedNamespaces, *podRegex, []string{} , tapApi.TrafficFilteringOptions{}, false); err != nil { + c.JSON(http.StatusBadRequest, err) + cancel() + return + } + + cancelTapperSyncer = cancel + globalTapConfig = tapConfig + + c.JSON(http.StatusOK, "OK") +} + +func GetTapConfig(c *gin.Context) { + if globalTapConfig != nil { + c.JSON(http.StatusOK, globalTapConfig) + } + + c.JSON(http.StatusBadRequest, "Not config found") +} + +func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, istio bool) (*kubernetes.MizuTapperSyncer, error) { + tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{ + TargetNamespaces: targetNamespaces, + PodFilterRegex: podFilterRegex, + MizuResourcesNamespace: config.Config.MizuResourcesNamespace, + AgentImage: config.Config.AgentImage, + TapperResources: config.Config.TapperResources, + ImagePullPolicy: v1.PullPolicy(config.Config.PullPolicy), + LogLevel: config.Config.LogLevel, + IgnoredUserAgents: ignoredUserAgents, + MizuApiFilteringOptions: mizuApiFilteringOptions, + MizuServiceAccountExists: true, //assume service account exists since install mode will not function without it anyway + Istio: istio, + }, time.Now()) + + if err != nil { + return nil, err + } + + // handle tapperSyncer events (pod changes and errors) + go func() { + for { + select { + case syncerErr, ok := <-tapperSyncer.ErrorOut: + if !ok { + logger.Log.Debug("mizuTapperSyncer err channel closed, ending listener loop") + return + } + logger.Log.Fatalf("fatal tap syncer error: %v", syncerErr) + case _, ok := <-tapperSyncer.TapPodChangesOut: + if !ok { + logger.Log.Debug("mizuTapperSyncer pod changes channel closed, ending listener loop") + return + } + + providers.TapStatus = shared.TapStatus{Pods: kubernetes.GetPodInfosForPods(tapperSyncer.CurrentlyTappedPods)} + broadcastTappedPodsStatus() + case tapperStatus, ok := <-tapperSyncer.TapperStatusChangedOut: + if !ok { + logger.Log.Debug("mizuTapperSyncer tapper status changed channel closed, ending listener loop") + return + } + + addTapperStatus(tapperStatus) + broadcastTappedPodsStatus() + case <-ctx.Done(): + logger.Log.Debug("mizuTapperSyncer event listener loop exiting due to context done") + return + } + } + }() + + return tapperSyncer, nil +} diff --git a/agent/pkg/controllers/status_controller.go b/agent/pkg/controllers/status_controller.go index 8819238db..7ffef36b6 100644 --- a/agent/pkg/controllers/status_controller.go +++ b/agent/pkg/controllers/status_controller.go @@ -54,21 +54,28 @@ func broadcastTappedPodsStatus() { } } +func addTapperStatus(tapperStatus shared.TapperStatus) { + if providers.TappersStatus == nil { + providers.TappersStatus = make(map[string]shared.TapperStatus) + } + + providers.TappersStatus[tapperStatus.NodeName] = tapperStatus +} + func PostTapperStatus(c *gin.Context) { tapperStatus := &shared.TapperStatus{} if err := c.Bind(tapperStatus); err != nil { c.JSON(http.StatusBadRequest, err) return } + if err := validation.Validate(tapperStatus); err != nil { c.JSON(http.StatusBadRequest, err) return } + logger.Log.Infof("[Status] POST request, tapper status: %v", tapperStatus) - if providers.TappersStatus == nil { - providers.TappersStatus = make(map[string]shared.TapperStatus) - } - providers.TappersStatus[tapperStatus.NodeName] = *tapperStatus + addTapperStatus(*tapperStatus) broadcastTappedPodsStatus() } diff --git a/agent/pkg/models/models.go b/agent/pkg/models/models.go index 84f8e9842..fc55d5ac6 100644 --- a/agent/pkg/models/models.go +++ b/agent/pkg/models/models.go @@ -16,6 +16,10 @@ func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error { return v.UnmarshalData(r) } +type StandaloneTapConfig struct { + TappedNamespaces map[string]bool `json:"tappedNamespaces"` +} + type EntriesRequest struct { LeftOff int `form:"leftOff" validate:"required,min=-1"` Direction int `form:"direction" validate:"required,oneof='1' '-1'"` diff --git a/agent/pkg/providers/status_provider.go b/agent/pkg/providers/status_provider.go index 9ed1a2a46..2f90b6d75 100644 --- a/agent/pkg/providers/status_provider.go +++ b/agent/pkg/providers/status_provider.go @@ -20,7 +20,6 @@ var ( TappersStatus map[string]shared.TapperStatus authStatus *models.AuthStatus RecentTLSLinks = cache.New(tlsLinkRetainmentTime, tlsLinkRetainmentTime) - ExpectedTapperAmount = -1 //only relevant in install mode as cli manages tappers otherwise tappersCountLock = sync.Mutex{} ) diff --git a/agent/pkg/routes/standalone_routes.go b/agent/pkg/routes/standalone_routes.go new file mode 100644 index 000000000..1245c6506 --- /dev/null +++ b/agent/pkg/routes/standalone_routes.go @@ -0,0 +1,13 @@ +package routes + +import ( + "github.com/gin-gonic/gin" + "mizuserver/pkg/controllers" +) + +func StandaloneRoutes(ginApp *gin.Engine) { + routeGroup := ginApp.Group("/standalone") + + routeGroup.POST("/tapConfig", controllers.PostTapConfig) + routeGroup.GET("/tapConfig", controllers.GetTapConfig) +} diff --git a/shared/kubernetes/mizuTapperSyncer.go b/shared/kubernetes/mizuTapperSyncer.go index 2f9748a57..8b79540a9 100644 --- a/shared/kubernetes/mizuTapperSyncer.go +++ b/shared/kubernetes/mizuTapperSyncer.go @@ -18,7 +18,6 @@ const updateTappersDelay = 5 * time.Second type TappedPodChangeEvent struct { Added []core.Pod Removed []core.Pod - ExpectedTapperAmount int } // MizuTapperSyncer uses a k8s pod watch to update tapper daemonsets when targeted pods are removed or created @@ -93,6 +92,10 @@ func (tapperSyncer *MizuTapperSyncer) watchTapperPods() { continue } + if tapperSyncer.startTime.After(pod.CreationTimestamp.Time) { + continue + } + logger.Log.Debugf("Watching tapper pods loop, tapper: %v, node: %v, status: %v", pod.Name, pod.Spec.NodeName, pod.Status.Phase) if pod.Spec.NodeName != "" { tapperStatus := shared.TapperStatus{TapperName: pod.Name, NodeName: pod.Spec.NodeName, Status: string(pod.Status.Phase)} @@ -147,7 +150,7 @@ func (tapperSyncer *MizuTapperSyncer) watchTapperEvents() { pod, err1 := tapperSyncer.kubernetesProvider.GetPod(tapperSyncer.context, tapperSyncer.config.MizuResourcesNamespace, event.Regarding.Name) if err1 != nil { - logger.Log.Debugf(fmt.Sprintf("Failed to get tapper pod %s", event.Regarding.Name)) + logger.Log.Debugf(fmt.Sprintf("Couldn't get tapper pod %s", event.Regarding.Name)) continue } @@ -284,7 +287,6 @@ func (tapperSyncer *MizuTapperSyncer) updateCurrentlyTappedPods() (err error, ch tapperSyncer.TapPodChangesOut <- TappedPodChangeEvent{ Added: addedPods, Removed: removedPods, - ExpectedTapperAmount: len(tapperSyncer.nodeToTappedPodMap), } return nil, true } From 282baee881584b553fa0953273bf3f4061771cac Mon Sep 17 00:00:00 2001 From: lirazyehezkel <61656597+lirazyehezkel@users.noreply.github.com> Date: Tue, 4 Jan 2022 17:51:07 +0200 Subject: [PATCH 24/72] is standalone variable (#585) --- ui/public/index.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ui/public/index.html b/ui/public/index.html index 8485870fb..a8268a85a 100644 --- a/ui/public/index.html +++ b/ui/public/index.html @@ -25,6 +25,14 @@ Learn how to configure a non-root public URL by running `npm run build`. --> MIZU + From dbc18b96485095ddfd8576011c2b7bf4b2762afc Mon Sep 17 00:00:00 2001 From: Adam Kol <93466081+AdamKol-up9@users.noreply.github.com> Date: Tue, 4 Jan 2022 18:02:02 +0200 Subject: [PATCH 25/72] Cypress: first-time-using cypress files + new GuiPort test on the UI (#584) --- .gitignore | 9 +++++++++ acceptanceTests/cypress.json | 8 ++++++++ acceptanceTests/cypress/integration/tests/GuiPort.js | 8 ++++++++ acceptanceTests/tap_test.go | 2 ++ acceptanceTests/testsUtils.go | 12 ++++++++++++ 5 files changed, 39 insertions(+) create mode 100644 acceptanceTests/cypress.json create mode 100644 acceptanceTests/cypress/integration/tests/GuiPort.js diff --git a/.gitignore b/.gitignore index 18b659878..35366c80f 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,12 @@ pprof/* # Nohup Files - https://man7.org/linux/man-pages/man1/nohup.1p.html nohup.* + +# Cypress tests +cypress.env.json +*/cypress/downloads +*/cypress/fixtures +*/cypress/plugins +*/cypress/screenshots +*/cypress/videos +*/cypress/support diff --git a/acceptanceTests/cypress.json b/acceptanceTests/cypress.json new file mode 100644 index 000000000..21abdfe9d --- /dev/null +++ b/acceptanceTests/cypress.json @@ -0,0 +1,8 @@ +{ + "watchForFileChanges":false, + "viewportWidth": 1920, + "viewportHeight": 1080, + "video": false, + "screenshotOnRunFailure": false, + "testFiles": ["tests/GuiPort.js"] +} diff --git a/acceptanceTests/cypress/integration/tests/GuiPort.js b/acceptanceTests/cypress/integration/tests/GuiPort.js new file mode 100644 index 000000000..bf0199a4f --- /dev/null +++ b/acceptanceTests/cypress/integration/tests/GuiPort.js @@ -0,0 +1,8 @@ +it('check', function () { + cy.visit(`http://localhost:${Cypress.env('port')}/`) + + cy.get('.header').should('be.visible') + cy.get('.TrafficPageHeader').should('be.visible') + cy.get('.TrafficPage-ListContainer').should('be.visible') + cy.get('.TrafficPage-Container').should('be.visible') +}) diff --git a/acceptanceTests/tap_test.go b/acceptanceTests/tap_test.go index 965b77c87..52a9410fa 100644 --- a/acceptanceTests/tap_test.go +++ b/acceptanceTests/tap_test.go @@ -138,6 +138,8 @@ func TestTapGuiPort(t *testing.T) { t.Errorf("failed to start tap pods on time, err: %v", err) return } + + runCypressTests(t, fmt.Sprintf("npx cypress run --spec \"cypress/integration/tests/GuiPort.js\" --env port=%d", guiPort)) }) } } diff --git a/acceptanceTests/testsUtils.go b/acceptanceTests/testsUtils.go index e1aee5806..2a2701d28 100644 --- a/acceptanceTests/testsUtils.go +++ b/acceptanceTests/testsUtils.go @@ -13,6 +13,7 @@ import ( "strings" "sync" "syscall" + "testing" "time" "github.com/gorilla/websocket" @@ -143,6 +144,17 @@ func getDefaultViewCommandArgs() []string { return append([]string{viewCommand}, defaultCmdArgs...) } +func runCypressTests(t *testing.T, cypressRunCmd string) { + cypressCmd := exec.Command("bash", "-c", cypressRunCmd) + t.Logf("running command: %v", cypressCmd.String()) + out, err := cypressCmd.Output() + if err != nil { + t.Errorf("%s", out) + return + } + t.Logf("%s", out) +} + func retriesExecute(retriesCount int, executeFunc func() error) error { var lastError interface{} From bf686892123755b16eea4eece8bc393a2a76c080 Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Tue, 4 Jan 2022 18:46:03 +0200 Subject: [PATCH 26/72] renamed standalone route to config route (#586) --- agent/main.go | 2 +- .../{standalone_controller.go => config_controller.go} | 4 ++-- agent/pkg/models/models.go | 2 +- agent/pkg/routes/{standalone_routes.go => config_routes.go} | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) rename agent/pkg/controllers/{standalone_controller.go => config_controller.go} (97%) rename agent/pkg/routes/{standalone_routes.go => config_routes.go} (70%) diff --git a/agent/main.go b/agent/main.go index 40b92fd54..c6be71870 100644 --- a/agent/main.go +++ b/agent/main.go @@ -261,7 +261,7 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) { api.WebSocketRoutes(app, &eventHandlers, startTime) if config.Config.StandaloneMode { - routes.StandaloneRoutes(app) + routes.ConfigRoutes(app) } routes.QueryRoutes(app) diff --git a/agent/pkg/controllers/standalone_controller.go b/agent/pkg/controllers/config_controller.go similarity index 97% rename from agent/pkg/controllers/standalone_controller.go rename to agent/pkg/controllers/config_controller.go index ea9468772..922c4c54c 100644 --- a/agent/pkg/controllers/standalone_controller.go +++ b/agent/pkg/controllers/config_controller.go @@ -16,12 +16,12 @@ import ( "time" ) -var globalTapConfig *models.StandaloneTapConfig +var globalTapConfig *models.TapConfig var cancelTapperSyncer context.CancelFunc var kubernetesProvider *kubernetes.Provider func PostTapConfig(c *gin.Context) { - tapConfig := &models.StandaloneTapConfig{} + tapConfig := &models.TapConfig{} if err := c.Bind(tapConfig); err != nil { c.JSON(http.StatusBadRequest, err) diff --git a/agent/pkg/models/models.go b/agent/pkg/models/models.go index fc55d5ac6..cf714519e 100644 --- a/agent/pkg/models/models.go +++ b/agent/pkg/models/models.go @@ -16,7 +16,7 @@ func GetEntry(r *tapApi.MizuEntry, v tapApi.DataUnmarshaler) error { return v.UnmarshalData(r) } -type StandaloneTapConfig struct { +type TapConfig struct { TappedNamespaces map[string]bool `json:"tappedNamespaces"` } diff --git a/agent/pkg/routes/standalone_routes.go b/agent/pkg/routes/config_routes.go similarity index 70% rename from agent/pkg/routes/standalone_routes.go rename to agent/pkg/routes/config_routes.go index 1245c6506..5a8c3c94b 100644 --- a/agent/pkg/routes/standalone_routes.go +++ b/agent/pkg/routes/config_routes.go @@ -5,8 +5,8 @@ import ( "mizuserver/pkg/controllers" ) -func StandaloneRoutes(ginApp *gin.Engine) { - routeGroup := ginApp.Group("/standalone") +func ConfigRoutes(ginApp *gin.Engine) { + routeGroup := ginApp.Group("/config") routeGroup.POST("/tapConfig", controllers.PostTapConfig) routeGroup.GET("/tapConfig", controllers.GetTapConfig) From 8e20ca797b29a692c8eb0c0a5be3776e7564054c Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Wed, 5 Jan 2022 11:15:42 +0200 Subject: [PATCH 27/72] Added endpoint for getting tapped namespaces (#587) --- agent/pkg/controllers/config_controller.go | 49 +++++++++------ cli/cmd/installRunner.go | 6 +- cli/resources/createResources.go | 17 ++---- shared/kubernetes/provider.go | 71 ++++++++++++---------- 4 files changed, 81 insertions(+), 62 deletions(-) diff --git a/agent/pkg/controllers/config_controller.go b/agent/pkg/controllers/config_controller.go index 922c4c54c..11c931bb8 100644 --- a/agent/pkg/controllers/config_controller.go +++ b/agent/pkg/controllers/config_controller.go @@ -16,9 +16,8 @@ import ( "time" ) -var globalTapConfig *models.TapConfig +var globalTapConfig = &models.TapConfig{} var cancelTapperSyncer context.CancelFunc -var kubernetesProvider *kubernetes.Provider func PostTapConfig(c *gin.Context) { tapConfig := &models.TapConfig{} @@ -37,17 +36,6 @@ func PostTapConfig(c *gin.Context) { broadcastTappedPodsStatus() } - if kubernetesProvider == nil { - var err error - kubernetesProvider, err = kubernetes.NewProviderInCluster() - if err != nil { - c.JSON(http.StatusBadRequest, err) - return - } - } - - ctx, cancel := context.WithCancel(context.Background()) - var tappedNamespaces []string for namespace, tapped := range tapConfig.TappedNamespaces { if tapped { @@ -57,8 +45,14 @@ func PostTapConfig(c *gin.Context) { podRegex, _ := regexp.Compile(".*") - if _, err := startMizuTapperSyncer(ctx, kubernetesProvider, tappedNamespaces, *podRegex, []string{} , tapApi.TrafficFilteringOptions{}, false); err != nil { - c.JSON(http.StatusBadRequest, err) + kubernetesProvider, err := kubernetes.NewProviderInCluster() + if err != nil { + c.JSON(http.StatusInternalServerError, err) + return + } + ctx, cancel := context.WithCancel(context.Background()) + if _, err := startMizuTapperSyncer(ctx, kubernetesProvider, tappedNamespaces, *podRegex, []string{}, tapApi.TrafficFilteringOptions{}, false); err != nil { + c.JSON(http.StatusInternalServerError, err) cancel() return } @@ -70,11 +64,30 @@ func PostTapConfig(c *gin.Context) { } func GetTapConfig(c *gin.Context) { - if globalTapConfig != nil { - c.JSON(http.StatusOK, globalTapConfig) + kubernetesProvider, err := kubernetes.NewProviderInCluster() + if err != nil { + c.JSON(http.StatusInternalServerError, err) + return + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + namespaces, err := kubernetesProvider.ListAllNamespaces(ctx) + if err != nil { + c.JSON(http.StatusInternalServerError, err) + return } - c.JSON(http.StatusBadRequest, "Not config found") + for _, namespace := range namespaces { + if namespace.Name == config.Config.MizuResourcesNamespace { + continue + } + + if _, ok := globalTapConfig.TappedNamespaces[namespace.Name]; !ok { + globalTapConfig.TappedNamespaces[namespace.Name] = false + } + } + + c.JSON(http.StatusOK, globalTapConfig) } func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, istio bool) (*kubernetes.MizuTapperSyncer, error) { diff --git a/cli/cmd/installRunner.go b/cli/cmd/installRunner.go index 962d32098..5062fd557 100644 --- a/cli/cmd/installRunner.go +++ b/cli/cmd/installRunner.go @@ -39,7 +39,11 @@ func runMizuInstall() { return } - if err = resources.CreateInstallMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, nil, defaultMaxEntriesDBSizeBytes, defaultResources, config.Config.ImagePullPolicy(), config.Config.LogLevel(), false); err != nil { + if err = resources.CreateInstallMizuResources(ctx, kubernetesProvider, serializedValidationRules, + serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), + config.Config.MizuResourcesNamespace, config.Config.AgentImage, + nil, defaultMaxEntriesDBSizeBytes, defaultResources, config.Config.ImagePullPolicy(), + config.Config.LogLevel(), false); err != nil { var statusError *k8serrors.StatusError if errors.As(err, &statusError) { if statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists { diff --git a/cli/resources/createResources.go b/cli/resources/createResources.go index 80af4a513..852b08f6c 100644 --- a/cli/resources/createResources.go +++ b/cli/resources/createResources.go @@ -25,7 +25,7 @@ func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes. logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to create resources required for policy validation. Mizu will not validate policy rules. error: %v", errormessage.FormatError(err))) } - mizuServiceAccountExists, err := createRBACIfNecessary(ctx, kubernetesProvider, isNsRestrictedMode, mizuResourcesNamespace) + mizuServiceAccountExists, err := createRBACIfNecessary(ctx, kubernetesProvider, isNsRestrictedMode, mizuResourcesNamespace, []string{"pods", "services", "endpoints"}) if err != nil { logger.Log.Warningf(uiUtils.Warning, fmt.Sprintf("Failed to ensure the resources required for IP resolving. Mizu will not resolve target IPs to names. error: %v", errormessage.FormatError(err))) } @@ -65,19 +65,12 @@ func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes. } func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, noPersistentVolumeClaim bool) error { - if !isNsRestrictedMode { - if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil { - return err - } - logger.Log.Infof("Created mizu namespace") - } - if err := createMizuConfigmap(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, mizuResourcesNamespace); err != nil { return err } logger.Log.Infof("Created config map") - _, err := createRBACIfNecessary(ctx, kubernetesProvider, isNsRestrictedMode, mizuResourcesNamespace) + _, err := createRBACIfNecessary(ctx, kubernetesProvider, isNsRestrictedMode, mizuResourcesNamespace, []string{"pods", "services", "endpoints", "namespaces"}) if err != nil { return err } @@ -124,9 +117,9 @@ func createMizuConfigmap(ctx context.Context, kubernetesProvider *kubernetes.Pro return err } -func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider, isNsRestrictedMode bool, mizuResourcesNamespace string) (bool, error) { +func createRBACIfNecessary(ctx context.Context, kubernetesProvider *kubernetes.Provider, isNsRestrictedMode bool, mizuResourcesNamespace string, resources []string) (bool, error) { if !isNsRestrictedMode { - if err := kubernetesProvider.CreateMizuRBAC(ctx, mizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.ClusterRoleName, kubernetes.ClusterRoleBindingName, mizu.RBACVersion); err != nil { + if err := kubernetesProvider.CreateMizuRBAC(ctx, mizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.ClusterRoleName, kubernetes.ClusterRoleBindingName, mizu.RBACVersion, resources); err != nil { return false, err } } else { @@ -176,7 +169,7 @@ func tryToCreatePersistentVolumeClaim(ctx context.Context, kubernetesProvider *k return false } - if _, err = kubernetesProvider.CreatePersistentVolumeClaim(ctx, opts.Namespace, kubernetes.PersistentVolumeClaimName, opts.MaxEntriesDBSizeBytes + mizu.InstallModePersistentVolumeSizeBufferBytes); err != nil { + if _, err = kubernetesProvider.CreatePersistentVolumeClaim(ctx, opts.Namespace, kubernetes.PersistentVolumeClaimName, opts.MaxEntriesDBSizeBytes+mizu.InstallModePersistentVolumeSizeBufferBytes); err != nil { logger.Log.Warningf(uiUtils.Yellow, "An error has occured while creating a persistent volume claim for mizu, this means mizu data will be lost on mizu-api-server pod restart") logger.Log.Debugf("error creating persistent volume claim: %v", err) return false diff --git a/shared/kubernetes/provider.go b/shared/kubernetes/provider.go index d0a28cc48..44827da78 100644 --- a/shared/kubernetes/provider.go +++ b/shared/kubernetes/provider.go @@ -43,8 +43,8 @@ type Provider struct { kubernetesConfig clientcmd.ClientConfig clientConfig restclient.Config Namespace string - managedBy string - createdBy string + managedBy string + createdBy string } const ( @@ -252,9 +252,9 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun pod := &core.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: opts.PodName, + Name: opts.PodName, Labels: map[string]string{ - "app": opts.PodName, + "app": opts.PodName, LabelManagedBy: provider.managedBy, LabelCreatedBy: provider.createdBy, }, @@ -369,41 +369,41 @@ func (provider *Provider) doesResourceExist(resource interface{}, err error) (bo return resource != nil, nil } -func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, serviceAccountName string, clusterRoleName string, clusterRoleBindingName string, version string) error { +func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, serviceAccountName string, clusterRoleName string, clusterRoleBindingName string, version string, resources []string) error { serviceAccount := &core.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ - Name: serviceAccountName, + Name: serviceAccountName, Labels: map[string]string{ "mizu-cli-version": version, - LabelManagedBy: provider.managedBy, - LabelCreatedBy: provider.createdBy, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, }, }, } clusterRole := &rbac.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ - Name: clusterRoleName, + Name: clusterRoleName, Labels: map[string]string{ "mizu-cli-version": version, - LabelManagedBy: provider.managedBy, - LabelCreatedBy: provider.createdBy, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, }, }, Rules: []rbac.PolicyRule{ { APIGroups: []string{"", "extensions", "apps"}, - Resources: []string{"pods", "services", "endpoints"}, + Resources: resources, Verbs: []string{"list", "get", "watch"}, }, }, } clusterRoleBinding := &rbac.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: clusterRoleBindingName, + Name: clusterRoleBindingName, Labels: map[string]string{ "mizu-cli-version": version, - LabelManagedBy: provider.managedBy, - LabelCreatedBy: provider.createdBy, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, }, }, RoleRef: rbac.RoleRef{ @@ -437,21 +437,21 @@ func (provider *Provider) CreateMizuRBAC(ctx context.Context, namespace string, func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context, namespace string, serviceAccountName string, roleName string, roleBindingName string, version string) error { serviceAccount := &core.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ - Name: serviceAccountName, + Name: serviceAccountName, Labels: map[string]string{ "mizu-cli-version": version, - LabelManagedBy: provider.managedBy, - LabelCreatedBy: provider.createdBy, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, }, }, } role := &rbac.Role{ ObjectMeta: metav1.ObjectMeta{ - Name: roleName, + Name: roleName, Labels: map[string]string{ "mizu-cli-version": version, - LabelManagedBy: provider.managedBy, - LabelCreatedBy: provider.createdBy, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, }, }, Rules: []rbac.PolicyRule{ @@ -464,11 +464,11 @@ func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context, } roleBinding := &rbac.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: roleBindingName, + Name: roleBindingName, Labels: map[string]string{ "mizu-cli-version": version, - LabelManagedBy: provider.managedBy, - LabelCreatedBy: provider.createdBy, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, }, }, RoleRef: rbac.RoleRef{ @@ -502,11 +502,11 @@ func (provider *Provider) CreateMizuRBACNamespaceRestricted(ctx context.Context, func (provider *Provider) CreateDaemonsetRBAC(ctx context.Context, namespace string, serviceAccountName string, roleName string, roleBindingName string, version string) error { role := &rbac.Role{ ObjectMeta: metav1.ObjectMeta{ - Name: roleName, + Name: roleName, Labels: map[string]string{ "mizu-cli-version": version, - LabelManagedBy: provider.managedBy, - LabelCreatedBy: provider.createdBy, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, }, }, Rules: []rbac.PolicyRule{ @@ -524,11 +524,11 @@ func (provider *Provider) CreateDaemonsetRBAC(ctx context.Context, namespace str } roleBinding := &rbac.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: roleBindingName, + Name: roleBindingName, Labels: map[string]string{ "mizu-cli-version": version, - LabelManagedBy: provider.managedBy, - LabelCreatedBy: provider.createdBy, + LabelManagedBy: provider.managedBy, + LabelCreatedBy: provider.createdBy, }, }, RoleRef: rbac.RoleRef{ @@ -805,7 +805,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac podTemplate := applyconfcore.PodTemplateSpec() podTemplate.WithLabels(map[string]string{ - "app": tapperPodName, + "app": tapperPodName, LabelManagedBy: provider.managedBy, LabelCreatedBy: provider.createdBy, }) @@ -869,6 +869,15 @@ func (provider *Provider) ListAllRunningPodsMatchingRegex(ctx context.Context, r return matchingPods, nil } +func (provider *Provider) ListAllNamespaces(ctx context.Context) ([]core.Namespace, error) { + namespaces, err := provider.clientSet.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + + return namespaces.Items, err +} + func (provider *Provider) GetPodLogs(ctx context.Context, namespace string, podName string) (string, error) { podLogOpts := core.PodLogOptions{} req := provider.clientSet.CoreV1().Pods(namespace).GetLogs(podName, &podLogOpts) From 833d08bb4095bcf40a8196d1ef9cc60e30f6a475 Mon Sep 17 00:00:00 2001 From: RamiBerm <54766858+RamiBerm@users.noreply.github.com> Date: Wed, 5 Jan 2022 17:50:27 +0200 Subject: [PATCH 28/72] TRA-4075 integrate kratos user management (#583) * WIP * WIP * WIP * WIP * Update App.tsx and Header.tsx * Update createResources.go, provider.go, and 2 more files... * WIP * fix eof newlines * Fix ts imports, add readiness probe to kratos to prevent mizu being used while kratos isnt ready * cleaned code * fix install create namespace * Update package-lock.json * Update provider.go * Update provider.go * Update provider.go * Update install_controller.go * Update kratos.yml * Update start.sh * Update provider.go * Update provider.go * Update main.go, socket_routes.go, and 8 more files... * Update App.tsx * Update installRunner.go * Update App.tsx --- acceptanceTests/go.sum | 7 +- agent/go.mod | 1 + agent/go.sum | 86 +++++++++- agent/kratos/Dockerfile | 14 ++ agent/kratos/build-push-featurebranch.sh | 28 +++ agent/kratos/identity.schema.json | 43 +++++ agent/kratos/kratos.yml | 84 +++++++++ agent/kratos/start.sh | 4 + agent/main.go | 21 +-- agent/pkg/api/socket_routes.go | 6 +- agent/pkg/controllers/install_controller.go | 18 ++ agent/pkg/controllers/user_controller.go | 39 +++++ agent/pkg/middlewares/cors.go | 19 ++ agent/pkg/middlewares/requiresAuth.go | 49 ++++++ agent/pkg/models/models.go | 4 + agent/pkg/providers/install_provider.go | 18 ++ agent/pkg/providers/user_provider.go | 162 ++++++++++++++++++ agent/pkg/routes/entries_routes.go | 2 + agent/pkg/routes/install_routes.go | 13 ++ agent/pkg/routes/metadata_routes.go | 3 +- agent/pkg/routes/query_routes.go | 2 + agent/pkg/routes/status_routes.go | 5 +- agent/pkg/routes/user_routes.go | 15 ++ cli/cmd/installRunner.go | 7 +- cli/cmd/tapRunner.go | 6 +- cli/resources/createResources.go | 11 +- shared/kubernetes/provider.go | 97 +++++++---- shared/kubernetes/proxy.go | 1 - ui/package-lock.json | 2 +- ui/src/App.sass | 14 +- ui/src/App.tsx | 108 ++++++++++-- .../AuthPresentation/AuthPresentation.tsx | 2 +- ui/src/components/EntriesList.tsx | 2 +- ui/src/components/Header/Header.tsx | 2 - ui/src/components/InstallPage.tsx | 60 +++++++ ui/src/components/LoadingOverlay.tsx | 25 +++ ui/src/components/LoginPage.tsx | 49 ++++++ ui/src/components/TLSWarning/TLSWarning.tsx | 2 +- ui/src/components/TrafficPage.tsx | 15 +- ui/src/components/style/LoadingOverlay.sass | 54 ++++++ ui/src/consts.ts | 1 + ui/src/helpers/api.js | 103 +++++++++-- 42 files changed, 1098 insertions(+), 106 deletions(-) create mode 100644 agent/kratos/Dockerfile create mode 100755 agent/kratos/build-push-featurebranch.sh create mode 100644 agent/kratos/identity.schema.json create mode 100755 agent/kratos/kratos.yml create mode 100755 agent/kratos/start.sh create mode 100644 agent/pkg/controllers/install_controller.go create mode 100644 agent/pkg/controllers/user_controller.go create mode 100644 agent/pkg/middlewares/cors.go create mode 100644 agent/pkg/middlewares/requiresAuth.go create mode 100644 agent/pkg/providers/install_provider.go create mode 100644 agent/pkg/providers/user_provider.go create mode 100644 agent/pkg/routes/install_routes.go create mode 100644 agent/pkg/routes/user_routes.go create mode 100644 ui/src/components/InstallPage.tsx create mode 100644 ui/src/components/LoadingOverlay.tsx create mode 100644 ui/src/components/LoginPage.tsx create mode 100644 ui/src/components/style/LoadingOverlay.sass create mode 100644 ui/src/consts.ts diff --git a/acceptanceTests/go.sum b/acceptanceTests/go.sum index 02e707750..825dbcd0f 100644 --- a/acceptanceTests/go.sum +++ b/acceptanceTests/go.sum @@ -75,6 +75,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -198,7 +199,6 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -321,6 +321,7 @@ github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -364,6 +365,7 @@ github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= @@ -373,6 +375,7 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -579,6 +582,7 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -693,4 +697,5 @@ sigs.k8s.io/kustomize/kyaml v0.10.17/go.mod h1:mlQFagmkm1P+W4lZJbJ/yaxMd8PqMRSC4 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/agent/go.mod b/agent/go.mod index a4ccbb693..2d58364fb 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -15,6 +15,7 @@ require ( github.com/gorilla/websocket v1.4.2 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 + github.com/ory/kratos-client-go v0.8.2-alpha.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/up9inc/basenine/client/go v0.0.0-20211215185650-10083bb9a1b3 github.com/up9inc/mizu/shared v0.0.0 diff --git a/agent/go.sum b/agent/go.sum index ed288cfc1..d73bf64fa 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -8,20 +8,30 @@ cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0 h1:Dg9iHVQfrhq82rUNu9ZxUDrJLaxFUe/HlCVaLyRruq8= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -76,6 +86,7 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -101,7 +112,9 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= @@ -123,14 +136,11 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm github.com/gin-contrib/static v0.0.1 h1:JVxuvHPuUfkoul12N7dtQw7KRn/pSMq7Ue1Va9Swm1U= github.com/gin-contrib/static v0.0.1/go.mod h1:CSxeF+wep05e0kCOsqWdAWbSszmc31zTIbD8TvWl7Hs= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/gin-gonic/gin v1.7.2 h1:Tg03T9yM2xa8j6I3Z3oqLaQRSmKvxPd6g/2HJ6zICFA= -github.com/gin-gonic/gin v1.7.2/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-errors/errors v1.4.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -215,11 +225,14 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -236,7 +249,9 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -247,11 +262,14 @@ github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -376,6 +394,8 @@ github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWEr github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231 h1:fa50YL1pzKW+1SsBnJDOHppJN9stOEwS+CRWyUtyYGU= github.com/orcaman/concurrent-map v0.0.0-20210106121528-16402b402231/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= +github.com/ory/kratos-client-go v0.8.2-alpha.1 h1:YlKhGOSZjounlB9iY4xSWlqHbyLYkeLzlLk8ZL7/nEM= +github.com/ory/kratos-client-go v0.8.2-alpha.1/go.mod h1:dOQIsar76K07wMPJD/6aMhrWyY+sFGEagLDLso1CpsA= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= @@ -462,7 +482,9 @@ github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6Ut github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -472,6 +494,7 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -539,6 +562,7 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -548,6 +572,13 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= @@ -556,14 +587,17 @@ golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAG golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558 h1:D7nTwh4J0i+5mW4Zjzn5omvlr6YBcWywE6KOcatyNxY= +golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -599,7 +633,13 @@ golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -659,9 +699,19 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -678,13 +728,20 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -702,16 +759,31 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -720,6 +792,7 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -756,6 +829,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.21.2 h1:vz7DqmRsXTCSa6pNxXwQ1IYeAZgdIsua+DZU+o+SX3Y= k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU= k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc= diff --git a/agent/kratos/Dockerfile b/agent/kratos/Dockerfile new file mode 100644 index 000000000..086a1a992 --- /dev/null +++ b/agent/kratos/Dockerfile @@ -0,0 +1,14 @@ +FROM oryd/kratos:v0.8.0-sqlite + +USER root + +RUN apk add sqlite + +RUN mkdir -p /etc/config/kratos + +COPY ./kratos.yml /etc/config/kratos/kratos.yml +COPY ./identity.schema.json /etc/config/kratos/identity.schema.json +COPY ./start.sh /opt/start.sh +RUN chmod +x /opt/start.sh + +ENTRYPOINT ["/opt/start.sh"] diff --git a/agent/kratos/build-push-featurebranch.sh b/agent/kratos/build-push-featurebranch.sh new file mode 100755 index 000000000..069114c23 --- /dev/null +++ b/agent/kratos/build-push-featurebranch.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -e + +GCP_PROJECT=up9-docker-hub +REPOSITORY=gcr.io/$GCP_PROJECT +SERVER_NAME=mizu-kratos +GIT_BRANCH=$(git branch | grep \* | cut -d ' ' -f2 | tr '[:upper:]' '[:lower:]') + +DOCKER_REPO=$REPOSITORY/$SERVER_NAME/$GIT_BRANCH +SEM_VER=${SEM_VER=0.0.0} + +DOCKER_TAGGED_BUILDS=("$DOCKER_REPO:latest" "$DOCKER_REPO:$SEM_VER") + +if [ "$GIT_BRANCH" = 'develop' -o "$GIT_BRANCH" = 'master' -o "$GIT_BRANCH" = 'main' ] +then + echo "Pushing to $GIT_BRANCH is allowed only via CI" + exit 1 +fi + +echo "building ${DOCKER_TAGGED_BUILDS[@]}" +DOCKER_TAGS_ARGS=$(echo ${DOCKER_TAGGED_BUILDS[@]/#/-t }) # "-t FIRST_TAG -t SECOND_TAG ..." +docker build $DOCKER_TAGS_ARGS --build-arg SEM_VER=${SEM_VER} --build-arg BUILD_TIMESTAMP=${BUILD_TIMESTAMP} --build-arg GIT_BRANCH=${GIT_BRANCH} --build-arg COMMIT_HASH=${COMMIT_HASH} . + +for DOCKER_TAG in "${DOCKER_TAGGED_BUILDS[@]}" +do + echo pushing "$DOCKER_TAG" + docker push "$DOCKER_TAG" +done diff --git a/agent/kratos/identity.schema.json b/agent/kratos/identity.schema.json new file mode 100644 index 000000000..318322299 --- /dev/null +++ b/agent/kratos/identity.schema.json @@ -0,0 +1,43 @@ +{ + "$id": "https://schemas.ory.sh/presets/kratos/quickstart/email-password/identity.schema.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Person", + "type": "object", + "properties": { + "traits": { + "type": "object", + "properties": { + "username": { + "type": "string", + "format": "username", + "title": "Username", + "minLength": 3, + "ory.sh/kratos": { + "credentials": { + "password": { + "identifier": true + } + } + } + }, + "name": { + "type": "object", + "properties": { + "first": { + "title": "First Name", + "type": "string" + }, + "last": { + "title": "Last Name", + "type": "string" + } + } + } + }, + "required": [ + "username" + ], + "additionalProperties": false + } + } +} diff --git a/agent/kratos/kratos.yml b/agent/kratos/kratos.yml new file mode 100755 index 000000000..a2362c24a --- /dev/null +++ b/agent/kratos/kratos.yml @@ -0,0 +1,84 @@ +version: v0.8.2-alpha.1 + +dsn: sqlite:///app/data/kratos.sqlite?_fk=true + +serve: + public: + base_url: http://127.0.0.1:4433/ + cors: + enabled: true + admin: + base_url: http://kratos:4434/ + +selfservice: + default_browser_return_url: http://127.0.0.1:4455/ + whitelisted_return_urls: + - http://127.0.0.1:4455 + + methods: + password: + enabled: true + + flows: + error: + ui_url: http://127.0.0.1:4455/error + + settings: + ui_url: http://127.0.0.1:4455/settings + privileged_session_max_age: 15m + + recovery: + enabled: true + ui_url: http://127.0.0.1:4455/recovery + + verification: + enabled: false + ui_url: http://127.0.0.1:4455/verification + after: + default_browser_return_url: http://127.0.0.1:4455/ + + logout: + after: + default_browser_return_url: http://127.0.0.1:4455/login + + login: + ui_url: http://127.0.0.1:4455/login + lifespan: 10m + + registration: + lifespan: 10m + ui_url: http://127.0.0.1:4455/registration + after: + password: + hooks: + - + hook: session + +log: + level: info + format: text + leak_sensitive_values: true + +secrets: + cookie: + - PLEASE-CHANGE-ME-I-AM-VERY-INSECURE + cipher: + - 32-LONG-SECRET-NOT-SECURE-AT-ALL + +ciphers: + algorithm: xchacha20-poly1305 + +hashers: + argon2: + parallelism: 1 + memory: 128MB + iterations: 2 + salt_length: 16 + key_length: 16 + +identity: + default_schema_url: file:///etc/config/kratos/identity.schema.json + +courier: + smtp: + connection_uri: smtps://test:test@mailslurper:1025/?skip_ssl_verify=true diff --git a/agent/kratos/start.sh b/agent/kratos/start.sh new file mode 100755 index 000000000..d81af188e --- /dev/null +++ b/agent/kratos/start.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +kratos migrate sql sqlite:///app/data/kratos.sqlite?_fk=true --yes # this initializes the db +kratos serve -c /etc/config/kratos/kratos.yml --watch-courier # start kratos diff --git a/agent/main.go b/agent/main.go index c6be71870..e8d815ff3 100644 --- a/agent/main.go +++ b/agent/main.go @@ -9,6 +9,7 @@ import ( "mizuserver/pkg/api" "mizuserver/pkg/config" "mizuserver/pkg/controllers" + "mizuserver/pkg/middlewares" "mizuserver/pkg/models" "mizuserver/pkg/routes" "mizuserver/pkg/up9" @@ -256,12 +257,14 @@ func hostApi(socketHarOutputChannel chan<- *tapApi.OutputChannelItem) { } app.Use(static.ServeRoot("/", "./site")) - app.Use(CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before + app.Use(middlewares.CORSMiddleware()) // This has to be called after the static middleware, does not work if its called before api.WebSocketRoutes(app, &eventHandlers, startTime) if config.Config.StandaloneMode { routes.ConfigRoutes(app) + routes.UserRoutes(app) + routes.InstallRoutes(app) } routes.QueryRoutes(app) @@ -284,22 +287,6 @@ func DisableRootStaticCache() gin.HandlerFunc { } } -func CORSMiddleware() gin.HandlerFunc { - return func(c *gin.Context) { - c.Writer.Header().Set("Access-Control-Allow-Origin", "*") - c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") - c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With") - c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT") - - if c.Request.Method == "OPTIONS" { - c.AbortWithStatus(204) - return - } - - c.Next() - } -} - func setUIMode() error { read, err := ioutil.ReadFile(uiIndexPath) if err != nil { diff --git a/agent/pkg/api/socket_routes.go b/agent/pkg/api/socket_routes.go index 89e8f45f0..41a1064e6 100644 --- a/agent/pkg/api/socket_routes.go +++ b/agent/pkg/api/socket_routes.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "mizuserver/pkg/middlewares" "mizuserver/pkg/models" "net/http" "sync" @@ -47,8 +48,9 @@ func init() { func WebSocketRoutes(app *gin.Engine, eventHandlers EventHandlers, startTime int64) { app.GET("/ws", func(c *gin.Context) { websocketHandler(c.Writer, c.Request, eventHandlers, false, startTime) - }) - app.GET("/wsTapper", func(c *gin.Context) { + }, middlewares.RequiresAuth()) + + app.GET("/wsTapper", func(c *gin.Context) { // TODO: add m2m authentication to this route websocketHandler(c.Writer, c.Request, eventHandlers, true, startTime) }) } diff --git a/agent/pkg/controllers/install_controller.go b/agent/pkg/controllers/install_controller.go new file mode 100644 index 000000000..843546bd4 --- /dev/null +++ b/agent/pkg/controllers/install_controller.go @@ -0,0 +1,18 @@ +package controllers + +import ( + "mizuserver/pkg/providers" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/up9inc/mizu/shared/logger" +) + +func IsSetupNecessary(c *gin.Context) { + if IsInstallNeeded, err := providers.IsInstallNeeded(); err != nil { + logger.Log.Errorf("unknown internal while checking if install is needed %s", err) + c.AbortWithStatusJSON(500, gin.H{"error": "internal error occured while checking if install is needed"}) + } else { + c.JSON(http.StatusOK, IsInstallNeeded) + } +} diff --git a/agent/pkg/controllers/user_controller.go b/agent/pkg/controllers/user_controller.go new file mode 100644 index 000000000..6232dfa1e --- /dev/null +++ b/agent/pkg/controllers/user_controller.go @@ -0,0 +1,39 @@ +package controllers + +import ( + "mizuserver/pkg/providers" + + "github.com/gin-gonic/gin" + "github.com/up9inc/mizu/shared/logger" +) + +func Login(c *gin.Context) { + if token, err := providers.PerformLogin(c.PostForm("username"), c.PostForm("password"), c.Request.Context()); err != nil { + c.AbortWithStatusJSON(401, gin.H{"error": "bad login"}) + } else { + c.JSON(200, gin.H{"token": token}) + } +} + +func Logout(c *gin.Context) { + token := c.GetHeader("x-session-token") + if err := providers.Logout(token, c.Request.Context()); err != nil { + c.AbortWithStatusJSON(401, gin.H{"error": "error occured while logging out, the session might still be valid"}) + } else { + c.JSON(200, "") + } +} + +func Register(c *gin.Context) { + if token, _, err, formErrorMessages := providers.RegisterUser(c.PostForm("username"), c.PostForm("password"), c.Request.Context()); err != nil { + if formErrorMessages != nil { + logger.Log.Infof("user attempted to register but had form errors %v %v", formErrorMessages, err) + c.AbortWithStatusJSON(400, formErrorMessages) + } else { + logger.Log.Errorf("unknown internal error registering user %s", err) + c.AbortWithStatusJSON(500, gin.H{"error": "internal error occured while registering"}) + } + } else { + c.JSON(200, gin.H{"token": token}) + } +} diff --git a/agent/pkg/middlewares/cors.go b/agent/pkg/middlewares/cors.go new file mode 100644 index 000000000..0e6d7d36e --- /dev/null +++ b/agent/pkg/middlewares/cors.go @@ -0,0 +1,19 @@ +package middlewares + +import "github.com/gin-gonic/gin" + +func CORSMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + c.Writer.Header().Set("Access-Control-Allow-Origin", "*") + c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") + c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With") + c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT") + + if c.Request.Method == "OPTIONS" { + c.AbortWithStatus(204) + return + } + + c.Next() + } +} diff --git a/agent/pkg/middlewares/requiresAuth.go b/agent/pkg/middlewares/requiresAuth.go new file mode 100644 index 000000000..08c5079b4 --- /dev/null +++ b/agent/pkg/middlewares/requiresAuth.go @@ -0,0 +1,49 @@ +package middlewares + +import ( + "mizuserver/pkg/config" + "mizuserver/pkg/providers" + "time" + + "github.com/gin-gonic/gin" + "github.com/patrickmn/go-cache" + "github.com/up9inc/mizu/shared/logger" +) + +const cachedValidTokensRetainmentTime = time.Minute * 1 + +var cachedValidTokens = cache.New(cachedValidTokensRetainmentTime, cachedValidTokensRetainmentTime) + +func RequiresAuth() gin.HandlerFunc { + return func(c *gin.Context) { + // auth is irrelevant for ephermeral mizu + if !config.Config.StandaloneMode { + c.Next() + return + } + + token := c.GetHeader("x-session-token") + if token == "" { + c.AbortWithStatusJSON(401, gin.H{"error": "token header is empty"}) + return + } + + if _, isTokenCached := cachedValidTokens.Get(token); isTokenCached { + c.Next() + return + } + + if isTokenValid, err := providers.VerifyToken(token, c.Request.Context()); err != nil { + logger.Log.Errorf("error verifying token %s", err) + c.AbortWithStatusJSON(401, gin.H{"error": "unknown auth error occured"}) + return + } else if !isTokenValid { + c.AbortWithStatusJSON(401, gin.H{"error": "invalid token"}) + return + } + + cachedValidTokens.Set(token, true, cachedValidTokensRetainmentTime) + + c.Next() + } +} diff --git a/agent/pkg/models/models.go b/agent/pkg/models/models.go index cf714519e..3ed193db2 100644 --- a/agent/pkg/models/models.go +++ b/agent/pkg/models/models.go @@ -159,3 +159,7 @@ func RunValidationRulesState(harEntry har.Entry, service string) (tapApi.Applica statusPolicyToSend, latency, numberOfRules := rules.PassedValidationRules(resultPolicyToSend) return tapApi.ApplicableRules{Status: statusPolicyToSend, Latency: latency, NumberOfRules: numberOfRules}, resultPolicyToSend, isEnabled } + +type InstallState struct { + Completed bool `json:"completed"` +} diff --git a/agent/pkg/providers/install_provider.go b/agent/pkg/providers/install_provider.go new file mode 100644 index 000000000..5e2590a9d --- /dev/null +++ b/agent/pkg/providers/install_provider.go @@ -0,0 +1,18 @@ +package providers + +import ( + "context" + "mizuserver/pkg/config" +) + +func IsInstallNeeded() (bool, error) { + if !config.Config.StandaloneMode { // install not needed in ephermeral mizu + return false, nil + } + + if anyUserExists, err := AnyUserExists(context.Background()); err != nil { + return false, err + } else { + return !anyUserExists, nil + } +} diff --git a/agent/pkg/providers/user_provider.go b/agent/pkg/providers/user_provider.go new file mode 100644 index 000000000..71b994c19 --- /dev/null +++ b/agent/pkg/providers/user_provider.go @@ -0,0 +1,162 @@ +package providers + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/cookiejar" + + ory "github.com/ory/kratos-client-go" + "github.com/up9inc/mizu/shared/logger" +) + +var client = getKratosClient("http://127.0.0.1:4433", "http://127.0.0.1:4434") + +// returns session token if successful +func RegisterUser(username string, password string, ctx context.Context) (token *string, identityId string, err error, formErrorMessages map[string][]ory.UiText) { + flow, _, err := client.V0alpha2Api.InitializeSelfServiceRegistrationFlowWithoutBrowser(ctx).Execute() + if err != nil { + return nil, "", err, nil + } + + result, _, err := client.V0alpha2Api.SubmitSelfServiceRegistrationFlow(ctx).Flow(flow.Id).SubmitSelfServiceRegistrationFlowBody( + ory.SubmitSelfServiceRegistrationFlowWithPasswordMethodBodyAsSubmitSelfServiceRegistrationFlowBody(&ory.SubmitSelfServiceRegistrationFlowWithPasswordMethodBody{ + Method: "password", + Password: password, + Traits: map[string]interface{}{"username": username}, + }), + ).Execute() + + if err != nil { + parsedKratosError, parsingErr := parseKratosRegistrationFormError(err) + if parsingErr != nil { + logger.Log.Debugf("error parsing kratos error: %v", parsingErr) + return nil, "", err, nil + } else { + return nil, "", err, parsedKratosError + } + } + + return result.SessionToken, result.Identity.Id, nil, nil +} + +func PerformLogin(username string, password string, ctx context.Context) (*string, error) { + flow, _, err := client.V0alpha2Api.InitializeSelfServiceLoginFlowWithoutBrowser(ctx).Execute() + if err != nil { + return nil, err + } + + result, _, err := client.V0alpha2Api.SubmitSelfServiceLoginFlow(ctx).Flow(flow.Id).SubmitSelfServiceLoginFlowBody( + ory.SubmitSelfServiceLoginFlowWithPasswordMethodBodyAsSubmitSelfServiceLoginFlowBody(&ory.SubmitSelfServiceLoginFlowWithPasswordMethodBody{ + Method: "password", + Password: password, + PasswordIdentifier: username, + }), + ).Execute() + + if err != nil { + return nil, err + } + if result == nil { + return nil, errors.New("unknown error occured during login") + } + + return result.SessionToken, nil +} + +func VerifyToken(token string, ctx context.Context) (bool, error) { + flow, _, err := client.V0alpha2Api.ToSession(ctx).XSessionToken(token).Execute() + if err != nil { + return false, err + } + + if flow == nil { + return false, nil + } + + return true, nil +} + +func DeleteUser(identityId string, ctx context.Context) error { + result, err := client.V0alpha2Api.AdminDeleteIdentity(ctx, identityId).Execute() + if err != nil { + return err + } + if result == nil { + return errors.New("unknown error occured during user deletion") + } + + if result.StatusCode < 200 || result.StatusCode > 299 { + return errors.New(fmt.Sprintf("user deletion returned bad status %d", result.StatusCode)) + } else { + return nil + } +} + +func AnyUserExists(ctx context.Context) (bool, error) { + request := client.V0alpha2Api.AdminListIdentities(ctx) + request.PerPage(1) + + if result, _, err := request.Execute(); err != nil { + return false, err + } else { + return len(result) > 0, nil + } +} + +func Logout(token string, ctx context.Context) error { + logoutRequest := client.V0alpha2Api.SubmitSelfServiceLogoutFlowWithoutBrowser(ctx) + logoutRequest.SubmitSelfServiceLogoutFlowWithoutBrowserBody(ory.SubmitSelfServiceLogoutFlowWithoutBrowserBody{ + SessionToken: token, + }) + if response, err := logoutRequest.Execute(); err != nil { + return err + } else if response == nil || response.StatusCode < 200 || response.StatusCode > 299 { + return errors.New("unknown error occured during logout") + } + + return nil +} + +func getKratosClient(url string, adminUrl string) *ory.APIClient { + conf := ory.NewConfiguration() + conf.Servers = ory.ServerConfigurations{{URL: url}} + + // this ensures kratos client uses the admin url for admin actions (any new admin action we use will have to be added here) + conf.OperationServers = map[string]ory.ServerConfigurations{ + "V0alpha2ApiService.AdminDeleteIdentity": {{URL: adminUrl}}, + "V0alpha2ApiService.AdminListIdentities": {{URL: adminUrl}}, + } + + cj, _ := cookiejar.New(nil) + conf.HTTPClient = &http.Client{Jar: cj} + return ory.NewAPIClient(conf) +} + +// returns map of form value key to error message +func parseKratosRegistrationFormError(err error) (map[string][]ory.UiText, error) { + var openApiError *ory.GenericOpenAPIError + if errors.As(err, &openApiError) { + var registrationFlowModel *ory.SelfServiceRegistrationFlow + if jsonErr := json.Unmarshal(openApiError.Body(), ®istrationFlowModel); jsonErr != nil { + return nil, jsonErr + } else { + formMessages := registrationFlowModel.Ui.Nodes + parsedMessages := make(map[string][]ory.UiText) + + for _, message := range formMessages { + if len(message.Messages) > 0 { + if _, ok := parsedMessages[message.Group]; !ok { + parsedMessages[message.Group] = make([]ory.UiText, 0) + } + parsedMessages[message.Group] = append(parsedMessages[message.Group], message.Messages...) + } + } + return parsedMessages, nil + } + } else { + return nil, errors.New("error is not a generic openapi error") + } +} diff --git a/agent/pkg/routes/entries_routes.go b/agent/pkg/routes/entries_routes.go index d931196a9..75956d6f1 100644 --- a/agent/pkg/routes/entries_routes.go +++ b/agent/pkg/routes/entries_routes.go @@ -2,6 +2,7 @@ package routes import ( "mizuserver/pkg/controllers" + "mizuserver/pkg/middlewares" "github.com/gin-gonic/gin" ) @@ -9,6 +10,7 @@ import ( // EntriesRoutes defines the group of har entries routes. func EntriesRoutes(ginApp *gin.Engine) { routeGroup := ginApp.Group("/entries") + routeGroup.Use(middlewares.RequiresAuth()) routeGroup.GET("/", controllers.GetEntries) // get entries (base/thin entries) and metadata routeGroup.GET("/:id", controllers.GetEntry) // get single (full) entry diff --git a/agent/pkg/routes/install_routes.go b/agent/pkg/routes/install_routes.go new file mode 100644 index 000000000..83b28cdd8 --- /dev/null +++ b/agent/pkg/routes/install_routes.go @@ -0,0 +1,13 @@ +package routes + +import ( + "mizuserver/pkg/controllers" + + "github.com/gin-gonic/gin" +) + +func InstallRoutes(ginApp *gin.Engine) { + routeGroup := ginApp.Group("/install") + + routeGroup.GET("/isNeeded", controllers.IsSetupNecessary) +} diff --git a/agent/pkg/routes/metadata_routes.go b/agent/pkg/routes/metadata_routes.go index 4f32ff771..9a3c21b91 100644 --- a/agent/pkg/routes/metadata_routes.go +++ b/agent/pkg/routes/metadata_routes.go @@ -1,8 +1,9 @@ package routes import ( - "github.com/gin-gonic/gin" "mizuserver/pkg/controllers" + + "github.com/gin-gonic/gin" ) // MetadataRoutes defines the group of metadata routes. diff --git a/agent/pkg/routes/query_routes.go b/agent/pkg/routes/query_routes.go index 2807bd0b6..7ad8d0490 100644 --- a/agent/pkg/routes/query_routes.go +++ b/agent/pkg/routes/query_routes.go @@ -2,12 +2,14 @@ package routes import ( "mizuserver/pkg/controllers" + "mizuserver/pkg/middlewares" "github.com/gin-gonic/gin" ) func QueryRoutes(ginApp *gin.Engine) { routeGroup := ginApp.Group("/query") + routeGroup.Use(middlewares.RequiresAuth()) routeGroup.POST("/validate", controllers.PostValidate) } diff --git a/agent/pkg/routes/status_routes.go b/agent/pkg/routes/status_routes.go index dd601213d..4d2a6a90f 100644 --- a/agent/pkg/routes/status_routes.go +++ b/agent/pkg/routes/status_routes.go @@ -1,12 +1,15 @@ package routes import ( - "github.com/gin-gonic/gin" "mizuserver/pkg/controllers" + "mizuserver/pkg/middlewares" + + "github.com/gin-gonic/gin" ) func StatusRoutes(ginApp *gin.Engine) { routeGroup := ginApp.Group("/status") + routeGroup.Use(middlewares.RequiresAuth()) routeGroup.GET("/health", controllers.HealthCheck) diff --git a/agent/pkg/routes/user_routes.go b/agent/pkg/routes/user_routes.go new file mode 100644 index 000000000..6b2090dd3 --- /dev/null +++ b/agent/pkg/routes/user_routes.go @@ -0,0 +1,15 @@ +package routes + +import ( + "mizuserver/pkg/controllers" + + "github.com/gin-gonic/gin" +) + +func UserRoutes(ginApp *gin.Engine) { + routeGroup := ginApp.Group("/user") + + routeGroup.POST("/login", controllers.Login) + routeGroup.POST("/logout", controllers.Logout) + routeGroup.POST("/register", controllers.Register) +} diff --git a/cli/cmd/installRunner.go b/cli/cmd/installRunner.go index 5062fd557..9c5108bd3 100644 --- a/cli/cmd/installRunner.go +++ b/cli/cmd/installRunner.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/creasty/defaults" "github.com/up9inc/mizu/cli/config" "github.com/up9inc/mizu/cli/errormessage" @@ -21,6 +22,10 @@ func runMizuInstall() { return } + if config.Config.IsNsRestrictedMode() { + logger.Log.Error("install is not supported in restricted namespace mode") + } + ctx, cancel := context.WithCancel(context.Background()) defer cancel() // cancel will be called when this function exits @@ -47,7 +52,7 @@ func runMizuInstall() { var statusError *k8serrors.StatusError if errors.As(err, &statusError) { if statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists { - logger.Log.Info("Mizu is already running in this namespace, change the `mizu-resources-namespace` configuration or run `mizu clean` to remove the currently running Mizu instance") + logger.Log.Info("Mizu is already running in this namespace, run `mizu clean` to remove the currently running Mizu instance") } } else { defer resources.CleanUpMizuResources(ctx, cancel, kubernetesProvider, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace) diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 44ac2532f..5cc51556c 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -4,13 +4,14 @@ import ( "context" "errors" "fmt" - "github.com/up9inc/mizu/cli/resources" - "github.com/up9inc/mizu/cli/utils" "io/ioutil" "regexp" "strings" "time" + "github.com/up9inc/mizu/cli/resources" + "github.com/up9inc/mizu/cli/utils" + "github.com/getkin/kin-openapi/openapi3" "gopkg.in/yaml.v3" core "k8s.io/api/core/v1" @@ -124,6 +125,7 @@ func RunMizuTap() { logger.Log.Infof("Waiting for Mizu Agent to start...") if state.mizuServiceAccountExists, err = resources.CreateTapMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, getSyncEntriesConfig(), config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel()); err != nil { + logger.Log.Errorf("error %v", err) var statusError *k8serrors.StatusError if errors.As(err, &statusError) { if statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists { diff --git a/cli/resources/createResources.go b/cli/resources/createResources.go index 852b08f6c..ce238a9ea 100644 --- a/cli/resources/createResources.go +++ b/cli/resources/createResources.go @@ -3,6 +3,7 @@ package resources import ( "context" "fmt" + "github.com/op/go-logging" "github.com/up9inc/mizu/cli/errormessage" "github.com/up9inc/mizu/cli/mizu" @@ -65,6 +66,12 @@ func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes. } func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, noPersistentVolumeClaim bool) error { + if !isNsRestrictedMode { + if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil { + return err + } + } + if err := createMizuConfigmap(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, mizuResourcesNamespace); err != nil { return err } @@ -137,7 +144,7 @@ func createMizuApiServerDeployment(ctx context.Context, kubernetesProvider *kube volumeClaimCreated = tryToCreatePersistentVolumeClaim(ctx, kubernetesProvider, opts) } - pod, err := kubernetesProvider.GetMizuApiServerPodObject(opts, volumeClaimCreated, kubernetes.PersistentVolumeClaimName) + pod, err := kubernetesProvider.GetMizuApiServerPodObject(opts, volumeClaimCreated, kubernetes.PersistentVolumeClaimName, true) if err != nil { return err } @@ -179,7 +186,7 @@ func tryToCreatePersistentVolumeClaim(ctx context.Context, kubernetesProvider *k } func createMizuApiServerPod(ctx context.Context, kubernetesProvider *kubernetes.Provider, opts *kubernetes.ApiServerOptions) error { - pod, err := kubernetesProvider.GetMizuApiServerPodObject(opts, false, "") + pod, err := kubernetesProvider.GetMizuApiServerPodObject(opts, false, "", false) if err != nil { return err } diff --git a/shared/kubernetes/provider.go b/shared/kubernetes/provider.go index 44827da78..072ea00f3 100644 --- a/shared/kubernetes/provider.go +++ b/shared/kubernetes/provider.go @@ -186,7 +186,7 @@ type ApiServerOptions struct { LogLevel logging.Level } -func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, mountVolumeClaim bool, volumeClaimName string) (*core.Pod, error) { +func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, mountVolumeClaim bool, volumeClaimName string, createAuthContainer bool) (*core.Pod, error) { var marshaledSyncEntriesConfig []byte if opts.SyncEntriesConfig != nil { var err error @@ -250,6 +250,69 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun }) } + containers := []core.Container{ + { + Name: opts.PodName, + Image: opts.PodImage, + ImagePullPolicy: opts.ImagePullPolicy, + VolumeMounts: volumeMounts, + Command: command, + Env: []core.EnvVar{ + { + Name: shared.SyncEntriesConfigEnvVar, + Value: string(marshaledSyncEntriesConfig), + }, + { + Name: shared.LogLevelEnvVar, + Value: opts.LogLevel.String(), + }, + }, + Resources: core.ResourceRequirements{ + Limits: core.ResourceList{ + "cpu": cpuLimit, + "memory": memLimit, + }, + Requests: core.ResourceList{ + "cpu": cpuRequests, + "memory": memRequests, + }, + }, + }, + } + + if createAuthContainer { + containers = append(containers, core.Container{ + Name: "kratos", + Image: "gcr.io/up9-docker-hub/mizu-kratos/stable:0.0.0", + ImagePullPolicy: opts.ImagePullPolicy, + VolumeMounts: volumeMounts, + ReadinessProbe: &core.Probe{ + FailureThreshold: 3, + Handler: core.Handler{ + HTTPGet: &core.HTTPGetAction{ + Path: "/health/ready", + Port: intstr.FromInt(4433), + Scheme: core.URISchemeHTTP, + }, + }, + PeriodSeconds: 1, + SuccessThreshold: 1, + TimeoutSeconds: 1, + }, + Resources: core.ResourceRequirements{ + Limits: core.ResourceList{ + "cpu": cpuLimit, + "memory": memLimit, + }, + Requests: core.ResourceList{ + "cpu": cpuRequests, + "memory": memRequests, + }, + }, + }) + + } + pod := &core.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: opts.PodName, @@ -260,35 +323,7 @@ func (provider *Provider) GetMizuApiServerPodObject(opts *ApiServerOptions, moun }, }, Spec: core.PodSpec{ - Containers: []core.Container{ - { - Name: opts.PodName, - Image: opts.PodImage, - ImagePullPolicy: opts.ImagePullPolicy, - VolumeMounts: volumeMounts, - Command: command, - Env: []core.EnvVar{ - { - Name: shared.SyncEntriesConfigEnvVar, - Value: string(marshaledSyncEntriesConfig), - }, - { - Name: shared.LogLevelEnvVar, - Value: opts.LogLevel.String(), - }, - }, - Resources: core.ResourceRequirements{ - Limits: core.ResourceList{ - "cpu": cpuLimit, - "memory": memLimit, - }, - Requests: core.ResourceList{ - "cpu": cpuRequests, - "memory": memRequests, - }, - }, - }, - }, + Containers: containers, Volumes: volumes, DNSPolicy: core.DNSClusterFirstWithHostNet, TerminationGracePeriodSeconds: new(int64), @@ -343,7 +378,7 @@ func (provider *Provider) CreateService(ctx context.Context, namespace string, s }, }, Spec: core.ServiceSpec{ - Ports: []core.ServicePort{{TargetPort: intstr.FromInt(shared.DefaultApiServerPort), Port: 80}}, + Ports: []core.ServicePort{{TargetPort: intstr.FromInt(shared.DefaultApiServerPort), Port: 80, Name: "api"}}, Type: core.ServiceTypeClusterIP, Selector: map[string]string{"app": appLabelValue}, }, diff --git a/shared/kubernetes/proxy.go b/shared/kubernetes/proxy.go index 138bce331..4cb425e96 100644 --- a/shared/kubernetes/proxy.go +++ b/shared/kubernetes/proxy.go @@ -31,7 +31,6 @@ func StartProxy(kubernetesProvider *Provider, proxyHost string, mizuPort uint16, mux.Handle(k8sProxyApiPrefix, getRerouteHttpHandlerMizuAPI(proxyHandler, mizuNamespace, mizuServiceName)) mux.Handle("/static/", getRerouteHttpHandlerMizuStatic(proxyHandler, mizuNamespace, mizuServiceName)) - l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", proxyHost, int(mizuPort))) if err != nil { return err diff --git a/ui/package-lock.json b/ui/package-lock.json index 97b475cc3..085ebaf6a 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -18279,4 +18279,4 @@ "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==" } } -} +} \ No newline at end of file diff --git a/ui/src/App.sass b/ui/src/App.sass index c9c6ae28a..47ecb56ef 100644 --- a/ui/src/App.sass +++ b/ui/src/App.sass @@ -1,7 +1,9 @@ @import './variables.module' -.mizuApp +body background-color: $main-background-color + +.mizuApp color: $font-color width: 100% @@ -23,3 +25,13 @@ font-size: 11px font-weight: bold color: $light-blue-color + + .centeredForm + max-width: 500px + text-align: center + display: flex + flex-direction: column + margin: 0 auto + + .form-input, .form-button + margin-top: 20px diff --git a/ui/src/App.tsx b/ui/src/App.tsx index 8e6ab26e1..7371e18b9 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -1,16 +1,71 @@ -import React, {useState} from 'react'; +import React, {useEffect, useState} from 'react'; import './App.sass'; import {TrafficPage} from "./components/TrafficPage"; import {TLSWarning} from "./components/TLSWarning/TLSWarning"; import {Header} from "./components/Header/Header"; +import { ToastContainer, toast } from 'react-toastify'; +import 'react-toastify/dist/ReactToastify.css'; +import Api from "./helpers/api"; +import LoadingOverlay from './components/LoadingOverlay'; +import LoginPage from './components/LoginPage'; +import InstallPage from './components/InstallPage'; + +const api = Api.getInstance(); + + // TODO: move to state management +export enum Page { + Traffic, + Setup, + Login +} + +// TODO: move to state management +export interface MizuContextModel { + page: Page; + setPage: (page: Page) => void; +} + +// TODO: move to state management +export const MizuContext = React.createContext(null); const App = () => { - + const [isLoading, setIsLoading] = useState(true); const [analyzeStatus, setAnalyzeStatus] = useState(null); const [showTLSWarning, setShowTLSWarning] = useState(false); const [userDismissedTLSWarning, setUserDismissedTLSWarning] = useState(false); const [addressesWithTLS, setAddressesWithTLS] = useState(new Set()); + const [page, setPage] = useState(Page.Traffic); // TODO: move to state management + + const determinePage = async () => { // TODO: move to state management + if (window['isEnt'] !== true) { + setPage(Page.Traffic); + setIsLoading(false); + return; + } + + try { + const isInstallNeeded = await api.isInstallNeeded(); + if (isInstallNeeded) { + setPage(Page.Setup); + } else { + const isAuthNeeded = await api.isAuthenticationNeeded(); + if(isAuthNeeded) { + setPage(Page.Login); + } + } + } catch (e) { + toast.error("Error occured while checking Mizu API status, see console for mode details"); + console.error(e); + } finally { + setIsLoading(false); + } + } + + useEffect(() => { + determinePage(); + }, []); + const onTLSDetected = (destAddress: string) => { addressesWithTLS.add(destAddress); setAddressesWithTLS(new Set(addressesWithTLS)); @@ -20,16 +75,49 @@ const App = () => { } }; + let pageComponent: any; + + switch (page) { // TODO: move to state management / proper routing + case Page.Traffic: + pageComponent = ; + break; + case Page.Setup: + pageComponent = ; + break; + case Page.Login: + pageComponent = ; + break; + default: + pageComponent =
Unknown Error
; + } + + if (isLoading) { + return ; + } + return (
-
- - + +
+ {pageComponent} + + +
); } diff --git a/ui/src/components/AuthPresentation/AuthPresentation.tsx b/ui/src/components/AuthPresentation/AuthPresentation.tsx index 8efa739ea..d069f2912 100644 --- a/ui/src/components/AuthPresentation/AuthPresentation.tsx +++ b/ui/src/components/AuthPresentation/AuthPresentation.tsx @@ -2,7 +2,7 @@ import React, {useEffect, useState} from "react"; import Api from "../../helpers/api"; import './AuthPresentation.sass'; -const api = new Api(); +const api = Api.getInstance(); export const AuthPresentation = () => { diff --git a/ui/src/components/EntriesList.tsx b/ui/src/components/EntriesList.tsx index 6b3a6917b..9562c6fef 100644 --- a/ui/src/components/EntriesList.tsx +++ b/ui/src/components/EntriesList.tsx @@ -35,7 +35,7 @@ interface EntriesListProps { setTruncatedTimestamp: any; } -const api = new Api(); +const api = Api.getInstance(); export const EntriesList: React.FC = ({entries, setEntries, query, listEntryREF, onSnapBrokenEvent, isSnappedToBottom, setIsSnappedToBottom, queriedCurrent, setQueriedCurrent, queriedTotal, setQueriedTotal, startTime, noMoreDataTop, setNoMoreDataTop, focusedEntryId, setFocusedEntryId, updateQuery, leftOffTop, setLeftOffTop, isWebSocketConnectionClosed, ws, openWebSocket, leftOffBottom, truncatedTimestamp, setTruncatedTimestamp}) => { const [loadMoreTop, setLoadMoreTop] = useState(false); diff --git a/ui/src/components/Header/Header.tsx b/ui/src/components/Header/Header.tsx index 7fa7d4a9d..84a119d28 100644 --- a/ui/src/components/Header/Header.tsx +++ b/ui/src/components/Header/Header.tsx @@ -6,9 +6,7 @@ import logo from '../assets/Mizu-logo.svg'; interface HeaderProps { analyzeStatus: any } - export const Header: React.FC = ({analyzeStatus}) => { - return
logo
diff --git a/ui/src/components/InstallPage.tsx b/ui/src/components/InstallPage.tsx new file mode 100644 index 000000000..a950d0f27 --- /dev/null +++ b/ui/src/components/InstallPage.tsx @@ -0,0 +1,60 @@ +import { Button, TextField } from "@material-ui/core"; +import React, { useContext, useState } from "react"; +import { MizuContext, Page } from "../App"; +import { adminUsername } from "../consts"; +import Api, { FormValidationErrorType } from "../helpers/api"; +import { toast } from 'react-toastify'; +import LoadingOverlay from "./LoadingOverlay"; + +const api = Api.getInstance(); + +export const InstallPage: React.FC = () => { + + const [isLoading, setIsLoading] = useState(false); + const [password, setPassword] = useState(""); + const [passwordConfirm, setPasswordConfirm] = useState(""); + + const {setPage} = useContext(MizuContext); + + const onFormSubmit = async () => { + if (password.length < 8) { + toast.error("Password must be at least 8 characters long"); + return; + } else if (password !== passwordConfirm) { + toast.error("Passwords do not match"); + return; + } + + try { + setIsLoading(true); + await api.register(adminUsername, password); + if (!await api.isAuthenticationNeeded()) { + toast.success("admin user created successfully"); + setPage(Page.Traffic); + } + } catch (e) { + if (e.type === FormValidationErrorType) { + for (const messages of Object.values(e.messages) as any[]) { + for (const message of messages) { + toast.error(message.text); + } + } + } + console.error(e); + } finally { + setIsLoading(false); + } + + } + + return
+ {isLoading && } +

Welcome to Mizu, please set up the admin user to continue

+ + setPassword(e.target.value)}/> + setPasswordConfirm(e.target.value)}/> + +
; +}; + +export default InstallPage; diff --git a/ui/src/components/LoadingOverlay.tsx b/ui/src/components/LoadingOverlay.tsx new file mode 100644 index 000000000..b0802ebaf --- /dev/null +++ b/ui/src/components/LoadingOverlay.tsx @@ -0,0 +1,25 @@ +import React, {useEffect, useState} from "react"; +import './style/LoadingOverlay.sass'; + +const SpinnerShowDelayMs = 350; + +interface LoadingOverlayProps { + delay?: number +} + +const LoadingOverlay: React.FC = ({delay}) => { + + const [isVisible, setIsVisible] = useState(false); + + useEffect(() => { + setTimeout(() => { + setIsVisible(true); + }, delay ?? SpinnerShowDelayMs); + }, []); + + return
} {tappingStatus && } -
) }; diff --git a/ui/src/components/style/LoadingOverlay.sass b/ui/src/components/style/LoadingOverlay.sass new file mode 100644 index 000000000..26af44cab --- /dev/null +++ b/ui/src/components/style/LoadingOverlay.sass @@ -0,0 +1,54 @@ +.loading-overlay-container + position: absolute + display: flex + width: 100% + height: 100% + top: 0 + left: 0 + align-items: center + justify-content: center + background-color: rgba(25, 25, 25, 0.5) + +.loading-overlay-spinner + width: 60px + height: 60px + top: 50% + left: 50% + z-index: 1000 + -webkit-animation: rotation .6s infinite linear + -moz-animation: rotation .6s infinite linear + -o-animation: rotation .6s infinite linear + animation: rotation .6s infinite linear + border-left: 6px solid rgba(0, 174, 239, 0.15) + border-right: 6px solid rgba(0, 174, 239, 0.15) + border-bottom: 6px solid rgba(0, 174, 239, 0.15) + border-top: 6px solid rgba(0, 174, 239, 0.8) + border-radius: 100% + +@-webkit-keyframes rotation + from + -webkit-transform: rotate(0deg) + + to + -webkit-transform: rotate(359deg) + +@-moz-keyframes rotation + from + -moz-transform: rotate(0deg) + + to + -moz-transform: rotate(359deg) + +@-o-keyframes rotation + from + -o-transform: rotate(0deg) + + to + -o-transform: rotate(359deg) + +@keyframes rotation + from + transform: rotate(0deg) + + to + transform: rotate(359deg) diff --git a/ui/src/consts.ts b/ui/src/consts.ts new file mode 100644 index 000000000..1e62b4dc5 --- /dev/null +++ b/ui/src/consts.ts @@ -0,0 +1 @@ +export const adminUsername = "admin"; diff --git a/ui/src/helpers/api.js b/ui/src/helpers/api.js index 524b58334..3d6ee3d98 100644 --- a/ui/src/helpers/api.js +++ b/ui/src/helpers/api.js @@ -4,23 +4,27 @@ import * as axios from "axios"; export const MizuWebsocketURL = process.env.REACT_APP_OVERRIDE_WS_URL ? process.env.REACT_APP_OVERRIDE_WS_URL : window.location.protocol === 'https:' ? `wss://${window.location.host}/ws` : `ws://${window.location.host}/ws`; +export const FormValidationErrorType = "formError"; + const CancelToken = axios.CancelToken; +// When working locally cp `cp .env.example .env` +const apiURL = process.env.REACT_APP_OVERRIDE_API_URL ? process.env.REACT_APP_OVERRIDE_API_URL : `${window.location.origin}/`; + export default class Api { + static instance; + + static getInstance() { + if (!Api.instance) { + Api.instance = new Api(); + } + return Api.instance; + } constructor() { + this.token = localStorage.getItem("token"); - // When working locally cp `cp .env.example .env` - const apiURL = process.env.REACT_APP_OVERRIDE_API_URL ? process.env.REACT_APP_OVERRIDE_API_URL : `${window.location.origin}/`; - - this.client = axios.create({ - baseURL: apiURL, - timeout: 31000, - headers: { - Accept: "application/json", - } - }); - + this.client = this.getAxiosClient(); this.source = null; } @@ -79,4 +83,81 @@ export default class Api { return response.data; } + + isInstallNeeded = async () => { + const response = await this.client.get("/install/isNeeded"); + return response.data; + } + + isAuthenticationNeeded = async () => { + try { + await this.client.get("/status/tap"); + return false; + } catch (e) { + if (e.response.status == 401) { + return true; + } + throw e; + } + } + + register = async (username, password) => { + const form = new FormData(); + form.append('username', username); + form.append('password', password); + + try { + const response = await this.client.post(`/user/register`, form); + this.persistToken(response.data.token); + return response; + } catch (e) { + if (e.response.status === 400) { + throw { + 'type': FormValidationErrorType, + 'messages': e.response.data + }; + } else { + throw e; + } + } + } + + login = async (username, password) => { + const form = new FormData(); + form.append('username', username); + form.append('password', password); + + const response = await this.client.post(`/user/login`, form); + if (response.status >= 200 && response.status < 300) { + this.persistToken(response.data.token); + } + + return response; + } + + persistToken = (token) => { + this.token = token; + this.client = this.getAxiosClient(); + localStorage.setItem('token', token); + } + + logout = async () => { + await this.client.post(`/user/logout`); + this.persistToken(null); + } + + getAxiosClient = () => { + const headers = { + Accept: "application/json" + } + + if (this.token) { + headers['x-session-token'] = `${this.token}`; // we use `x-session-token` instead of `Authorization` because the latter is reserved by kubectl proxy, making mizu view not work + } + return axios.create({ + baseURL: apiURL, + timeout: 31000, + headers + }); + } } From b88bdb90f67d503c515e75c139662ce5beacba7c Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Thu, 6 Jan 2022 12:04:58 +0200 Subject: [PATCH 29/72] Fixed standalone config, small refactor (#589) --- agent/pkg/controllers/config_controller.go | 12 ++++++++---- agent/pkg/routes/config_routes.go | 2 ++ cli/cmd/install.go | 9 +++++++++ cli/cmd/installRunner.go | 4 ---- cli/cmd/tapRunner.go | 1 - cli/resources/createResources.go | 20 ++++++++++++-------- shared/kubernetes/consts.go | 4 ++-- 7 files changed, 33 insertions(+), 19 deletions(-) diff --git a/agent/pkg/controllers/config_controller.go b/agent/pkg/controllers/config_controller.go index 11c931bb8..bd7b0cc96 100644 --- a/agent/pkg/controllers/config_controller.go +++ b/agent/pkg/controllers/config_controller.go @@ -50,7 +50,9 @@ func PostTapConfig(c *gin.Context) { c.JSON(http.StatusInternalServerError, err) return } + ctx, cancel := context.WithCancel(context.Background()) + if _, err := startMizuTapperSyncer(ctx, kubernetesProvider, tappedNamespaces, *podRegex, []string{}, tapApi.TrafficFilteringOptions{}, false); err != nil { c.JSON(http.StatusInternalServerError, err) cancel() @@ -69,25 +71,27 @@ func GetTapConfig(c *gin.Context) { c.JSON(http.StatusInternalServerError, err) return } + ctx, cancel := context.WithCancel(context.Background()) defer cancel() + namespaces, err := kubernetesProvider.ListAllNamespaces(ctx) if err != nil { c.JSON(http.StatusInternalServerError, err) return } + tappedNamespaces := make(map[string]bool) for _, namespace := range namespaces { if namespace.Name == config.Config.MizuResourcesNamespace { continue } - if _, ok := globalTapConfig.TappedNamespaces[namespace.Name]; !ok { - globalTapConfig.TappedNamespaces[namespace.Name] = false - } + tappedNamespaces[namespace.Name] = globalTapConfig.TappedNamespaces[namespace.Name] } - c.JSON(http.StatusOK, globalTapConfig) + tapConfig := models.TapConfig{TappedNamespaces: tappedNamespaces} + c.JSON(http.StatusOK, tapConfig) } func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, istio bool) (*kubernetes.MizuTapperSyncer, error) { diff --git a/agent/pkg/routes/config_routes.go b/agent/pkg/routes/config_routes.go index 5a8c3c94b..c13c191e0 100644 --- a/agent/pkg/routes/config_routes.go +++ b/agent/pkg/routes/config_routes.go @@ -3,10 +3,12 @@ package routes import ( "github.com/gin-gonic/gin" "mizuserver/pkg/controllers" + "mizuserver/pkg/middlewares" ) func ConfigRoutes(ginApp *gin.Engine) { routeGroup := ginApp.Group("/config") + routeGroup.Use(middlewares.RequiresAuth()) routeGroup.POST("/tapConfig", controllers.PostTapConfig) routeGroup.GET("/tapConfig", controllers.GetTapConfig) diff --git a/cli/cmd/install.go b/cli/cmd/install.go index dfca9bf1a..e17b1258b 100644 --- a/cli/cmd/install.go +++ b/cli/cmd/install.go @@ -1,7 +1,9 @@ package cmd import ( + "fmt" "github.com/spf13/cobra" + "github.com/up9inc/mizu/cli/config" "github.com/up9inc/mizu/cli/telemetry" ) @@ -13,6 +15,13 @@ var installCmd = &cobra.Command{ runMizuInstall() return nil }, + PreRunE: func(cmd *cobra.Command, args []string) error { + if config.Config.IsNsRestrictedMode() { + return fmt.Errorf("install is not supported in restricted namespace mode") + } + + return nil + }, } func init() { diff --git a/cli/cmd/installRunner.go b/cli/cmd/installRunner.go index 9c5108bd3..7167f36b4 100644 --- a/cli/cmd/installRunner.go +++ b/cli/cmd/installRunner.go @@ -22,10 +22,6 @@ func runMizuInstall() { return } - if config.Config.IsNsRestrictedMode() { - logger.Log.Error("install is not supported in restricted namespace mode") - } - ctx, cancel := context.WithCancel(context.Background()) defer cancel() // cancel will be called when this function exits diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 5cc51556c..d23a775ea 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -125,7 +125,6 @@ func RunMizuTap() { logger.Log.Infof("Waiting for Mizu Agent to start...") if state.mizuServiceAccountExists, err = resources.CreateTapMizuResources(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, config.Config.IsNsRestrictedMode(), config.Config.MizuResourcesNamespace, config.Config.AgentImage, getSyncEntriesConfig(), config.Config.Tap.MaxEntriesDBSizeBytes(), config.Config.Tap.ApiServerResources, config.Config.ImagePullPolicy(), config.Config.LogLevel()); err != nil { - logger.Log.Errorf("error %v", err) var statusError *k8serrors.StatusError if errors.As(err, &statusError) { if statusError.ErrStatus.Reason == metav1.StatusReasonAlreadyExists { diff --git a/cli/resources/createResources.go b/cli/resources/createResources.go index ce238a9ea..0e3b7c2dd 100644 --- a/cli/resources/createResources.go +++ b/cli/resources/createResources.go @@ -66,25 +66,29 @@ func CreateTapMizuResources(ctx context.Context, kubernetesProvider *kubernetes. } func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kubernetes.Provider, serializedValidationRules string, serializedContract string, serializedMizuConfig string, isNsRestrictedMode bool, mizuResourcesNamespace string, agentImage string, syncEntriesConfig *shared.SyncEntriesConfig, maxEntriesDBSizeBytes int64, apiServerResources shared.Resources, imagePullPolicy core.PullPolicy, logLevel logging.Level, noPersistentVolumeClaim bool) error { - if !isNsRestrictedMode { - if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil { - return err - } + if err := createMizuNamespace(ctx, kubernetesProvider, mizuResourcesNamespace); err != nil { + return err } + logger.Log.Infof("namespace/%v created", mizuResourcesNamespace) if err := createMizuConfigmap(ctx, kubernetesProvider, serializedValidationRules, serializedContract, serializedMizuConfig, mizuResourcesNamespace); err != nil { return err } - logger.Log.Infof("Created config map") + logger.Log.Infof("configmap/%v created", kubernetes.ConfigMapName) _, err := createRBACIfNecessary(ctx, kubernetesProvider, isNsRestrictedMode, mizuResourcesNamespace, []string{"pods", "services", "endpoints", "namespaces"}) if err != nil { return err } + logger.Log.Infof("serviceaccount/%v created", kubernetes.ServiceAccountName) + logger.Log.Infof("clusterrole.rbac.authorization.k8s.io/%v created", kubernetes.ClusterRoleName) + logger.Log.Infof("clusterrolebinding.rbac.authorization.k8s.io/%v created", kubernetes.ClusterRoleBindingName) + if err := kubernetesProvider.CreateDaemonsetRBAC(ctx, mizuResourcesNamespace, kubernetes.ServiceAccountName, kubernetes.DaemonRoleName, kubernetes.DaemonRoleBindingName, mizu.RBACVersion); err != nil { return err } - logger.Log.Infof("Created RBAC") + logger.Log.Infof("role.rbac.authorization.k8s.io/%v created", kubernetes.DaemonRoleName) + logger.Log.Infof("rolebinding.rbac.authorization.k8s.io/%v created", kubernetes.DaemonRoleBindingName) serviceAccountName := kubernetes.ServiceAccountName opts := &kubernetes.ApiServerOptions{ @@ -103,13 +107,13 @@ func CreateInstallMizuResources(ctx context.Context, kubernetesProvider *kuberne if err := createMizuApiServerDeployment(ctx, kubernetesProvider, opts, noPersistentVolumeClaim); err != nil { return err } - logger.Log.Infof("Created Api Server deployment") + logger.Log.Infof("deployment.apps/%v created", kubernetes.ApiServerPodName) _, err = kubernetesProvider.CreateService(ctx, mizuResourcesNamespace, kubernetes.ApiServerPodName, kubernetes.ApiServerPodName) if err != nil { return err } - logger.Log.Infof("Created Api Server service") + logger.Log.Infof("service/%v created", kubernetes.ApiServerPodName) return nil } diff --git a/shared/kubernetes/consts.go b/shared/kubernetes/consts.go index de84d5714..449bb5c6d 100644 --- a/shared/kubernetes/consts.go +++ b/shared/kubernetes/consts.go @@ -4,9 +4,9 @@ const ( MizuResourcesPrefix = "mizu-" ApiServerPodName = MizuResourcesPrefix + "api-server" ClusterRoleBindingName = MizuResourcesPrefix + "cluster-role-binding" - DaemonRoleBindingName = MizuResourcesPrefix + "cluster-role-binding-daemon" + DaemonRoleBindingName = MizuResourcesPrefix + "role-binding-daemon" ClusterRoleName = MizuResourcesPrefix + "cluster-role" - DaemonRoleName = MizuResourcesPrefix + "cluster-role-daemon" + DaemonRoleName = MizuResourcesPrefix + "role-daemon" K8sAllNamespaces = "" RoleBindingName = MizuResourcesPrefix + "role-binding" RoleName = MizuResourcesPrefix + "role" From 63122cb0a70c108049544ab4f4c69c2247cbebe2 Mon Sep 17 00:00:00 2001 From: RamiBerm <54766858+RamiBerm@users.noreply.github.com> Date: Thu, 6 Jan 2022 12:15:46 +0200 Subject: [PATCH 30/72] TRA-4147 simpler kratos password policy (#590) --- agent/kratos/Dockerfile | 2 +- ui/src/components/InstallPage.tsx | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/agent/kratos/Dockerfile b/agent/kratos/Dockerfile index 086a1a992..31dbf628d 100644 --- a/agent/kratos/Dockerfile +++ b/agent/kratos/Dockerfile @@ -1,4 +1,4 @@ -FROM oryd/kratos:v0.8.0-sqlite +FROM gcr.io/up9-docker-hub/mizu-kratos-base/simple-password-policy:latest USER root diff --git a/ui/src/components/InstallPage.tsx b/ui/src/components/InstallPage.tsx index a950d0f27..1606d319f 100644 --- a/ui/src/components/InstallPage.tsx +++ b/ui/src/components/InstallPage.tsx @@ -17,8 +17,8 @@ export const InstallPage: React.FC = () => { const {setPage} = useContext(MizuContext); const onFormSubmit = async () => { - if (password.length < 8) { - toast.error("Password must be at least 8 characters long"); + if (password.length < 4) { + toast.error("Password must be at least 4 characters long"); return; } else if (password !== passwordConfirm) { toast.error("Passwords do not match"); From 5d36d9184d42974609df03c50352662f5eca8729 Mon Sep 17 00:00:00 2001 From: RamiBerm <54766858+RamiBerm@users.noreply.github.com> Date: Thu, 6 Jan 2022 12:39:03 +0200 Subject: [PATCH 31/72] fix enterprise cors (#592) --- agent/pkg/middlewares/cors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/pkg/middlewares/cors.go b/agent/pkg/middlewares/cors.go index 0e6d7d36e..04afb297a 100644 --- a/agent/pkg/middlewares/cors.go +++ b/agent/pkg/middlewares/cors.go @@ -6,7 +6,7 @@ func CORSMiddleware() gin.HandlerFunc { return func(c *gin.Context) { c.Writer.Header().Set("Access-Control-Allow-Origin", "*") c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") - c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With") + c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With, x-session-token") c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT") if c.Request.Method == "OPTIONS" { From 14650aa3f47353ab7d9121ac6fa209ae96b2825b Mon Sep 17 00:00:00 2001 From: Igor Gov Date: Thu, 6 Jan 2022 12:47:50 +0200 Subject: [PATCH 32/72] Updating mizu base docker image alpine 3.14 -> 3.15 (#591) Co-authored-by: Igor Gov --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 51a42eae9..20468ad99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,7 +50,7 @@ RUN chmod +x ./basenine_linux_amd64 COPY devops/build_extensions.sh .. RUN cd .. && /bin/bash build_extensions.sh -FROM alpine:3.14 +FROM alpine:3.15 RUN apk add bash libpcap-dev From 2c4a5d06ab93987c0b101d45e89b11387f36b9b1 Mon Sep 17 00:00:00 2001 From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com> Date: Thu, 6 Jan 2022 13:21:58 +0200 Subject: [PATCH 33/72] Init tapped namespaces map (#593) --- agent/pkg/controllers/config_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/pkg/controllers/config_controller.go b/agent/pkg/controllers/config_controller.go index bd7b0cc96..7adc5a761 100644 --- a/agent/pkg/controllers/config_controller.go +++ b/agent/pkg/controllers/config_controller.go @@ -16,7 +16,7 @@ import ( "time" ) -var globalTapConfig = &models.TapConfig{} +var globalTapConfig = &models.TapConfig{TappedNamespaces: make(map[string]bool)} var cancelTapperSyncer context.CancelFunc func PostTapConfig(c *gin.Context) { From 2110afc514785b5cf20d2e1f7afc34c429b9a12c Mon Sep 17 00:00:00 2001 From: RamiBerm <54766858+RamiBerm@users.noreply.github.com> Date: Thu, 6 Jan 2022 14:01:14 +0200 Subject: [PATCH 34/72] TRA-4075 fix logout (#595) --- agent/pkg/controllers/user_controller.go | 2 +- agent/pkg/providers/user_provider.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/agent/pkg/controllers/user_controller.go b/agent/pkg/controllers/user_controller.go index 6232dfa1e..80453d5be 100644 --- a/agent/pkg/controllers/user_controller.go +++ b/agent/pkg/controllers/user_controller.go @@ -18,7 +18,7 @@ func Login(c *gin.Context) { func Logout(c *gin.Context) { token := c.GetHeader("x-session-token") if err := providers.Logout(token, c.Request.Context()); err != nil { - c.AbortWithStatusJSON(401, gin.H{"error": "error occured while logging out, the session might still be valid"}) + c.AbortWithStatusJSON(500, gin.H{"error": "error occured while logging out, the session might still be valid"}) } else { c.JSON(200, "") } diff --git a/agent/pkg/providers/user_provider.go b/agent/pkg/providers/user_provider.go index 71b994c19..1e2ba0322 100644 --- a/agent/pkg/providers/user_provider.go +++ b/agent/pkg/providers/user_provider.go @@ -108,7 +108,7 @@ func AnyUserExists(ctx context.Context) (bool, error) { func Logout(token string, ctx context.Context) error { logoutRequest := client.V0alpha2Api.SubmitSelfServiceLogoutFlowWithoutBrowser(ctx) - logoutRequest.SubmitSelfServiceLogoutFlowWithoutBrowserBody(ory.SubmitSelfServiceLogoutFlowWithoutBrowserBody{ + logoutRequest = logoutRequest.SubmitSelfServiceLogoutFlowWithoutBrowserBody(ory.SubmitSelfServiceLogoutFlowWithoutBrowserBody{ SessionToken: token, }) if response, err := logoutRequest.Execute(); err != nil { From cb5344090a99a19684c743246583e294d9729976 Mon Sep 17 00:00:00 2001 From: lirazyehezkel <61656597+lirazyehezkel@users.noreply.github.com> Date: Thu, 6 Jan 2022 14:50:50 +0200 Subject: [PATCH 35/72] TRA-4089 Mizu enterprise frame (#594) * ent app * mizu ent frame * apis * design settings modal by Javier * fix warnings * fix warnings * text change * redirect after logout * cr fixes --- ui/package-lock.json | 405 ++++++++++++++++++ ui/package.json | 1 + ui/src/App.sass | 19 - ui/src/App.tsx | 110 +---- ui/src/EntApp.tsx | 107 +++++ ui/src/components/Filters.tsx | 7 +- ui/src/components/Header/EntHeader.tsx | 78 ++++ ui/src/components/Header/Header.sass | 23 + ui/src/components/Header/Header.tsx | 1 + ui/src/components/InstallPage.tsx | 9 +- ui/src/components/LoadingOverlay.tsx | 10 +- ui/src/components/LoginPage.tsx | 2 +- .../components/SettingsModal/SettingModal.tsx | 150 +++++++ .../SettingsModal/SettingsModal.sass | 61 +++ ui/src/components/TLSWarning/TLSWarning.tsx | 2 +- ui/src/components/TrafficPage.tsx | 14 +- ui/src/components/UI/Checkbox.tsx | 4 +- ui/src/components/assets/MizuEntLogo.svg | 11 + ui/src/components/assets/logout.png | Bin 0 -> 7623 bytes ui/src/components/assets/settings.svg | 3 + ui/src/components/assets/user-circle.svg | 4 + ui/src/helpers/api.js | 15 +- ui/src/helpers/commonStyle.ts | 30 ++ ui/src/index.sass | 1 - ui/src/index.tsx | 18 +- ui/src/variables.module.scss | 2 + 26 files changed, 948 insertions(+), 139 deletions(-) create mode 100644 ui/src/EntApp.tsx create mode 100644 ui/src/components/Header/EntHeader.tsx create mode 100644 ui/src/components/Header/Header.sass create mode 100644 ui/src/components/SettingsModal/SettingModal.tsx create mode 100644 ui/src/components/SettingsModal/SettingsModal.sass create mode 100644 ui/src/components/assets/MizuEntLogo.svg create mode 100644 ui/src/components/assets/logout.png create mode 100644 ui/src/components/assets/settings.svg create mode 100644 ui/src/components/assets/user-circle.svg create mode 100644 ui/src/helpers/commonStyle.ts diff --git a/ui/package-lock.json b/ui/package-lock.json index 085ebaf6a..08d70cc0d 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -1161,11 +1161,51 @@ "resolved": "https://registry.npmjs.org/@csstools/normalize.css/-/normalize.css-10.1.0.tgz", "integrity": "sha512-ij4wRiunFfaJxjB0BdrYHIH8FxBJpOwNPhhAcunlmPdXudL1WQV1qoP9un6JsEBAgQH+7UXyyjh0g7jTxXK6tg==" }, + "@emotion/cache": { + "version": "11.7.1", + "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.7.1.tgz", + "integrity": "sha512-r65Zy4Iljb8oyjtLeCuBH8Qjiy107dOYC6SJq7g7GV5UCQWMObY4SJDPGFjiiVpPrOJ2hmJOoBiYTC7hwx9E2A==", + "requires": { + "@emotion/memoize": "^0.7.4", + "@emotion/sheet": "^1.1.0", + "@emotion/utils": "^1.0.0", + "@emotion/weak-memoize": "^0.2.5", + "stylis": "4.0.13" + } + }, "@emotion/hash": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.8.0.tgz", "integrity": "sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow==" }, + "@emotion/is-prop-valid": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@emotion/is-prop-valid/-/is-prop-valid-1.1.1.tgz", + "integrity": "sha512-bW1Tos67CZkOURLc0OalnfxtSXQJMrAMV0jZTVGJUPSOd4qgjF3+tTD5CwJM13PHA8cltGW1WGbbvV9NpvUZPw==", + "requires": { + "@emotion/memoize": "^0.7.4" + } + }, + "@emotion/memoize": { + "version": "0.7.5", + "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.7.5.tgz", + "integrity": "sha512-igX9a37DR2ZPGYtV6suZ6whr8pTFtyHL3K/oLUotxpSVO2ASaprmAe2Dkq7tBo7CRY7MMDrAa9nuQP9/YG8FxQ==" + }, + "@emotion/sheet": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.1.0.tgz", + "integrity": "sha512-u0AX4aSo25sMAygCuQTzS+HsImZFuS8llY8O7b9MDRzbJM0kVJlAz6KNDqcG7pOuQZJmj/8X/rAW+66kMnMW+g==" + }, + "@emotion/utils": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.0.0.tgz", + "integrity": "sha512-mQC2b3XLDs6QCW+pDQDiyO/EdGZYOygE8s5N5rrzjSI4M3IejPE/JPndCBwRT9z982aqQNi6beWs1UeayrQxxA==" + }, + "@emotion/weak-memoize": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.2.5.tgz", + "integrity": "sha512-6U71C2Wp7r5XtFtQzYrW5iKFT67OixrSxjI4MptCHzdSVlgabczzqLe0ZSgnub/5Kp4hSbpDB1tMytZY9pwxxA==" + }, "@eslint/eslintrc": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.0.tgz", @@ -1898,6 +1938,335 @@ "react-is": "^16.8.0 || ^17.0.0" } }, + "@mui/base": { + "version": "5.0.0-alpha.62", + "resolved": "https://registry.npmjs.org/@mui/base/-/base-5.0.0-alpha.62.tgz", + "integrity": "sha512-ItmdSZwHKQbLbAsS3sWguR7OHqYqh2cYWahoVmHb13Kc6bMdmVUTY4x57IlDSU712B0yuA0Q/gPTq7xADKnFow==", + "requires": { + "@babel/runtime": "^7.16.3", + "@emotion/is-prop-valid": "^1.1.1", + "@mui/utils": "^5.2.3", + "@popperjs/core": "^2.4.4", + "clsx": "^1.1.1", + "prop-types": "^15.7.2", + "react-is": "^17.0.2" + }, + "dependencies": { + "@babel/runtime": { + "version": "7.16.5", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.5.tgz", + "integrity": "sha512-TXWihFIS3Pyv5hzR7j6ihmeLkZfrXGxAr5UfSl8CHf+6q/wpiYDkUau0czckpYG8QmnCIuPpdLtuA9VmuGGyMA==", + "requires": { + "regenerator-runtime": "^0.13.4" + } + }, + "react-is": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", + "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==" + } + } + }, + "@mui/icons-material": { + "version": "5.2.5", + "resolved": "https://registry.npmjs.org/@mui/icons-material/-/icons-material-5.2.5.tgz", + "integrity": "sha512-uQiUz+l0xy+2jExyKyU19MkMAR2F7bQFcsQ5hdqAtsB14Jw2zlmIAD55mV6f0NxKCut7Rx6cA3ZpfzlzAfoK8Q==", + "requires": { + "@babel/runtime": "^7.16.3" + }, + "dependencies": { + "@babel/runtime": { + "version": "7.16.5", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.5.tgz", + "integrity": "sha512-TXWihFIS3Pyv5hzR7j6ihmeLkZfrXGxAr5UfSl8CHf+6q/wpiYDkUau0czckpYG8QmnCIuPpdLtuA9VmuGGyMA==", + "requires": { + "regenerator-runtime": "^0.13.4" + } + } + } + }, + "@mui/material": { + "version": "5.2.6", + "resolved": "https://registry.npmjs.org/@mui/material/-/material-5.2.6.tgz", + "integrity": "sha512-yF2bRqyJMo6bYXT7TPA9IU/XLaXHi47Xvmj8duQa5ha3bCpFMXLfGoZcAUl6ZDjjGEz1nCFS+c1qx219xD/aeQ==", + "requires": { + "@babel/runtime": "^7.16.3", + "@mui/base": "5.0.0-alpha.62", + "@mui/system": "^5.2.6", + "@mui/types": "^7.1.0", + "@mui/utils": "^5.2.3", + "@types/react-transition-group": "^4.4.4", + "clsx": "^1.1.1", + "csstype": "^3.0.10", + "hoist-non-react-statics": "^3.3.2", + "prop-types": "^15.7.2", + "react-is": "^17.0.2", + "react-transition-group": "^4.4.2" + }, + "dependencies": { + "@babel/runtime": { + "version": "7.16.5", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.5.tgz", + "integrity": "sha512-TXWihFIS3Pyv5hzR7j6ihmeLkZfrXGxAr5UfSl8CHf+6q/wpiYDkUau0czckpYG8QmnCIuPpdLtuA9VmuGGyMA==", + "requires": { + "regenerator-runtime": "^0.13.4" + } + }, + "@types/react-transition-group": { + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.4.tgz", + "integrity": "sha512-7gAPz7anVK5xzbeQW9wFBDg7G++aPLAFY0QaSMOou9rJZpbuI58WAuJrgu+qR92l61grlnCUe7AFX8KGahAgug==", + "requires": { + "@types/react": "*" + } + }, + "csstype": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.10.tgz", + "integrity": "sha512-2u44ZG2OcNUO9HDp/Jl8C07x6pU/eTR3ncV91SiK3dhG9TWvRVsCoJw14Ckx5DgWkzGA3waZWO3d7pgqpUI/XA==" + }, + "react-is": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", + "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==" + }, + "react-transition-group": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.2.tgz", + "integrity": "sha512-/RNYfRAMlZwDSr6z4zNKV6xu53/e2BuaBbGhbyYIXTrmgu/bGHzmqOs7mJSJBHy9Ud+ApHx3QjrkKSp1pxvlFg==", + "requires": { + "@babel/runtime": "^7.5.5", + "dom-helpers": "^5.0.1", + "loose-envify": "^1.4.0", + "prop-types": "^15.6.2" + } + } + } + }, + "@mui/private-theming": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@mui/private-theming/-/private-theming-5.2.3.tgz", + "integrity": "sha512-Lc1Cmu8lSsYZiXADi9PBb17Ho82ZbseHQujUFAcp6bCJ5x/d+87JYCIpCBMagPu/isRlFCwbziuXPmz7WOzJPQ==", + "requires": { + "@babel/runtime": "^7.16.3", + "@mui/utils": "^5.2.3", + "prop-types": "^15.7.2" + }, + "dependencies": { + "@babel/runtime": { + "version": "7.16.5", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.5.tgz", + "integrity": "sha512-TXWihFIS3Pyv5hzR7j6ihmeLkZfrXGxAr5UfSl8CHf+6q/wpiYDkUau0czckpYG8QmnCIuPpdLtuA9VmuGGyMA==", + "requires": { + "regenerator-runtime": "^0.13.4" + } + } + } + }, + "@mui/styled-engine": { + "version": "5.2.6", + "resolved": "https://registry.npmjs.org/@mui/styled-engine/-/styled-engine-5.2.6.tgz", + "integrity": "sha512-bqAhli8eGS6v2qxivy2/4K0Ag8o//jsu1G2G6QcieFiT6y7oIF/nd/6Tvw6OSm3roOTifVQWNKwkt1yFWhGS+w==", + "requires": { + "@babel/runtime": "^7.16.3", + "@emotion/cache": "^11.7.1", + "prop-types": "^15.7.2" + }, + "dependencies": { + "@babel/runtime": { + "version": "7.16.5", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.5.tgz", + "integrity": "sha512-TXWihFIS3Pyv5hzR7j6ihmeLkZfrXGxAr5UfSl8CHf+6q/wpiYDkUau0czckpYG8QmnCIuPpdLtuA9VmuGGyMA==", + "requires": { + "regenerator-runtime": "^0.13.4" + } + } + } + }, + "@mui/styles": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@mui/styles/-/styles-5.2.3.tgz", + "integrity": "sha512-Art4qjlEI9H2h34mLL8s+CE9nWZWZbuJLbNpievaIM6DGuayz3DYkJHcH5mXJYFPhTNoe9IQYbpyKofjE0YVag==", + "requires": { + "@babel/runtime": "^7.16.3", + "@emotion/hash": "^0.8.0", + "@mui/private-theming": "^5.2.3", + "@mui/types": "^7.1.0", + "@mui/utils": "^5.2.3", + "clsx": "^1.1.1", + "csstype": "^3.0.10", + "hoist-non-react-statics": "^3.3.2", + "jss": "^10.8.2", + "jss-plugin-camel-case": "^10.8.2", + "jss-plugin-default-unit": "^10.8.2", + "jss-plugin-global": "^10.8.2", + "jss-plugin-nested": "^10.8.2", + "jss-plugin-props-sort": "^10.8.2", + "jss-plugin-rule-value-function": "^10.8.2", + "jss-plugin-vendor-prefixer": "^10.8.2", + "prop-types": "^15.7.2" + }, + "dependencies": { + "@babel/runtime": { + "version": "7.16.5", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.5.tgz", + "integrity": "sha512-TXWihFIS3Pyv5hzR7j6ihmeLkZfrXGxAr5UfSl8CHf+6q/wpiYDkUau0czckpYG8QmnCIuPpdLtuA9VmuGGyMA==", + "requires": { + "regenerator-runtime": "^0.13.4" + } + }, + "csstype": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.10.tgz", + "integrity": "sha512-2u44ZG2OcNUO9HDp/Jl8C07x6pU/eTR3ncV91SiK3dhG9TWvRVsCoJw14Ckx5DgWkzGA3waZWO3d7pgqpUI/XA==" + }, + "jss": { + "version": "10.9.0", + "resolved": "https://registry.npmjs.org/jss/-/jss-10.9.0.tgz", + "integrity": "sha512-YpzpreB6kUunQBbrlArlsMpXYyndt9JATbt95tajx0t4MTJJcCJdd4hdNpHmOIDiUJrF/oX5wtVFrS3uofWfGw==", + "requires": { + "@babel/runtime": "^7.3.1", + "csstype": "^3.0.2", + "is-in-browser": "^1.1.3", + "tiny-warning": "^1.0.2" + } + }, + "jss-plugin-camel-case": { + "version": "10.9.0", + "resolved": "https://registry.npmjs.org/jss-plugin-camel-case/-/jss-plugin-camel-case-10.9.0.tgz", + "integrity": "sha512-UH6uPpnDk413/r/2Olmw4+y54yEF2lRIV8XIZyuYpgPYTITLlPOsq6XB9qeqv+75SQSg3KLocq5jUBXW8qWWww==", + "requires": { + "@babel/runtime": "^7.3.1", + "hyphenate-style-name": "^1.0.3", + "jss": "10.9.0" + } + }, + "jss-plugin-default-unit": { + "version": "10.9.0", + "resolved": "https://registry.npmjs.org/jss-plugin-default-unit/-/jss-plugin-default-unit-10.9.0.tgz", + "integrity": "sha512-7Ju4Q9wJ/MZPsxfu4T84mzdn7pLHWeqoGd/D8O3eDNNJ93Xc8PxnLmV8s8ZPNRYkLdxZqKtm1nPQ0BM4JRlq2w==", + "requires": { + "@babel/runtime": "^7.3.1", + "jss": "10.9.0" + } + }, + "jss-plugin-global": { + "version": "10.9.0", + "resolved": "https://registry.npmjs.org/jss-plugin-global/-/jss-plugin-global-10.9.0.tgz", + "integrity": "sha512-4G8PHNJ0x6nwAFsEzcuVDiBlyMsj2y3VjmFAx/uHk/R/gzJV+yRHICjT4MKGGu1cJq2hfowFWCyrr/Gg37FbgQ==", + "requires": { + "@babel/runtime": "^7.3.1", + "jss": "10.9.0" + } + }, + "jss-plugin-nested": { + "version": "10.9.0", + "resolved": "https://registry.npmjs.org/jss-plugin-nested/-/jss-plugin-nested-10.9.0.tgz", + "integrity": "sha512-2UJnDrfCZpMYcpPYR16oZB7VAC6b/1QLsRiAutOt7wJaaqwCBvNsosLEu/fUyKNQNGdvg2PPJFDO5AX7dwxtoA==", + "requires": { + "@babel/runtime": "^7.3.1", + "jss": "10.9.0", + "tiny-warning": "^1.0.2" + } + }, + "jss-plugin-props-sort": { + "version": "10.9.0", + "resolved": "https://registry.npmjs.org/jss-plugin-props-sort/-/jss-plugin-props-sort-10.9.0.tgz", + "integrity": "sha512-7A76HI8bzwqrsMOJTWKx/uD5v+U8piLnp5bvru7g/3ZEQOu1+PjHvv7bFdNO3DwNPC9oM0a//KwIJsIcDCjDzw==", + "requires": { + "@babel/runtime": "^7.3.1", + "jss": "10.9.0" + } + }, + "jss-plugin-rule-value-function": { + "version": "10.9.0", + "resolved": "https://registry.npmjs.org/jss-plugin-rule-value-function/-/jss-plugin-rule-value-function-10.9.0.tgz", + "integrity": "sha512-IHJv6YrEf8pRzkY207cPmdbBstBaE+z8pazhPShfz0tZSDtRdQua5jjg6NMz3IbTasVx9FdnmptxPqSWL5tyJg==", + "requires": { + "@babel/runtime": "^7.3.1", + "jss": "10.9.0", + "tiny-warning": "^1.0.2" + } + }, + "jss-plugin-vendor-prefixer": { + "version": "10.9.0", + "resolved": "https://registry.npmjs.org/jss-plugin-vendor-prefixer/-/jss-plugin-vendor-prefixer-10.9.0.tgz", + "integrity": "sha512-MbvsaXP7iiVdYVSEoi+blrW+AYnTDvHTW6I6zqi7JcwXdc6I9Kbm234nEblayhF38EftoenbM+5218pidmC5gA==", + "requires": { + "@babel/runtime": "^7.3.1", + "css-vendor": "^2.0.8", + "jss": "10.9.0" + } + } + } + }, + "@mui/system": { + "version": "5.2.6", + "resolved": "https://registry.npmjs.org/@mui/system/-/system-5.2.6.tgz", + "integrity": "sha512-PZ7bmpWOLikWgqn2zWv9/Xa7lxnRBOmfjoMH7c/IVYJs78W3971brXJ3xV9MEWWQcoqiYQeXzUJaNf4rFbKCBA==", + "requires": { + "@babel/runtime": "^7.16.3", + "@mui/private-theming": "^5.2.3", + "@mui/styled-engine": "^5.2.6", + "@mui/types": "^7.1.0", + "@mui/utils": "^5.2.3", + "clsx": "^1.1.1", + "csstype": "^3.0.10", + "prop-types": "^15.7.2" + }, + "dependencies": { + "@babel/runtime": { + "version": "7.16.5", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.5.tgz", + "integrity": "sha512-TXWihFIS3Pyv5hzR7j6ihmeLkZfrXGxAr5UfSl8CHf+6q/wpiYDkUau0czckpYG8QmnCIuPpdLtuA9VmuGGyMA==", + "requires": { + "regenerator-runtime": "^0.13.4" + } + }, + "csstype": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.0.10.tgz", + "integrity": "sha512-2u44ZG2OcNUO9HDp/Jl8C07x6pU/eTR3ncV91SiK3dhG9TWvRVsCoJw14Ckx5DgWkzGA3waZWO3d7pgqpUI/XA==" + } + } + }, + "@mui/types": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/@mui/types/-/types-7.1.0.tgz", + "integrity": "sha512-Hh7ALdq/GjfIwLvqH3XftuY3bcKhupktTm+S6qRIDGOtPtRuq2L21VWzOK4p7kblirK0XgGVH5BLwa6u8z/6QQ==" + }, + "@mui/utils": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@mui/utils/-/utils-5.2.3.tgz", + "integrity": "sha512-sQujlajIS0zQKcGIS6tZR0L1R+ib26B6UtuEn+cZqwKHsPo3feuS+SkdscYBdcCdMbrZs4gj8WIJHl2z6tbSzQ==", + "requires": { + "@babel/runtime": "^7.16.3", + "@types/prop-types": "^15.7.4", + "@types/react-is": "^16.7.1 || ^17.0.0", + "prop-types": "^15.7.2", + "react-is": "^17.0.2" + }, + "dependencies": { + "@babel/runtime": { + "version": "7.16.5", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.5.tgz", + "integrity": "sha512-TXWihFIS3Pyv5hzR7j6ihmeLkZfrXGxAr5UfSl8CHf+6q/wpiYDkUau0czckpYG8QmnCIuPpdLtuA9VmuGGyMA==", + "requires": { + "regenerator-runtime": "^0.13.4" + } + }, + "@types/prop-types": { + "version": "15.7.4", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.4.tgz", + "integrity": "sha512-rZ5drC/jWjrArrS8BR6SIr4cWpW09RNTYt9AMZo3Jwwif+iacXAqgVjm0B0Bv/S1jhDXKHqRVNCbACkJ89RAnQ==" + }, + "react-is": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", + "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==" + } + } + }, "@nodelib/fs.scandir": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.4.tgz", @@ -1957,6 +2326,11 @@ } } }, + "@popperjs/core": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.0.tgz", + "integrity": "sha512-zrsUxjLOKAzdewIDRWy9nsV1GQsKBCWaGwsZQlCgr6/q+vjyZhFgqedLfFBuI9anTPEUT4APq9Mu0SZBTzIcGQ==" + }, "@rollup/plugin-node-resolve": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/@rollup/plugin-node-resolve/-/plugin-node-resolve-7.1.3.tgz", @@ -2479,6 +2853,14 @@ "@types/react": "*" } }, + "@types/react-is": { + "version": "17.0.3", + "resolved": "https://registry.npmjs.org/@types/react-is/-/react-is-17.0.3.tgz", + "integrity": "sha512-aBTIWg1emtu95bLTLx0cpkxwGW3ueZv71nE2YFBpL8k/z5czEW8yYpOo8Dp+UUAFAtKwNaOsh/ioSeQnWlZcfw==", + "requires": { + "@types/react": "*" + } + }, "@types/react-transition-group": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.1.tgz", @@ -4355,6 +4737,11 @@ } } }, + "classnames": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.1.tgz", + "integrity": "sha512-OlQdbZ7gLfGarSqxesMesDa5uz7KFbID8Kpq/SxIoNGDqY8lSYs0D+hhtBXhcdB3rcbXArFr7vlHheLk1voeNA==" + }, "clean-css": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-4.2.3.tgz", @@ -10689,6 +11076,19 @@ "object-visit": "^1.0.0" } }, + "material-ui-popup-state": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/material-ui-popup-state/-/material-ui-popup-state-2.0.0.tgz", + "integrity": "sha512-1sbb9xpMs7OxG0SOGfGO0ZnwiLtqZSoXda0/AqqJkpouT4e0nADXutQtDJFMa9GUMNAODVDlYnNmfqM+MhFjsg==", + "requires": { + "@babel/runtime": "^7.12.5", + "@mui/icons-material": "^5.0.0", + "@mui/material": "^5.0.0", + "@mui/styles": "^5.0.0", + "classnames": "^2.2.6", + "prop-types": "^15.7.2" + } + }, "md5.js": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", @@ -15733,6 +16133,11 @@ } } }, + "stylis": { + "version": "4.0.13", + "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.0.13.tgz", + "integrity": "sha512-xGPXiFVl4YED9Jh7Euv2V220mriG9u4B2TA6Ybjc1catrstKD2PpIdU3U0RKpkVBC2EhmL/F0sPCr9vrFTNRag==" + }, "supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", diff --git a/ui/package.json b/ui/package.json index 71b1ed470..64763c557 100644 --- a/ui/package.json +++ b/ui/package.json @@ -18,6 +18,7 @@ "highlight.js": "^11.3.1", "json-beautify": "^1.1.1", "jsonpath": "^1.1.1", + "material-ui-popup-state": "^2.0.0", "moment": "^2.29.1", "node-sass": "^5.0.0", "numeral": "^2.0.6", diff --git a/ui/src/App.sass b/ui/src/App.sass index 47ecb56ef..44fe3e268 100644 --- a/ui/src/App.sass +++ b/ui/src/App.sass @@ -7,25 +7,6 @@ body color: $font-color width: 100% - .header - height: 60px - display: flex - align-items: center - padding: 5px 24px - justify-content: space-between - - .title - letter-spacing: 2px - - img - height: 45px - - .description - margin-left: 10px - font-size: 11px - font-weight: bold - color: $light-blue-color - .centeredForm max-width: 500px text-align: center diff --git a/ui/src/App.tsx b/ui/src/App.tsx index 7371e18b9..65b8a7aff 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -1,71 +1,16 @@ -import React, {useEffect, useState} from 'react'; +import React, {useState} from 'react'; import './App.sass'; -import {TrafficPage} from "./components/TrafficPage"; import {TLSWarning} from "./components/TLSWarning/TLSWarning"; import {Header} from "./components/Header/Header"; -import { ToastContainer, toast } from 'react-toastify'; -import 'react-toastify/dist/ReactToastify.css'; -import Api from "./helpers/api"; -import LoadingOverlay from './components/LoadingOverlay'; -import LoginPage from './components/LoginPage'; -import InstallPage from './components/InstallPage'; - -const api = Api.getInstance(); - - // TODO: move to state management -export enum Page { - Traffic, - Setup, - Login -} - -// TODO: move to state management -export interface MizuContextModel { - page: Page; - setPage: (page: Page) => void; -} - -// TODO: move to state management -export const MizuContext = React.createContext(null); +import {TrafficPage} from "./components/TrafficPage"; const App = () => { - const [isLoading, setIsLoading] = useState(true); + const [analyzeStatus, setAnalyzeStatus] = useState(null); const [showTLSWarning, setShowTLSWarning] = useState(false); const [userDismissedTLSWarning, setUserDismissedTLSWarning] = useState(false); const [addressesWithTLS, setAddressesWithTLS] = useState(new Set()); - const [page, setPage] = useState(Page.Traffic); // TODO: move to state management - - const determinePage = async () => { // TODO: move to state management - if (window['isEnt'] !== true) { - setPage(Page.Traffic); - setIsLoading(false); - return; - } - - try { - const isInstallNeeded = await api.isInstallNeeded(); - if (isInstallNeeded) { - setPage(Page.Setup); - } else { - const isAuthNeeded = await api.isAuthenticationNeeded(); - if(isAuthNeeded) { - setPage(Page.Login); - } - } - } catch (e) { - toast.error("Error occured while checking Mizu API status, see console for mode details"); - console.error(e); - } finally { - setIsLoading(false); - } - } - - useEffect(() => { - determinePage(); - }, []); - const onTLSDetected = (destAddress: string) => { addressesWithTLS.add(destAddress); setAddressesWithTLS(new Set(addressesWithTLS)); @@ -75,49 +20,16 @@ const App = () => { } }; - let pageComponent: any; - - switch (page) { // TODO: move to state management / proper routing - case Page.Traffic: - pageComponent = ; - break; - case Page.Setup: - pageComponent = ; - break; - case Page.Login: - pageComponent = ; - break; - default: - pageComponent =
Unknown Error
; - } - - if (isLoading) { - return ; - } - return (
- -
- {pageComponent} - - - +
+ +
); } diff --git a/ui/src/EntApp.tsx b/ui/src/EntApp.tsx new file mode 100644 index 000000000..1103367cb --- /dev/null +++ b/ui/src/EntApp.tsx @@ -0,0 +1,107 @@ +import React, {useEffect, useState} from 'react'; +import './App.sass'; +import {TrafficPage} from "./components/TrafficPage"; +import {TLSWarning} from "./components/TLSWarning/TLSWarning"; +import {EntHeader} from "./components/Header/EntHeader"; +import Api from "./helpers/api"; +import {toast} from "react-toastify"; +import InstallPage from "./components/InstallPage"; +import LoginPage from "./components/LoginPage"; +import LoadingOverlay from "./components/LoadingOverlay"; + +const api = Api.getInstance(); + +// TODO: move to state management +export enum Page { + Traffic, + Setup, + Login +} + +// TODO: move to state management +export interface MizuContextModel { + page: Page; + setPage: (page: Page) => void; +} + +// TODO: move to state management +export const MizuContext = React.createContext(null); + +const EntApp = () => { + + const [isLoading, setIsLoading] = useState(true); + const [showTLSWarning, setShowTLSWarning] = useState(false); + const [userDismissedTLSWarning, setUserDismissedTLSWarning] = useState(false); + const [addressesWithTLS, setAddressesWithTLS] = useState(new Set()); + const [page, setPage] = useState(Page.Traffic); // TODO: move to state management + const [isFirstLogin, setIsFirstLogin] = useState(false); + + const determinePage = async () => { // TODO: move to state management + try { + const isInstallNeeded = await api.isInstallNeeded(); + if (isInstallNeeded) { + setPage(Page.Setup); + } else { + const isAuthNeeded = await api.isAuthenticationNeeded(); + if(isAuthNeeded) { + setPage(Page.Login); + } + } + } catch (e) { + toast.error("Error occured while checking Mizu API status, see console for mode details"); + console.error(e); + } finally { + setIsLoading(false); + } + } + + useEffect(() => { + determinePage(); + }, []); + + const onTLSDetected = (destAddress: string) => { + addressesWithTLS.add(destAddress); + setAddressesWithTLS(new Set(addressesWithTLS)); + + if (!userDismissedTLSWarning) { + setShowTLSWarning(true); + } + }; + + let pageComponent: any; + + switch (page) { // TODO: move to state management / proper routing + case Page.Traffic: + pageComponent = ; + break; + case Page.Setup: + pageComponent = setIsFirstLogin(true)}/>; + break; + case Page.Login: + pageComponent = ; + break; + default: + pageComponent =
Unknown Error
; + } + + if (isLoading) { + return ; + } + + return ( +
+ + {page === Page.Traffic && } + {pageComponent} + {page === Page.Traffic && } + +
+ ); +} + +export default EntApp; diff --git a/ui/src/components/Filters.tsx b/ui/src/components/Filters.tsx index 9e281c9b6..94ff6cb96 100644 --- a/ui/src/components/Filters.tsx +++ b/ui/src/components/Filters.tsx @@ -36,7 +36,7 @@ interface QueryFormProps { openWebSocket: (query: string, resetEntries: boolean) => void; } -const style = { +export const modalStyle = { position: 'absolute', top: '10%', left: '50%', @@ -45,6 +45,7 @@ const style = { bgcolor: 'background.paper', borderRadius: '5px', boxShadow: 24, + outline: "none", p: 4, color: '#000', }; @@ -153,11 +154,11 @@ export const QueryForm: React.FC = ({query, setQuery, background style={{overflow: 'auto'}} > - + Filtering Guide (Cheatsheet) - +

Mizu has a rich filtering syntax that let's you query the results both flexibly and efficiently.

Here are some examples that you can try;

diff --git a/ui/src/components/Header/EntHeader.tsx b/ui/src/components/Header/EntHeader.tsx new file mode 100644 index 000000000..d331ef144 --- /dev/null +++ b/ui/src/components/Header/EntHeader.tsx @@ -0,0 +1,78 @@ +import React, {useContext, useEffect, useState} from "react"; +import logo from '../assets/MizuEntLogo.svg'; +import './Header.sass'; +import userImg from '../assets/user-circle.svg'; +import settingImg from '../assets/settings.svg'; +import {Menu, MenuItem} from "@material-ui/core"; +import PopupState, {bindMenu, bindTrigger} from "material-ui-popup-state"; +import logoutIcon from '../assets/logout.png'; +import {SettingsModal} from "../SettingsModal/SettingModal"; +import Api from "../../helpers/api"; +import {toast} from "react-toastify"; +import {MizuContext, Page} from "../../EntApp"; + +const api = Api.getInstance(); + +interface EntHeaderProps { + isFirstLogin: boolean; + setIsFirstLogin: (flag: boolean) => void +} + +export const EntHeader: React.FC = ({isFirstLogin, setIsFirstLogin}) => { + + const [isSettingsModalOpen, setIsSettingsModalOpen] = useState(false); + + useEffect(() => { + if(isFirstLogin) { + setIsSettingsModalOpen(true) + } + }, [isFirstLogin]) + + const onSettingsModalClose = () => { + setIsSettingsModalOpen(false); + setIsFirstLogin(false); + } + + return
+
+
+ logo +
+
+
+ settings setIsSettingsModalOpen(true)}/> + +
+ +
; +} + +const ProfileButton = () => { + + const {setPage} = useContext(MizuContext); + + const logout = async (popupState) => { + try { + await api.logout(); + setPage(Page.Login); + } catch (e) { + toast.error("Something went wrong, please check the console"); + console.error(e); + } + popupState.close(); + } + + return ( + {(popupState) => ( + + user + + logout(popupState)}> + logout + Log Out + + + + )} + ); +} diff --git a/ui/src/components/Header/Header.sass b/ui/src/components/Header/Header.sass new file mode 100644 index 000000000..dfc2c4cad --- /dev/null +++ b/ui/src/components/Header/Header.sass @@ -0,0 +1,23 @@ +@import '../../variables.module' + +.header + height: 60px + display: flex + align-items: center + padding: 5px 24px + justify-content: space-between + + .title + letter-spacing: 2px + + img + height: 45px + + .description + margin-left: 10px + font-size: 11px + font-weight: bold + color: $light-blue-color + + .headerIcon + cursor: pointer diff --git a/ui/src/components/Header/Header.tsx b/ui/src/components/Header/Header.tsx index 84a119d28..84f0ccd3c 100644 --- a/ui/src/components/Header/Header.tsx +++ b/ui/src/components/Header/Header.tsx @@ -2,6 +2,7 @@ import React from "react"; import {AuthPresentation} from "../AuthPresentation/AuthPresentation"; import {AnalyzeButton} from "../AnalyzeButton/AnalyzeButton"; import logo from '../assets/Mizu-logo.svg'; +import './Header.sass'; interface HeaderProps { analyzeStatus: any diff --git a/ui/src/components/InstallPage.tsx b/ui/src/components/InstallPage.tsx index 1606d319f..65840b96e 100644 --- a/ui/src/components/InstallPage.tsx +++ b/ui/src/components/InstallPage.tsx @@ -1,6 +1,6 @@ import { Button, TextField } from "@material-ui/core"; import React, { useContext, useState } from "react"; -import { MizuContext, Page } from "../App"; +import { MizuContext, Page } from "../EntApp"; import { adminUsername } from "../consts"; import Api, { FormValidationErrorType } from "../helpers/api"; import { toast } from 'react-toastify'; @@ -8,7 +8,11 @@ import LoadingOverlay from "./LoadingOverlay"; const api = Api.getInstance(); -export const InstallPage: React.FC = () => { +interface InstallPageProps { + onFirstLogin: () => void; +} + +export const InstallPage: React.FC = ({onFirstLogin}) => { const [isLoading, setIsLoading] = useState(false); const [password, setPassword] = useState(""); @@ -31,6 +35,7 @@ export const InstallPage: React.FC = () => { if (!await api.isAuthenticationNeeded()) { toast.success("admin user created successfully"); setPage(Page.Traffic); + onFirstLogin(); } } catch (e) { if (e.type === FormValidationErrorType) { diff --git a/ui/src/components/LoadingOverlay.tsx b/ui/src/components/LoadingOverlay.tsx index b0802ebaf..27dcc2a02 100644 --- a/ui/src/components/LoadingOverlay.tsx +++ b/ui/src/components/LoadingOverlay.tsx @@ -11,11 +11,17 @@ const LoadingOverlay: React.FC = ({delay}) => { const [isVisible, setIsVisible] = useState(false); + // @ts-ignore useEffect(() => { + let isRelevant = true; + setTimeout(() => { - setIsVisible(true); + if(isRelevant) + setIsVisible(true); }, delay ?? SpinnerShowDelayMs); - }, []); + + return () => isRelevant = false; + }, [delay]); return