Added mizu install template (#884)

* Improve PR validation checks

.github/workflows/build.yml

@@ -15,15 +15,23 @@ jobs:
     name: CLI executable build
     runs-on: ubuntu-latest
     steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      
+      - name: Check modified files
+        id: modified_files
+        run: devops/check_modified_files.sh cli/
+
       - name: Set up Go 1.17
+        if: steps.modified_files.outputs.matched == 'true'
         uses: actions/setup-go@v2
         with:
           go-version: '1.17'
 
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
       - name: Build CLI
+        if: steps.modified_files.outputs.matched == 'true'
         run: make cli
 
   build-agent:
@@ -32,6 +40,23 @@ jobs:
     steps:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+      
+      - name: Check modified files
+        id: modified_files
+        run: devops/check_modified_files.sh agent/ shared/ tap/ ui/ Dockerfile
 
-      - name: Build Agent
-        run: make agent-docker
+      - name: Set up Docker Buildx
+        if: steps.modified_files.outputs.matched == 'true'
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build
+        uses: docker/build-push-action@v2
+        if: steps.modified_files.outputs.matched == 'true'
+        with:
+          context: .
+          push: false
+          tags: up9inc/mizu:devlatest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/test.yml

@@ -19,14 +19,16 @@ jobs:
     name: Unit Tests
     runs-on: ubuntu-latest
     steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 2
+
       - name: Set up Go 1.17
         uses: actions/setup-go@v2
         with:
           go-version: '^1.17'
 
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
       - name: Install libpcap
         shell: bash
         run: |
@@ -40,16 +42,31 @@ jobs:
       - name: 'Set up Cloud SDK'
         uses: 'google-github-actions/setup-gcloud@v0'
 
+      - name: Check CLI modified files
+        id: cli_modified_files
+        run: devops/check_modified_files.sh cli/
+
       - name: CLI Test
+        if: github.event_name == 'push' || steps.cli_modified_files.outputs.matched == 'true'
         run: make test-cli
 
+      - name: Check Agent modified files
+        id: agent_modified_files
+        run: devops/check_modified_files.sh agent/
+
       - name: Agent Test
+        if: github.event_name == 'push' || steps.agent_modified_files.outputs.matched == 'true'
         run: make test-agent
 
       - name: Shared Test
         run: make test-shared
+      
+      - name: Check extensions modified files
+        id: ext_modified_files
+        run: devops/check_modified_files.sh tap/extensions/
 
       - name: Extensions Test
+        if: github.event_name == 'push' || steps.ext_modified_files.outputs.matched == 'true'
         run: make test-extensions
 
       - name: Upload coverage to Codecov

cli/apiserver/provider.go

@@ -4,11 +4,10 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
+	"github.com/up9inc/mizu/cli/utils"
 	"io/ioutil"
 	"net/http"
 	"net/url"
-	"strings"
 	"time"
 
 	"github.com/up9inc/mizu/shared/kubernetes"
@@ -59,7 +58,7 @@ func (provider *Provider) TestConnection() error {
 
 func (provider *Provider) isReachable() (bool, error) {
 	echoUrl := fmt.Sprintf("%s/echo", provider.url)
-	if _, err := provider.get(echoUrl); err != nil {
+	if _, err := utils.Get(echoUrl, provider.client); err != nil {
 		return false, err
 	} else {
 		return true, nil
@@ -72,7 +71,7 @@ func (provider *Provider) ReportTapperStatus(tapperStatus shared.TapperStatus) e
 	if jsonValue, err := json.Marshal(tapperStatus); err != nil {
 		return fmt.Errorf("failed Marshal the tapper status %w", err)
 	} else {
-		if _, err := provider.post(tapperStatusUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
+		if _, err := utils.Post(tapperStatusUrl, "application/json", bytes.NewBuffer(jsonValue), provider.client); err != nil {
 			return fmt.Errorf("failed sending to API server the tapped pods %w", err)
 		} else {
 			logger.Log.Debugf("Reported to server API about tapper status: %v", tapperStatus)
@@ -89,7 +88,7 @@ func (provider *Provider) ReportTappedPods(pods []core.Pod) error {
 	if jsonValue, err := json.Marshal(podInfos); err != nil {
 		return fmt.Errorf("failed Marshal the tapped pods %w", err)
 	} else {
-		if _, err := provider.post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue)); err != nil {
+		if _, err := utils.Post(tappedPodsUrl, "application/json", bytes.NewBuffer(jsonValue), provider.client); err != nil {
 			return fmt.Errorf("failed sending to API server the tapped pods %w", err)
 		} else {
 			logger.Log.Debugf("Reported to server API about %d taped pods successfully", len(podInfos))
@@ -101,7 +100,7 @@ func (provider *Provider) ReportTappedPods(pods []core.Pod) error {
 func (provider *Provider) GetGeneralStats() (map[string]interface{}, error) {
 	generalStatsUrl := fmt.Sprintf("%s/status/general", provider.url)
 
-	response, requestErr := provider.get(generalStatsUrl)
+	response, requestErr := utils.Get(generalStatsUrl, provider.client)
 	if requestErr != nil {
 		return nil, fmt.Errorf("failed to get general stats for telemetry, err: %w", requestErr)
 	}
@@ -126,7 +125,7 @@ func (provider *Provider) GetVersion() (string, error) {
 		Method: http.MethodGet,
 		URL:    versionUrl,
 	}
-	statusResp, err := provider.do(req)
+	statusResp, err := utils.Do(req, provider.client)
 	if err != nil {
 		return "", err
 	}
@@ -139,40 +138,3 @@ func (provider *Provider) GetVersion() (string, error) {
 
 	return versionResponse.Ver, nil
 }
-
-// When err is nil, resp always contains a non-nil resp.Body.
-// Caller should close resp.Body when done reading from it.
-func (provider *Provider) get(url string) (*http.Response, error) {
-	return provider.checkError(provider.client.Get(url))
-}
-
-// When err is nil, resp always contains a non-nil resp.Body.
-// Caller should close resp.Body when done reading from it.
-func (provider *Provider) post(url, contentType string, body io.Reader) (*http.Response, error) {
-	return provider.checkError(provider.client.Post(url, contentType, body))
-}
-
-// When err is nil, resp always contains a non-nil resp.Body.
-// Caller should close resp.Body when done reading from it.
-func (provider *Provider) do(req *http.Request) (*http.Response, error) {
-	return provider.checkError(provider.client.Do(req))
-}
-
-func (provider *Provider) checkError(response *http.Response, errInOperation error) (*http.Response, error) {
-	if (errInOperation != nil) {
-		return response, errInOperation
-	// Check only if status != 200 (and not status >= 300). Agent APIs return only 200 on success.
-	} else if response.StatusCode != http.StatusOK {
-		body, err := ioutil.ReadAll(response.Body)
-		response.Body.Close()
-		response.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind
-		if err != nil {
-			return response, err
-		}
-
-		errorMsg := strings.ReplaceAll((string(body)), "\n", ";")
-		return response, fmt.Errorf("got response with status code: %d, body: %s", response.StatusCode, errorMsg)
-	}
-
-	return response, nil
-}

cli/bucket/provider.go

@@ -0,0 +1,42 @@
+package bucket
+
+import (
+	"fmt"
+	"github.com/up9inc/mizu/cli/utils"
+	"io/ioutil"
+	"net/http"
+	"time"
+)
+
+type Provider struct {
+	url     string
+	client  *http.Client
+}
+
+const DefaultTimeout = 2 * time.Second
+
+func NewProvider(url string, timeout time.Duration) *Provider {
+	return &Provider{
+		url:     url,
+		client: &http.Client{
+			Timeout: timeout,
+		},
+	}
+}
+
+func (provider *Provider) GetInstallTemplate(templateName string) (string, error) {
+	url := fmt.Sprintf("%s/%v", provider.url, templateName)
+	response, err := utils.Get(url, provider.client)
+	if err != nil {
+		return "", err
+	}
+
+	defer response.Body.Close()
+
+	installTemplate, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return "", err
+	}
+
+	return string(installTemplate), nil
+}

cli/cmd/common.go

@@ -36,8 +36,8 @@ func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, ctx con
 		return
 	}
 
-	apiProvider = apiserver.NewProvider(GetApiServerUrl(port), apiserver.DefaultRetries, apiserver.DefaultTimeout)
-	if err := apiProvider.TestConnection(); err != nil {
+	provider := apiserver.NewProvider(GetApiServerUrl(port), apiserver.DefaultRetries, apiserver.DefaultTimeout)
+	if err := provider.TestConnection(); err != nil {
 		logger.Log.Debugf("Couldn't connect using proxy, stopping proxy and trying to create port-forward")
 		if err := httpServer.Shutdown(ctx); err != nil {
 			logger.Log.Debugf("Error occurred while stopping proxy %v", errormessage.FormatError(err))
@@ -51,8 +51,8 @@ func startProxyReportErrorIfAny(kubernetesProvider *kubernetes.Provider, ctx con
 			return
 		}
 
-		apiProvider = apiserver.NewProvider(GetApiServerUrl(port), apiserver.DefaultRetries, apiserver.DefaultTimeout)
-		if err := apiProvider.TestConnection(); err != nil {
+		provider = apiserver.NewProvider(GetApiServerUrl(port), apiserver.DefaultRetries, apiserver.DefaultTimeout)
+		if err := provider.TestConnection(); err != nil {
 			logger.Log.Errorf(uiUtils.Error, fmt.Sprintf("Couldn't connect to API server, for more info check logs at %s", fsUtils.GetLogFilePath()))
 			cancel()
 			return

cli/cmd/install.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"github.com/spf13/cobra"
 	"github.com/up9inc/mizu/cli/telemetry"
-	"github.com/up9inc/mizu/shared/logger"
 )
 
 var installCmd = &cobra.Command{
@@ -11,14 +10,7 @@ var installCmd = &cobra.Command{
 	Short: "Installs mizu components",
 	RunE: func(cmd *cobra.Command, args []string) error {
 		go telemetry.ReportRun("install", nil)
-		logger.Log.Infof("This command has been deprecated, please use helm as described below.\n\n")
-
-		logger.Log.Infof("To install stable build of Mizu on your cluster using helm, run the following command:")
-		logger.Log.Infof("    helm install mizu up9mizu --repo https://static.up9.com/mizu/helm --namespace=mizu --create-namespace\n\n")
-
-		logger.Log.Infof("To install development build of Mizu on your cluster using helm, run the following command:")
-		logger.Log.Infof("    helm install mizu up9mizu --repo https://static.up9.com/mizu/helm-develop --namespace=mizu --create-namespace\n")
-
+		runMizuInstall()
 		return nil
 	},
 }

cli/cmd/installRunner.go

@@ -0,0 +1,19 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/up9inc/mizu/cli/bucket"
+	"github.com/up9inc/mizu/cli/config"
+	"github.com/up9inc/mizu/shared/logger"
+)
+
+func runMizuInstall() {
+	bucketProvider := bucket.NewProvider(config.Config.Install.TemplateUrl, bucket.DefaultTimeout)
+	installTemplate, err := bucketProvider.GetInstallTemplate(config.Config.Install.TemplateName)
+	if err != nil {
+		logger.Log.Errorf("Failed getting install template, err: %v", err)
+		return
+	}
+
+	fmt.Print(installTemplate)
+}

cli/config/configStruct.go

@@ -23,6 +23,7 @@ const (
 type ConfigStruct struct {
 	Tap                    configStructs.TapConfig     `yaml:"tap"`
 	Check                  configStructs.CheckConfig   `yaml:"check"`
+	Install                configStructs.InstallConfig `yaml:"install"`
 	Version                configStructs.VersionConfig `yaml:"version"`
 	View                   configStructs.ViewConfig    `yaml:"view"`
 	Logs                   configStructs.LogsConfig    `yaml:"logs"`

cli/config/configStructs/installConfig.go

@@ -0,0 +1,6 @@
+package configStructs
+
+type InstallConfig struct {
+	TemplateUrl  string `yaml:"template-url" default:"https://storage.googleapis.com/static.up9.io/mizu/helm-template"`
+	TemplateName string `yaml:"template-name" default:"helm-template.yaml"`
+}

cli/utils/httpUtils.go

@@ -0,0 +1,47 @@
+package utils
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+// Get - When err is nil, resp always contains a non-nil resp.Body.
+// Caller should close resp.Body when done reading from it.
+func Get(url string, client *http.Client) (*http.Response, error) {
+	return checkError(client.Get(url))
+}
+
+// Post - When err is nil, resp always contains a non-nil resp.Body.
+// Caller should close resp.Body when done reading from it.
+func Post(url, contentType string, body io.Reader, client *http.Client) (*http.Response, error) {
+	return checkError(client.Post(url, contentType, body))
+}
+
+// Do - When err is nil, resp always contains a non-nil resp.Body.
+// Caller should close resp.Body when done reading from it.
+func Do(req *http.Request, client *http.Client) (*http.Response, error) {
+	return checkError(client.Do(req))
+}
+
+func checkError(response *http.Response, errInOperation error) (*http.Response, error) {
+	if errInOperation != nil {
+		return response, errInOperation
+		// Check only if status != 200 (and not status >= 300). Agent APIs return only 200 on success.
+	} else if response.StatusCode != http.StatusOK {
+		body, err := ioutil.ReadAll(response.Body)
+		response.Body.Close()
+		response.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind
+		if err != nil {
+			return response, err
+		}
+
+		errorMsg := strings.ReplaceAll(string(body), "\n", ";")
+		return response, fmt.Errorf("got response with status code: %d, body: %s", response.StatusCode, errorMsg)
+	}
+
+	return response, nil
+}

devops/check_modified_files.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+paths_arr=( "$@" )
+
+printf "\n========== List modified files ==========\n"
+echo "$(git diff --name-only HEAD^ HEAD)"
+
+printf "\n========== List paths to match and check existence ==========\n"
+for path in ${paths_arr[*]}
+do
+  if [ -f "$path" ] || [ -d "$path" ]; then
+      echo "$path - found"
+  else
+      echo "$path - does not found - exiting with failure"
+      exit 1
+  fi
+done
+
+printf "\n========== Check paths of modified files ==========\n"
+git diff --name-only HEAD^ HEAD > files.txt
+matched=false
+while IFS= read -r file
+do
+  for path in ${paths_arr[*]}
+  do
+      if [[ $file == $path* ]]; then
+        echo "$file - match path: $path"
+        matched=true
+        break
+      fi
+  done
+  if [[ $matched == true ]]; then
+      break
+  else
+      echo "$file - does not match any given path"
+  fi
+done < files.txt
+
+printf "\n========== Result ==========\n"
+if [[ $matched = true ]]; then
+  echo "match found"
+  echo "::set-output name=matched::true"
+else
+  echo "no match found"
+  echo "::set-output name=matched::false"
+fi

shared/kubernetes/mizuTapperSyncer.go

@@ -325,15 +325,22 @@ func (tapperSyncer *MizuTapperSyncer) updateMizuTappers() error {
 			tapperSyncer.config.MizuApiFilteringOptions,
 			tapperSyncer.config.LogLevel,
 			tapperSyncer.config.ServiceMesh,
-			tapperSyncer.config.Tls,
-		); err != nil {
+			tapperSyncer.config.Tls); err != nil {
 			return err
 		}
+
 		logger.Log.Debugf("Successfully created %v tappers", len(tapperSyncer.nodeToTappedPodMap))
 	} else {
-		if err := tapperSyncer.kubernetesProvider.RemoveDaemonSet(tapperSyncer.context, tapperSyncer.config.MizuResourcesNamespace, TapperDaemonSetName); err != nil {
+		if err := tapperSyncer.kubernetesProvider.ResetMizuTapperDaemonSet(
+			tapperSyncer.context,
+			tapperSyncer.config.MizuResourcesNamespace,
+			TapperDaemonSetName,
+			tapperSyncer.config.AgentImage,
+			TapperPodName); err != nil {
 			return err
 		}
+
+		logger.Log.Debugf("Successfully reset tapper daemon set")
 	}
 
 	return nil

shared/kubernetes/provider.go

@@ -449,9 +449,9 @@ func (provider *Provider) CanI(ctx context.Context, namespace string, resource s
 		Spec: auth.SelfSubjectAccessReviewSpec{
 			ResourceAttributes: &auth.ResourceAttributes{
 				Namespace: namespace,
-				Resource: resource,
-				Verb: verb,
-				Group: group,
+				Resource:  resource,
+				Verb:      verb,
+				Group:     group,
 			},
 		},
 	}
@@ -995,6 +995,55 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 	return err
 }
 
+func (provider *Provider) ResetMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string) error {
+	agentContainer := applyconfcore.Container()
+	agentContainer.WithName(tapperPodName)
+	agentContainer.WithImage(podImage)
+
+	nodeSelectorRequirement := applyconfcore.NodeSelectorRequirement()
+	nodeSelectorRequirement.WithKey("mizu-non-existing-label")
+	nodeSelectorRequirement.WithOperator(core.NodeSelectorOpExists)
+	nodeSelectorTerm := applyconfcore.NodeSelectorTerm()
+	nodeSelectorTerm.WithMatchExpressions(nodeSelectorRequirement)
+	nodeSelector := applyconfcore.NodeSelector()
+	nodeSelector.WithNodeSelectorTerms(nodeSelectorTerm)
+	nodeAffinity := applyconfcore.NodeAffinity()
+	nodeAffinity.WithRequiredDuringSchedulingIgnoredDuringExecution(nodeSelector)
+	affinity := applyconfcore.Affinity()
+	affinity.WithNodeAffinity(nodeAffinity)
+
+	podSpec := applyconfcore.PodSpec()
+	podSpec.WithContainers(agentContainer)
+	podSpec.WithAffinity(affinity)
+
+	podTemplate := applyconfcore.PodTemplateSpec()
+	podTemplate.WithLabels(map[string]string{
+		"app":          tapperPodName,
+		LabelManagedBy: provider.managedBy,
+		LabelCreatedBy: provider.createdBy,
+	})
+	podTemplate.WithSpec(podSpec)
+
+	labelSelector := applyconfmeta.LabelSelector()
+	labelSelector.WithMatchLabels(map[string]string{"app": tapperPodName})
+
+	applyOptions := metav1.ApplyOptions{
+		Force:        true,
+		FieldManager: fieldManagerName,
+	}
+
+	daemonSet := applyconfapp.DaemonSet(daemonSetName, namespace)
+	daemonSet.
+		WithLabels(map[string]string{
+			LabelManagedBy: provider.managedBy,
+			LabelCreatedBy: provider.createdBy,
+		}).
+		WithSpec(applyconfapp.DaemonSetSpec().WithSelector(labelSelector).WithTemplate(podTemplate))
+
+	_, err := provider.clientSet.AppsV1().DaemonSets(namespace).Apply(ctx, daemonSet, applyOptions)
+	return err
+}
+
 func (provider *Provider) listPodsImpl(ctx context.Context, regex *regexp.Regexp, namespaces []string, listOptions metav1.ListOptions) ([]core.Pod, error) {
 	var pods []core.Pod
 	for _, namespace := range namespaces {
@@ -1038,7 +1087,7 @@ func (provider *Provider) ListAllRunningPodsMatchingRegex(ctx context.Context, r
 	return matchingPods, nil
 }
 
-func(provider *Provider) ListPodsByAppLabel(ctx context.Context, namespaces string, labelName string) ([]core.Pod, error) {
+func (provider *Provider) ListPodsByAppLabel(ctx context.Context, namespaces string, labelName string) ([]core.Pod, error) {
 	pods, err := provider.clientSet.CoreV1().Pods(namespaces).List(ctx, metav1.ListOptions{LabelSelector: fmt.Sprintf("app=%s", labelName)})
 	if err != nil {
 		return nil, err

tap/tlstapper/bpf-builder/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3
+FROM alpine:3.14
 
 RUN apk --no-cache update && apk --no-cache add clang llvm libbpf-dev go linux-headers
 

tap/tlstapper/tls_process_discoverer.go

@@ -132,8 +132,22 @@ func extractCgroup(lines []string) string {
 	return ""
 }
 
+// cgroup in the /proc/<pid>/cgroup may look something like
+//
+//  /system.slice/docker-<ID>.scope
+//  /system.slice/containerd-<ID>.scope
+//  /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod3beae8e0_164d_4689_a087_efd902d8c2ab.slice/docker-<ID>.scope
+//  /kubepods/besteffort/pod7709c1d5-447c-428f-bed9-8ddec35c93f4/<ID>
+//
+// This function extract the <ID> out of the cgroup path, the <ID> should match 
+//	the "Container ID:" field when running kubectl describe pod <POD>
+//
 func normalizeCgroup(cgrouppath string) string {
 	basename := strings.TrimSpace(path.Base(cgrouppath))
+	
+	if strings.Contains(basename, "-") {
+		basename = basename[strings.Index(basename, "-") + 1:]
+	}
 
 	if strings.Contains(basename, ".") {
 		return strings.TrimSuffix(basename, filepath.Ext(basename))

tap/tlstapper/tlstapper_bpfeb.go

@@ -1,5 +1,4 @@
 // Code generated by bpf2go; DO NOT EDIT.
-//go:build arm64be || armbe || mips || mips64 || mips64p32 || ppc64 || s390 || s390x || sparc || sparc64
 // +build arm64be armbe mips mips64 mips64p32 ppc64 s390 s390x sparc sparc64
 
 package tlstapper

tap/tlstapper/tlstapper_bpfel.go

@@ -1,5 +1,4 @@
 // Code generated by bpf2go; DO NOT EDIT.
-//go:build 386 || amd64 || amd64p32 || arm || arm64 || mips64le || mips64p32le || mipsle || ppc64le || riscv64
 // +build 386 amd64 amd64p32 arm arm64 mips64le mips64p32le mipsle ppc64le riscv64
 
 package tlstapper