Feat: upgrade cluster gateway to support remote debug (#2673)

.github/workflows/e2e-multicluster-test.yml

@@ -86,9 +86,12 @@ jobs:
           make e2e-setup-core
           make
           make setup-runtime-e2e-cluster
+          make vela-cli
 
       - name: Run e2e multicluster tests
-        run: make e2e-multicluster-test
+        run: |
+          export PATH=$(pwd)/bin:$PATH
+          make e2e-multicluster-test
 
       - name: Stop kubevela, get profile
         run: make end-e2e-core

charts/vela-core/templates/cluster-gateway.yaml

@@ -107,6 +107,64 @@ spec:
 {{ end }}
 ---
 {{ if and .Values.multicluster.enabled .Values.multicluster.clusterGateway.secureTLS.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ template "kubevela.fullname" . }}-cluster-gateway-admission
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  labels:
+    app: {{ template "kubevela.name" . }}-cluster-gateway-admission
+    {{- include "kubevela.labels" . | nindent 4 }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - create
+{{- end }}
+---
+{{ if and .Values.multicluster.enabled .Values.multicluster.clusterGateway.secureTLS.enabled }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ template "kubevela.fullname" . }}-cluster-gateway-admission
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  labels:
+    app: {{ template "kubevela.name" . }}-cluster-gateway-admission
+    {{- include "kubevela.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ template "kubevela.fullname" . }}-cluster-gateway-admission
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "kubevela.fullname" . }}-cluster-gateway-admission
+    namespace: {{ .Release.Namespace }}
+{{- end }}
+---
+{{ if and .Values.multicluster.enabled .Values.multicluster.clusterGateway.secureTLS.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "kubevela.fullname" . }}-cluster-gateway-admission
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+  labels:
+    app: {{ template "kubevela.name" . }}-cluster-gateway-admission
+    {{- include "kubevela.labels" . | nindent 4 }}
+{{- end }}
+---
+{{ if and .Values.multicluster.enabled .Values.multicluster.clusterGateway.secureTLS.enabled }}
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -142,7 +200,7 @@ spec:
           - --key-name=apiserver.key
           - --cert-name=apiserver.crt
       restartPolicy: OnFailure
-      serviceAccountName: {{ template "kubevela.fullname" . }}-admission
+      serviceAccountName: {{ template "kubevela.fullname" . }}-cluster-gateway-admission
       securityContext:
         runAsGroup: 2000
         runAsNonRoot: true

charts/vela-core/values.yaml

@@ -122,7 +122,7 @@ multicluster:
     port: 9443
     image:
       repository: oamdev/cluster-gateway
-      tag: v1.1.3
+      tag: v1.1.6
       pullPolicy: Always
     resources:
       limits:

references/cli/exec.go

@@ -132,6 +132,7 @@ func NewExecCommand(c common.Args, ioStreams util.IOStreams) *cobra.Command {
 	cmd.Flags().Duration(podRunningTimeoutFlag, defaultPodExecTimeout,
 		"The length of time (like 5s, 2m, or 3h, higher than zero) to wait until at least one pod is running",
 	)
+	cmd.Flags().StringP(FlagNamespace, "n", "", "Specify which namespace to get. If empty, uses namespace in env.")
 
 	return cmd
 }
@@ -141,12 +142,18 @@ func (o *VelaExecOptions) Init(ctx context.Context, c *cobra.Command, argsIn []s
 	o.Cmd = c
 	o.Args = argsIn
 
-	env, err := GetFlagEnvOrCurrent(o.Cmd, o.VelaC)
+	namespace, err := c.Flags().GetString(FlagNamespace)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "failed to get `%s`", FlagNamespace)
 	}
-	o.Env = env
-	app, err := appfile.LoadApplication(env.Namespace, o.Args[0], o.VelaC)
+	if namespace == "" {
+		env, err := GetFlagEnvOrCurrent(o.Cmd, o.VelaC)
+		if err != nil {
+			return err
+		}
+		namespace = env.Namespace
+	}
+	app, err := appfile.LoadApplication(namespace, o.Args[0], o.VelaC)
 	if err != nil {
 		return err
 	}
@@ -173,6 +180,10 @@ func (o *VelaExecOptions) Init(ctx context.Context, c *cobra.Command, argsIn []s
 		return err
 	}
 	o.ClientSet = k8sClient
+
+	o.kcExecOptions.In = c.InOrStdin()
+	o.kcExecOptions.Out = c.OutOrStdout()
+	o.kcExecOptions.ErrOut = c.OutOrStderr()
 	return nil
 }
 

references/cli/portforward.go

@@ -25,6 +25,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	corev1 "k8s.io/api/core/v1"
 	types2 "k8s.io/apimachinery/pkg/types"
@@ -117,6 +118,7 @@ func NewPortForwardCommand(c common.Args, ioStreams util.IOStreams) *cobra.Comma
 		"The length of time (like 5s, 2m, or 3h, higher than zero) to wait until at least one pod is running",
 	)
 	cmd.Flags().BoolVar(&o.routeTrait, "route", false, "forward ports from route trait service")
+	cmd.Flags().StringP(FlagNamespace, "n", "", "Specify which namespace to get. If empty, uses namespace in env.")
 	return cmd
 }
 
@@ -126,12 +128,18 @@ func (o *VelaPortForwardOptions) Init(ctx context.Context, cmd *cobra.Command, a
 	o.Cmd = cmd
 	o.Args = argsIn
 
-	env, err := GetFlagEnvOrCurrent(o.Cmd, o.VelaC)
+	namespace, err := cmd.Flags().GetString(FlagNamespace)
 	if err != nil {
-		return err
+		return errors.Wrapf(err, "failed to get `%s`", FlagNamespace)
 	}
-	o.Env = env
-	app, err := appfile.LoadApplication(env.Namespace, o.Args[0], o.VelaC)
+	if namespace == "" {
+		env, err := GetFlagEnvOrCurrent(o.Cmd, o.VelaC)
+		if err != nil {
+			return err
+		}
+		namespace = env.Namespace
+	}
+	app, err := appfile.LoadApplication(namespace, o.Args[0], o.VelaC)
 	if err != nil {
 		return err
 	}

test/e2e-multicluster-test/multicluster_cli_test.go

@@ -0,0 +1,106 @@
+/*
+Copyright 2021 The KubeVela Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_multicluster_test
+
+import (
+	"context"
+	"io/ioutil"
+	"net/http"
+	"os/exec"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gexec"
+	v1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
+
+	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
+)
+
+var _ = Describe("Test multicluster CLI commands", func() {
+
+	var namespace string
+	var hubCtx context.Context
+	var workerCtx context.Context
+	var app *v1beta1.Application
+
+	BeforeEach(func() {
+		hubCtx, workerCtx, namespace = initializeContextAndNamespace()
+		app = &v1beta1.Application{}
+		bs, err := ioutil.ReadFile("./testdata/app/example-vela-cli-tool-test-app.yaml")
+		Expect(err).Should(Succeed())
+		appYaml := strings.ReplaceAll(string(bs), "TEST_NAMESPACE", namespace)
+		Expect(yaml.Unmarshal([]byte(appYaml), app)).Should(Succeed())
+		app.SetNamespace(namespace)
+		Expect(k8sClient.Create(hubCtx, app)).Should(Succeed())
+		Expect(err).Should(Succeed())
+		Eventually(func(g Gomega) {
+			pods := &v1.PodList{}
+			g.Expect(k8sClient.List(workerCtx, pods, client.InNamespace(namespace))).Should(Succeed())
+			g.Expect(len(pods.Items)).Should(Equal(1))
+			g.Expect(pods.Items[0].Status.Phase).Should(Equal(v1.PodRunning))
+			g.Expect(k8sClient.Get(hubCtx, client.ObjectKeyFromObject(app), app)).Should(Succeed())
+			g.Expect(len(app.Status.AppliedResources)).ShouldNot(Equal(0))
+		}, 2*time.Minute).Should(Succeed())
+	})
+
+	AfterEach(func() {
+		Expect(k8sClient.Get(hubCtx, client.ObjectKeyFromObject(app), app)).Should(Succeed())
+		Expect(k8sClient.Delete(hubCtx, app)).Should(Succeed())
+		cleanUpNamespace(hubCtx, workerCtx, namespace)
+	})
+
+	Context("Test debugging tools in multicluster", func() {
+
+		It("Test vela exec", func() {
+			command := exec.Command("vela", "exec", app.Name, "-n", namespace, "-i=false", "-t=false", "--", "pwd")
+			outputs, err := command.CombinedOutput()
+			Expect(err).Should(Succeed())
+			Expect(string(outputs)).Should(ContainSubstring("/"))
+		})
+
+		It("Test vela port-forward", func() {
+			stopChannel := make(chan struct{}, 1)
+			go func() {
+				defer GinkgoRecover()
+				command := exec.Command("vela", "port-forward", app.Name, "-n", namespace)
+				session, err := gexec.Start(command, ioutil.Discard, ioutil.Discard)
+				Expect(err).Should(Succeed())
+				<-stopChannel
+				session.Terminate()
+			}()
+			defer func() {
+				stopChannel <- struct{}{}
+			}()
+			var resp *http.Response
+			var err error
+			Eventually(func(g Gomega) {
+				resp, err = http.Get("http://127.0.0.1:8000")
+				g.Expect(err).Should(Succeed())
+			}, time.Minute).Should(Succeed())
+			bs := make([]byte, 128)
+			_, err = resp.Body.Read(bs)
+			Expect(err).Should(Succeed())
+			Expect(string(bs)).Should(ContainSubstring("Hello World"))
+		})
+
+	})
+
+})

test/e2e-multicluster-test/suite_test.go

@@ -75,14 +75,14 @@ var _ = BeforeSuite(func() {
 var _ = AfterSuite(func() {
 	Eventually(func(g Gomega) {
 		apps := &v1beta1.ApplicationList{}
-		Expect(k8sClient.List(context.Background(), apps)).Should(Succeed())
+		g.Expect(k8sClient.List(context.Background(), apps)).Should(Succeed())
 		for _, app := range apps.Items {
-			Expect(k8sClient.Delete(context.Background(), app.DeepCopy())).Should(Succeed())
+			g.Expect(k8sClient.Delete(context.Background(), app.DeepCopy())).Should(Succeed())
 		}
-		Expect(len(apps.Items)).Should(Equal(0))
+		g.Expect(len(apps.Items)).Should(Equal(0))
 	}, 3*time.Minute).Should(Succeed())
 	Eventually(func(g Gomega) {
 		_, err := execCommand("cluster", "detach", WorkerClusterName)
-		Expect(err).Should(Succeed())
+		g.Expect(err).Should(Succeed())
 	}, time.Minute).Should(Succeed())
 })

test/e2e-multicluster-test/testdata/app/example-vela-cli-tool-test-app.yaml

@@ -0,0 +1,28 @@
+apiVersion: core.oam.dev/v1beta1
+kind: Application
+metadata:
+  name: cli-test-app
+  namespace: TEST_NAMESPACE
+spec:
+  components:
+    - name: exec-podinfo
+      type: webservice
+      properties:
+        image: crccheck/hello-world
+        port: 8000
+  policies:
+    - name: env-policy
+      type: env-binding
+      properties:
+        envs:
+          - name: test
+            placement:
+              clusterSelector:
+                name: cluster-worker
+  workflow:
+    steps:
+      - name: deploy-test
+        type: deploy2env
+        properties:
+          policy: env-policy
+          env: test