diff --git a/.github/workflows/e2e-multicluster-test.yml b/.github/workflows/e2e-multicluster-test.yml index f6852efc7..2179f98b6 100644 --- a/.github/workflows/e2e-multicluster-test.yml +++ b/.github/workflows/e2e-multicluster-test.yml @@ -86,9 +86,12 @@ jobs: make e2e-setup-core make make setup-runtime-e2e-cluster + make vela-cli - name: Run e2e multicluster tests - run: make e2e-multicluster-test + run: | + export PATH=$(pwd)/bin:$PATH + make e2e-multicluster-test - name: Stop kubevela, get profile run: make end-e2e-core diff --git a/charts/vela-core/templates/cluster-gateway.yaml b/charts/vela-core/templates/cluster-gateway.yaml index 9590b5a45..b0943f13a 100644 --- a/charts/vela-core/templates/cluster-gateway.yaml +++ b/charts/vela-core/templates/cluster-gateway.yaml @@ -107,6 +107,64 @@ spec: {{ end }} --- {{ if and .Values.multicluster.enabled .Values.multicluster.clusterGateway.secureTLS.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "kubevela.fullname" . }}-cluster-gateway-admission + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kubevela.name" . }}-cluster-gateway-admission + {{- include "kubevela.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +{{- end }} +--- +{{ if and .Values.multicluster.enabled .Values.multicluster.clusterGateway.secureTLS.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "kubevela.fullname" . }}-cluster-gateway-admission + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kubevela.name" . }}-cluster-gateway-admission + {{- include "kubevela.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "kubevela.fullname" . }}-cluster-gateway-admission +subjects: + - kind: ServiceAccount + name: {{ template "kubevela.fullname" . }}-cluster-gateway-admission + namespace: {{ .Release.Namespace }} +{{- end }} +--- +{{ if and .Values.multicluster.enabled .Values.multicluster.clusterGateway.secureTLS.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "kubevela.fullname" . }}-cluster-gateway-admission + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + labels: + app: {{ template "kubevela.name" . }}-cluster-gateway-admission + {{- include "kubevela.labels" . | nindent 4 }} +{{- end }} +--- +{{ if and .Values.multicluster.enabled .Values.multicluster.clusterGateway.secureTLS.enabled }} apiVersion: batch/v1 kind: Job metadata: @@ -142,7 +200,7 @@ spec: - --key-name=apiserver.key - --cert-name=apiserver.crt restartPolicy: OnFailure - serviceAccountName: {{ template "kubevela.fullname" . }}-admission + serviceAccountName: {{ template "kubevela.fullname" . }}-cluster-gateway-admission securityContext: runAsGroup: 2000 runAsNonRoot: true diff --git a/charts/vela-core/values.yaml b/charts/vela-core/values.yaml index 95885b762..327fd4615 100644 --- a/charts/vela-core/values.yaml +++ b/charts/vela-core/values.yaml @@ -122,7 +122,7 @@ multicluster: port: 9443 image: repository: oamdev/cluster-gateway - tag: v1.1.3 + tag: v1.1.6 pullPolicy: Always resources: limits: diff --git a/references/cli/exec.go b/references/cli/exec.go index 3f1765b80..75681f94e 100644 --- a/references/cli/exec.go +++ b/references/cli/exec.go @@ -132,6 +132,7 @@ func NewExecCommand(c common.Args, ioStreams util.IOStreams) *cobra.Command { cmd.Flags().Duration(podRunningTimeoutFlag, defaultPodExecTimeout, "The length of time (like 5s, 2m, or 3h, higher than zero) to wait until at least one pod is running", ) + cmd.Flags().StringP(FlagNamespace, "n", "", "Specify which namespace to get. If empty, uses namespace in env.") return cmd } @@ -141,12 +142,18 @@ func (o *VelaExecOptions) Init(ctx context.Context, c *cobra.Command, argsIn []s o.Cmd = c o.Args = argsIn - env, err := GetFlagEnvOrCurrent(o.Cmd, o.VelaC) + namespace, err := c.Flags().GetString(FlagNamespace) if err != nil { - return err + return errors.Wrapf(err, "failed to get `%s`", FlagNamespace) } - o.Env = env - app, err := appfile.LoadApplication(env.Namespace, o.Args[0], o.VelaC) + if namespace == "" { + env, err := GetFlagEnvOrCurrent(o.Cmd, o.VelaC) + if err != nil { + return err + } + namespace = env.Namespace + } + app, err := appfile.LoadApplication(namespace, o.Args[0], o.VelaC) if err != nil { return err } @@ -173,6 +180,10 @@ func (o *VelaExecOptions) Init(ctx context.Context, c *cobra.Command, argsIn []s return err } o.ClientSet = k8sClient + + o.kcExecOptions.In = c.InOrStdin() + o.kcExecOptions.Out = c.OutOrStdout() + o.kcExecOptions.ErrOut = c.OutOrStderr() return nil } diff --git a/references/cli/portforward.go b/references/cli/portforward.go index 2832ab997..f575de767 100644 --- a/references/cli/portforward.go +++ b/references/cli/portforward.go @@ -25,6 +25,7 @@ import ( "strconv" "strings" + "github.com/pkg/errors" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" types2 "k8s.io/apimachinery/pkg/types" @@ -117,6 +118,7 @@ func NewPortForwardCommand(c common.Args, ioStreams util.IOStreams) *cobra.Comma "The length of time (like 5s, 2m, or 3h, higher than zero) to wait until at least one pod is running", ) cmd.Flags().BoolVar(&o.routeTrait, "route", false, "forward ports from route trait service") + cmd.Flags().StringP(FlagNamespace, "n", "", "Specify which namespace to get. If empty, uses namespace in env.") return cmd } @@ -126,12 +128,18 @@ func (o *VelaPortForwardOptions) Init(ctx context.Context, cmd *cobra.Command, a o.Cmd = cmd o.Args = argsIn - env, err := GetFlagEnvOrCurrent(o.Cmd, o.VelaC) + namespace, err := cmd.Flags().GetString(FlagNamespace) if err != nil { - return err + return errors.Wrapf(err, "failed to get `%s`", FlagNamespace) } - o.Env = env - app, err := appfile.LoadApplication(env.Namespace, o.Args[0], o.VelaC) + if namespace == "" { + env, err := GetFlagEnvOrCurrent(o.Cmd, o.VelaC) + if err != nil { + return err + } + namespace = env.Namespace + } + app, err := appfile.LoadApplication(namespace, o.Args[0], o.VelaC) if err != nil { return err } diff --git a/test/e2e-multicluster-test/multicluster_cli_test.go b/test/e2e-multicluster-test/multicluster_cli_test.go new file mode 100644 index 000000000..d9d1154b0 --- /dev/null +++ b/test/e2e-multicluster-test/multicluster_cli_test.go @@ -0,0 +1,106 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_multicluster_test + +import ( + "context" + "io/ioutil" + "net/http" + "os/exec" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" + + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" +) + +var _ = Describe("Test multicluster CLI commands", func() { + + var namespace string + var hubCtx context.Context + var workerCtx context.Context + var app *v1beta1.Application + + BeforeEach(func() { + hubCtx, workerCtx, namespace = initializeContextAndNamespace() + app = &v1beta1.Application{} + bs, err := ioutil.ReadFile("./testdata/app/example-vela-cli-tool-test-app.yaml") + Expect(err).Should(Succeed()) + appYaml := strings.ReplaceAll(string(bs), "TEST_NAMESPACE", namespace) + Expect(yaml.Unmarshal([]byte(appYaml), app)).Should(Succeed()) + app.SetNamespace(namespace) + Expect(k8sClient.Create(hubCtx, app)).Should(Succeed()) + Expect(err).Should(Succeed()) + Eventually(func(g Gomega) { + pods := &v1.PodList{} + g.Expect(k8sClient.List(workerCtx, pods, client.InNamespace(namespace))).Should(Succeed()) + g.Expect(len(pods.Items)).Should(Equal(1)) + g.Expect(pods.Items[0].Status.Phase).Should(Equal(v1.PodRunning)) + g.Expect(k8sClient.Get(hubCtx, client.ObjectKeyFromObject(app), app)).Should(Succeed()) + g.Expect(len(app.Status.AppliedResources)).ShouldNot(Equal(0)) + }, 2*time.Minute).Should(Succeed()) + }) + + AfterEach(func() { + Expect(k8sClient.Get(hubCtx, client.ObjectKeyFromObject(app), app)).Should(Succeed()) + Expect(k8sClient.Delete(hubCtx, app)).Should(Succeed()) + cleanUpNamespace(hubCtx, workerCtx, namespace) + }) + + Context("Test debugging tools in multicluster", func() { + + It("Test vela exec", func() { + command := exec.Command("vela", "exec", app.Name, "-n", namespace, "-i=false", "-t=false", "--", "pwd") + outputs, err := command.CombinedOutput() + Expect(err).Should(Succeed()) + Expect(string(outputs)).Should(ContainSubstring("/")) + }) + + It("Test vela port-forward", func() { + stopChannel := make(chan struct{}, 1) + go func() { + defer GinkgoRecover() + command := exec.Command("vela", "port-forward", app.Name, "-n", namespace) + session, err := gexec.Start(command, ioutil.Discard, ioutil.Discard) + Expect(err).Should(Succeed()) + <-stopChannel + session.Terminate() + }() + defer func() { + stopChannel <- struct{}{} + }() + var resp *http.Response + var err error + Eventually(func(g Gomega) { + resp, err = http.Get("http://127.0.0.1:8000") + g.Expect(err).Should(Succeed()) + }, time.Minute).Should(Succeed()) + bs := make([]byte, 128) + _, err = resp.Body.Read(bs) + Expect(err).Should(Succeed()) + Expect(string(bs)).Should(ContainSubstring("Hello World")) + }) + + }) + +}) diff --git a/test/e2e-multicluster-test/suite_test.go b/test/e2e-multicluster-test/suite_test.go index ad27694da..b54ba68d1 100644 --- a/test/e2e-multicluster-test/suite_test.go +++ b/test/e2e-multicluster-test/suite_test.go @@ -75,14 +75,14 @@ var _ = BeforeSuite(func() { var _ = AfterSuite(func() { Eventually(func(g Gomega) { apps := &v1beta1.ApplicationList{} - Expect(k8sClient.List(context.Background(), apps)).Should(Succeed()) + g.Expect(k8sClient.List(context.Background(), apps)).Should(Succeed()) for _, app := range apps.Items { - Expect(k8sClient.Delete(context.Background(), app.DeepCopy())).Should(Succeed()) + g.Expect(k8sClient.Delete(context.Background(), app.DeepCopy())).Should(Succeed()) } - Expect(len(apps.Items)).Should(Equal(0)) + g.Expect(len(apps.Items)).Should(Equal(0)) }, 3*time.Minute).Should(Succeed()) Eventually(func(g Gomega) { _, err := execCommand("cluster", "detach", WorkerClusterName) - Expect(err).Should(Succeed()) + g.Expect(err).Should(Succeed()) }, time.Minute).Should(Succeed()) }) diff --git a/test/e2e-multicluster-test/testdata/app/example-vela-cli-tool-test-app.yaml b/test/e2e-multicluster-test/testdata/app/example-vela-cli-tool-test-app.yaml new file mode 100644 index 000000000..f2cc12c4c --- /dev/null +++ b/test/e2e-multicluster-test/testdata/app/example-vela-cli-tool-test-app.yaml @@ -0,0 +1,28 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: cli-test-app + namespace: TEST_NAMESPACE +spec: + components: + - name: exec-podinfo + type: webservice + properties: + image: crccheck/hello-world + port: 8000 + policies: + - name: env-policy + type: env-binding + properties: + envs: + - name: test + placement: + clusterSelector: + name: cluster-worker + workflow: + steps: + - name: deploy-test + type: deploy2env + properties: + policy: env-policy + env: test