diff --git a/charts/vela-core/values.yaml b/charts/vela-core/values.yaml index 7ee2c6bd3..eae8f91e7 100644 --- a/charts/vela-core/values.yaml +++ b/charts/vela-core/values.yaml @@ -118,7 +118,7 @@ multicluster: port: 9443 image: repository: oamdev/cluster-gateway - tag: v1.1.1 + tag: v1.1.2 pullPolicy: Always resources: limits: diff --git a/charts/vela-minimal/values.yaml b/charts/vela-minimal/values.yaml index e9ee83f32..407bcf374 100644 --- a/charts/vela-minimal/values.yaml +++ b/charts/vela-minimal/values.yaml @@ -117,7 +117,7 @@ multicluster: port: 9443 image: repository: oamdev/cluster-gateway - tag: v1.1.1 + tag: v1.1.2 pullPolicy: Always resources: limits: diff --git a/go.mod b/go.mod index dd731bc3f..8f7ca34ed 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/imdario/mergo v0.3.12 github.com/kyokomi/emoji v2.2.4+incompatible github.com/mitchellh/hashstructure/v2 v2.0.1 - github.com/oam-dev/cluster-gateway v0.0.0-20210907072424-2f8720b116f8 + github.com/oam-dev/cluster-gateway v1.1.2 github.com/oam-dev/terraform-config-inspect v0.0.0-20210418082552-fc72d929aa28 github.com/oam-dev/terraform-controller v0.2.0 github.com/olekukonko/tablewriter v0.0.5 diff --git a/go.sum b/go.sum index 58bdfcabf..4bf5acbb3 100644 --- a/go.sum +++ b/go.sum @@ -580,7 +580,6 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.6/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= -github.com/go-openapi/swag v0.19.11/go.mod h1:Uc0gKkdR+ojzsEpjh39QChyu92vPgIr72POcgHMAgSY= github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/validate v0.17.2/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= @@ -1201,8 +1200,8 @@ github.com/nishanths/predeclared v0.2.1/go.mod h1:HvkGJcA3naj4lOwnFXFDkFxVtSqQMB github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oam-dev/cluster-gateway v0.0.0-20210907072424-2f8720b116f8 h1:aTfUhMr6DC+jTiBvCzv+fcgqBvIoGLvrnmXjIUgib7I= -github.com/oam-dev/cluster-gateway v0.0.0-20210907072424-2f8720b116f8/go.mod h1:ZB7/tubU3MtdQ9bzQjksjO8UK+9e0Wrw3HAqKdY/lZY= +github.com/oam-dev/cluster-gateway v1.1.2 h1:sxC8Uyx/d3Yu8nIFSz31i+4JKhJfDAS9XVIPEWa1y+Q= +github.com/oam-dev/cluster-gateway v1.1.2/go.mod h1:EjPUZwTYBe+gFtPV/yGohLE19fDr3CUg4tfSRY72fkM= github.com/oam-dev/stern v1.13.0-alpha h1:EVjM8Qvh6LssB6t4RZrjf9DtCq1cz+/cy6OF7fpy9wk= github.com/oam-dev/stern v1.13.0-alpha/go.mod h1:AOkvfFUv0Arz7GBi0jz7S0Jsu4K/kdvSjNsnRt1+BIg= github.com/oam-dev/terraform-config-inspect v0.0.0-20210418082552-fc72d929aa28 h1:tD8HiFKnt0jnwdTWjeqUnfnUYLD/+Nsmj8ZGIxqDWiU= @@ -2523,8 +2522,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyz sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22 h1:fmRfl9WJ4ApJn7LxNuED4m0t18qivVQOxP6aAYG9J6c= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-runtime v1.0.3-0.20210906132642-810075b08b5f h1:RM+QxWPIZi3mo3i4fiUSvyFELyKr7jjMSJ5EEmt+i6M= -sigs.k8s.io/apiserver-runtime v1.0.3-0.20210906132642-810075b08b5f/go.mod h1:ynhMgoDTl6cihO5OpeV95iG+3bzKfHwWQU2VSbmoNmk= +sigs.k8s.io/apiserver-runtime v1.0.3-0.20210913073608-0663f60bfee2 h1:c6RYHA1wUg9IEsfjnxg0WsPwvDC2Qw2eryXKXgSEF1c= +sigs.k8s.io/apiserver-runtime v1.0.3-0.20210913073608-0663f60bfee2/go.mod h1:gvPfh5FX3Wi3kIRpkh7qvY0i/DQl3SDpRtvqMGZE3Vo= sigs.k8s.io/controller-runtime v0.6.0/go.mod h1:CpYf5pdNY/B352A1TFLAS2JVSlnGQ5O2cftPHndTroo= sigs.k8s.io/controller-runtime v0.6.2/go.mod h1:vhcq/rlnENJ09SIRp3EveTaZ0yqH526hjf9iJdbUJ/E= sigs.k8s.io/controller-runtime v0.9.2/go.mod h1:TxzMCHyEUpaeuOiZx/bIdc2T81vfs/aKdvJt9wuu0zk= diff --git a/pkg/multicluster/utils.go b/pkg/multicluster/utils.go index 3659769e2..31428e53c 100644 --- a/pkg/multicluster/utils.go +++ b/pkg/multicluster/utils.go @@ -25,6 +25,7 @@ import ( "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1" errors2 "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" @@ -34,6 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/oam-dev/kubevela/pkg/utils/common" + errors3 "github.com/oam-dev/kubevela/pkg/utils/errors" ) type contextKey string @@ -52,11 +54,6 @@ var ( ClusterGatewaySecretNamespace string ) -// Context create context with multi-cluster -func Context(ctx context.Context, obj *unstructured.Unstructured) context.Context { - return ContextWithClusterName(ctx, obj.GetLabels()[ClusterLabelKey]) -} - // ContextWithClusterName create context with multi-cluster by cluster name func ContextWithClusterName(ctx context.Context, clusterName string) context.Context { return context.WithValue(ctx, ClusterContextKey, clusterName) @@ -129,5 +126,34 @@ func Initialize(restConfig *rest.Config) error { ClusterGatewaySecretNamespace = svc.Namespace klog.Infof("find cluster gateway service %s/%s:%d", svc.Namespace, svc.Name, *svc.Port) restConfig.Wrap(NewSecretModeMultiClusterRoundTripper) + if err = UpgradeExistingClusterSecret(context.Background(), c); err != nil { + // this error do not affect the running of current version + klog.ErrorS(err, "error encountered while grading existing cluster secret to the latest version") + } + return nil +} + +// UpgradeExistingClusterSecret upgrade outdated cluster secrets in v1.1.1 to latest +func UpgradeExistingClusterSecret(ctx context.Context, c client.Client) error { + const outdatedClusterCredentialLabelKey = "cluster.core.oam.dev/cluster-credential" + secrets := &v1.SecretList{} + if err := c.List(ctx, secrets, client.InNamespace(ClusterGatewaySecretNamespace), client.HasLabels{outdatedClusterCredentialLabelKey}); err != nil { + if err != nil { + return errors2.Wrapf(err, "failed to find outdated cluster secrets to do upgrade") + } + } + errs := errors3.ErrorList{} + for _, item := range secrets.Items { + credType := item.Labels[v1alpha1.LabelKeyClusterCredentialType] + if credType == "" && item.Type == v1.SecretTypeTLS { + item.Labels[v1alpha1.LabelKeyClusterCredentialType] = string(v1alpha1.CredentialTypeX509Certificate) + if err := c.Update(ctx, item.DeepCopy()); err != nil { + errs.Append(errors2.Wrapf(err, "failed to update outdated secret %s", item.Name)) + } + } + } + if errs.HasError() { + return errs + } return nil } diff --git a/pkg/multicluster/utils_test.go b/pkg/multicluster/utils_test.go new file mode 100644 index 000000000..9d551bbf9 --- /dev/null +++ b/pkg/multicluster/utils_test.go @@ -0,0 +1,63 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package multicluster + +import ( + "context" + "testing" + + "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1" + v1 "k8s.io/api/core/v1" + v12 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/oam-dev/kubevela/pkg/utils/common" +) + +func TestUpgradeExistingClusterSecret(t *testing.T) { + oldClusterGatewaySecretNamespace := ClusterGatewaySecretNamespace + ClusterGatewaySecretNamespace = "default" + defer func() { + ClusterGatewaySecretNamespace = oldClusterGatewaySecretNamespace + }() + ctx := context.Background() + c := fake.NewClientBuilder().WithScheme(common.Scheme).Build() + secret := &v1.Secret{ + ObjectMeta: v12.ObjectMeta{ + Name: "example-outdated-cluster-secret", + Namespace: "default", + Labels: map[string]string{ + "cluster.core.oam.dev/cluster-credential": "tls", + }, + }, + Type: v1.SecretTypeTLS, + } + if err := c.Create(ctx, secret); err != nil { + t.Fatalf("failed to create fake outdated cluster secret, err: %v", err) + } + if err := UpgradeExistingClusterSecret(ctx, c); err != nil { + t.Fatalf("expect no error while upgrading outdated cluster secret but encounter error: %v", err) + } + newSecret := &v1.Secret{} + if err := c.Get(ctx, client.ObjectKeyFromObject(secret), newSecret); err != nil { + t.Fatalf("found error while getting updated cluster secret: %v", err) + } + if newSecret.Labels[v1alpha1.LabelKeyClusterCredentialType] != string(v1alpha1.CredentialTypeX509Certificate) { + t.Fatalf("updated secret label should has credential type x509") + } +} diff --git a/references/cli/cluster.go b/references/cli/cluster.go index 3c6ae08fe..1e285952d 100644 --- a/references/cli/cluster.go +++ b/references/cli/cluster.go @@ -19,8 +19,8 @@ package cli import ( "context" "fmt" - "strings" + v1alpha12 "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1" "github.com/pkg/errors" "github.com/spf13/cobra" v1 "k8s.io/api/core/v1" @@ -42,8 +42,6 @@ import ( ) const ( - // ClusterCredentialLabelKey identifies the cluster secrets - ClusterCredentialLabelKey = "cluster.core.oam.dev/cluster-credential" // FlagClusterName specifies the cluster name FlagClusterName = "name" ) @@ -99,12 +97,12 @@ func NewClusterListCommand(c *common.Args) *cobra.Command { Args: cobra.ExactValidArgs(0), RunE: func(cmd *cobra.Command, args []string) error { secrets := v1.SecretList{} - if err := c.Client.List(context.Background(), &secrets, client.HasLabels{ClusterCredentialLabelKey}, client.InNamespace(multicluster.ClusterGatewaySecretNamespace)); err != nil { + if err := c.Client.List(context.Background(), &secrets, client.HasLabels{v1alpha12.LabelKeyClusterCredentialType}, client.InNamespace(multicluster.ClusterGatewaySecretNamespace)); err != nil { return errors.Wrapf(err, "failed to get cluster secrets") } table := newUITable().AddRow("CLUSTER", "TYPE", "ENDPOINT") for _, secret := range secrets.Items { - table.AddRow(secret.Name, secret.GetLabels()[ClusterCredentialLabelKey], string(secret.Data["endpoint"])) + table.AddRow(secret.Name, secret.GetLabels()[v1alpha12.LabelKeyClusterCredentialType], string(secret.Data["endpoint"])) } if len(table.Rows) == 1 { cmd.Println("No managed cluster found.") @@ -198,20 +196,28 @@ func NewClusterJoinCommand(c *common.Args) *cobra.Command { if err := ensureClusterNotExists(c.Client, clusterName); err != nil { return errors.Wrapf(err, "cannot use cluster name %s", clusterName) } - credentialType := v1.SecretTypeTLS + var credentialType v1alpha12.CredentialType data := map[string][]byte{ "endpoint": []byte(cluster.Server), "ca.crt": cluster.CertificateAuthorityData, - "tls.crt": authInfo.ClientCertificateData, - "tls.key": authInfo.ClientKeyData, + } + if len(authInfo.Token) > 0 { + credentialType = v1alpha12.CredentialTypeServiceAccountToken + data["token"] = []byte(authInfo.Token) + } else { + credentialType = v1alpha12.CredentialTypeX509Certificate + data["tls.crt"] = authInfo.ClientCertificateData + data["tls.key"] = authInfo.ClientKeyData } secret := &v1.Secret{ ObjectMeta: v12.ObjectMeta{ Name: clusterName, Namespace: multicluster.ClusterGatewaySecretNamespace, - Labels: map[string]string{ClusterCredentialLabelKey: strings.Split(string(credentialType), "/")[1]}, + Labels: map[string]string{ + v1alpha12.LabelKeyClusterCredentialType: string(credentialType), + }, }, - Type: credentialType, + Type: v1.SecretTypeOpaque, Data: data, } if err := c.Client.Create(context.Background(), secret); err != nil { @@ -235,8 +241,8 @@ func getMutableClusterSecret(c client.Client, clusterName string) (*v1.Secret, e return nil, errors.Wrapf(err, "failed to find target cluster secret %s", clusterName) } labels := clusterSecret.GetLabels() - if labels == nil || labels[ClusterCredentialLabelKey] == "" { - return nil, fmt.Errorf("invalid cluster secret %s: cluster credential type label %s is not set", clusterName, ClusterCredentialLabelKey) + if labels == nil || labels[v1alpha12.LabelKeyClusterCredentialType] == "" { + return nil, fmt.Errorf("invalid cluster secret %s: cluster credential type label %s is not set", clusterName, v1alpha12.LabelKeyClusterCredentialType) } ebs := &v1alpha1.EnvBindingList{} if err := c.List(context.Background(), ebs); err != nil { diff --git a/test/e2e-multicluster-test/multicluster_test.go b/test/e2e-multicluster-test/multicluster_test.go index 5b606be05..aa5f3513d 100644 --- a/test/e2e-multicluster-test/multicluster_test.go +++ b/test/e2e-multicluster-test/multicluster_test.go @@ -19,14 +19,17 @@ package e2e_multicluster_test import ( "context" "fmt" + "os" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" v13 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + v14 "k8s.io/api/rbac/v1" v12 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" @@ -34,6 +37,31 @@ import ( "github.com/oam-dev/kubevela/pkg/utils/common" ) +func initializeContext() (hubCtx context.Context, workerCtx context.Context) { + hubCtx = context.Background() + workerCtx = multicluster.ContextWithClusterName(hubCtx, WorkerClusterName) + return +} + +func initializeContextAndNamespace() (hubCtx context.Context, workerCtx context.Context, namespace string) { + hubCtx, workerCtx = initializeContext() + // initialize test namespace + namespace = fmt.Sprintf("test-%d", time.Now().UnixNano()) + ns := &v1.Namespace{ObjectMeta: v12.ObjectMeta{Name: namespace}} + Expect(k8sClient.Create(hubCtx, ns.DeepCopy())).Should(Succeed()) + Expect(k8sClient.Create(workerCtx, ns.DeepCopy())).Should(Succeed()) + return +} + +func cleanUpNamespace(hubCtx context.Context, workerCtx context.Context, namespace string) { + hubNs := &v1.Namespace{} + Expect(k8sClient.Get(hubCtx, types.NamespacedName{Name: namespace}, hubNs)).Should(Succeed()) + Expect(k8sClient.Delete(hubCtx, hubNs)).Should(Succeed()) + workerNs := &v1.Namespace{} + Expect(k8sClient.Get(workerCtx, types.NamespacedName{Name: namespace}, workerNs)).Should(Succeed()) + Expect(k8sClient.Delete(workerCtx, workerNs)).Should(Succeed()) +} + var _ = Describe("Test multicluster scenario", func() { Context("Test vela cluster command", func() { @@ -60,6 +88,59 @@ var _ = Describe("Test multicluster scenario", func() { Expect(out).ShouldNot(ContainSubstring(newClusterName)) }) + It("Test generate service account kubeconfig", func() { + _, workerCtx := initializeContext() + // create service account kubeconfig in worker cluster + key := time.Now().UnixNano() + serviceAccountName := fmt.Sprintf("test-service-account-%d", key) + serviceAccount := &v1.ServiceAccount{ + ObjectMeta: v12.ObjectMeta{Namespace: "kube-system", Name: serviceAccountName}, + } + Expect(k8sClient.Create(workerCtx, serviceAccount)).Should(Succeed()) + defer func() { + Expect(k8sClient.Get(workerCtx, types.NamespacedName{Namespace: "kube-system", Name: serviceAccountName}, serviceAccount)).Should(Succeed()) + Expect(k8sClient.Delete(workerCtx, serviceAccount)).Should(Succeed()) + }() + clusterRoleBindingName := fmt.Sprintf("test-cluster-role-binding-%d", key) + clusterRoleBinding := &v14.ClusterRoleBinding{ + ObjectMeta: v12.ObjectMeta{Name: clusterRoleBindingName}, + Subjects: []v14.Subject{{Kind: "ServiceAccount", Name: serviceAccountName, Namespace: "kube-system"}}, + RoleRef: v14.RoleRef{Name: "cluster-admin", APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole"}, + } + Expect(k8sClient.Create(workerCtx, clusterRoleBinding)).Should(Succeed()) + defer func() { + Expect(k8sClient.Get(workerCtx, types.NamespacedName{Namespace: "kube-system", Name: clusterRoleBindingName}, clusterRoleBinding)).Should(Succeed()) + Expect(k8sClient.Delete(workerCtx, clusterRoleBinding)).Should(Succeed()) + }() + serviceAccount = &v1.ServiceAccount{} + Expect(k8sClient.Get(workerCtx, types.NamespacedName{Name: serviceAccountName, Namespace: "kube-system"}, serviceAccount)).Should(Succeed()) + Expect(len(serviceAccount.Secrets)).Should(Equal(1)) + secret := &v1.Secret{} + Expect(k8sClient.Get(workerCtx, types.NamespacedName{Name: serviceAccount.Secrets[0].Name, Namespace: "kube-system"}, secret)).Should(Succeed()) + token, ok := secret.Data["token"] + Expect(ok).Should(BeTrue()) + config, err := clientcmd.LoadFromFile(WorkerClusterKubeConfigPath) + Expect(err).Should(Succeed()) + currentContext, ok := config.Contexts[config.CurrentContext] + Expect(ok).Should(BeTrue()) + authInfo, ok := config.AuthInfos[currentContext.AuthInfo] + Expect(ok).Should(BeTrue()) + authInfo.Token = string(token) + authInfo.ClientKeyData = nil + authInfo.ClientCertificateData = nil + kubeconfigFilePath := fmt.Sprintf("/tmp/worker.sa-%d.kubeconfig", key) + Expect(clientcmd.WriteToFile(*config, kubeconfigFilePath)).Should(Succeed()) + defer func() { + Expect(os.Remove(kubeconfigFilePath)).Should(Succeed()) + }() + // try to join cluster with service account token based kubeconfig + clusterName := fmt.Sprintf("cluster-sa-%d", key) + _, err = execCommand("cluster", "join", kubeconfigFilePath, "--name", clusterName) + Expect(err).Should(Succeed()) + _, err = execCommand("cluster", "detach", clusterName) + Expect(err).Should(Succeed()) + }) + }) Context("Test EnvBinding Application", func() { @@ -69,23 +150,11 @@ var _ = Describe("Test multicluster scenario", func() { var workerCtx context.Context BeforeEach(func() { - hubCtx = context.Background() - workerCtx = multicluster.ContextWithClusterName(hubCtx, WorkerClusterName) - // initialize test namespace - namespace = fmt.Sprintf("test-%d", time.Now().UnixNano()) - ns := &v1.Namespace{ObjectMeta: v12.ObjectMeta{Name: namespace}} - Expect(k8sClient.Create(hubCtx, ns.DeepCopy())).Should(Succeed()) - Expect(k8sClient.Create(workerCtx, ns.DeepCopy())).Should(Succeed()) + hubCtx, workerCtx, namespace = initializeContextAndNamespace() }) AfterEach(func() { - // clean up test namespaces - hubNs := &v1.Namespace{} - Expect(k8sClient.Get(hubCtx, types.NamespacedName{Name: namespace}, hubNs)).Should(Succeed()) - Expect(k8sClient.Delete(hubCtx, hubNs)).Should(Succeed()) - workerNs := &v1.Namespace{} - Expect(k8sClient.Get(workerCtx, types.NamespacedName{Name: namespace}, workerNs)).Should(Succeed()) - Expect(k8sClient.Delete(workerCtx, workerNs)).Should(Succeed()) + cleanUpNamespace(hubCtx, workerCtx, namespace) }) It("Test create EnvBinding Application", func() {