From 70f050082571ea2e3130991dc87eb53f69d44cef Mon Sep 17 00:00:00 2001 From: Jianbo Sun Date: Thu, 20 Oct 2022 20:31:29 +0800 Subject: [PATCH] Feat: support context.clusterVersion for definition graceful upgrade (#4890) * Feat: support context.clusterVersion for definition graceful upgrade Signed-off-by: Jianbo Sun * Fix: add test for context.clusterVersion Signed-off-by: Jianbo Sun * Fix: use control plane context cluster Signed-off-by: Jianbo Sun Signed-off-by: Jianbo Sun --- apis/types/multicluster.go | 14 ++ .../templates/defwithtemplate/gateway.yaml | 9 +- .../templates/defwithtemplate/gateway.yaml | 9 +- cmd/core/main.go | 6 + pkg/apiserver/domain/service/cluster.go | 14 +- .../v1alpha2/application/generator.go | 3 +- .../v1alpha2/application/suite_test.go | 2 + pkg/cue/definition/template_test.go | 19 +++ pkg/cue/process/handle.go | 22 +++ pkg/cue/process/handle_test.go | 12 ++ pkg/cue/process/keyword.go | 2 + pkg/multicluster/cluster_management.go | 11 ++ .../cluster_metrics_management_test.go | 2 + pkg/multicluster/virtual_cluster.go | 135 ++++++++++++++++++ pkg/multicluster/virtual_cluster_test.go | 9 ++ pkg/workflow/operation/operation.go | 4 +- references/cli/cluster.go | 9 +- test/e2e-multicluster-test/suite_test.go | 7 + .../definitions/internal/trait/gateway.cue | 9 +- 19 files changed, 277 insertions(+), 21 deletions(-) diff --git a/apis/types/multicluster.go b/apis/types/multicluster.go index 1418ac042..1b2b55090 100644 --- a/apis/types/multicluster.go +++ b/apis/types/multicluster.go @@ -39,4 +39,18 @@ const ( var ( // AnnotationClusterAlias the annotation key for cluster alias AnnotationClusterAlias = config.MetaApiGroupName + "/cluster-alias" + + // AnnotationClusterVersion the annotation key for cluster version + AnnotationClusterVersion = config.MetaApiGroupName + "/cluster-version" ) + +// ClusterVersion defines the Version info of managed clusters. +type ClusterVersion struct { + Major string `json:"major"` + Minor string `json:"minor"` + GitVersion string `json:"gitVersion,omitempty"` + Platform string `json:"platform,omitempty"` +} + +// ControlPlaneClusterVersion will be the default value of cluster info if managed cluster version get error, it will have value when vela-core started. +var ControlPlaneClusterVersion ClusterVersion diff --git a/charts/vela-core/templates/defwithtemplate/gateway.yaml b/charts/vela-core/templates/defwithtemplate/gateway.yaml index f69a7a527..0ace001d0 100644 --- a/charts/vela-core/templates/defwithtemplate/gateway.yaml +++ b/charts/vela-core/templates/defwithtemplate/gateway.yaml @@ -31,8 +31,13 @@ spec: } } outputs: ingress: { - apiVersion: "networking.k8s.io/v1" - kind: "Ingress" + if context.clusterVersion.minor < 19 { + apiVersion: "networking.k8s.io/v1beta1" + } + if context.clusterVersion.minor >= 19 { + apiVersion: "networking.k8s.io/v1" + } + kind: "Ingress" metadata: { name: context.name annotations: { diff --git a/charts/vela-minimal/templates/defwithtemplate/gateway.yaml b/charts/vela-minimal/templates/defwithtemplate/gateway.yaml index f69a7a527..0ace001d0 100644 --- a/charts/vela-minimal/templates/defwithtemplate/gateway.yaml +++ b/charts/vela-minimal/templates/defwithtemplate/gateway.yaml @@ -31,8 +31,13 @@ spec: } } outputs: ingress: { - apiVersion: "networking.k8s.io/v1" - kind: "Ingress" + if context.clusterVersion.minor < 19 { + apiVersion: "networking.k8s.io/v1beta1" + } + if context.clusterVersion.minor >= 19 { + apiVersion: "networking.k8s.io/v1" + } + kind: "Ingress" metadata: { name: context.name annotations: { diff --git a/cmd/core/main.go b/cmd/core/main.go index feacee75b..cf91d46a6 100644 --- a/cmd/core/main.go +++ b/cmd/core/main.go @@ -202,6 +202,7 @@ func main() { } } } + ctrl.SetLogger(klogr.New()) if utilfeature.DefaultMutableFeatureGate.Enabled(features.ApplyOnce) { @@ -279,6 +280,11 @@ func main() { os.Exit(1) } + if err = multicluster.InitClusterInfo(restConfig); err != nil { + klog.ErrorS(err, "Init control plane cluster info") + os.Exit(1) + } + if driver := os.Getenv(system.StorageDriverEnv); len(driver) == 0 { // first use system environment, err := os.Setenv(system.StorageDriverEnv, storageDriver) diff --git a/pkg/apiserver/domain/service/cluster.go b/pkg/apiserver/domain/service/cluster.go index 08702a8a5..789539a38 100644 --- a/pkg/apiserver/domain/service/cluster.go +++ b/pkg/apiserver/domain/service/cluster.go @@ -28,6 +28,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/kubevela/pkg/util/rand" @@ -69,9 +70,10 @@ type ClusterService interface { } type clusterServiceImpl struct { - Store datastore.DataStore `inject:"datastore"` - K8sClient client.Client `inject:"kubeClient"` - caches *utils2.MemoryCacheStore + Store datastore.DataStore `inject:"datastore"` + K8sClient client.Client `inject:"kubeClient"` + KubeConfig *rest.Config `inject:"kubeConfig"` + caches *utils2.MemoryCacheStore } // NewClusterService new cluster service @@ -115,7 +117,7 @@ func (c *clusterServiceImpl) rollbackJoinedKubeCluster(ctx context.Context, clus } func (c *clusterServiceImpl) rollbackDetachedKubeCluster(ctx context.Context, cluster *model.Cluster) { - if _, e := joinClusterByKubeConfigString(ctx, c.K8sClient, cluster.Name, cluster.KubeConfig); e != nil { + if _, e := joinClusterByKubeConfigString(context.WithValue(ctx, multicluster.KubeConfigContext, c.KubeConfig), c.K8sClient, cluster.Name, cluster.KubeConfig); e != nil { log.Logger.Errorf("failed to rollback detached cluster %s in kubevela: %s", utils.Sanitize(cluster.Name), e.Error()) } } @@ -260,7 +262,7 @@ func (c *clusterServiceImpl) createKubeCluster(ctx context.Context, req apis.Cre return nil, err } if req.KubeConfig != "" { - cluster.APIServerURL, err = joinClusterByKubeConfigString(ctx, c.K8sClient, req.Name, req.KubeConfig) + cluster.APIServerURL, err = joinClusterByKubeConfigString(context.WithValue(ctx, multicluster.KubeConfigContext, c.KubeConfig), c.K8sClient, req.Name, req.KubeConfig) if err != nil { return nil, err } @@ -331,7 +333,7 @@ func (c *clusterServiceImpl) ModifyKubeCluster(ctx context.Context, req apis.Cre return nil, bcode.ErrKubeConfigSecretNotSupport } newClusterTempName := newCluster.Name + "_tmp_" + rand.RandomString(8) - newCluster.APIServerURL, err = joinClusterByKubeConfigString(ctx, c.K8sClient, newCluster.Name, newCluster.KubeConfig) + newCluster.APIServerURL, err = joinClusterByKubeConfigString(context.WithValue(ctx, multicluster.KubeConfigContext, c.KubeConfig), c.K8sClient, newCluster.Name, newCluster.KubeConfig) if err != nil { return nil, errors.Wrapf(err, "failed to join new cluster %s", newCluster.Name) } diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/generator.go b/pkg/controller/core.oam.dev/v1alpha2/application/generator.go index b6697bc96..12e4b2072 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/generator.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/generator.go @@ -431,6 +431,8 @@ func (h *AppHandler) prepareWorkloadAndManifests(ctx context.Context, if cluster, ok := pkgmulticluster.ClusterFrom(ctx); ok && cluster != "" { ctxData.Cluster = cluster } + // cluster info are secrets stored in the control plane cluster + ctxData.ClusterVersion = multicluster.GetVersionInfoFromObject(pkgmulticluster.WithCluster(ctx, types.ClusterLocalName), h.r.Client, ctxData.Cluster) }) if err != nil { return nil, nil, errors.WithMessage(err, "GenerateComponentManifest") @@ -441,7 +443,6 @@ func (h *AppHandler) prepareWorkloadAndManifests(ctx context.Context, if err := h.HandleComponentsRevision(contextWithComponent(ctx, &comp), []*types.ComponentManifest{manifest}); err != nil { return nil, nil, errors.WithMessage(err, "HandleComponentsRevision") } - return wl, manifest, nil } diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/suite_test.go b/pkg/controller/core.oam.dev/v1alpha2/application/suite_test.go index ce0c4cc54..dc280bd27 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/suite_test.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/suite_test.go @@ -55,6 +55,7 @@ import ( "github.com/oam-dev/kubevela/apis/standard.oam.dev/v1alpha1" "github.com/oam-dev/kubevela/pkg/appfile" "github.com/oam-dev/kubevela/pkg/features" + "github.com/oam-dev/kubevela/pkg/multicluster" "github.com/oam-dev/kubevela/pkg/oam/discoverymapper" // +kubebuilder:scaffold:imports ) @@ -169,6 +170,7 @@ var _ = BeforeSuite(func(done Done) { }() close(done) Expect(utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=true", features.LegacyComponentRevision))).Should(Succeed()) + multicluster.InitClusterInfo(cfg) }, 120) var _ = AfterSuite(func() { diff --git a/pkg/cue/definition/template_test.go b/pkg/cue/definition/template_test.go index af35649f2..fa943363a 100644 --- a/pkg/cue/definition/template_test.go +++ b/pkg/cue/definition/template_test.go @@ -216,6 +216,24 @@ parameter: { }}, hasCompileErr: true, }, + "cluster version info": { + workloadTemplate: ` +output:{ + if context.clusterVersion.minor < 19 { + apiVersion: "networking.k8s.io/v1beta1" + } + if context.clusterVersion.minor >= 19 { + apiVersion: "networking.k8s.io/v1" + } + "kind": "Ingress", +} +`, + params: map[string]interface{}{}, + expectObj: &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "networking.k8s.io/v1", + "kind": "Ingress", + }}, + }, } for _, v := range testCases { @@ -224,6 +242,7 @@ parameter: { CompName: "test", Namespace: "default", AppRevisionName: "myapp-v1", + ClusterVersion: types.ClusterVersion{Minor: "19+"}, }) wt := NewWorkloadAbstractEngine("testWorkload", &packages.PackageDiscover{}) err := wt.Complete(ctx, v.workloadTemplate, v.params) diff --git a/pkg/cue/process/handle.go b/pkg/cue/process/handle.go index 2f03c1ee1..8a5a5ca9e 100644 --- a/pkg/cue/process/handle.go +++ b/pkg/cue/process/handle.go @@ -18,10 +18,13 @@ package process import ( "context" + "strconv" + "strings" "github.com/kubevela/workflow/pkg/cue/process" "github.com/oam-dev/kubevela/apis/core.oam.dev/common" + "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/oam/util" ) @@ -44,6 +47,8 @@ type ContextData struct { AppLabels map[string]string AppAnnotations map[string]string + + ClusterVersion types.ClusterVersion } // NewContext creates a new process context @@ -68,5 +73,22 @@ func NewContext(data ContextData) process.Context { revNum, _ := util.ExtractRevisionNum(data.AppRevisionName, "-") ctx.PushData(ContextAppRevisionNum, revNum) ctx.PushData(ContextCluster, data.Cluster) + ctx.PushData(ContextClusterVersion, parseClusterVersion(data.ClusterVersion)) return ctx } + +func parseClusterVersion(cv types.ClusterVersion) map[string]interface{} { + // no minor found, use control plane cluster version instead. + if cv.Minor == "" { + cv = types.ControlPlaneClusterVersion + } + minorS := strings.TrimSpace(cv.Minor) + minorS = strings.TrimRight(minorS, ".+-/?!") + minor, _ := strconv.ParseInt(minorS, 10, 64) + return map[string]interface{}{ + "major": cv.Major, + "gitVersion": cv.GitVersion, + "platform": cv.Platform, + "minor": minor, + } +} diff --git a/pkg/cue/process/handle_test.go b/pkg/cue/process/handle_test.go index e2433a70e..b106eb550 100644 --- a/pkg/cue/process/handle_test.go +++ b/pkg/cue/process/handle_test.go @@ -25,6 +25,8 @@ import ( "github.com/kubevela/workflow/pkg/cue/model" "github.com/kubevela/workflow/pkg/cue/model/value" "github.com/kubevela/workflow/pkg/cue/process" + + "github.com/oam-dev/kubevela/apis/types" ) func TestContext(t *testing.T) { @@ -167,3 +169,13 @@ image: "myserver" assert.Equal(t, nil, err) assert.Equal(t, "{\"bool\":false,\"int\":10,\"map\":{\"key\":\"value\"},\"slice\":[\"str1\",\"str2\",\"str3\"],\"string\":\"mytxt\"}", string(arbitraryData)) } + +func TestParseClusterVersion(t *testing.T) { + types.ControlPlaneClusterVersion = types.ClusterVersion{Minor: "18+"} + got := parseClusterVersion(types.ClusterVersion{}) + assert.Equal(t, got["minor"], int64(18)) + + types.ControlPlaneClusterVersion = types.ClusterVersion{Minor: "22-"} + got = parseClusterVersion(types.ClusterVersion{}) + assert.Equal(t, got["minor"], int64(22)) +} diff --git a/pkg/cue/process/keyword.go b/pkg/cue/process/keyword.go index 4752756c9..d7ca7244b 100644 --- a/pkg/cue/process/keyword.go +++ b/pkg/cue/process/keyword.go @@ -41,6 +41,8 @@ const ( ContextNamespace = "namespace" // ContextCluster is the cluster currently focusing on ContextCluster = "cluster" + // ContextClusterVersion is the version object info of cluster + ContextClusterVersion = "clusterVersion" // ContextPublishVersion is the publish version of the app ContextPublishVersion = "publishVersion" // ContextWorkflowName is the name of the workflow diff --git a/pkg/multicluster/cluster_management.go b/pkg/multicluster/cluster_management.go index a423022f3..aff337834 100644 --- a/pkg/multicluster/cluster_management.go +++ b/pkg/multicluster/cluster_management.go @@ -48,6 +48,12 @@ import ( cmdutil "github.com/oam-dev/kubevela/pkg/utils/util" ) +// ContextKey defines the key in context +type ContextKey string + +// KubeConfigContext marks the kubeConfig object in context +const KubeConfigContext ContextKey = "kubeConfig" + // KubeClusterConfig info for cluster management type KubeClusterConfig struct { FilePath string @@ -403,6 +409,11 @@ func JoinClusterByKubeConfig(ctx context.Context, cli client.Client, kubeconfigP return clusterConfig, err } } + if cfg, ok := ctx.Value(KubeConfigContext).(*rest.Config); ok { + if err = SetClusterVersionInfo(ctx, cfg, clusterName); err != nil { + return nil, err + } + } return clusterConfig, nil } diff --git a/pkg/multicluster/cluster_metrics_management_test.go b/pkg/multicluster/cluster_metrics_management_test.go index da4d00eb4..79a9affcb 100644 --- a/pkg/multicluster/cluster_metrics_management_test.go +++ b/pkg/multicluster/cluster_metrics_management_test.go @@ -45,6 +45,7 @@ const ( ) func TestRefresh(t *testing.T) { + ClusterGatewaySecretNamespace = "default" fakeClient := NewFakeClient(fake.NewClientBuilder(). WithScheme(common.Scheme). WithRuntimeObjects(FakeManagedCluster("managed-cluster")). @@ -138,6 +139,7 @@ func FakeNode(name string, cpu string, memory string) *corev1.Node { func FakeSecret(name string) *corev1.Secret { secret := &corev1.Secret{} secret.Name = name + secret.Namespace = ClusterGatewaySecretNamespace secret.Labels = map[string]string{ clustercommon.LabelKeyClusterCredentialType: "ServiceAccountToken", } diff --git a/pkg/multicluster/virtual_cluster.go b/pkg/multicluster/virtual_cluster.go index d44eda658..e43039810 100644 --- a/pkg/multicluster/virtual_cluster.go +++ b/pkg/multicluster/virtual_cluster.go @@ -18,17 +18,23 @@ package multicluster import ( "context" + "encoding/json" "fmt" "strings" + "github.com/oam-dev/cluster-gateway/pkg/generated/clientset/versioned" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" apilabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/selection" apitypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "k8s.io/kubectl/pkg/scheme" clusterv1 "open-cluster-management.io/api/cluster/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -37,9 +43,35 @@ import ( clustercommon "github.com/oam-dev/cluster-gateway/pkg/common" "github.com/oam-dev/kubevela/apis/types" + "github.com/oam-dev/kubevela/pkg/utils/common" velaerrors "github.com/oam-dev/kubevela/pkg/utils/errors" ) +// InitClusterInfo will initialize control plane cluster info +func InitClusterInfo(cfg *rest.Config) error { + ctx := context.Background() + var err error + types.ControlPlaneClusterVersion, err = GetVersionInfoFromCluster(ctx, ClusterLocalName, cfg) + if err != nil { + return err + } + client, err := client.New(cfg, client.Options{Scheme: common.Scheme}) + if err != nil { + return err + } + clusters, err := prismclusterv1alpha1.NewClusterClient(client).List(ctx) + if err != nil { + return errors.Wrap(err, "fail to get registered clusters") + } + for _, cluster := range clusters.Items { + if err = SetClusterVersionInfo(ctx, cfg, cluster.Name); err != nil { + klog.Warningf("set cluster version for %s: %v, skip it...", cluster.Name, err) + continue + } + } + return nil +} + // VirtualCluster contains base info of cluster, it unifies the difference between different cluster implementations // like cluster secret or ocm managed cluster type VirtualCluster struct { @@ -137,6 +169,9 @@ func GetVirtualCluster(ctx context.Context, c client.Client, clusterName string) if clusterName == ClusterLocalName { return NewVirtualClusterFromLocal(), nil } + if ClusterGatewaySecretNamespace == "" { + ClusterGatewaySecretNamespace = types.DefaultKubeVelaNS + } secret := &corev1.Secret{} err = c.Get(ctx, apitypes.NamespacedName{ Name: clusterName, @@ -275,3 +310,103 @@ func NewClusterNameMapper(ctx context.Context, c client.Client) (ClusterNameMapp } return cm, nil } + +// SetClusterVersionInfo update cluster version info into virtual cluster object +func SetClusterVersionInfo(ctx context.Context, cfg *rest.Config, clusterName string) error { + cli, err := client.New(cfg, client.Options{Scheme: common.Scheme}) + if err != nil { + return err + } + if clusterName == ClusterLocalName { + return nil + } + vc, err := GetVirtualCluster(ctx, cli, clusterName) + if err != nil { + return errors.Wrap(err, "get virtual cluster") + } + _, err = getClusterVersionFromObject(vc.Object) + if err == nil { + // info already exist + return nil + } + cv, err := GetVersionInfoFromCluster(ctx, clusterName, cfg) + if err != nil { + return err + } + setClusterVersion(vc.Object, cv) + klog.Infof("joining cluster %s with version: %s", clusterName, cv.GitVersion) + return cli.Update(ctx, vc.Object) +} + +// GetVersionInfoFromObject will get cluster version info from virtual cluster, it will fall back to control plane cluster version if any error occur +func GetVersionInfoFromObject(ctx context.Context, cli client.Client, clusterName string) types.ClusterVersion { + vc, err := GetVirtualCluster(ctx, cli, clusterName) + if err != nil { + klog.Warningf("get virtual cluster for %s err %v, using control plane cluster version", clusterName, err) + return types.ControlPlaneClusterVersion + } + cv, err := getClusterVersionFromObject(vc.Object) + if err != nil { + klog.Warningf("get version info for %s err %v, using control plane cluster version", clusterName, err) + return types.ControlPlaneClusterVersion + } + return cv +} + +func setClusterVersion(o client.Object, info types.ClusterVersion) { + if o == nil { + return + } + ann := o.GetAnnotations() + if ann == nil { + ann = map[string]string{} + } + content, _ := json.Marshal(info) + ann[types.AnnotationClusterVersion] = string(content) + o.SetAnnotations(ann) +} + +func getClusterVersionFromObject(o client.Object) (types.ClusterVersion, error) { + if o == nil { + return types.ControlPlaneClusterVersion, nil + } + var cv types.ClusterVersion + ann := o.GetAnnotations() + if ann == nil { + return cv, errors.New("no cluster version info") + } + versionRaw := ann[types.AnnotationClusterVersion] + err := json.Unmarshal([]byte(versionRaw), &cv) + return cv, err +} + +// GetVersionInfoFromCluster will add remote cluster version info into secret annotation +func GetVersionInfoFromCluster(ctx context.Context, clusterName string, cfg *rest.Config) (types.ClusterVersion, error) { + var cv types.ClusterVersion + content, err := RequestRawK8sAPIForCluster(ctx, "version", clusterName, cfg) + if err != nil { + return cv, err + } + if err = json.Unmarshal(content, &cv); err != nil { + return cv, err + } + return cv, nil +} + +// RequestRawK8sAPIForCluster will request multi-cluster K8s API with raw client, such as /healthz, /version, etc +func RequestRawK8sAPIForCluster(ctx context.Context, path, clusterName string, cfg *rest.Config) ([]byte, error) { + cfg.GroupVersion = &schema.GroupVersion{Group: "", Version: "v1"} + cfg.NegotiatedSerializer = scheme.Codecs + defer func() { + cfg.GroupVersion = nil + cfg.NegotiatedSerializer = nil + }() + if clusterName == ClusterLocalName { + restClient, err := rest.RESTClientFor(cfg) + if err != nil { + return nil, errors.Wrap(err, "fail to get local cluster") + } + return restClient.Get().AbsPath(path).DoRaw(ctx) + } + return versioned.NewForConfigOrDie(cfg).ClusterV1alpha1().ClusterGateways().RESTClient(clusterName).Get().AbsPath(path).DoRaw(ctx) +} diff --git a/pkg/multicluster/virtual_cluster_test.go b/pkg/multicluster/virtual_cluster_test.go index c7424f41d..1d344d6bc 100644 --- a/pkg/multicluster/virtual_cluster_test.go +++ b/pkg/multicluster/virtual_cluster_test.go @@ -146,6 +146,15 @@ var _ = Describe("Test Virtual Cluster", func() { _, err = NewClusterNameMapper(ctx, cli) Expect(err).ShouldNot(Succeed()) }) + It("Test Cluster Version Get and Set", func() { + ClusterGatewaySecretNamespace = "vela-system2" + ctx := context.Background() + Expect(k8sClient.Create(ctx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ClusterGatewaySecretNamespace}})).Should(Succeed()) + cv, err := GetVersionInfoFromCluster(ctx, "local", cfg) + Expect(err).Should(BeNil()) + Expect(cv.Minor).Should(Not(BeEquivalentTo(""))) + Expect(cv.Major).Should(BeEquivalentTo("1")) + }) }) diff --git a/pkg/workflow/operation/operation.go b/pkg/workflow/operation/operation.go index 2e427da1a..16bba6eb8 100644 --- a/pkg/workflow/operation/operation.go +++ b/pkg/workflow/operation/operation.go @@ -95,7 +95,7 @@ func (wo wfOperator) Resume(ctx context.Context, app *v1beta1.Application) error return err } if !rolloutResumed && !app.Status.Workflow.Suspend { - return wo.writeOutput("the workflow is not suspending") + return wo.writeOutputF("workflow %s is not suspended.\n", app.Name) } if app.Status.Workflow.Suspend { @@ -224,7 +224,7 @@ func (wo wfOperator) Rollback(ctx context.Context, app *v1beta1.Application) err } if rollback { - err = wo.writeOutput("Successfully rollback rollout") + err = wo.writeOutput("Successfully rollback app.\n") if err != nil { return err } diff --git a/references/cli/cluster.go b/references/cli/cluster.go index 05c863410..fd673ebbd 100644 --- a/references/cli/cluster.go +++ b/references/cli/cluster.go @@ -32,7 +32,6 @@ import ( pkgmulticluster "github.com/kubevela/pkg/multicluster" prismclusterv1alpha1 "github.com/kubevela/prism/pkg/apis/cluster/v1alpha1" "github.com/oam-dev/cluster-gateway/pkg/config" - "github.com/oam-dev/cluster-gateway/pkg/generated/clientset/versioned" "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/multicluster" @@ -183,7 +182,8 @@ func NewClusterJoinCommand(c *common.Args, ioStreams cmdutil.IOStreams) *cobra.C } managedClusterKubeConfig := args[0] - clusterConfig, err := multicluster.JoinClusterByKubeConfig(context.Background(), client, managedClusterKubeConfig, clusterName, + ctx := context.WithValue(context.Background(), multicluster.KubeConfigContext, restConfig) + clusterConfig, err := multicluster.JoinClusterByKubeConfig(ctx, client, managedClusterKubeConfig, clusterName, multicluster.JoinClusterCreateNamespaceOption(createNamespace), multicluster.JoinClusterEngineOption(clusterManagementType), multicluster.JoinClusterOCMOptions{ @@ -292,14 +292,11 @@ func NewClusterProbeCommand(c *common.Args) *cobra.Command { Args: cobra.ExactValidArgs(1), RunE: func(cmd *cobra.Command, args []string) error { clusterName := args[0] - if clusterName == multicluster.ClusterLocalName { - return errors.New("you must specify a remote cluster name") - } config, err := c.GetConfig() if err != nil { return err } - content, err := versioned.NewForConfigOrDie(config).ClusterV1alpha1().ClusterGateways().RESTClient(clusterName).Get().AbsPath("healthz").DoRaw(context.TODO()) + content, err := multicluster.RequestRawK8sAPIForCluster(context.TODO(), "healthz", clusterName, config) if err != nil { return errors.Wrapf(err, "failed connect cluster %s", clusterName) } diff --git a/test/e2e-multicluster-test/suite_test.go b/test/e2e-multicluster-test/suite_test.go index bc43c8a6a..2525d0bd6 100644 --- a/test/e2e-multicluster-test/suite_test.go +++ b/test/e2e-multicluster-test/suite_test.go @@ -33,6 +33,7 @@ import ( "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/apis/types" + multicluster2 "github.com/oam-dev/kubevela/pkg/multicluster" oamutil "github.com/oam-dev/kubevela/pkg/oam/util" "github.com/oam-dev/kubevela/pkg/utils/common" "github.com/oam-dev/kubevela/pkg/utils/util" @@ -79,6 +80,12 @@ var _ = BeforeSuite(func() { // join worker cluster _, err = execCommand("cluster", "join", WorkerClusterKubeConfigPath, "--name", WorkerClusterName) Expect(err).Should(Succeed()) + cv, err := multicluster2.GetVersionInfoFromCluster(context.Background(), WorkerClusterName, config) + Expect(err).Should(Succeed()) + Expect(cv.Minor).Should(Not(BeEquivalentTo(""))) + Expect(cv.Major).Should(BeEquivalentTo("1")) + ocv := multicluster2.GetVersionInfoFromObject(context.Background(), k8sClient, WorkerClusterName) + Expect(ocv).Should(BeEquivalentTo(cv)) }) var _ = AfterSuite(func() { diff --git a/vela-templates/definitions/internal/trait/gateway.cue b/vela-templates/definitions/internal/trait/gateway.cue index 05dfb8d19..3008bd23e 100644 --- a/vela-templates/definitions/internal/trait/gateway.cue +++ b/vela-templates/definitions/internal/trait/gateway.cue @@ -56,8 +56,13 @@ template: { } outputs: ingress: { - apiVersion: "networking.k8s.io/v1" - kind: "Ingress" + if context.clusterVersion.minor < 19 { + apiVersion: "networking.k8s.io/v1beta1" + } + if context.clusterVersion.minor >= 19 { + apiVersion: "networking.k8s.io/v1" + } + kind: "Ingress" metadata: { name: context.name annotations: {