Feat: support context.clusterVersion for definition graceful upgrade (#4890)

* Feat: support context.clusterVersion for definition graceful upgrade

Signed-off-by: Jianbo Sun <jianbo.sjb@alibaba-inc.com>

* Fix: add test for context.clusterVersion

Signed-off-by: Jianbo Sun <jianbo.sjb@alibaba-inc.com>

* Fix: use control plane context cluster

Signed-off-by: Jianbo Sun <jianbo.sjb@alibaba-inc.com>

Signed-off-by: Jianbo Sun <jianbo.sjb@alibaba-inc.com>

apis/types/multicluster.go

@@ -39,4 +39,18 @@ const (
 var (
 	// AnnotationClusterAlias the annotation key for cluster alias
 	AnnotationClusterAlias = config.MetaApiGroupName + "/cluster-alias"
+
+	// AnnotationClusterVersion the annotation key for cluster version
+	AnnotationClusterVersion = config.MetaApiGroupName + "/cluster-version"
 )
+
+// ClusterVersion defines the Version info of managed clusters.
+type ClusterVersion struct {
+	Major      string `json:"major"`
+	Minor      string `json:"minor"`
+	GitVersion string `json:"gitVersion,omitempty"`
+	Platform   string `json:"platform,omitempty"`
+}
+
+// ControlPlaneClusterVersion will be the default value of cluster info if managed cluster version get error, it will have value when vela-core started.
+var ControlPlaneClusterVersion ClusterVersion

charts/vela-core/templates/defwithtemplate/gateway.yaml

@@ -31,8 +31,13 @@ spec:
         	}
         }
         outputs: ingress: {
-        	apiVersion: "networking.k8s.io/v1"
-        	kind:       "Ingress"
+        	if context.clusterVersion.minor < 19 {
+        		apiVersion: "networking.k8s.io/v1beta1"
+        	}
+        	if context.clusterVersion.minor >= 19 {
+        		apiVersion: "networking.k8s.io/v1"
+        	}
+        	kind: "Ingress"
         	metadata: {
         		name: context.name
         		annotations: {

charts/vela-minimal/templates/defwithtemplate/gateway.yaml

@@ -31,8 +31,13 @@ spec:
         	}
         }
         outputs: ingress: {
-        	apiVersion: "networking.k8s.io/v1"
-        	kind:       "Ingress"
+        	if context.clusterVersion.minor < 19 {
+        		apiVersion: "networking.k8s.io/v1beta1"
+        	}
+        	if context.clusterVersion.minor >= 19 {
+        		apiVersion: "networking.k8s.io/v1"
+        	}
+        	kind: "Ingress"
         	metadata: {
         		name: context.name
         		annotations: {

cmd/core/main.go

@@ -202,6 +202,7 @@ func main() {
 			}
 		}
 	}
+
 	ctrl.SetLogger(klogr.New())
 
 	if utilfeature.DefaultMutableFeatureGate.Enabled(features.ApplyOnce) {
@@ -279,6 +280,11 @@ func main() {
 		os.Exit(1)
 	}
 
+	if err = multicluster.InitClusterInfo(restConfig); err != nil {
+		klog.ErrorS(err, "Init control plane cluster info")
+		os.Exit(1)
+	}
+
 	if driver := os.Getenv(system.StorageDriverEnv); len(driver) == 0 {
 		// first use system environment,
 		err := os.Setenv(system.StorageDriverEnv, storageDriver)

pkg/apiserver/domain/service/cluster.go

@@ -28,6 +28,7 @@ import (
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/kubevela/pkg/util/rand"
@@ -69,9 +70,10 @@ type ClusterService interface {
 }
 
 type clusterServiceImpl struct {
-	Store     datastore.DataStore `inject:"datastore"`
-	K8sClient client.Client       `inject:"kubeClient"`
-	caches    *utils2.MemoryCacheStore
+	Store      datastore.DataStore `inject:"datastore"`
+	K8sClient  client.Client       `inject:"kubeClient"`
+	KubeConfig *rest.Config        `inject:"kubeConfig"`
+	caches     *utils2.MemoryCacheStore
 }
 
 // NewClusterService new cluster service
@@ -115,7 +117,7 @@ func (c *clusterServiceImpl) rollbackJoinedKubeCluster(ctx context.Context, clus
 }
 
 func (c *clusterServiceImpl) rollbackDetachedKubeCluster(ctx context.Context, cluster *model.Cluster) {
-	if _, e := joinClusterByKubeConfigString(ctx, c.K8sClient, cluster.Name, cluster.KubeConfig); e != nil {
+	if _, e := joinClusterByKubeConfigString(context.WithValue(ctx, multicluster.KubeConfigContext, c.KubeConfig), c.K8sClient, cluster.Name, cluster.KubeConfig); e != nil {
 		log.Logger.Errorf("failed to rollback detached cluster %s in kubevela: %s", utils.Sanitize(cluster.Name), e.Error())
 	}
 }
@@ -260,7 +262,7 @@ func (c *clusterServiceImpl) createKubeCluster(ctx context.Context, req apis.Cre
 		return nil, err
 	}
 	if req.KubeConfig != "" {
-		cluster.APIServerURL, err = joinClusterByKubeConfigString(ctx, c.K8sClient, req.Name, req.KubeConfig)
+		cluster.APIServerURL, err = joinClusterByKubeConfigString(context.WithValue(ctx, multicluster.KubeConfigContext, c.KubeConfig), c.K8sClient, req.Name, req.KubeConfig)
 		if err != nil {
 			return nil, err
 		}
@@ -331,7 +333,7 @@ func (c *clusterServiceImpl) ModifyKubeCluster(ctx context.Context, req apis.Cre
 			return nil, bcode.ErrKubeConfigSecretNotSupport
 		}
 		newClusterTempName := newCluster.Name + "_tmp_" + rand.RandomString(8)
-		newCluster.APIServerURL, err = joinClusterByKubeConfigString(ctx, c.K8sClient, newCluster.Name, newCluster.KubeConfig)
+		newCluster.APIServerURL, err = joinClusterByKubeConfigString(context.WithValue(ctx, multicluster.KubeConfigContext, c.KubeConfig), c.K8sClient, newCluster.Name, newCluster.KubeConfig)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to join new cluster %s", newCluster.Name)
 		}

pkg/controller/core.oam.dev/v1alpha2/application/generator.go

@@ -431,6 +431,8 @@ func (h *AppHandler) prepareWorkloadAndManifests(ctx context.Context,
 		if cluster, ok := pkgmulticluster.ClusterFrom(ctx); ok && cluster != "" {
 			ctxData.Cluster = cluster
 		}
+		// cluster info are secrets stored in the control plane cluster
+		ctxData.ClusterVersion = multicluster.GetVersionInfoFromObject(pkgmulticluster.WithCluster(ctx, types.ClusterLocalName), h.r.Client, ctxData.Cluster)
 	})
 	if err != nil {
 		return nil, nil, errors.WithMessage(err, "GenerateComponentManifest")
@@ -441,7 +443,6 @@ func (h *AppHandler) prepareWorkloadAndManifests(ctx context.Context,
 	if err := h.HandleComponentsRevision(contextWithComponent(ctx, &comp), []*types.ComponentManifest{manifest}); err != nil {
 		return nil, nil, errors.WithMessage(err, "HandleComponentsRevision")
 	}
-
 	return wl, manifest, nil
 }
 

pkg/controller/core.oam.dev/v1alpha2/application/suite_test.go

@@ -55,6 +55,7 @@ import (
 	"github.com/oam-dev/kubevela/apis/standard.oam.dev/v1alpha1"
 	"github.com/oam-dev/kubevela/pkg/appfile"
 	"github.com/oam-dev/kubevela/pkg/features"
+	"github.com/oam-dev/kubevela/pkg/multicluster"
 	"github.com/oam-dev/kubevela/pkg/oam/discoverymapper"
 	// +kubebuilder:scaffold:imports
 )
@@ -169,6 +170,7 @@ var _ = BeforeSuite(func(done Done) {
 	}()
 	close(done)
 	Expect(utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=true", features.LegacyComponentRevision))).Should(Succeed())
+	multicluster.InitClusterInfo(cfg)
 }, 120)
 
 var _ = AfterSuite(func() {

pkg/cue/definition/template_test.go

@@ -216,6 +216,24 @@ parameter: {
 			}},
 			hasCompileErr: true,
 		},
+		"cluster version info": {
+			workloadTemplate: `
+output:{
+  if context.clusterVersion.minor <  19 {
+    apiVersion: "networking.k8s.io/v1beta1"
+  }
+  if context.clusterVersion.minor >= 19 {
+    apiVersion: "networking.k8s.io/v1"
+  }
+  "kind":       "Ingress",
+}
+`,
+			params: map[string]interface{}{},
+			expectObj: &unstructured.Unstructured{Object: map[string]interface{}{
+				"apiVersion": "networking.k8s.io/v1",
+				"kind":       "Ingress",
+			}},
+		},
 	}
 
 	for _, v := range testCases {
@@ -224,6 +242,7 @@ parameter: {
 			CompName:        "test",
 			Namespace:       "default",
 			AppRevisionName: "myapp-v1",
+			ClusterVersion:  types.ClusterVersion{Minor: "19+"},
 		})
 		wt := NewWorkloadAbstractEngine("testWorkload", &packages.PackageDiscover{})
 		err := wt.Complete(ctx, v.workloadTemplate, v.params)

pkg/cue/process/handle.go

@@ -18,10 +18,13 @@ package process
 
 import (
 	"context"
+	"strconv"
+	"strings"
 
 	"github.com/kubevela/workflow/pkg/cue/process"
 
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
+	"github.com/oam-dev/kubevela/apis/types"
 	"github.com/oam-dev/kubevela/pkg/oam/util"
 )
 
@@ -44,6 +47,8 @@ type ContextData struct {
 
 	AppLabels      map[string]string
 	AppAnnotations map[string]string
+
+	ClusterVersion types.ClusterVersion
 }
 
 // NewContext creates a new process context
@@ -68,5 +73,22 @@ func NewContext(data ContextData) process.Context {
 	revNum, _ := util.ExtractRevisionNum(data.AppRevisionName, "-")
 	ctx.PushData(ContextAppRevisionNum, revNum)
 	ctx.PushData(ContextCluster, data.Cluster)
+	ctx.PushData(ContextClusterVersion, parseClusterVersion(data.ClusterVersion))
 	return ctx
 }
+
+func parseClusterVersion(cv types.ClusterVersion) map[string]interface{} {
+	// no minor found, use control plane cluster version instead.
+	if cv.Minor == "" {
+		cv = types.ControlPlaneClusterVersion
+	}
+	minorS := strings.TrimSpace(cv.Minor)
+	minorS = strings.TrimRight(minorS, ".+-/?!")
+	minor, _ := strconv.ParseInt(minorS, 10, 64)
+	return map[string]interface{}{
+		"major":      cv.Major,
+		"gitVersion": cv.GitVersion,
+		"platform":   cv.Platform,
+		"minor":      minor,
+	}
+}

pkg/cue/process/handle_test.go

@@ -25,6 +25,8 @@ import (
 	"github.com/kubevela/workflow/pkg/cue/model"
 	"github.com/kubevela/workflow/pkg/cue/model/value"
 	"github.com/kubevela/workflow/pkg/cue/process"
+
+	"github.com/oam-dev/kubevela/apis/types"
 )
 
 func TestContext(t *testing.T) {
@@ -167,3 +169,13 @@ image: "myserver"
 	assert.Equal(t, nil, err)
 	assert.Equal(t, "{\"bool\":false,\"int\":10,\"map\":{\"key\":\"value\"},\"slice\":[\"str1\",\"str2\",\"str3\"],\"string\":\"mytxt\"}", string(arbitraryData))
 }
+
+func TestParseClusterVersion(t *testing.T) {
+	types.ControlPlaneClusterVersion = types.ClusterVersion{Minor: "18+"}
+	got := parseClusterVersion(types.ClusterVersion{})
+	assert.Equal(t, got["minor"], int64(18))
+
+	types.ControlPlaneClusterVersion = types.ClusterVersion{Minor: "22-"}
+	got = parseClusterVersion(types.ClusterVersion{})
+	assert.Equal(t, got["minor"], int64(22))
+}

pkg/cue/process/keyword.go

@@ -41,6 +41,8 @@ const (
 	ContextNamespace = "namespace"
 	// ContextCluster is the cluster currently focusing on
 	ContextCluster = "cluster"
+	// ContextClusterVersion is the version object info of cluster
+	ContextClusterVersion = "clusterVersion"
 	// ContextPublishVersion is the publish version of the app
 	ContextPublishVersion = "publishVersion"
 	// ContextWorkflowName is the name of the workflow

pkg/multicluster/cluster_management.go

@@ -48,6 +48,12 @@ import (
 	cmdutil "github.com/oam-dev/kubevela/pkg/utils/util"
 )
 
+// ContextKey defines the key in context
+type ContextKey string
+
+// KubeConfigContext marks the kubeConfig object in context
+const KubeConfigContext ContextKey = "kubeConfig"
+
 // KubeClusterConfig info for cluster management
 type KubeClusterConfig struct {
 	FilePath        string
@@ -403,6 +409,11 @@ func JoinClusterByKubeConfig(ctx context.Context, cli client.Client, kubeconfigP
 			return clusterConfig, err
 		}
 	}
+	if cfg, ok := ctx.Value(KubeConfigContext).(*rest.Config); ok {
+		if err = SetClusterVersionInfo(ctx, cfg, clusterName); err != nil {
+			return nil, err
+		}
+	}
 	return clusterConfig, nil
 }
 

pkg/multicluster/cluster_metrics_management_test.go

@@ -45,6 +45,7 @@ const (
 )
 
 func TestRefresh(t *testing.T) {
+	ClusterGatewaySecretNamespace = "default"
 	fakeClient := NewFakeClient(fake.NewClientBuilder().
 		WithScheme(common.Scheme).
 		WithRuntimeObjects(FakeManagedCluster("managed-cluster")).
@@ -138,6 +139,7 @@ func FakeNode(name string, cpu string, memory string) *corev1.Node {
 func FakeSecret(name string) *corev1.Secret {
 	secret := &corev1.Secret{}
 	secret.Name = name
+	secret.Namespace = ClusterGatewaySecretNamespace
 	secret.Labels = map[string]string{
 		clustercommon.LabelKeyClusterCredentialType: "ServiceAccountToken",
 	}

pkg/multicluster/virtual_cluster.go

@@ -18,17 +18,23 @@ package multicluster
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 
+	"github.com/oam-dev/cluster-gateway/pkg/generated/clientset/versioned"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	apilabels "k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/selection"
 	apitypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/rest"
+	"k8s.io/klog/v2"
+	"k8s.io/kubectl/pkg/scheme"
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -37,9 +43,35 @@ import (
 	clustercommon "github.com/oam-dev/cluster-gateway/pkg/common"
 
 	"github.com/oam-dev/kubevela/apis/types"
+	"github.com/oam-dev/kubevela/pkg/utils/common"
 	velaerrors "github.com/oam-dev/kubevela/pkg/utils/errors"
 )
 
+// InitClusterInfo will initialize control plane cluster info
+func InitClusterInfo(cfg *rest.Config) error {
+	ctx := context.Background()
+	var err error
+	types.ControlPlaneClusterVersion, err = GetVersionInfoFromCluster(ctx, ClusterLocalName, cfg)
+	if err != nil {
+		return err
+	}
+	client, err := client.New(cfg, client.Options{Scheme: common.Scheme})
+	if err != nil {
+		return err
+	}
+	clusters, err := prismclusterv1alpha1.NewClusterClient(client).List(ctx)
+	if err != nil {
+		return errors.Wrap(err, "fail to get registered clusters")
+	}
+	for _, cluster := range clusters.Items {
+		if err = SetClusterVersionInfo(ctx, cfg, cluster.Name); err != nil {
+			klog.Warningf("set cluster version for %s: %v, skip it...", cluster.Name, err)
+			continue
+		}
+	}
+	return nil
+}
+
 // VirtualCluster contains base info of cluster, it unifies the difference between different cluster implementations
 // like cluster secret or ocm managed cluster
 type VirtualCluster struct {
@@ -137,6 +169,9 @@ func GetVirtualCluster(ctx context.Context, c client.Client, clusterName string)
 	if clusterName == ClusterLocalName {
 		return NewVirtualClusterFromLocal(), nil
 	}
+	if ClusterGatewaySecretNamespace == "" {
+		ClusterGatewaySecretNamespace = types.DefaultKubeVelaNS
+	}
 	secret := &corev1.Secret{}
 	err = c.Get(ctx, apitypes.NamespacedName{
 		Name:      clusterName,
@@ -275,3 +310,103 @@ func NewClusterNameMapper(ctx context.Context, c client.Client) (ClusterNameMapp
 	}
 	return cm, nil
 }
+
+// SetClusterVersionInfo update cluster version info into virtual cluster object
+func SetClusterVersionInfo(ctx context.Context, cfg *rest.Config, clusterName string) error {
+	cli, err := client.New(cfg, client.Options{Scheme: common.Scheme})
+	if err != nil {
+		return err
+	}
+	if clusterName == ClusterLocalName {
+		return nil
+	}
+	vc, err := GetVirtualCluster(ctx, cli, clusterName)
+	if err != nil {
+		return errors.Wrap(err, "get virtual cluster")
+	}
+	_, err = getClusterVersionFromObject(vc.Object)
+	if err == nil {
+		// info already exist
+		return nil
+	}
+	cv, err := GetVersionInfoFromCluster(ctx, clusterName, cfg)
+	if err != nil {
+		return err
+	}
+	setClusterVersion(vc.Object, cv)
+	klog.Infof("joining cluster %s with version: %s", clusterName, cv.GitVersion)
+	return cli.Update(ctx, vc.Object)
+}
+
+// GetVersionInfoFromObject will get cluster version info from virtual cluster, it will fall back to control plane cluster version if any error occur
+func GetVersionInfoFromObject(ctx context.Context, cli client.Client, clusterName string) types.ClusterVersion {
+	vc, err := GetVirtualCluster(ctx, cli, clusterName)
+	if err != nil {
+		klog.Warningf("get virtual cluster for %s err %v, using control plane cluster version", clusterName, err)
+		return types.ControlPlaneClusterVersion
+	}
+	cv, err := getClusterVersionFromObject(vc.Object)
+	if err != nil {
+		klog.Warningf("get version info for %s err %v, using control plane cluster version", clusterName, err)
+		return types.ControlPlaneClusterVersion
+	}
+	return cv
+}
+
+func setClusterVersion(o client.Object, info types.ClusterVersion) {
+	if o == nil {
+		return
+	}
+	ann := o.GetAnnotations()
+	if ann == nil {
+		ann = map[string]string{}
+	}
+	content, _ := json.Marshal(info)
+	ann[types.AnnotationClusterVersion] = string(content)
+	o.SetAnnotations(ann)
+}
+
+func getClusterVersionFromObject(o client.Object) (types.ClusterVersion, error) {
+	if o == nil {
+		return types.ControlPlaneClusterVersion, nil
+	}
+	var cv types.ClusterVersion
+	ann := o.GetAnnotations()
+	if ann == nil {
+		return cv, errors.New("no cluster version info")
+	}
+	versionRaw := ann[types.AnnotationClusterVersion]
+	err := json.Unmarshal([]byte(versionRaw), &cv)
+	return cv, err
+}
+
+// GetVersionInfoFromCluster will add remote cluster version info into secret annotation
+func GetVersionInfoFromCluster(ctx context.Context, clusterName string, cfg *rest.Config) (types.ClusterVersion, error) {
+	var cv types.ClusterVersion
+	content, err := RequestRawK8sAPIForCluster(ctx, "version", clusterName, cfg)
+	if err != nil {
+		return cv, err
+	}
+	if err = json.Unmarshal(content, &cv); err != nil {
+		return cv, err
+	}
+	return cv, nil
+}
+
+// RequestRawK8sAPIForCluster will request multi-cluster K8s API with raw client, such as /healthz, /version, etc
+func RequestRawK8sAPIForCluster(ctx context.Context, path, clusterName string, cfg *rest.Config) ([]byte, error) {
+	cfg.GroupVersion = &schema.GroupVersion{Group: "", Version: "v1"}
+	cfg.NegotiatedSerializer = scheme.Codecs
+	defer func() {
+		cfg.GroupVersion = nil
+		cfg.NegotiatedSerializer = nil
+	}()
+	if clusterName == ClusterLocalName {
+		restClient, err := rest.RESTClientFor(cfg)
+		if err != nil {
+			return nil, errors.Wrap(err, "fail to get local cluster")
+		}
+		return restClient.Get().AbsPath(path).DoRaw(ctx)
+	}
+	return versioned.NewForConfigOrDie(cfg).ClusterV1alpha1().ClusterGateways().RESTClient(clusterName).Get().AbsPath(path).DoRaw(ctx)
+}

pkg/multicluster/virtual_cluster_test.go

@@ -146,6 +146,15 @@ var _ = Describe("Test Virtual Cluster", func() {
 		_, err = NewClusterNameMapper(ctx, cli)
 		Expect(err).ShouldNot(Succeed())
 	})
+	It("Test Cluster Version Get and Set", func() {
+		ClusterGatewaySecretNamespace = "vela-system2"
+		ctx := context.Background()
+		Expect(k8sClient.Create(ctx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ClusterGatewaySecretNamespace}})).Should(Succeed())
+		cv, err := GetVersionInfoFromCluster(ctx, "local", cfg)
+		Expect(err).Should(BeNil())
+		Expect(cv.Minor).Should(Not(BeEquivalentTo("")))
+		Expect(cv.Major).Should(BeEquivalentTo("1"))
+	})
 
 })
 

pkg/workflow/operation/operation.go

@@ -95,7 +95,7 @@ func (wo wfOperator) Resume(ctx context.Context, app *v1beta1.Application) error
 		return err
 	}
 	if !rolloutResumed && !app.Status.Workflow.Suspend {
-		return wo.writeOutput("the workflow is not suspending")
+		return wo.writeOutputF("workflow %s is not suspended.\n", app.Name)
 	}
 
 	if app.Status.Workflow.Suspend {
@@ -224,7 +224,7 @@ func (wo wfOperator) Rollback(ctx context.Context, app *v1beta1.Application) err
 	}
 
 	if rollback {
-		err = wo.writeOutput("Successfully rollback rollout")
+		err = wo.writeOutput("Successfully rollback app.\n")
 		if err != nil {
 			return err
 		}

references/cli/cluster.go

@@ -32,7 +32,6 @@ import (
 	pkgmulticluster "github.com/kubevela/pkg/multicluster"
 	prismclusterv1alpha1 "github.com/kubevela/prism/pkg/apis/cluster/v1alpha1"
 	"github.com/oam-dev/cluster-gateway/pkg/config"
-	"github.com/oam-dev/cluster-gateway/pkg/generated/clientset/versioned"
 
 	"github.com/oam-dev/kubevela/apis/types"
 	"github.com/oam-dev/kubevela/pkg/multicluster"
@@ -183,7 +182,8 @@ func NewClusterJoinCommand(c *common.Args, ioStreams cmdutil.IOStreams) *cobra.C
 			}
 
 			managedClusterKubeConfig := args[0]
-			clusterConfig, err := multicluster.JoinClusterByKubeConfig(context.Background(), client, managedClusterKubeConfig, clusterName,
+			ctx := context.WithValue(context.Background(), multicluster.KubeConfigContext, restConfig)
+			clusterConfig, err := multicluster.JoinClusterByKubeConfig(ctx, client, managedClusterKubeConfig, clusterName,
 				multicluster.JoinClusterCreateNamespaceOption(createNamespace),
 				multicluster.JoinClusterEngineOption(clusterManagementType),
 				multicluster.JoinClusterOCMOptions{
@@ -292,14 +292,11 @@ func NewClusterProbeCommand(c *common.Args) *cobra.Command {
 		Args:  cobra.ExactValidArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			clusterName := args[0]
-			if clusterName == multicluster.ClusterLocalName {
-				return errors.New("you must specify a remote cluster name")
-			}
 			config, err := c.GetConfig()
 			if err != nil {
 				return err
 			}
-			content, err := versioned.NewForConfigOrDie(config).ClusterV1alpha1().ClusterGateways().RESTClient(clusterName).Get().AbsPath("healthz").DoRaw(context.TODO())
+			content, err := multicluster.RequestRawK8sAPIForCluster(context.TODO(), "healthz", clusterName, config)
 			if err != nil {
 				return errors.Wrapf(err, "failed connect cluster %s", clusterName)
 			}

test/e2e-multicluster-test/suite_test.go

@@ -33,6 +33,7 @@ import (
 
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
 	"github.com/oam-dev/kubevela/apis/types"
+	multicluster2 "github.com/oam-dev/kubevela/pkg/multicluster"
 	oamutil "github.com/oam-dev/kubevela/pkg/oam/util"
 	"github.com/oam-dev/kubevela/pkg/utils/common"
 	"github.com/oam-dev/kubevela/pkg/utils/util"
@@ -79,6 +80,12 @@ var _ = BeforeSuite(func() {
 	// join worker cluster
 	_, err = execCommand("cluster", "join", WorkerClusterKubeConfigPath, "--name", WorkerClusterName)
 	Expect(err).Should(Succeed())
+	cv, err := multicluster2.GetVersionInfoFromCluster(context.Background(), WorkerClusterName, config)
+	Expect(err).Should(Succeed())
+	Expect(cv.Minor).Should(Not(BeEquivalentTo("")))
+	Expect(cv.Major).Should(BeEquivalentTo("1"))
+	ocv := multicluster2.GetVersionInfoFromObject(context.Background(), k8sClient, WorkerClusterName)
+	Expect(ocv).Should(BeEquivalentTo(cv))
 })
 
 var _ = AfterSuite(func() {

vela-templates/definitions/internal/trait/gateway.cue

@@ -56,8 +56,13 @@ template: {
 	}
 
 	outputs: ingress: {
-		apiVersion: "networking.k8s.io/v1"
-		kind:       "Ingress"
+		if context.clusterVersion.minor < 19 {
+			apiVersion: "networking.k8s.io/v1beta1"
+		}
+		if context.clusterVersion.minor >= 19 {
+			apiVersion: "networking.k8s.io/v1"
+		}
+		kind: "Ingress"
 		metadata: {
 			name: context.name
 			annotations: {