Feat: support remote cluster client to allow user use CRD resource in managed cluster without installing CRD to hub cluster (#6008)

Signed-off-by: Somefive <yd219913@alibaba-inc.com>

go.mod

@@ -53,7 +53,7 @@ require (
 	github.com/hashicorp/hcl/v2 v2.16.2
 	github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174
 	github.com/imdario/mergo v0.3.15
-	github.com/kubevela/pkg v1.8.1-0.20230421100856-841accb6dd92
+	github.com/kubevela/pkg v1.8.1-0.20230522085329-7d5e1241a86d
 	github.com/kubevela/workflow v0.5.1-0.20230412142834-be9e5a10baf0
 	github.com/kyokomi/emoji v2.2.4+incompatible
 	github.com/magiconair/properties v1.8.7
@@ -211,6 +211,7 @@ require (
 	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/jellydator/ttlcache/v3 v3.0.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jmoiron/sqlx v1.3.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect

go.sum

@@ -905,6 +905,8 @@ github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mq
 github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/jellydator/ttlcache/v3 v3.0.1 h1:cHgCSMS7TdQcoprXnWUptJZzyFsqs18Lt8VVhRuZYVU=
+github.com/jellydator/ttlcache/v3 v3.0.1/go.mod h1:WwTaEmcXQ3MTjOm4bsZoDFiCu/hMvNWLO1w67RXz6h4=
 github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
@@ -982,8 +984,8 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kubevela/pkg v1.8.1-0.20230421100856-841accb6dd92 h1:wbsdJ6p6OhwUMRLpqCTFI1WxdjLxRrfWnNSHv/PVzsI=
-github.com/kubevela/pkg v1.8.1-0.20230421100856-841accb6dd92/go.mod h1:dGT23SGBw16frh5ReEYMFraxjppkX6jPCltBgTeLUhE=
+github.com/kubevela/pkg v1.8.1-0.20230522085329-7d5e1241a86d h1:QMmTg33lUZEfTz94eYJKa6Nb7GDcEOmuXsXRt/dA5vk=
+github.com/kubevela/pkg v1.8.1-0.20230522085329-7d5e1241a86d/go.mod h1:3ZWrl2+zb5ROdC2NJPPrL/4sun4M10wYfRP/9gF9WJE=
 github.com/kubevela/workflow v0.5.1-0.20230412142834-be9e5a10baf0 h1:/ZPmjKpd/+fpCjJfNfUnE7jdESuCcZeP+fyTUAU9an0=
 github.com/kubevela/workflow v0.5.1-0.20230412142834-be9e5a10baf0/go.mod h1:0GhIWFIPP+Zt31m4Aslx9mihoyNz3HrOvCV69ljMIBo=
 github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=

test/e2e-multicluster-test/multicluster_test.go

@@ -34,6 +34,7 @@ import (
 	rbacv1 "k8s.io/api/rbac/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/clientcmd"
@@ -1216,5 +1217,34 @@ var _ = Describe("Test multicluster scenario", func() {
 				g.Expect(string(secret.Data["key"])).Should(Equal("vvv\n"))
 			}).WithPolling(2 * time.Second).WithTimeout(20 * time.Second).Should(Succeed())
 		})
+
+		It("Test application apply resources into managed cluster without installing CRD on the control plane", func() {
+			ctx := context.Background()
+			crd := &unstructured.Unstructured{}
+			bs, err := os.ReadFile("./testdata/kube/sample-crd.yaml")
+			Expect(err).Should(Succeed())
+			Expect(yaml.Unmarshal(bs, crd)).Should(Succeed())
+			Expect(client.IgnoreAlreadyExists(k8sClient.Create(workerCtx, crd))).Should(Succeed())
+
+			app := &v1beta1.Application{}
+			bs, err = os.ReadFile("./testdata/app/app-remote-resource.yaml")
+			Expect(err).Should(Succeed())
+			Expect(yaml.Unmarshal(bs, app)).Should(Succeed())
+			app.SetNamespace(namespace)
+			Eventually(func(g Gomega) {
+				g.Expect(k8sClient.Create(ctx, app)).Should(Succeed())
+			}).WithPolling(2 * time.Second).WithTimeout(5 * time.Second).Should(Succeed())
+			appKey := client.ObjectKeyFromObject(app)
+			Eventually(func(g Gomega) {
+				_app := &v1beta1.Application{}
+				g.Expect(k8sClient.Get(ctx, appKey, _app)).Should(Succeed())
+				g.Expect(_app.Status.Phase).Should(Equal(common.ApplicationRunning))
+			}).WithPolling(2 * time.Second).WithTimeout(20 * time.Second).Should(Succeed())
+			obj := &unstructured.Unstructured{}
+			obj.SetAPIVersion("sample.custom.io/v1alpha1")
+			obj.SetKind("Foo")
+			Expect(k8sClient.Get(workerCtx, appKey, obj)).Should(Succeed())
+			Expect(obj.Object["spec"].(map[string]interface{})["key"]).Should(Equal("value"))
+		})
 	})
 })

test/e2e-multicluster-test/suite_test.go

@@ -72,7 +72,7 @@ var _ = BeforeSuite(func() {
 	options := client.Options{Scheme: common.Scheme}
 	config := config.GetConfigOrDie()
 	config.Wrap(multicluster.NewTransportWrapper())
-	k8sClient, err = client.New(config, options)
+	k8sClient, err = multicluster.NewClient(config, multicluster.ClientOptions{Options: options})
 	Expect(err).Should(Succeed())
 	k8sCli, err = kubernetes.NewForConfig(config)
 	Expect(err).Should(Succeed())

test/e2e-multicluster-test/testdata/app/app-remote-resource.yaml

@@ -0,0 +1,21 @@
+apiVersion: core.oam.dev/v1beta1
+kind: Application
+metadata:
+  name: remote
+spec:
+  components:
+    - type: k8s-objects
+      name: remote
+      properties:
+        objects:
+          - apiVersion: sample.custom.io/v1alpha1
+            kind: Foo
+            metadata:
+              name: remote
+            spec:
+              key: value
+  policies:
+    - type: topology
+      name: topology
+      properties:
+        clusters: ["cluster-worker"]

test/e2e-multicluster-test/testdata/kube/sample-crd.yaml

@@ -0,0 +1,24 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: foos.sample.custom.io
+spec:
+  group: sample.custom.io
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      schema:
+        # schema used for validation
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                key:
+                  type: string
+  names:
+    kind: Foo
+    plural: foos
+  scope: Namespaced