Feat: Support trait with custom resource be dispatched to hubcluster. (#3774)

Signed-off-by: fourierr <maxiangboo@qq.com>

apis/core.oam.dev/v1beta1/core_types.go

@@ -157,6 +157,9 @@ type TraitDefinitionSpec struct {
 	// SkipRevisionAffect defines the update this trait will not generate a new application Revision
 	// +optional
 	SkipRevisionAffect bool `json:"skipRevisionAffect,omitempty"`
+	// ControlPlaneOnly defines which cluster is dispatched to
+	// +optional
+	ControlPlaneOnly bool `json:"controlPlaneOnly,omitempty"`
 }
 
 // TraitDefinitionStatus is the status of TraitDefinition

charts/oam-runtime/crds/core.oam.dev_traitdefinitions.yaml

@@ -372,6 +372,10 @@ spec:
                 items:
                   type: string
                 type: array
+              controlPlaneOnly:
+                description: ControlPlaneOnly defines which cluster is dispatched
+                  to
+                type: boolean
               definitionRef:
                 description: Reference to the CustomResourceDefinition that defines
                   this trait kind.

charts/vela-core/crds/core.oam.dev_applicationrevisions.yaml

@@ -3585,6 +3585,10 @@ spec:
                           items:
                             type: string
                           type: array
+                        controlPlaneOnly:
+                          description: ControlPlaneOnly defines which cluster is dispatched
+                            to
+                          type: boolean
                         definitionRef:
                           description: Reference to the CustomResourceDefinition that
                             defines this trait kind.

charts/vela-core/crds/core.oam.dev_definitionrevisions.yaml

@@ -724,6 +724,10 @@ spec:
                         items:
                           type: string
                         type: array
+                      controlPlaneOnly:
+                        description: ControlPlaneOnly defines which cluster is dispatched
+                          to
+                        type: boolean
                       definitionRef:
                         description: Reference to the CustomResourceDefinition that
                           defines this trait kind.

charts/vela-core/crds/core.oam.dev_traitdefinitions.yaml

@@ -372,6 +372,10 @@ spec:
                 items:
                   type: string
                 type: array
+              controlPlaneOnly:
+                description: ControlPlaneOnly defines which cluster is dispatched
+                  to
+                type: boolean
               definitionRef:
                 description: Reference to the CustomResourceDefinition that defines
                   this trait kind.

charts/vela-minimal/crds/core.oam.dev_applicationrevisions.yaml

@@ -3585,6 +3585,10 @@ spec:
                           items:
                             type: string
                           type: array
+                        controlPlaneOnly:
+                          description: ControlPlaneOnly defines which cluster is dispatched
+                            to
+                          type: boolean
                         definitionRef:
                           description: Reference to the CustomResourceDefinition that
                             defines this trait kind.

charts/vela-minimal/crds/core.oam.dev_definitionrevisions.yaml

@@ -724,6 +724,10 @@ spec:
                         items:
                           type: string
                         type: array
+                      controlPlaneOnly:
+                        description: ControlPlaneOnly defines which cluster is dispatched
+                          to
+                        type: boolean
                       definitionRef:
                         description: Reference to the CustomResourceDefinition that
                           defines this trait kind.

charts/vela-minimal/crds/core.oam.dev_traitdefinitions.yaml

@@ -372,6 +372,10 @@ spec:
                 items:
                   type: string
                 type: array
+              controlPlaneOnly:
+                description: ControlPlaneOnly defines which cluster is dispatched
+                  to
+                type: boolean
               definitionRef:
                 description: Reference to the CustomResourceDefinition that defines
                   this trait kind.

legacy/charts/vela-core-legacy/crds/core.oam.dev_applicationrevisions.yaml

@@ -3585,6 +3585,10 @@ spec:
                           items:
                             type: string
                           type: array
+                        controlPlaneOnly:
+                          description: ControlPlaneOnly defines which cluster is dispatched
+                            to
+                          type: boolean
                         definitionRef:
                           description: Reference to the CustomResourceDefinition that
                             defines this trait kind.

legacy/charts/vela-core-legacy/crds/core.oam.dev_definitionrevisions.yaml

@@ -724,6 +724,10 @@ spec:
                         items:
                           type: string
                         type: array
+                      controlPlaneOnly:
+                        description: ControlPlaneOnly defines which cluster is dispatched
+                          to
+                        type: boolean
                       definitionRef:
                         description: Reference to the CustomResourceDefinition that
                           defines this trait kind.

legacy/charts/vela-core-legacy/crds/core.oam.dev_traitdefinitions.yaml

@@ -372,6 +372,10 @@ spec:
                 items:
                   type: string
                 type: array
+              controlPlaneOnly:
+                description: ControlPlaneOnly defines which cluster is dispatched
+                  to
+                type: boolean
               definitionRef:
                 description: Reference to the CustomResourceDefinition that defines
                   this trait kind.

pkg/controller/core.oam.dev/v1alpha2/application/application_controller_test.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	v1 "k8s.io/api/apps/v1"
+	autoscalingv1 "k8s.io/api/autoscaling/v1"
 	corev1 "k8s.io/api/core/v1"
 	v1beta12 "k8s.io/api/networking/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -293,6 +294,31 @@ var _ = Describe("Test Application Controller", func() {
 		},
 	}
 
+	appWithControlPlaneOnly := &v1beta1.Application{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Application",
+			APIVersion: "core.oam.dev/v1beta1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "app-controlplaneonly",
+		},
+		Spec: v1beta1.ApplicationSpec{
+			Components: []common.ApplicationComponent{
+				{
+					Name:       "app-controlplaneonly-component",
+					Type:       "worker",
+					Properties: &runtime.RawExtension{Raw: []byte("{\"cmd\":[\"sleep\",\"1000\"],\"image\":\"busybox\"}")},
+				},
+			},
+		},
+	}
+	appWithControlPlaneOnly.Spec.Components[0].Traits = []common.ApplicationTrait{
+		{
+			Type:       "hubcpuscaler",
+			Properties: &runtime.RawExtension{Raw: []byte("{\"min\": 1,\"max\": 10,\"cpuPercent\": 60}")},
+		},
+	}
+
 	appWithMountToEnvs := &v1beta1.Application{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       "Application",
@@ -339,6 +365,8 @@ var _ = Describe("Test Application Controller", func() {
 
 	importEnv := &v1alpha2.TraitDefinition{}
 
+	importHubCpuScaler := &v1beta1.TraitDefinition{}
+
 	webserverwd := &v1alpha2.ComponentDefinition{}
 	webserverwdJson, _ := yaml.YAMLToJSON([]byte(webComponentDefYaml))
 
@@ -380,6 +408,11 @@ var _ = Describe("Test Application Controller", func() {
 		Expect(json.Unmarshal(envJson, importEnv)).Should(BeNil())
 		Expect(k8sClient.Create(ctx, importEnv.DeepCopy())).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{}))
 
+		hubCpuScalerJson, hubCpuScalerErr := yaml.YAMLToJSON([]byte(hubCpuScalerYaml))
+		Expect(hubCpuScalerErr).ShouldNot(HaveOccurred())
+		Expect(json.Unmarshal(hubCpuScalerJson, importHubCpuScaler)).Should(BeNil())
+		Expect(k8sClient.Create(ctx, importHubCpuScaler.DeepCopy())).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{}))
+
 		Expect(json.Unmarshal(tDDefJson, td)).Should(BeNil())
 		Expect(k8sClient.Create(ctx, td.DeepCopy())).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{}))
 
@@ -2716,6 +2749,60 @@ var _ = Describe("Test Application Controller", func() {
 
 		Expect(k8sClient.Delete(ctx, app)).Should(BeNil())
 	})
+
+	It("test application with controlPlaneOnly trait ", func() {
+
+		ns := &corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "vela-test-with-controlplaneonly",
+			},
+		}
+		Expect(k8sClient.Create(ctx, ns)).Should(BeNil())
+
+		appWithControlPlaneOnly.SetNamespace(ns.Name)
+		app := appWithControlPlaneOnly.DeepCopy()
+		Expect(k8sClient.Create(ctx, app)).Should(BeNil())
+
+		appKey := client.ObjectKey{
+			Name:      app.Name,
+			Namespace: app.Namespace,
+		}
+		testutil.ReconcileOnceAfterFinalizer(reconciler, reconcile.Request{NamespacedName: appKey})
+
+		By("Check App running successfully")
+		curApp := &v1beta1.Application{}
+		Expect(k8sClient.Get(ctx, appKey, curApp)).Should(BeNil())
+		Expect(curApp.Status.Phase).Should(Equal(common.ApplicationRunning))
+
+		appRevision := &v1beta1.ApplicationRevision{}
+		Expect(k8sClient.Get(ctx, client.ObjectKey{
+			Namespace: app.Namespace,
+			Name:      curApp.Status.LatestRevision.Name,
+		}, appRevision)).Should(BeNil())
+
+		By("Check affiliated resource tracker is created")
+		expectRTName := fmt.Sprintf("%s-%s", appRevision.GetName(), appRevision.GetNamespace())
+		Eventually(func() error {
+			return k8sClient.Get(ctx, client.ObjectKey{Name: expectRTName}, &v1beta1.ResourceTracker{})
+		}, 10*time.Second, 500*time.Millisecond).Should(Succeed())
+
+		By("Check AppRevision Created with the expected workload spec")
+		appRev := &v1beta1.ApplicationRevision{}
+		Eventually(func() error {
+			return k8sClient.Get(ctx, client.ObjectKey{Name: app.Name + "-v1", Namespace: app.GetNamespace()}, appRev)
+		}, 10*time.Second, 500*time.Millisecond).Should(Succeed())
+
+		By("Check secret Created with the expected trait-storage spec")
+		hpa := &autoscalingv1.HorizontalPodAutoscaler{}
+		Expect(k8sClient.Get(ctx, client.ObjectKey{
+			Namespace: app.GetNamespace(),
+			Name:      app.Spec.Components[0].Name,
+		}, hpa)).Should(BeNil())
+
+		Expect(k8sClient.Delete(ctx, cm)).Should(BeNil())
+		Expect(k8sClient.Delete(ctx, hpa)).Should(BeNil())
+		Expect(k8sClient.Delete(ctx, app)).Should(BeNil())
+	})
 })
 
 const (
@@ -4081,6 +4168,51 @@ spec:
         })
         errs: [ for c in patch.spec.template.spec.containers if c.err != _|_ {c.err}]
 
+`
+
+	hubCpuScalerYaml = `apiVersion: core.oam.dev/v1beta1
+kind: TraitDefinition
+metadata:
+  annotations:
+    definition.oam.dev/description: Automatically scale the component based on CPU usage.
+  labels:
+    custom.definition.oam.dev/ui-hidden: "true"
+  name: hubcpuscaler
+  namespace: vela-system
+spec:
+  appliesToWorkloads:
+    - deployments.apps
+  controlPlaneOnly: true
+  schematic:
+    cue:
+      template: |
+        outputs: hubcpuscaler: {
+        	apiVersion: "autoscaling/v1"
+        	kind:       "HorizontalPodAutoscaler"
+        	metadata: name: context.name
+        	spec: {
+        		scaleTargetRef: {
+        			apiVersion: parameter.targetAPIVersion
+        			kind:       parameter.targetKind
+        			name:       context.name
+        		}
+        		minReplicas:                    parameter.min
+        		maxReplicas:                    parameter.max
+        		targetCPUUtilizationPercentage: parameter.cpuUtil
+        	}
+        }
+        parameter: {
+        	// +usage=Specify the minimal number of replicas to which the autoscaler can scale down
+        	min: *1 | int
+        	// +usage=Specify the maximum number of of replicas to which the autoscaler can scale up
+        	max: *10 | int
+        	// +usage=Specify the average CPU utilization, for example, 50 means the CPU usage is 50%
+        	cpuUtil: *50 | int
+        	// +usage=Specify the apiVersion of scale target
+        	targetAPIVersion: *"apps/v1" | string
+        	// +usage=Specify the kind of scale target
+        	targetKind: *"Deployment" | string
+        }
 `
 )
 

pkg/controller/core.oam.dev/v1alpha2/application/apply.go

@@ -89,6 +89,9 @@ func (h *AppHandler) Dispatch(ctx context.Context, cluster string, owner common.
 		if mf == nil {
 			continue
 		}
+		if oam.GetCluster(mf) != "" {
+			cluster = oam.GetCluster(mf)
+		}
 		ref := common.ClusterObjectReference{
 			Cluster: cluster,
 			Creator: owner,

pkg/controller/core.oam.dev/v1alpha2/application/generator.go

@@ -169,7 +169,7 @@ func (h *AppHandler) renderComponentFunc(appParser *appfile.Parser, appRev *v1be
 		if err != nil {
 			return nil, nil, err
 		}
-		return renderComponentsAndTraits(h.r.Client, manifest, appRev, overrideNamespace, env)
+		return renderComponentsAndTraits(h.r.Client, manifest, appRev, clusterName, overrideNamespace, env)
 	}
 }
 
@@ -193,7 +193,7 @@ func (h *AppHandler) applyComponentFunc(appParser *appfile.Parser, appRev *v1bet
 		}
 		wl.Ctx.SetCtx(ctx)
 
-		readyWorkload, readyTraits, err := renderComponentsAndTraits(h.r.Client, manifest, appRev, overrideNamespace, env)
+		readyWorkload, readyTraits, err := renderComponentsAndTraits(h.r.Client, manifest, appRev, clusterName, overrideNamespace, env)
 		if err != nil {
 			return nil, nil, false, err
 		}
@@ -224,6 +224,21 @@ func (h *AppHandler) applyComponentFunc(appParser *appfile.Parser, appRev *v1bet
 	}
 }
 
+// overrideTraits will override cluster field to be local for traits which are control plane only
+func overrideTraits(appRev *v1beta1.ApplicationRevision, readyTraits []*unstructured.Unstructured) []*unstructured.Unstructured {
+	traits := readyTraits
+	for index, readyTrait := range readyTraits {
+		for _, trait := range appRev.Spec.TraitDefinitions {
+			if trait.Spec.ControlPlaneOnly && trait.Name == readyTrait.GetLabels()[oam.TraitTypeLabel] {
+				oam.SetCluster(traits[index], "local")
+				traits[index].SetNamespace(appRev.GetNamespace())
+				break
+			}
+		}
+	}
+	return traits
+}
+
 func (h *AppHandler) prepareWorkloadAndManifests(ctx context.Context,
 	appParser *appfile.Parser,
 	comp common.ApplicationComponent,
@@ -249,17 +264,24 @@ func (h *AppHandler) prepareWorkloadAndManifests(ctx context.Context,
 	return wl, manifest, nil
 }
 
-func renderComponentsAndTraits(client client.Client, manifest *types.ComponentManifest, appRev *v1beta1.ApplicationRevision, overrideNamespace string, env string) (*unstructured.Unstructured, []*unstructured.Unstructured, error) {
+func renderComponentsAndTraits(client client.Client, manifest *types.ComponentManifest, appRev *v1beta1.ApplicationRevision, clusterName string, overrideNamespace string, env string) (*unstructured.Unstructured, []*unstructured.Unstructured, error) {
 	readyWorkload, readyTraits, err := assemble.PrepareBeforeApply(manifest, appRev, []assemble.WorkloadOption{assemble.DiscoveryHelmBasedWorkload(context.TODO(), client)})
 	if err != nil {
 		return nil, nil, errors.WithMessage(err, "assemble resources before apply fail")
 	}
+	if clusterName != "" {
+		oam.SetCluster(readyWorkload, clusterName)
+		for _, readyTrait := range readyTraits {
+			oam.SetCluster(readyTrait, clusterName)
+		}
+	}
 	if overrideNamespace != "" {
 		readyWorkload.SetNamespace(overrideNamespace)
 		for _, readyTrait := range readyTraits {
 			readyTrait.SetNamespace(overrideNamespace)
 		}
 	}
+	readyTraits = overrideTraits(appRev, readyTraits)
 	if env != "" {
 		meta.AddLabels(readyWorkload, map[string]string{oam.LabelAppEnv: env})
 		for _, readyTrait := range readyTraits {

pkg/multicluster/utils.go

@@ -85,7 +85,9 @@ func ResourcesWithClusterName(clusterName string, objs ...*unstructured.Unstruct
 	var _objs []*unstructured.Unstructured
 	for _, obj := range objs {
 		if obj != nil {
-			oam.SetCluster(obj, clusterName)
+			if oam.GetCluster(obj) == "" {
+				oam.SetCluster(obj, clusterName)
+			}
 			_objs = append(_objs, obj)
 		}
 	}