diff --git a/apis/core.oam.dev/v1beta1/core_types.go b/apis/core.oam.dev/v1beta1/core_types.go index c49ff6578..82cbdde9b 100644 --- a/apis/core.oam.dev/v1beta1/core_types.go +++ b/apis/core.oam.dev/v1beta1/core_types.go @@ -157,6 +157,9 @@ type TraitDefinitionSpec struct { // SkipRevisionAffect defines the update this trait will not generate a new application Revision // +optional SkipRevisionAffect bool `json:"skipRevisionAffect,omitempty"` + // ControlPlaneOnly defines which cluster is dispatched to + // +optional + ControlPlaneOnly bool `json:"controlPlaneOnly,omitempty"` } // TraitDefinitionStatus is the status of TraitDefinition diff --git a/charts/oam-runtime/crds/core.oam.dev_traitdefinitions.yaml b/charts/oam-runtime/crds/core.oam.dev_traitdefinitions.yaml index ebe8e4ccd..ff8b3c4e2 100644 --- a/charts/oam-runtime/crds/core.oam.dev_traitdefinitions.yaml +++ b/charts/oam-runtime/crds/core.oam.dev_traitdefinitions.yaml @@ -372,6 +372,10 @@ spec: items: type: string type: array + controlPlaneOnly: + description: ControlPlaneOnly defines which cluster is dispatched + to + type: boolean definitionRef: description: Reference to the CustomResourceDefinition that defines this trait kind. diff --git a/charts/vela-core/crds/core.oam.dev_applicationrevisions.yaml b/charts/vela-core/crds/core.oam.dev_applicationrevisions.yaml index 4fbb238c4..d36d6a820 100644 --- a/charts/vela-core/crds/core.oam.dev_applicationrevisions.yaml +++ b/charts/vela-core/crds/core.oam.dev_applicationrevisions.yaml @@ -3585,6 +3585,10 @@ spec: items: type: string type: array + controlPlaneOnly: + description: ControlPlaneOnly defines which cluster is dispatched + to + type: boolean definitionRef: description: Reference to the CustomResourceDefinition that defines this trait kind. diff --git a/charts/vela-core/crds/core.oam.dev_definitionrevisions.yaml b/charts/vela-core/crds/core.oam.dev_definitionrevisions.yaml index fb6c5650f..d1608bc82 100644 --- a/charts/vela-core/crds/core.oam.dev_definitionrevisions.yaml +++ b/charts/vela-core/crds/core.oam.dev_definitionrevisions.yaml @@ -724,6 +724,10 @@ spec: items: type: string type: array + controlPlaneOnly: + description: ControlPlaneOnly defines which cluster is dispatched + to + type: boolean definitionRef: description: Reference to the CustomResourceDefinition that defines this trait kind. diff --git a/charts/vela-core/crds/core.oam.dev_traitdefinitions.yaml b/charts/vela-core/crds/core.oam.dev_traitdefinitions.yaml index ebe8e4ccd..ff8b3c4e2 100644 --- a/charts/vela-core/crds/core.oam.dev_traitdefinitions.yaml +++ b/charts/vela-core/crds/core.oam.dev_traitdefinitions.yaml @@ -372,6 +372,10 @@ spec: items: type: string type: array + controlPlaneOnly: + description: ControlPlaneOnly defines which cluster is dispatched + to + type: boolean definitionRef: description: Reference to the CustomResourceDefinition that defines this trait kind. diff --git a/charts/vela-minimal/crds/core.oam.dev_applicationrevisions.yaml b/charts/vela-minimal/crds/core.oam.dev_applicationrevisions.yaml index 4fbb238c4..d36d6a820 100644 --- a/charts/vela-minimal/crds/core.oam.dev_applicationrevisions.yaml +++ b/charts/vela-minimal/crds/core.oam.dev_applicationrevisions.yaml @@ -3585,6 +3585,10 @@ spec: items: type: string type: array + controlPlaneOnly: + description: ControlPlaneOnly defines which cluster is dispatched + to + type: boolean definitionRef: description: Reference to the CustomResourceDefinition that defines this trait kind. diff --git a/charts/vela-minimal/crds/core.oam.dev_definitionrevisions.yaml b/charts/vela-minimal/crds/core.oam.dev_definitionrevisions.yaml index fb6c5650f..d1608bc82 100644 --- a/charts/vela-minimal/crds/core.oam.dev_definitionrevisions.yaml +++ b/charts/vela-minimal/crds/core.oam.dev_definitionrevisions.yaml @@ -724,6 +724,10 @@ spec: items: type: string type: array + controlPlaneOnly: + description: ControlPlaneOnly defines which cluster is dispatched + to + type: boolean definitionRef: description: Reference to the CustomResourceDefinition that defines this trait kind. diff --git a/charts/vela-minimal/crds/core.oam.dev_traitdefinitions.yaml b/charts/vela-minimal/crds/core.oam.dev_traitdefinitions.yaml index ebe8e4ccd..ff8b3c4e2 100644 --- a/charts/vela-minimal/crds/core.oam.dev_traitdefinitions.yaml +++ b/charts/vela-minimal/crds/core.oam.dev_traitdefinitions.yaml @@ -372,6 +372,10 @@ spec: items: type: string type: array + controlPlaneOnly: + description: ControlPlaneOnly defines which cluster is dispatched + to + type: boolean definitionRef: description: Reference to the CustomResourceDefinition that defines this trait kind. diff --git a/legacy/charts/vela-core-legacy/crds/core.oam.dev_applicationrevisions.yaml b/legacy/charts/vela-core-legacy/crds/core.oam.dev_applicationrevisions.yaml index 8281e121b..5e7e2a27c 100644 --- a/legacy/charts/vela-core-legacy/crds/core.oam.dev_applicationrevisions.yaml +++ b/legacy/charts/vela-core-legacy/crds/core.oam.dev_applicationrevisions.yaml @@ -3585,6 +3585,10 @@ spec: items: type: string type: array + controlPlaneOnly: + description: ControlPlaneOnly defines which cluster is dispatched + to + type: boolean definitionRef: description: Reference to the CustomResourceDefinition that defines this trait kind. diff --git a/legacy/charts/vela-core-legacy/crds/core.oam.dev_definitionrevisions.yaml b/legacy/charts/vela-core-legacy/crds/core.oam.dev_definitionrevisions.yaml index a33e02b27..287b7ed57 100644 --- a/legacy/charts/vela-core-legacy/crds/core.oam.dev_definitionrevisions.yaml +++ b/legacy/charts/vela-core-legacy/crds/core.oam.dev_definitionrevisions.yaml @@ -724,6 +724,10 @@ spec: items: type: string type: array + controlPlaneOnly: + description: ControlPlaneOnly defines which cluster is dispatched + to + type: boolean definitionRef: description: Reference to the CustomResourceDefinition that defines this trait kind. diff --git a/legacy/charts/vela-core-legacy/crds/core.oam.dev_traitdefinitions.yaml b/legacy/charts/vela-core-legacy/crds/core.oam.dev_traitdefinitions.yaml index a9075f6a9..4439d4d64 100644 --- a/legacy/charts/vela-core-legacy/crds/core.oam.dev_traitdefinitions.yaml +++ b/legacy/charts/vela-core-legacy/crds/core.oam.dev_traitdefinitions.yaml @@ -372,6 +372,10 @@ spec: items: type: string type: array + controlPlaneOnly: + description: ControlPlaneOnly defines which cluster is dispatched + to + type: boolean definitionRef: description: Reference to the CustomResourceDefinition that defines this trait kind. diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller_test.go b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller_test.go index ee9caa0cf..99ab9d2c6 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/application_controller_test.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/application_controller_test.go @@ -32,6 +32,7 @@ import ( "github.com/google/go-cmp/cmp" v1 "k8s.io/api/apps/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" v1beta12 "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -293,6 +294,31 @@ var _ = Describe("Test Application Controller", func() { }, } + appWithControlPlaneOnly := &v1beta1.Application{ + TypeMeta: metav1.TypeMeta{ + Kind: "Application", + APIVersion: "core.oam.dev/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "app-controlplaneonly", + }, + Spec: v1beta1.ApplicationSpec{ + Components: []common.ApplicationComponent{ + { + Name: "app-controlplaneonly-component", + Type: "worker", + Properties: &runtime.RawExtension{Raw: []byte("{\"cmd\":[\"sleep\",\"1000\"],\"image\":\"busybox\"}")}, + }, + }, + }, + } + appWithControlPlaneOnly.Spec.Components[0].Traits = []common.ApplicationTrait{ + { + Type: "hubcpuscaler", + Properties: &runtime.RawExtension{Raw: []byte("{\"min\": 1,\"max\": 10,\"cpuPercent\": 60}")}, + }, + } + appWithMountToEnvs := &v1beta1.Application{ TypeMeta: metav1.TypeMeta{ Kind: "Application", @@ -339,6 +365,8 @@ var _ = Describe("Test Application Controller", func() { importEnv := &v1alpha2.TraitDefinition{} + importHubCpuScaler := &v1beta1.TraitDefinition{} + webserverwd := &v1alpha2.ComponentDefinition{} webserverwdJson, _ := yaml.YAMLToJSON([]byte(webComponentDefYaml)) @@ -380,6 +408,11 @@ var _ = Describe("Test Application Controller", func() { Expect(json.Unmarshal(envJson, importEnv)).Should(BeNil()) Expect(k8sClient.Create(ctx, importEnv.DeepCopy())).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + hubCpuScalerJson, hubCpuScalerErr := yaml.YAMLToJSON([]byte(hubCpuScalerYaml)) + Expect(hubCpuScalerErr).ShouldNot(HaveOccurred()) + Expect(json.Unmarshal(hubCpuScalerJson, importHubCpuScaler)).Should(BeNil()) + Expect(k8sClient.Create(ctx, importHubCpuScaler.DeepCopy())).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) + Expect(json.Unmarshal(tDDefJson, td)).Should(BeNil()) Expect(k8sClient.Create(ctx, td.DeepCopy())).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) @@ -2716,6 +2749,60 @@ var _ = Describe("Test Application Controller", func() { Expect(k8sClient.Delete(ctx, app)).Should(BeNil()) }) + + It("test application with controlPlaneOnly trait ", func() { + + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vela-test-with-controlplaneonly", + }, + } + Expect(k8sClient.Create(ctx, ns)).Should(BeNil()) + + appWithControlPlaneOnly.SetNamespace(ns.Name) + app := appWithControlPlaneOnly.DeepCopy() + Expect(k8sClient.Create(ctx, app)).Should(BeNil()) + + appKey := client.ObjectKey{ + Name: app.Name, + Namespace: app.Namespace, + } + testutil.ReconcileOnceAfterFinalizer(reconciler, reconcile.Request{NamespacedName: appKey}) + + By("Check App running successfully") + curApp := &v1beta1.Application{} + Expect(k8sClient.Get(ctx, appKey, curApp)).Should(BeNil()) + Expect(curApp.Status.Phase).Should(Equal(common.ApplicationRunning)) + + appRevision := &v1beta1.ApplicationRevision{} + Expect(k8sClient.Get(ctx, client.ObjectKey{ + Namespace: app.Namespace, + Name: curApp.Status.LatestRevision.Name, + }, appRevision)).Should(BeNil()) + + By("Check affiliated resource tracker is created") + expectRTName := fmt.Sprintf("%s-%s", appRevision.GetName(), appRevision.GetNamespace()) + Eventually(func() error { + return k8sClient.Get(ctx, client.ObjectKey{Name: expectRTName}, &v1beta1.ResourceTracker{}) + }, 10*time.Second, 500*time.Millisecond).Should(Succeed()) + + By("Check AppRevision Created with the expected workload spec") + appRev := &v1beta1.ApplicationRevision{} + Eventually(func() error { + return k8sClient.Get(ctx, client.ObjectKey{Name: app.Name + "-v1", Namespace: app.GetNamespace()}, appRev) + }, 10*time.Second, 500*time.Millisecond).Should(Succeed()) + + By("Check secret Created with the expected trait-storage spec") + hpa := &autoscalingv1.HorizontalPodAutoscaler{} + Expect(k8sClient.Get(ctx, client.ObjectKey{ + Namespace: app.GetNamespace(), + Name: app.Spec.Components[0].Name, + }, hpa)).Should(BeNil()) + + Expect(k8sClient.Delete(ctx, cm)).Should(BeNil()) + Expect(k8sClient.Delete(ctx, hpa)).Should(BeNil()) + Expect(k8sClient.Delete(ctx, app)).Should(BeNil()) + }) }) const ( @@ -4081,6 +4168,51 @@ spec: }) errs: [ for c in patch.spec.template.spec.containers if c.err != _|_ {c.err}] +` + + hubCpuScalerYaml = `apiVersion: core.oam.dev/v1beta1 +kind: TraitDefinition +metadata: + annotations: + definition.oam.dev/description: Automatically scale the component based on CPU usage. + labels: + custom.definition.oam.dev/ui-hidden: "true" + name: hubcpuscaler + namespace: vela-system +spec: + appliesToWorkloads: + - deployments.apps + controlPlaneOnly: true + schematic: + cue: + template: | + outputs: hubcpuscaler: { + apiVersion: "autoscaling/v1" + kind: "HorizontalPodAutoscaler" + metadata: name: context.name + spec: { + scaleTargetRef: { + apiVersion: parameter.targetAPIVersion + kind: parameter.targetKind + name: context.name + } + minReplicas: parameter.min + maxReplicas: parameter.max + targetCPUUtilizationPercentage: parameter.cpuUtil + } + } + parameter: { + // +usage=Specify the minimal number of replicas to which the autoscaler can scale down + min: *1 | int + // +usage=Specify the maximum number of of replicas to which the autoscaler can scale up + max: *10 | int + // +usage=Specify the average CPU utilization, for example, 50 means the CPU usage is 50% + cpuUtil: *50 | int + // +usage=Specify the apiVersion of scale target + targetAPIVersion: *"apps/v1" | string + // +usage=Specify the kind of scale target + targetKind: *"Deployment" | string + } ` ) diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/apply.go b/pkg/controller/core.oam.dev/v1alpha2/application/apply.go index 1be10d584..23aa3aa92 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/apply.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/apply.go @@ -89,6 +89,9 @@ func (h *AppHandler) Dispatch(ctx context.Context, cluster string, owner common. if mf == nil { continue } + if oam.GetCluster(mf) != "" { + cluster = oam.GetCluster(mf) + } ref := common.ClusterObjectReference{ Cluster: cluster, Creator: owner, diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/generator.go b/pkg/controller/core.oam.dev/v1alpha2/application/generator.go index 384d1d639..74223acad 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/generator.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/generator.go @@ -169,7 +169,7 @@ func (h *AppHandler) renderComponentFunc(appParser *appfile.Parser, appRev *v1be if err != nil { return nil, nil, err } - return renderComponentsAndTraits(h.r.Client, manifest, appRev, overrideNamespace, env) + return renderComponentsAndTraits(h.r.Client, manifest, appRev, clusterName, overrideNamespace, env) } } @@ -193,7 +193,7 @@ func (h *AppHandler) applyComponentFunc(appParser *appfile.Parser, appRev *v1bet } wl.Ctx.SetCtx(ctx) - readyWorkload, readyTraits, err := renderComponentsAndTraits(h.r.Client, manifest, appRev, overrideNamespace, env) + readyWorkload, readyTraits, err := renderComponentsAndTraits(h.r.Client, manifest, appRev, clusterName, overrideNamespace, env) if err != nil { return nil, nil, false, err } @@ -224,6 +224,21 @@ func (h *AppHandler) applyComponentFunc(appParser *appfile.Parser, appRev *v1bet } } +// overrideTraits will override cluster field to be local for traits which are control plane only +func overrideTraits(appRev *v1beta1.ApplicationRevision, readyTraits []*unstructured.Unstructured) []*unstructured.Unstructured { + traits := readyTraits + for index, readyTrait := range readyTraits { + for _, trait := range appRev.Spec.TraitDefinitions { + if trait.Spec.ControlPlaneOnly && trait.Name == readyTrait.GetLabels()[oam.TraitTypeLabel] { + oam.SetCluster(traits[index], "local") + traits[index].SetNamespace(appRev.GetNamespace()) + break + } + } + } + return traits +} + func (h *AppHandler) prepareWorkloadAndManifests(ctx context.Context, appParser *appfile.Parser, comp common.ApplicationComponent, @@ -249,17 +264,24 @@ func (h *AppHandler) prepareWorkloadAndManifests(ctx context.Context, return wl, manifest, nil } -func renderComponentsAndTraits(client client.Client, manifest *types.ComponentManifest, appRev *v1beta1.ApplicationRevision, overrideNamespace string, env string) (*unstructured.Unstructured, []*unstructured.Unstructured, error) { +func renderComponentsAndTraits(client client.Client, manifest *types.ComponentManifest, appRev *v1beta1.ApplicationRevision, clusterName string, overrideNamespace string, env string) (*unstructured.Unstructured, []*unstructured.Unstructured, error) { readyWorkload, readyTraits, err := assemble.PrepareBeforeApply(manifest, appRev, []assemble.WorkloadOption{assemble.DiscoveryHelmBasedWorkload(context.TODO(), client)}) if err != nil { return nil, nil, errors.WithMessage(err, "assemble resources before apply fail") } + if clusterName != "" { + oam.SetCluster(readyWorkload, clusterName) + for _, readyTrait := range readyTraits { + oam.SetCluster(readyTrait, clusterName) + } + } if overrideNamespace != "" { readyWorkload.SetNamespace(overrideNamespace) for _, readyTrait := range readyTraits { readyTrait.SetNamespace(overrideNamespace) } } + readyTraits = overrideTraits(appRev, readyTraits) if env != "" { meta.AddLabels(readyWorkload, map[string]string{oam.LabelAppEnv: env}) for _, readyTrait := range readyTraits { diff --git a/pkg/multicluster/utils.go b/pkg/multicluster/utils.go index f971c9e35..a9985d2df 100644 --- a/pkg/multicluster/utils.go +++ b/pkg/multicluster/utils.go @@ -85,7 +85,9 @@ func ResourcesWithClusterName(clusterName string, objs ...*unstructured.Unstruct var _objs []*unstructured.Unstructured for _, obj := range objs { if obj != nil { - oam.SetCluster(obj, clusterName) + if oam.GetCluster(obj) == "" { + oam.SetCluster(obj, clusterName) + } _objs = append(_objs, obj) } }