Feat: automatically populate the provider and region configuration for cloud app (#2841)

* Feat: automatically populate the provider and region configuration for cloud app.

* Fix: fix e2e test bug

* Fix: component name and type inconsistency

* Fix: fix unit test bug

* Fix: fix unit test bug

Co-authored-by: barnettZQG <yiyun.pro>

pkg/apiserver/rest/usecase/application.go

@@ -96,10 +96,11 @@ type applicationUsecaseImpl struct {
 	workflowUsecase       WorkflowUsecase
 	envBindingUsecase     EnvBindingUsecase
 	deliveryTargetUsecase DeliveryTargetUsecase
+	definitionUsecase     DefinitionUsecase
 }
 
 // NewApplicationUsecase new application usecase
-func NewApplicationUsecase(ds datastore.DataStore, workflowUsecase WorkflowUsecase, envBindingUsecase EnvBindingUsecase, deliveryTargetUsecase DeliveryTargetUsecase) ApplicationUsecase {
+func NewApplicationUsecase(ds datastore.DataStore, workflowUsecase WorkflowUsecase, envBindingUsecase EnvBindingUsecase, deliveryTargetUsecase DeliveryTargetUsecase, definitionUsecase DefinitionUsecase) ApplicationUsecase {
 	kubecli, err := clients.GetKubeClient()
 	if err != nil {
 		log.Logger.Fatalf("get kubeclient failure %s", err.Error())
@@ -111,6 +112,7 @@ func NewApplicationUsecase(ds datastore.DataStore, workflowUsecase WorkflowUseca
 		deliveryTargetUsecase: deliveryTargetUsecase,
 		kubeClient:            kubecli,
 		apply:                 apply.NewAPIApplicator(kubecli),
+		definitionUsecase:     definitionUsecase,
 	}
 }
 
@@ -309,7 +311,7 @@ func (c *applicationUsecaseImpl) CreateApplication(ctx context.Context, req apis
 	return base, nil
 }
 
-func (c *applicationUsecaseImpl) genPolicyByEnv(ctx context.Context, app *model.Application, envName string) (v1beta1.AppPolicy, error) {
+func (c *applicationUsecaseImpl) genPolicyByEnv(ctx context.Context, app *model.Application, envName string, components []*model.ApplicationComponent) (v1beta1.AppPolicy, error) {
 	appPolicy := v1beta1.AppPolicy{}
 	envBinding, err := c.envBindingUsecase.GetEnvBinding(ctx, app, envName)
 	if err != nil {
@@ -324,7 +326,7 @@ func (c *applicationUsecaseImpl) genPolicyByEnv(ctx context.Context, app *model.
 		if err != nil || target == nil {
 			return appPolicy, bcode.ErrFoundEnvbindingDeliveryTarget
 		}
-		envBindingSpec.Envs = append(envBindingSpec.Envs, createTargetClusterEnv(envBinding, target))
+		envBindingSpec.Envs = append(envBindingSpec.Envs, c.createTargetClusterEnv(ctx, app, envBinding, target, components))
 	}
 	properties, err := model.NewJSONStructByStruct(envBindingSpec)
 	if err != nil {
@@ -733,9 +735,10 @@ func (c *applicationUsecaseImpl) renderOAMApplication(ctx context.Context, appMo
 	if err != nil {
 		return nil, err
 	}
-
+	var componentModels []*model.ApplicationComponent
 	for _, entity := range components {
 		component := entity.(*model.ApplicationComponent)
+		componentModels = append(componentModels, component)
 		var traits []common.ApplicationTrait
 		for _, trait := range component.Traits {
 			aTrait := common.ApplicationTrait{
@@ -775,7 +778,7 @@ func (c *applicationUsecaseImpl) renderOAMApplication(ctx context.Context, appMo
 		app.Spec.Policies = append(app.Spec.Policies, apolicy)
 	}
 	if workflow.EnvName != "" {
-		envPolicy, err := c.genPolicyByEnv(ctx, appModel, workflow.EnvName)
+		envPolicy, err := c.genPolicyByEnv(ctx, appModel, workflow.EnvName, componentModels)
 		if err != nil {
 			return nil, err
 		}
@@ -1214,7 +1217,7 @@ func (c *applicationUsecaseImpl) Statistics(ctx context.Context, app *model.Appl
 	}, nil
 }
 
-func createTargetClusterEnv(envBind *model.EnvBinding, target *model.DeliveryTarget) v1alpha1.EnvConfig {
+func (c *applicationUsecaseImpl) createTargetClusterEnv(ctx context.Context, app *model.Application, envBind *model.EnvBinding, target *model.DeliveryTarget, components []*model.ApplicationComponent) v1alpha1.EnvConfig {
 	placement := v1alpha1.EnvPlacement{}
 	var componentSelector *v1alpha1.EnvSelector
 	if envBind.ComponentSelector != nil {
@@ -1226,10 +1229,53 @@ func createTargetClusterEnv(envBind *model.EnvBinding, target *model.DeliveryTar
 		placement.ClusterSelector = &common.ClusterSelector{Name: target.Cluster.ClusterName}
 		placement.NamespaceSelector = &v1alpha1.NamespaceSelector{Name: target.Cluster.Namespace}
 	}
+	var componentPatchs []v1alpha1.EnvComponentPatch
+	// init cloud application region and provider info
+	for _, component := range components {
+		definition, err := c.definitionUsecase.GetComponentDefinition(ctx, component.Type)
+		if err != nil {
+			log.Logger.Errorf("get component definition %s failure %s", component.Type, err.Error())
+			continue
+		}
+		if definition != nil {
+			if definition.Spec.Workload.Type == TerraformWorkfloadType {
+				properties := model.JSONStruct{
+					"providerRef": map[string]interface{}{
+						"name":      "",
+						"namespace": "default",
+					},
+					"region": "",
+					"writeConnectionSecretToRef": map[string]interface{}{
+						"name":      fmt.Sprintf("%s-%s", component.Name, envBind.Name),
+						"namespace": app.Namespace,
+					},
+				}
+				if region, ok := target.Variable["region"]; ok {
+					properties["region"] = region
+				}
+				if providerName, ok := target.Variable["providerName"]; ok {
+					properties["providerRef"].(map[string]interface{})["name"] = providerName
+				}
+				if providerNamespace, ok := target.Variable["providerNamespace"]; ok {
+					properties["providerRef"].(map[string]interface{})["namespace"] = providerNamespace
+				}
+				log.Logger.Info(properties)
+				componentPatchs = append(componentPatchs, v1alpha1.EnvComponentPatch{
+					Name:       converComponentName(component.Name, envBind.Name),
+					Properties: properties.RawExtension(),
+					Type:       component.Type,
+				})
+			}
+		}
+	}
+
 	return v1alpha1.EnvConfig{
 		Name:      genPolicyEnvName(target.Name),
 		Placement: placement,
 		Selector:  componentSelector,
+		Patch: v1alpha1.EnvPatch{
+			Components: componentPatchs,
+		},
 	}
 }
 

pkg/apiserver/rest/usecase/application_test.go

@@ -26,12 +26,16 @@ import (
 	"github.com/google/go-cmp/cmp/cmpopts"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	k8stypes "k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/common"
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
+	"github.com/oam-dev/kubevela/apis/types"
 	"github.com/oam-dev/kubevela/pkg/apiserver/model"
 	v1 "github.com/oam-dev/kubevela/pkg/apiserver/rest/apis/v1"
 	"github.com/oam-dev/kubevela/pkg/apiserver/rest/utils/bcode"
@@ -58,6 +62,7 @@ var _ = Describe("Test application usecase function", func() {
 			apply:                 apply.NewAPIApplicator(k8sClient),
 			kubeClient:            k8sClient,
 			envBindingUsecase:     envBindingUsecase,
+			definitionUsecase:     definitionUsecase,
 			deliveryTargetUsecase: deliveryTargetUsecase,
 		}
 	})
@@ -491,6 +496,54 @@ var _ = Describe("Test application usecase function", func() {
 		Expect(err).Should(BeNil())
 		Expect(resp.Total).Should(Equal(int64(3)))
 	})
+
+	It("Test createTargetClusterEnv function", func() {
+		var namespace corev1.Namespace
+		err := k8sClient.Get(context.TODO(), k8stypes.NamespacedName{Name: types.DefaultKubeVelaNS}, &namespace)
+		if apierrors.IsNotFound(err) {
+			err := k8sClient.Create(context.TODO(), &corev1.Namespace{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: types.DefaultKubeVelaNS,
+				},
+			})
+			Expect(err).Should(BeNil())
+		} else {
+			Expect(err).Should(BeNil())
+		}
+		definition := &v1beta1.ComponentDefinition{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "aliyun-rds",
+				Namespace: types.DefaultKubeVelaNS,
+			},
+			Spec: v1beta1.ComponentDefinitionSpec{
+				Workload: common.WorkloadTypeDescriptor{
+					Type: TerraformWorkfloadType,
+				},
+			},
+		}
+		err = k8sClient.Create(context.TODO(), definition)
+		Expect(err).Should(BeNil())
+		envConfig := appUsecase.createTargetClusterEnv(context.TODO(), &model.Application{
+			Namespace: "prod",
+		}, &model.EnvBinding{
+			TargetNames: []string{"prod"},
+		}, &model.DeliveryTarget{
+			Name: "prod",
+			Variable: map[string]interface{}{
+				"region":       "hangzhou",
+				"providerName": "aliyun",
+			},
+		}, []*model.ApplicationComponent{
+			{
+				Name: "component1",
+				Type: "aliyun-rds",
+			},
+		})
+		Expect(cmp.Diff(len(envConfig.Patch.Components), 1)).Should(BeEmpty())
+		Expect(cmp.Diff(strings.Contains(string(envConfig.Patch.Components[0].Properties.Raw), "aliyun"), true)).Should(BeEmpty())
+		err = k8sClient.Delete(context.TODO(), definition)
+		Expect(err).Should(BeNil())
+	})
 })
 
 func createTestSuspendApp(ctx context.Context, appName, envName, revisionVersion, wfName, recordName string, kubeClient client.Client) (*v1beta1.Application, error) {

pkg/apiserver/rest/usecase/definition_test.go

@@ -60,11 +60,17 @@ var _ = Describe("Test namespace usecase functions", func() {
 		Expect(err).Should(Succeed())
 		err = k8sClient.Create(context.Background(), &cd)
 		Expect(err).Should(Succeed())
-		components, err := definitionUsecase.ListDefinitions(context.TODO(), "", "component", "")
+		definitions, err := definitionUsecase.ListDefinitions(context.TODO(), "", "component", "")
 		Expect(err).Should(BeNil())
-		Expect(cmp.Diff(len(components), 1)).Should(BeEmpty())
-		Expect(cmp.Diff(components[0].Name, "webservice-test")).Should(BeEmpty())
-		Expect(components[0].Description).ShouldNot(BeEmpty())
+		var selectDefinition *v1.DefinitionBase
+		for i, definition := range definitions {
+			if definition.WorkloadType == "deployments.apps" {
+				selectDefinition = definitions[i]
+			}
+		}
+		Expect(selectDefinition).ShouldNot(BeNil())
+		Expect(cmp.Diff(selectDefinition.Name, "webservice-test")).Should(BeEmpty())
+		Expect(selectDefinition.Description).ShouldNot(BeEmpty())
 
 		By("List trait definitions")
 		myingress, err := ioutil.ReadFile("./testdata/myingress-td.yaml")

pkg/apiserver/rest/usecase/testdata/ui-schema.yaml

@@ -51,19 +51,6 @@
     - valueFrom
     label: Add By Secret
   subParameters:
-  - description: Environment variable name
-    jsonKey: name
-    label: Name
-    sort: 100
-    uiType: Input
-    validate:
-      required: true
-  - description: The value of the environment variable
-    jsonKey: value
-    label: Value
-    sort: 100
-    uiType: Input
-    validate: {}
   - description: Specifies a source the value of this var should come from
     jsonKey: valueFrom
     label: Secret Selector
@@ -94,6 +81,19 @@
         required: true
     uiType: InnerGroup
     validate: {}
+  - description: Environment variable name
+    jsonKey: name
+    label: Name
+    sort: 100
+    uiType: Input
+    validate:
+      required: true
+  - description: The value of the environment variable
+    jsonKey: value
+    label: Value
+    sort: 100
+    uiType: Input
+    validate: {}
   uiType: Structs
   validate: {}
 - description: Instructions for assessing whether the container is in a suitable state
@@ -182,6 +182,14 @@
     label: HttpGet
     sort: 100
     subParameters:
+    - description: The endpoint, relative to the port, to which the HTTP GET request
+        should be directed.
+      jsonKey: path
+      label: Path
+      sort: 100
+      uiType: Input
+      validate:
+        required: true
     - description: The TCP socket within the container to which the HTTP GET request
         should be directed.
       jsonKey: port
@@ -211,14 +219,6 @@
           required: true
       uiType: Structs
       validate: {}
-    - description: The endpoint, relative to the port, to which the HTTP GET request
-        should be directed.
-      jsonKey: path
-      label: Path
-      sort: 100
-      uiType: Input
-      validate:
-        required: true
     uiType: Group
     validate: {}
   - description: Number of seconds after the container is started before the first
@@ -237,53 +237,6 @@
   label: LivenessProbe
   sort: 15
   subParameters:
-  - description: Instructions for assessing container health by executing an HTTP
-      GET request. Either this attribute or the exec attribute or the tcpSocket attribute
-      MUST be specified. This attribute is mutually exclusive with both the exec attribute
-      and the tcpSocket attribute.
-    jsonKey: httpGet
-    label: HttpGet
-    sort: 100
-    subParameters:
-    - description: ""
-      jsonKey: httpHeaders
-      label: HttpHeaders
-      sort: 100
-      subParameters:
-      - description: ""
-        jsonKey: name
-        label: Name
-        sort: 100
-        uiType: Input
-        validate:
-          required: true
-      - description: ""
-        jsonKey: value
-        label: Value
-        sort: 100
-        uiType: Input
-        validate:
-          required: true
-      uiType: Structs
-      validate: {}
-    - description: The endpoint, relative to the port, to which the HTTP GET request
-        should be directed.
-      jsonKey: path
-      label: Path
-      sort: 100
-      uiType: Input
-      validate:
-        required: true
-    - description: The TCP socket within the container to which the HTTP GET request
-        should be directed.
-      jsonKey: port
-      label: Port
-      sort: 100
-      uiType: Number
-      validate:
-        required: true
-    uiType: Group
-    validate: {}
   - description: Number of seconds after the container is started before the first
       probe is initiated.
     jsonKey: initialDelaySeconds
@@ -365,8 +318,61 @@
     validate:
       defaultValue: 3
       required: true
+  - description: Instructions for assessing container health by executing an HTTP
+      GET request. Either this attribute or the exec attribute or the tcpSocket attribute
+      MUST be specified. This attribute is mutually exclusive with both the exec attribute
+      and the tcpSocket attribute.
+    jsonKey: httpGet
+    label: HttpGet
+    sort: 100
+    subParameters:
+    - description: ""
+      jsonKey: httpHeaders
+      label: HttpHeaders
+      sort: 100
+      subParameters:
+      - description: ""
+        jsonKey: name
+        label: Name
+        sort: 100
+        uiType: Input
+        validate:
+          required: true
+      - description: ""
+        jsonKey: value
+        label: Value
+        sort: 100
+        uiType: Input
+        validate:
+          required: true
+      uiType: Structs
+      validate: {}
+    - description: The endpoint, relative to the port, to which the HTTP GET request
+        should be directed.
+      jsonKey: path
+      label: Path
+      sort: 100
+      uiType: Input
+      validate:
+        required: true
+    - description: The TCP socket within the container to which the HTTP GET request
+        should be directed.
+      jsonKey: port
+      label: Port
+      sort: 100
+      uiType: Number
+      validate:
+        required: true
+    uiType: Group
+    validate: {}
   uiType: Group
   validate: {}
+- description: Specify image pull secrets for your service
+  jsonKey: imagePullSecrets
+  label: ImagePullSecrets
+  sort: 100
+  uiType: Strings
+  validate: {}
 - description: Which port do you want customer traffic sent to
   disable: true
   jsonKey: port
@@ -376,22 +382,6 @@
   validate:
     defaultValue: 80
     required: true
-- description: Specify image pull secrets for your service
-  jsonKey: imagePullSecrets
-  label: ImagePullSecrets
-  sort: 100
-  uiType: Strings
-  validate: {}
-- description: If addRevisionLabel is true, the appRevision label will be added to
-    the underlying pods
-  disable: true
-  jsonKey: addRevisionLabel
-  label: AddRevisionLabel
-  sort: 100
-  uiType: Switch
-  validate:
-    defaultValue: false
-    required: true
 - description: Declare volumes and volumeMounts
   disable: true
   jsonKey: volumes
@@ -430,3 +420,13 @@
       required: true
   uiType: Structs
   validate: {}
+- description: If addRevisionLabel is true, the appRevision label will be added to
+    the underlying pods
+  disable: true
+  jsonKey: addRevisionLabel
+  label: AddRevisionLabel
+  sort: 100
+  uiType: Switch
+  validate:
+    defaultValue: false
+    required: true

pkg/apiserver/rest/usecase/testdata/webserver-cd.yaml

@@ -249,6 +249,7 @@ spec:
         	failureThreshold: *3 | int
         }
   workload:
+    type: deployments.apps
     definition:
       apiVersion: apps/v1
       kind: Deployment

pkg/apiserver/rest/webservice/webservice.go

@@ -67,7 +67,7 @@ func Init(ds datastore.DataStore) {
 	definitionUsecase := usecase.NewDefinitionUsecase()
 	addonUsecase := usecase.NewAddonUsecase(ds)
 	envBindingUsecase := usecase.NewEnvBindingUsecase(ds, workflowUsecase, definitionUsecase)
-	applicationUsecase := usecase.NewApplicationUsecase(ds, workflowUsecase, envBindingUsecase, deliveryTargetUsecase)
+	applicationUsecase := usecase.NewApplicationUsecase(ds, workflowUsecase, envBindingUsecase, deliveryTargetUsecase, definitionUsecase)
 	RegistWebService(NewClusterWebService(clusterUsecase))
 	RegistWebService(NewApplicationWebService(applicationUsecase, envBindingUsecase, workflowUsecase))
 	RegistWebService(NewNamespaceWebService(namespaceUsecase))