Feat: add support for envbinding with namespace selector (#2432)

* Feat: add support for envbinding with namespace selector

* Fix: service account kubeconfig e2e-test

* Docs: add comments to explain functions

* Docs: add envbinding example

docs/examples/envbinding/example.yaml

@@ -0,0 +1,78 @@
+apiVersion: core.oam.dev/v1beta1
+kind: Application
+metadata:
+  name: example-app
+  namespace: default
+spec:
+  components:
+    - name: hello-world-server
+      type: webservice
+      properties:
+        image: crccheck/hello-world
+        port: 8000
+      traits:
+        - type: scaler
+          properties:
+            replicas: 1
+    - name: data-worker
+      type: worker
+      properties:
+        image: busybox
+        cmd:
+          - sleep
+          - '1000000'
+  policies:
+    - name: example-multi-env-policy
+      type: env-binding
+      properties:
+        envs:
+          - name: test
+            placement: # selecting the namespace (in local cluster) to deploy to
+              namespaceSelector:
+                name: test
+            selector:
+              components:
+                - data-worker
+
+          - name: staging
+            placement: # selecting the cluster to deploy to
+              clusterSelector:
+                name: cluster-worker
+
+          - name: prod
+            placement: # selecting both namespace and cluster to deploy to
+              clusterSelector:
+                name: cluster-worker
+              namespaceSelector:
+                name: prod
+            patch: # overlay patch on above components
+              components:
+                - name: hello-world-server
+                  type: webservice
+                  traits:
+                    - type: scaler
+                      properties:
+                        replicas: 3
+
+  workflow:
+    steps:
+      # deploy to test env
+      - name: deploy-test
+        type: deploy2env
+        properties:
+          policy: example-multi-env-policy
+          env: test
+
+      # deploy to staging env
+      - name: deploy-staging
+        type: deploy2env
+        properties:
+          policy: example-multi-env-policy
+          env: staging
+
+      # deploy to prod env
+      - name: deploy-prod
+        type: deploy2env
+        properties:
+          policy: example-multi-env-policy
+          env: prod

pkg/controller/core.oam.dev/v1alpha1/envbinding/cluster_gateway_engine.go

@@ -32,6 +32,11 @@ import (
 	"github.com/oam-dev/kubevela/pkg/multicluster"
 )
 
+const (
+	// OverrideNamespaceLabelKey identifies the override namespace for patched Application
+	OverrideNamespaceLabelKey = "envbinding.oam.dev/override-namespace"
+)
+
 // ClusterGatewayEngine construct the multicluster engine of using cluster-gateway
 type ClusterGatewayEngine struct {
 	client.Client
@@ -47,34 +52,43 @@ func NewClusterGatewayEngine(cli client.Client, envBindingName string) ClusterMa
 	}
 }
 
-// TODO only support cluster name now, should support selector and namespace later
+// TODO only support single cluster name and namespace name now, should support label selector
 func (engine *ClusterGatewayEngine) prepare(ctx context.Context, configs []v1alpha1.EnvConfig) error {
 	engine.clusterDecisions = make(map[string]v1alpha1.ClusterDecision)
-	clusterNameToConfig := make(map[string]string)
+	locationToConfig := make(map[string]string)
 	for _, config := range configs {
+		var namespace, clusterName string
+		// check if namespace selector is valid
 		if config.Placement.NamespaceSelector != nil {
-			return errors.Errorf("invalid env %s: namespace selector in cluster-gateway is not supported now", config.Name)
+			if len(config.Placement.NamespaceSelector.Labels) != 0 {
+				return errors.Errorf("invalid env %s: namespace selector in cluster-gateway does not support label selector for now", config.Name)
+			}
+			namespace = config.Placement.NamespaceSelector.Name
 		}
-		if config.Placement.ClusterSelector == nil {
-			return errors.Errorf("invalid env %s: cluster selector must be set for now", config.Name)
+		// check if cluster selector is valid
+		if config.Placement.ClusterSelector != nil {
+			if len(config.Placement.ClusterSelector.Labels) != 0 {
+				return errors.Errorf("invalid env %s: cluster selector does not support label selector for now", config.Name)
+			}
+			clusterName = config.Placement.ClusterSelector.Name
 		}
-		if len(config.Placement.ClusterSelector.Labels) != 0 {
-			return errors.Errorf("invalid env %s: cluster selector does not support label selector for now", config.Name)
+		// set fallback cluster
+		if clusterName == "" {
+			clusterName = multicluster.ClusterLocalName
 		}
-		if len(config.Placement.ClusterSelector.Name) == 0 {
-			return errors.Errorf("invalid env %s: cluster selector must set cluster name for now", config.Name)
+		// check if current environment uses the same cluster and namespace as resource destination with other environment, if yes, a conflict occurs
+		location := clusterName + "/" + namespace
+		if dupConfigName, ok := locationToConfig[location]; ok {
+			return errors.Errorf("invalid env %s: location %s conflict with env %s", config.Name, location, dupConfigName)
 		}
-		clusterName := config.Placement.ClusterSelector.Name
-		if dupConfigName, ok := clusterNameToConfig[clusterName]; ok {
-			return errors.Errorf("invalid env %s: cluster name %s is conflict with env %s", config.Name, clusterName, dupConfigName)
-		}
-		clusterNameToConfig[clusterName] = config.Name
+		locationToConfig[clusterName] = config.Name
+		// check if target cluster exists
 		if clusterName != multicluster.ClusterLocalName {
 			if err := engine.Get(ctx, types.NamespacedName{Namespace: multicluster.ClusterGatewaySecretNamespace, Name: clusterName}, &v1.Secret{}); err != nil {
 				return errors.Wrapf(err, "failed to get cluster %s for env %s", clusterName, config.Name)
 			}
 		}
-		engine.clusterDecisions[config.Name] = v1alpha1.ClusterDecision{Env: config.Name, Cluster: clusterName}
+		engine.clusterDecisions[config.Name] = v1alpha1.ClusterDecision{Env: config.Name, Cluster: clusterName, Namespace: namespace}
 	}
 	return nil
 }
@@ -87,12 +101,14 @@ func (engine *ClusterGatewayEngine) schedule(ctx context.Context, apps []*EnvBin
 	for _, app := range apps {
 		app.ScheduledManifests = make(map[string]*unstructured.Unstructured)
 		clusterName := engine.clusterDecisions[app.envConfig.Name].Cluster
+		namespace := engine.clusterDecisions[app.envConfig.Name].Namespace
 		raw, err := runtime.DefaultUnstructuredConverter.ToUnstructured(app.PatchedApp)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to convert app [Env: %s](%s/%s) into unstructured", app.envConfig.Name, app.PatchedApp.Namespace, app.PatchedApp.Name)
 		}
 		patchedApp := &unstructured.Unstructured{Object: raw}
 		multicluster.SetClusterName(patchedApp, clusterName)
+		SetOverrideNamespace(patchedApp, namespace)
 		app.ScheduledManifests[patchedApp.GetName()] = patchedApp
 	}
 	var decisions []v1alpha1.ClusterDecision
@@ -101,3 +117,15 @@ func (engine *ClusterGatewayEngine) schedule(ctx context.Context, apps []*EnvBin
 	}
 	return decisions, nil
 }
+
+// SetOverrideNamespace set the override namespace for object in its label
+func SetOverrideNamespace(obj *unstructured.Unstructured, overrideNamespace string) {
+	if overrideNamespace != "" {
+		labels := obj.GetLabels()
+		if labels == nil {
+			labels = map[string]string{}
+		}
+		labels[OverrideNamespaceLabelKey] = overrideNamespace
+		obj.SetLabels(labels)
+	}
+}

pkg/controller/core.oam.dev/v1alpha2/application/generator.go

@@ -124,7 +124,7 @@ func convertStepProperties(step *v1beta1.WorkflowStep, app *v1beta1.Application)
 }
 
 func (h *AppHandler) applyComponentFunc(appParser *appfile.Parser, appRev *v1beta1.ApplicationRevision, af *appfile.Appfile, cli client.Client) oamProvider.ComponentApply {
-	return func(comp common.ApplicationComponent, patcher *value.Value, clusterName string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error) {
+	return func(comp common.ApplicationComponent, patcher *value.Value, clusterName string, overrideNamespace string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error) {
 		ctx := multicluster.ContextWithClusterName(context.Background(), clusterName)
 
 		wl, err := appParser.ParseWorkloadFromRevision(comp, appRev)
@@ -151,7 +151,12 @@ func (h *AppHandler) applyComponentFunc(appParser *appfile.Parser, appRev *v1bet
 		if err != nil {
 			return nil, nil, false, errors.WithMessage(err, "assemble resources before apply fail")
 		}
-
+		if overrideNamespace != "" {
+			readyWorkload.SetNamespace(overrideNamespace)
+			for _, readyTrait := range readyTraits {
+				readyTrait.SetNamespace(overrideNamespace)
+			}
+		}
 		skipStandardWorkload := skipApplyWorkload(wl)
 		if !skipStandardWorkload {
 			if err := h.Dispatch(ctx, clusterName, common.WorkflowResourceCreator, readyWorkload); err != nil {

pkg/stdlib/op.cue

@@ -118,6 +118,9 @@ import (
 				if patchedApp.metadata.labels != _|_ && patchedApp.metadata.labels["cluster.oam.dev/clusterName"] != _|_ {
 					cluster: patchedApp.metadata.labels["cluster.oam.dev/clusterName"]
 				}
+				if patchedApp.metadata.labels != _|_ && patchedApp.metadata.labels["envbinding.oam.dev/override-namespace"] != _|_ {
+					namespace: patchedApp.metadata.labels["envbinding.oam.dev/override-namespace"]
+				}
 			} @step(4)
 		}
 	}

pkg/workflow/providers/oam/apply.go

@@ -39,7 +39,7 @@ const (
 )
 
 // ComponentApply apply oam component.
-type ComponentApply func(comp common.ApplicationComponent, patcher *value.Value, clusterName string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error)
+type ComponentApply func(comp common.ApplicationComponent, patcher *value.Value, clusterName string, overrideNamespace string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error)
 
 type provider struct {
 	apply ComponentApply
@@ -62,7 +62,11 @@ func (p *provider) ApplyComponent(ctx wfContext.Context, v *value.Value, act wfT
 	if err != nil {
 		clusterName = ""
 	}
-	workload, traits, healthy, err := p.apply(comp, patcher, clusterName)
+	overrideNamespace, err := v.GetString("namespace")
+	if err != nil {
+		overrideNamespace = ""
+	}
+	workload, traits, healthy, err := p.apply(comp, patcher, clusterName, overrideNamespace)
 	if err != nil {
 		return err
 	}

pkg/workflow/providers/oam/apply_test.go

@@ -115,7 +115,7 @@ func TestLoadComponent(t *testing.T) {
 
 var testHealthy bool
 
-func simpleComponentApplyForTest(comp common.ApplicationComponent, _ *value.Value, _ string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error) {
+func simpleComponentApplyForTest(comp common.ApplicationComponent, _ *value.Value, _ string, _ string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error) {
 	workload := new(unstructured.Unstructured)
 	workload.UnmarshalJSON([]byte(`{
   "apiVersion": "v1",

test/e2e-multicluster-test/multicluster_test.go

@@ -19,7 +19,9 @@ package e2e_multicluster_test
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo"
@@ -31,10 +33,10 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
 
 	"github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1"
 	"github.com/oam-dev/kubevela/pkg/multicluster"
-	"github.com/oam-dev/kubevela/pkg/utils/common"
 )
 
 func initializeContext() (hubCtx context.Context, workerCtx context.Context) {
@@ -113,8 +115,10 @@ var _ = Describe("Test multicluster scenario", func() {
 				Expect(k8sClient.Delete(workerCtx, clusterRoleBinding)).Should(Succeed())
 			}()
 			serviceAccount = &v1.ServiceAccount{}
-			Expect(k8sClient.Get(workerCtx, types.NamespacedName{Name: serviceAccountName, Namespace: "kube-system"}, serviceAccount)).Should(Succeed())
-			Expect(len(serviceAccount.Secrets)).Should(Equal(1))
+			Eventually(func(g Gomega) {
+				Expect(k8sClient.Get(workerCtx, types.NamespacedName{Name: serviceAccountName, Namespace: "kube-system"}, serviceAccount)).Should(Succeed())
+				Expect(len(serviceAccount.Secrets)).Should(Equal(1))
+			}, time.Second*30).Should(Succeed())
 			secret := &v1.Secret{}
 			Expect(k8sClient.Get(workerCtx, types.NamespacedName{Name: serviceAccount.Secrets[0].Name, Namespace: "kube-system"}, secret)).Should(Succeed())
 			token, ok := secret.Data["token"]
@@ -146,37 +150,50 @@ var _ = Describe("Test multicluster scenario", func() {
 	Context("Test EnvBinding Application", func() {
 
 		var namespace string
+		var testNamespace string
+		var prodNamespace string
 		var hubCtx context.Context
 		var workerCtx context.Context
 
 		BeforeEach(func() {
 			hubCtx, workerCtx, namespace = initializeContextAndNamespace()
+			_, _, testNamespace = initializeContextAndNamespace()
+			_, _, prodNamespace = initializeContextAndNamespace()
 		})
 
 		AfterEach(func() {
 			cleanUpNamespace(hubCtx, workerCtx, namespace)
+			cleanUpNamespace(hubCtx, workerCtx, testNamespace)
+			cleanUpNamespace(hubCtx, workerCtx, prodNamespace)
 		})
 
 		It("Test create EnvBinding Application", func() {
 			// This test is going to cover multiple functions, including
-			// 1. Multiple stage deployment for two environment, involving suspend
-			// 2. A special cluster: local cluster
-			// 3. Component selector.
+			// 1. Multiple stage deployment for three environment
+			// 2. Namespace selector.
+			// 3. A special cluster: local cluster
+			// 4. Component selector.
 			app := &v1beta1.Application{}
-			Expect(common.ReadYamlToObject("./testdata/app/example-envbinding-app.yaml", app)).Should(BeNil())
+			bs, err := ioutil.ReadFile("./testdata/app/example-envbinding-app.yaml")
+			Expect(err).Should(Succeed())
+			appYaml := strings.ReplaceAll(strings.ReplaceAll(string(bs), "TEST_NAMESPACE", testNamespace), "PROD_NAMESPACE", prodNamespace)
+			Expect(yaml.Unmarshal([]byte(appYaml), app)).Should(Succeed())
 			app.SetNamespace(namespace)
-			err := k8sClient.Create(hubCtx, app)
+			err = k8sClient.Create(hubCtx, app)
 			Expect(err).Should(Succeed())
 			var hubDeployName string
 			Eventually(func(g Gomega) {
 				// check deployments in clusters
 				deploys := &v13.DeploymentList{}
-				g.Expect(k8sClient.List(hubCtx, deploys, client.InNamespace(namespace))).Should(Succeed())
+				g.Expect(k8sClient.List(hubCtx, deploys, client.InNamespace(testNamespace))).Should(Succeed())
 				g.Expect(len(deploys.Items)).Should(Equal(1))
 				hubDeployName = deploys.Items[0].Name
 				deploys = &v13.DeploymentList{}
 				g.Expect(k8sClient.List(workerCtx, deploys, client.InNamespace(namespace))).Should(Succeed())
 				g.Expect(len(deploys.Items)).Should(Equal(2))
+				deploys = &v13.DeploymentList{}
+				g.Expect(k8sClient.List(workerCtx, deploys, client.InNamespace(prodNamespace))).Should(Succeed())
+				g.Expect(len(deploys.Items)).Should(Equal(2))
 			}, 2*time.Minute).Should(Succeed())
 			Expect(hubDeployName).Should(Equal("data-worker"))
 			// delete application

test/e2e-multicluster-test/testdata/app/example-envbinding-app.yaml

@@ -26,18 +26,25 @@ spec:
       type: env-binding
       properties:
         envs:
-          - name: staging
-            placement: # selecting the cluster to deploy to
-              clusterSelector:
-                name: local
+          - name: test
+            placement: # selecting the namespace (in local cluster) to deploy to
+              namespaceSelector:
+                name: TEST_NAMESPACE
             selector:
               components:
                 - data-worker
 
-          - name: prod
-            placement:
+          - name: staging
+            placement: # selecting the cluster to deploy to
               clusterSelector:
                 name: cluster-worker
+
+          - name: prod
+            placement: # selecting both namespace and cluster to deploy to
+              clusterSelector:
+                name: cluster-worker
+              namespaceSelector:
+                name: PROD_NAMESPACE
             patch: # overlay patch on above components
               components:
                 - name: hello-world-server
@@ -49,6 +56,13 @@ spec:
 
   workflow:
     steps:
+      # deploy to test env
+      - name: deploy-test
+        type: deploy2env
+        properties:
+          policy: example-multi-env-policy
+          env: test
+
       # deploy to staging env
       - name: deploy-staging
         type: deploy2env
@@ -61,4 +75,4 @@ spec:
         type: deploy2env
         properties:
           policy: example-multi-env-policy
-          env: prod
+          env: prod