diff --git a/pkg/controller/core.oam.dev/v1alpha1/envbinding/cluster_gateway_engine.go b/pkg/controller/core.oam.dev/v1alpha1/envbinding/cluster_gateway_engine.go index d275a7206..8ccaae806 100644 --- a/pkg/controller/core.oam.dev/v1alpha1/envbinding/cluster_gateway_engine.go +++ b/pkg/controller/core.oam.dev/v1alpha1/envbinding/cluster_gateway_engine.go @@ -32,6 +32,11 @@ import ( "github.com/oam-dev/kubevela/pkg/multicluster" ) +const ( + // OverrideNamespaceLabelKey identifies the override namespace for patched Application + OverrideNamespaceLabelKey = "envbinding.oam.dev/override-namespace" +) + // ClusterGatewayEngine construct the multicluster engine of using cluster-gateway type ClusterGatewayEngine struct { client.Client @@ -47,34 +52,38 @@ func NewClusterGatewayEngine(cli client.Client, envBindingName string) ClusterMa } } -// TODO only support cluster name now, should support selector and namespace later +// TODO only support single cluster name and namespace name now, should support label selector func (engine *ClusterGatewayEngine) prepare(ctx context.Context, configs []v1alpha1.EnvConfig) error { engine.clusterDecisions = make(map[string]v1alpha1.ClusterDecision) - clusterNameToConfig := make(map[string]string) + locationToConfig := make(map[string]string) for _, config := range configs { + var namespace, clusterName string if config.Placement.NamespaceSelector != nil { - return errors.Errorf("invalid env %s: namespace selector in cluster-gateway is not supported now", config.Name) + if len(config.Placement.NamespaceSelector.Labels) != 0 { + return errors.Errorf("invalid env %s: namespace selector in cluster-gateway does not support label selector for now", config.Name) + } + namespace = config.Placement.NamespaceSelector.Name } - if config.Placement.ClusterSelector == nil { - return errors.Errorf("invalid env %s: cluster selector must be set for now", config.Name) + if config.Placement.ClusterSelector != nil { + if len(config.Placement.ClusterSelector.Labels) != 0 { + return errors.Errorf("invalid env %s: cluster selector does not support label selector for now", config.Name) + } + clusterName = config.Placement.ClusterSelector.Name } - if len(config.Placement.ClusterSelector.Labels) != 0 { - return errors.Errorf("invalid env %s: cluster selector does not support label selector for now", config.Name) + if clusterName == "" { + clusterName = multicluster.ClusterLocalName } - if len(config.Placement.ClusterSelector.Name) == 0 { - return errors.Errorf("invalid env %s: cluster selector must set cluster name for now", config.Name) + location := clusterName + "/" + namespace + if dupConfigName, ok := locationToConfig[location]; ok { + return errors.Errorf("invalid env %s: location %s conflict with env %s", config.Name, location, dupConfigName) } - clusterName := config.Placement.ClusterSelector.Name - if dupConfigName, ok := clusterNameToConfig[clusterName]; ok { - return errors.Errorf("invalid env %s: cluster name %s is conflict with env %s", config.Name, clusterName, dupConfigName) - } - clusterNameToConfig[clusterName] = config.Name + locationToConfig[clusterName] = config.Name if clusterName != multicluster.ClusterLocalName { if err := engine.Get(ctx, types.NamespacedName{Namespace: multicluster.ClusterGatewaySecretNamespace, Name: clusterName}, &v1.Secret{}); err != nil { return errors.Wrapf(err, "failed to get cluster %s for env %s", clusterName, config.Name) } } - engine.clusterDecisions[config.Name] = v1alpha1.ClusterDecision{Env: config.Name, Cluster: clusterName} + engine.clusterDecisions[config.Name] = v1alpha1.ClusterDecision{Env: config.Name, Cluster: clusterName, Namespace: namespace} } return nil } @@ -87,12 +96,14 @@ func (engine *ClusterGatewayEngine) schedule(ctx context.Context, apps []*EnvBin for _, app := range apps { app.ScheduledManifests = make(map[string]*unstructured.Unstructured) clusterName := engine.clusterDecisions[app.envConfig.Name].Cluster + namespace := engine.clusterDecisions[app.envConfig.Name].Namespace raw, err := runtime.DefaultUnstructuredConverter.ToUnstructured(app.PatchedApp) if err != nil { return nil, errors.Wrapf(err, "failed to convert app [Env: %s](%s/%s) into unstructured", app.envConfig.Name, app.PatchedApp.Namespace, app.PatchedApp.Name) } patchedApp := &unstructured.Unstructured{Object: raw} multicluster.SetClusterName(patchedApp, clusterName) + SetOverrideNamespace(patchedApp, namespace) app.ScheduledManifests[patchedApp.GetName()] = patchedApp } var decisions []v1alpha1.ClusterDecision @@ -101,3 +112,15 @@ func (engine *ClusterGatewayEngine) schedule(ctx context.Context, apps []*EnvBin } return decisions, nil } + +// SetOverrideNamespace set the override namespace for object in its label +func SetOverrideNamespace(obj *unstructured.Unstructured, overrideNamespace string) { + if overrideNamespace != "" { + labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{} + } + labels[OverrideNamespaceLabelKey] = overrideNamespace + obj.SetLabels(labels) + } +} diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/generator.go b/pkg/controller/core.oam.dev/v1alpha2/application/generator.go index 9d1b14e34..50376b057 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/generator.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/generator.go @@ -124,7 +124,7 @@ func convertStepProperties(step *v1beta1.WorkflowStep, app *v1beta1.Application) } func (h *AppHandler) applyComponentFunc(appParser *appfile.Parser, appRev *v1beta1.ApplicationRevision, af *appfile.Appfile, cli client.Client) oamProvider.ComponentApply { - return func(comp common.ApplicationComponent, patcher *value.Value, clusterName string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error) { + return func(comp common.ApplicationComponent, patcher *value.Value, clusterName string, overrideNamespace string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error) { ctx := multicluster.ContextWithClusterName(context.Background(), clusterName) wl, err := appParser.ParseWorkloadFromRevision(comp, appRev) @@ -151,7 +151,12 @@ func (h *AppHandler) applyComponentFunc(appParser *appfile.Parser, appRev *v1bet if err != nil { return nil, nil, false, errors.WithMessage(err, "assemble resources before apply fail") } - + if overrideNamespace != "" { + readyWorkload.SetNamespace(overrideNamespace) + for _, readyTrait := range readyTraits { + readyTrait.SetNamespace(overrideNamespace) + } + } skipStandardWorkload := skipApplyWorkload(wl) if !skipStandardWorkload { if err := h.Dispatch(ctx, clusterName, common.WorkflowResourceCreator, readyWorkload); err != nil { diff --git a/pkg/stdlib/op.cue b/pkg/stdlib/op.cue index 13e62aaa9..a241859df 100644 --- a/pkg/stdlib/op.cue +++ b/pkg/stdlib/op.cue @@ -118,6 +118,9 @@ import ( if patchedApp.metadata.labels != _|_ && patchedApp.metadata.labels["cluster.oam.dev/clusterName"] != _|_ { cluster: patchedApp.metadata.labels["cluster.oam.dev/clusterName"] } + if patchedApp.metadata.labels != _|_ && patchedApp.metadata.labels["envbinding.oam.dev/override-namespace"] != _|_ { + namespace: patchedApp.metadata.labels["envbinding.oam.dev/override-namespace"] + } } @step(4) } } diff --git a/pkg/workflow/providers/oam/apply.go b/pkg/workflow/providers/oam/apply.go index b21617b30..167beb767 100644 --- a/pkg/workflow/providers/oam/apply.go +++ b/pkg/workflow/providers/oam/apply.go @@ -39,7 +39,7 @@ const ( ) // ComponentApply apply oam component. -type ComponentApply func(comp common.ApplicationComponent, patcher *value.Value, clusterName string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error) +type ComponentApply func(comp common.ApplicationComponent, patcher *value.Value, clusterName string, overrideNamespace string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error) type provider struct { apply ComponentApply @@ -62,7 +62,11 @@ func (p *provider) ApplyComponent(ctx wfContext.Context, v *value.Value, act wfT if err != nil { clusterName = "" } - workload, traits, healthy, err := p.apply(comp, patcher, clusterName) + overrideNamespace, err := v.GetString("namespace") + if err != nil { + overrideNamespace = "" + } + workload, traits, healthy, err := p.apply(comp, patcher, clusterName, overrideNamespace) if err != nil { return err } diff --git a/pkg/workflow/providers/oam/apply_test.go b/pkg/workflow/providers/oam/apply_test.go index 65a87581b..d0cf9b664 100644 --- a/pkg/workflow/providers/oam/apply_test.go +++ b/pkg/workflow/providers/oam/apply_test.go @@ -115,7 +115,7 @@ func TestLoadComponent(t *testing.T) { var testHealthy bool -func simpleComponentApplyForTest(comp common.ApplicationComponent, _ *value.Value, _ string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error) { +func simpleComponentApplyForTest(comp common.ApplicationComponent, _ *value.Value, _ string, _ string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error) { workload := new(unstructured.Unstructured) workload.UnmarshalJSON([]byte(`{ "apiVersion": "v1", diff --git a/test/e2e-multicluster-test/multicluster_test.go b/test/e2e-multicluster-test/multicluster_test.go index aa5f3513d..85c09209a 100644 --- a/test/e2e-multicluster-test/multicluster_test.go +++ b/test/e2e-multicluster-test/multicluster_test.go @@ -19,7 +19,9 @@ package e2e_multicluster_test import ( "context" "fmt" + "io/ioutil" "os" + "strings" "time" . "github.com/onsi/ginkgo" @@ -31,10 +33,10 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/pkg/multicluster" - "github.com/oam-dev/kubevela/pkg/utils/common" ) func initializeContext() (hubCtx context.Context, workerCtx context.Context) { @@ -146,37 +148,50 @@ var _ = Describe("Test multicluster scenario", func() { Context("Test EnvBinding Application", func() { var namespace string + var testNamespace string + var prodNamespace string var hubCtx context.Context var workerCtx context.Context BeforeEach(func() { hubCtx, workerCtx, namespace = initializeContextAndNamespace() + _, _, testNamespace = initializeContextAndNamespace() + _, _, prodNamespace = initializeContextAndNamespace() }) AfterEach(func() { cleanUpNamespace(hubCtx, workerCtx, namespace) + cleanUpNamespace(hubCtx, workerCtx, testNamespace) + cleanUpNamespace(hubCtx, workerCtx, prodNamespace) }) It("Test create EnvBinding Application", func() { // This test is going to cover multiple functions, including - // 1. Multiple stage deployment for two environment, involving suspend - // 2. A special cluster: local cluster - // 3. Component selector. + // 1. Multiple stage deployment for three environment + // 2. Namespace selector. + // 3. A special cluster: local cluster + // 4. Component selector. app := &v1beta1.Application{} - Expect(common.ReadYamlToObject("./testdata/app/example-envbinding-app.yaml", app)).Should(BeNil()) + bs, err := ioutil.ReadFile("./testdata/app/example-envbinding-app.yaml") + Expect(err).Should(Succeed()) + appYaml := strings.ReplaceAll(strings.ReplaceAll(string(bs), "TEST_NAMESPACE", testNamespace), "PROD_NAMESPACE", prodNamespace) + Expect(yaml.Unmarshal([]byte(appYaml), app)).Should(Succeed()) app.SetNamespace(namespace) - err := k8sClient.Create(hubCtx, app) + err = k8sClient.Create(hubCtx, app) Expect(err).Should(Succeed()) var hubDeployName string Eventually(func(g Gomega) { // check deployments in clusters deploys := &v13.DeploymentList{} - g.Expect(k8sClient.List(hubCtx, deploys, client.InNamespace(namespace))).Should(Succeed()) + g.Expect(k8sClient.List(hubCtx, deploys, client.InNamespace(testNamespace))).Should(Succeed()) g.Expect(len(deploys.Items)).Should(Equal(1)) hubDeployName = deploys.Items[0].Name deploys = &v13.DeploymentList{} g.Expect(k8sClient.List(workerCtx, deploys, client.InNamespace(namespace))).Should(Succeed()) g.Expect(len(deploys.Items)).Should(Equal(2)) + deploys = &v13.DeploymentList{} + g.Expect(k8sClient.List(workerCtx, deploys, client.InNamespace(prodNamespace))).Should(Succeed()) + g.Expect(len(deploys.Items)).Should(Equal(2)) }, 2*time.Minute).Should(Succeed()) Expect(hubDeployName).Should(Equal("data-worker")) // delete application diff --git a/test/e2e-multicluster-test/testdata/app/example-envbinding-app.yaml b/test/e2e-multicluster-test/testdata/app/example-envbinding-app.yaml index b1ec3e775..8970dc6f2 100644 --- a/test/e2e-multicluster-test/testdata/app/example-envbinding-app.yaml +++ b/test/e2e-multicluster-test/testdata/app/example-envbinding-app.yaml @@ -26,18 +26,25 @@ spec: type: env-binding properties: envs: - - name: staging - placement: # selecting the cluster to deploy to - clusterSelector: - name: local + - name: test + placement: # selecting the namespace to deploy to + namespaceSelector: + name: TEST_NAMESPACE selector: components: - data-worker - - name: prod - placement: + - name: staging + placement: # selecting the cluster to deploy to clusterSelector: name: cluster-worker + + - name: prod + placement: # selecting both namespace and cluster to deploy to + clusterSelector: + name: cluster-worker + namespaceSelector: + name: PROD_NAMESPACE patch: # overlay patch on above components components: - name: hello-world-server @@ -49,6 +56,13 @@ spec: workflow: steps: + # deploy to test env + - name: deploy-test + type: deploy2env + properties: + policy: example-multi-env-policy + env: test + # deploy to staging env - name: deploy-staging type: deploy2env @@ -61,4 +75,4 @@ spec: type: deploy2env properties: policy: example-multi-env-policy - env: prod \ No newline at end of file + env: prod