From 472ed3a1463f50e60997b61a8a988fb2e0df21be Mon Sep 17 00:00:00 2001 From: Somefive Date: Mon, 11 Oct 2021 14:52:01 +0800 Subject: [PATCH] Feat: add support for envbinding with namespace selector (#2432) * Feat: add support for envbinding with namespace selector * Fix: service account kubeconfig e2e-test * Docs: add comments to explain functions * Docs: add envbinding example --- docs/examples/envbinding/example.yaml | 78 +++++++++++++++++++ .../envbinding/cluster_gateway_engine.go | 58 ++++++++++---- .../v1alpha2/application/generator.go | 9 ++- pkg/stdlib/op.cue | 3 + pkg/workflow/providers/oam/apply.go | 8 +- pkg/workflow/providers/oam/apply_test.go | 2 +- .../multicluster_test.go | 35 ++++++--- .../testdata/app/example-envbinding-app.yaml | 28 +++++-- 8 files changed, 185 insertions(+), 36 deletions(-) create mode 100644 docs/examples/envbinding/example.yaml diff --git a/docs/examples/envbinding/example.yaml b/docs/examples/envbinding/example.yaml new file mode 100644 index 000000000..f0c0a6a8d --- /dev/null +++ b/docs/examples/envbinding/example.yaml @@ -0,0 +1,78 @@ +apiVersion: core.oam.dev/v1beta1 +kind: Application +metadata: + name: example-app + namespace: default +spec: + components: + - name: hello-world-server + type: webservice + properties: + image: crccheck/hello-world + port: 8000 + traits: + - type: scaler + properties: + replicas: 1 + - name: data-worker + type: worker + properties: + image: busybox + cmd: + - sleep + - '1000000' + policies: + - name: example-multi-env-policy + type: env-binding + properties: + envs: + - name: test + placement: # selecting the namespace (in local cluster) to deploy to + namespaceSelector: + name: test + selector: + components: + - data-worker + + - name: staging + placement: # selecting the cluster to deploy to + clusterSelector: + name: cluster-worker + + - name: prod + placement: # selecting both namespace and cluster to deploy to + clusterSelector: + name: cluster-worker + namespaceSelector: + name: prod + patch: # overlay patch on above components + components: + - name: hello-world-server + type: webservice + traits: + - type: scaler + properties: + replicas: 3 + + workflow: + steps: + # deploy to test env + - name: deploy-test + type: deploy2env + properties: + policy: example-multi-env-policy + env: test + + # deploy to staging env + - name: deploy-staging + type: deploy2env + properties: + policy: example-multi-env-policy + env: staging + + # deploy to prod env + - name: deploy-prod + type: deploy2env + properties: + policy: example-multi-env-policy + env: prod diff --git a/pkg/controller/core.oam.dev/v1alpha1/envbinding/cluster_gateway_engine.go b/pkg/controller/core.oam.dev/v1alpha1/envbinding/cluster_gateway_engine.go index d275a7206..1405897bf 100644 --- a/pkg/controller/core.oam.dev/v1alpha1/envbinding/cluster_gateway_engine.go +++ b/pkg/controller/core.oam.dev/v1alpha1/envbinding/cluster_gateway_engine.go @@ -32,6 +32,11 @@ import ( "github.com/oam-dev/kubevela/pkg/multicluster" ) +const ( + // OverrideNamespaceLabelKey identifies the override namespace for patched Application + OverrideNamespaceLabelKey = "envbinding.oam.dev/override-namespace" +) + // ClusterGatewayEngine construct the multicluster engine of using cluster-gateway type ClusterGatewayEngine struct { client.Client @@ -47,34 +52,43 @@ func NewClusterGatewayEngine(cli client.Client, envBindingName string) ClusterMa } } -// TODO only support cluster name now, should support selector and namespace later +// TODO only support single cluster name and namespace name now, should support label selector func (engine *ClusterGatewayEngine) prepare(ctx context.Context, configs []v1alpha1.EnvConfig) error { engine.clusterDecisions = make(map[string]v1alpha1.ClusterDecision) - clusterNameToConfig := make(map[string]string) + locationToConfig := make(map[string]string) for _, config := range configs { + var namespace, clusterName string + // check if namespace selector is valid if config.Placement.NamespaceSelector != nil { - return errors.Errorf("invalid env %s: namespace selector in cluster-gateway is not supported now", config.Name) + if len(config.Placement.NamespaceSelector.Labels) != 0 { + return errors.Errorf("invalid env %s: namespace selector in cluster-gateway does not support label selector for now", config.Name) + } + namespace = config.Placement.NamespaceSelector.Name } - if config.Placement.ClusterSelector == nil { - return errors.Errorf("invalid env %s: cluster selector must be set for now", config.Name) + // check if cluster selector is valid + if config.Placement.ClusterSelector != nil { + if len(config.Placement.ClusterSelector.Labels) != 0 { + return errors.Errorf("invalid env %s: cluster selector does not support label selector for now", config.Name) + } + clusterName = config.Placement.ClusterSelector.Name } - if len(config.Placement.ClusterSelector.Labels) != 0 { - return errors.Errorf("invalid env %s: cluster selector does not support label selector for now", config.Name) + // set fallback cluster + if clusterName == "" { + clusterName = multicluster.ClusterLocalName } - if len(config.Placement.ClusterSelector.Name) == 0 { - return errors.Errorf("invalid env %s: cluster selector must set cluster name for now", config.Name) + // check if current environment uses the same cluster and namespace as resource destination with other environment, if yes, a conflict occurs + location := clusterName + "/" + namespace + if dupConfigName, ok := locationToConfig[location]; ok { + return errors.Errorf("invalid env %s: location %s conflict with env %s", config.Name, location, dupConfigName) } - clusterName := config.Placement.ClusterSelector.Name - if dupConfigName, ok := clusterNameToConfig[clusterName]; ok { - return errors.Errorf("invalid env %s: cluster name %s is conflict with env %s", config.Name, clusterName, dupConfigName) - } - clusterNameToConfig[clusterName] = config.Name + locationToConfig[clusterName] = config.Name + // check if target cluster exists if clusterName != multicluster.ClusterLocalName { if err := engine.Get(ctx, types.NamespacedName{Namespace: multicluster.ClusterGatewaySecretNamespace, Name: clusterName}, &v1.Secret{}); err != nil { return errors.Wrapf(err, "failed to get cluster %s for env %s", clusterName, config.Name) } } - engine.clusterDecisions[config.Name] = v1alpha1.ClusterDecision{Env: config.Name, Cluster: clusterName} + engine.clusterDecisions[config.Name] = v1alpha1.ClusterDecision{Env: config.Name, Cluster: clusterName, Namespace: namespace} } return nil } @@ -87,12 +101,14 @@ func (engine *ClusterGatewayEngine) schedule(ctx context.Context, apps []*EnvBin for _, app := range apps { app.ScheduledManifests = make(map[string]*unstructured.Unstructured) clusterName := engine.clusterDecisions[app.envConfig.Name].Cluster + namespace := engine.clusterDecisions[app.envConfig.Name].Namespace raw, err := runtime.DefaultUnstructuredConverter.ToUnstructured(app.PatchedApp) if err != nil { return nil, errors.Wrapf(err, "failed to convert app [Env: %s](%s/%s) into unstructured", app.envConfig.Name, app.PatchedApp.Namespace, app.PatchedApp.Name) } patchedApp := &unstructured.Unstructured{Object: raw} multicluster.SetClusterName(patchedApp, clusterName) + SetOverrideNamespace(patchedApp, namespace) app.ScheduledManifests[patchedApp.GetName()] = patchedApp } var decisions []v1alpha1.ClusterDecision @@ -101,3 +117,15 @@ func (engine *ClusterGatewayEngine) schedule(ctx context.Context, apps []*EnvBin } return decisions, nil } + +// SetOverrideNamespace set the override namespace for object in its label +func SetOverrideNamespace(obj *unstructured.Unstructured, overrideNamespace string) { + if overrideNamespace != "" { + labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{} + } + labels[OverrideNamespaceLabelKey] = overrideNamespace + obj.SetLabels(labels) + } +} diff --git a/pkg/controller/core.oam.dev/v1alpha2/application/generator.go b/pkg/controller/core.oam.dev/v1alpha2/application/generator.go index 9d1b14e34..50376b057 100644 --- a/pkg/controller/core.oam.dev/v1alpha2/application/generator.go +++ b/pkg/controller/core.oam.dev/v1alpha2/application/generator.go @@ -124,7 +124,7 @@ func convertStepProperties(step *v1beta1.WorkflowStep, app *v1beta1.Application) } func (h *AppHandler) applyComponentFunc(appParser *appfile.Parser, appRev *v1beta1.ApplicationRevision, af *appfile.Appfile, cli client.Client) oamProvider.ComponentApply { - return func(comp common.ApplicationComponent, patcher *value.Value, clusterName string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error) { + return func(comp common.ApplicationComponent, patcher *value.Value, clusterName string, overrideNamespace string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error) { ctx := multicluster.ContextWithClusterName(context.Background(), clusterName) wl, err := appParser.ParseWorkloadFromRevision(comp, appRev) @@ -151,7 +151,12 @@ func (h *AppHandler) applyComponentFunc(appParser *appfile.Parser, appRev *v1bet if err != nil { return nil, nil, false, errors.WithMessage(err, "assemble resources before apply fail") } - + if overrideNamespace != "" { + readyWorkload.SetNamespace(overrideNamespace) + for _, readyTrait := range readyTraits { + readyTrait.SetNamespace(overrideNamespace) + } + } skipStandardWorkload := skipApplyWorkload(wl) if !skipStandardWorkload { if err := h.Dispatch(ctx, clusterName, common.WorkflowResourceCreator, readyWorkload); err != nil { diff --git a/pkg/stdlib/op.cue b/pkg/stdlib/op.cue index 13e62aaa9..a241859df 100644 --- a/pkg/stdlib/op.cue +++ b/pkg/stdlib/op.cue @@ -118,6 +118,9 @@ import ( if patchedApp.metadata.labels != _|_ && patchedApp.metadata.labels["cluster.oam.dev/clusterName"] != _|_ { cluster: patchedApp.metadata.labels["cluster.oam.dev/clusterName"] } + if patchedApp.metadata.labels != _|_ && patchedApp.metadata.labels["envbinding.oam.dev/override-namespace"] != _|_ { + namespace: patchedApp.metadata.labels["envbinding.oam.dev/override-namespace"] + } } @step(4) } } diff --git a/pkg/workflow/providers/oam/apply.go b/pkg/workflow/providers/oam/apply.go index b21617b30..167beb767 100644 --- a/pkg/workflow/providers/oam/apply.go +++ b/pkg/workflow/providers/oam/apply.go @@ -39,7 +39,7 @@ const ( ) // ComponentApply apply oam component. -type ComponentApply func(comp common.ApplicationComponent, patcher *value.Value, clusterName string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error) +type ComponentApply func(comp common.ApplicationComponent, patcher *value.Value, clusterName string, overrideNamespace string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error) type provider struct { apply ComponentApply @@ -62,7 +62,11 @@ func (p *provider) ApplyComponent(ctx wfContext.Context, v *value.Value, act wfT if err != nil { clusterName = "" } - workload, traits, healthy, err := p.apply(comp, patcher, clusterName) + overrideNamespace, err := v.GetString("namespace") + if err != nil { + overrideNamespace = "" + } + workload, traits, healthy, err := p.apply(comp, patcher, clusterName, overrideNamespace) if err != nil { return err } diff --git a/pkg/workflow/providers/oam/apply_test.go b/pkg/workflow/providers/oam/apply_test.go index 65a87581b..d0cf9b664 100644 --- a/pkg/workflow/providers/oam/apply_test.go +++ b/pkg/workflow/providers/oam/apply_test.go @@ -115,7 +115,7 @@ func TestLoadComponent(t *testing.T) { var testHealthy bool -func simpleComponentApplyForTest(comp common.ApplicationComponent, _ *value.Value, _ string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error) { +func simpleComponentApplyForTest(comp common.ApplicationComponent, _ *value.Value, _ string, _ string) (*unstructured.Unstructured, []*unstructured.Unstructured, bool, error) { workload := new(unstructured.Unstructured) workload.UnmarshalJSON([]byte(`{ "apiVersion": "v1", diff --git a/test/e2e-multicluster-test/multicluster_test.go b/test/e2e-multicluster-test/multicluster_test.go index aa5f3513d..c284aa6ef 100644 --- a/test/e2e-multicluster-test/multicluster_test.go +++ b/test/e2e-multicluster-test/multicluster_test.go @@ -19,7 +19,9 @@ package e2e_multicluster_test import ( "context" "fmt" + "io/ioutil" "os" + "strings" "time" . "github.com/onsi/ginkgo" @@ -31,10 +33,10 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/pkg/multicluster" - "github.com/oam-dev/kubevela/pkg/utils/common" ) func initializeContext() (hubCtx context.Context, workerCtx context.Context) { @@ -113,8 +115,10 @@ var _ = Describe("Test multicluster scenario", func() { Expect(k8sClient.Delete(workerCtx, clusterRoleBinding)).Should(Succeed()) }() serviceAccount = &v1.ServiceAccount{} - Expect(k8sClient.Get(workerCtx, types.NamespacedName{Name: serviceAccountName, Namespace: "kube-system"}, serviceAccount)).Should(Succeed()) - Expect(len(serviceAccount.Secrets)).Should(Equal(1)) + Eventually(func(g Gomega) { + Expect(k8sClient.Get(workerCtx, types.NamespacedName{Name: serviceAccountName, Namespace: "kube-system"}, serviceAccount)).Should(Succeed()) + Expect(len(serviceAccount.Secrets)).Should(Equal(1)) + }, time.Second*30).Should(Succeed()) secret := &v1.Secret{} Expect(k8sClient.Get(workerCtx, types.NamespacedName{Name: serviceAccount.Secrets[0].Name, Namespace: "kube-system"}, secret)).Should(Succeed()) token, ok := secret.Data["token"] @@ -146,37 +150,50 @@ var _ = Describe("Test multicluster scenario", func() { Context("Test EnvBinding Application", func() { var namespace string + var testNamespace string + var prodNamespace string var hubCtx context.Context var workerCtx context.Context BeforeEach(func() { hubCtx, workerCtx, namespace = initializeContextAndNamespace() + _, _, testNamespace = initializeContextAndNamespace() + _, _, prodNamespace = initializeContextAndNamespace() }) AfterEach(func() { cleanUpNamespace(hubCtx, workerCtx, namespace) + cleanUpNamespace(hubCtx, workerCtx, testNamespace) + cleanUpNamespace(hubCtx, workerCtx, prodNamespace) }) It("Test create EnvBinding Application", func() { // This test is going to cover multiple functions, including - // 1. Multiple stage deployment for two environment, involving suspend - // 2. A special cluster: local cluster - // 3. Component selector. + // 1. Multiple stage deployment for three environment + // 2. Namespace selector. + // 3. A special cluster: local cluster + // 4. Component selector. app := &v1beta1.Application{} - Expect(common.ReadYamlToObject("./testdata/app/example-envbinding-app.yaml", app)).Should(BeNil()) + bs, err := ioutil.ReadFile("./testdata/app/example-envbinding-app.yaml") + Expect(err).Should(Succeed()) + appYaml := strings.ReplaceAll(strings.ReplaceAll(string(bs), "TEST_NAMESPACE", testNamespace), "PROD_NAMESPACE", prodNamespace) + Expect(yaml.Unmarshal([]byte(appYaml), app)).Should(Succeed()) app.SetNamespace(namespace) - err := k8sClient.Create(hubCtx, app) + err = k8sClient.Create(hubCtx, app) Expect(err).Should(Succeed()) var hubDeployName string Eventually(func(g Gomega) { // check deployments in clusters deploys := &v13.DeploymentList{} - g.Expect(k8sClient.List(hubCtx, deploys, client.InNamespace(namespace))).Should(Succeed()) + g.Expect(k8sClient.List(hubCtx, deploys, client.InNamespace(testNamespace))).Should(Succeed()) g.Expect(len(deploys.Items)).Should(Equal(1)) hubDeployName = deploys.Items[0].Name deploys = &v13.DeploymentList{} g.Expect(k8sClient.List(workerCtx, deploys, client.InNamespace(namespace))).Should(Succeed()) g.Expect(len(deploys.Items)).Should(Equal(2)) + deploys = &v13.DeploymentList{} + g.Expect(k8sClient.List(workerCtx, deploys, client.InNamespace(prodNamespace))).Should(Succeed()) + g.Expect(len(deploys.Items)).Should(Equal(2)) }, 2*time.Minute).Should(Succeed()) Expect(hubDeployName).Should(Equal("data-worker")) // delete application diff --git a/test/e2e-multicluster-test/testdata/app/example-envbinding-app.yaml b/test/e2e-multicluster-test/testdata/app/example-envbinding-app.yaml index b1ec3e775..4e77d8bf3 100644 --- a/test/e2e-multicluster-test/testdata/app/example-envbinding-app.yaml +++ b/test/e2e-multicluster-test/testdata/app/example-envbinding-app.yaml @@ -26,18 +26,25 @@ spec: type: env-binding properties: envs: - - name: staging - placement: # selecting the cluster to deploy to - clusterSelector: - name: local + - name: test + placement: # selecting the namespace (in local cluster) to deploy to + namespaceSelector: + name: TEST_NAMESPACE selector: components: - data-worker - - name: prod - placement: + - name: staging + placement: # selecting the cluster to deploy to clusterSelector: name: cluster-worker + + - name: prod + placement: # selecting both namespace and cluster to deploy to + clusterSelector: + name: cluster-worker + namespaceSelector: + name: PROD_NAMESPACE patch: # overlay patch on above components components: - name: hello-world-server @@ -49,6 +56,13 @@ spec: workflow: steps: + # deploy to test env + - name: deploy-test + type: deploy2env + properties: + policy: example-multi-env-policy + env: test + # deploy to staging env - name: deploy-staging type: deploy2env @@ -61,4 +75,4 @@ spec: type: deploy2env properties: policy: example-multi-env-policy - env: prod \ No newline at end of file + env: prod