From 9780ce3bef81d07b294ba63bfe6f2c552a707762 Mon Sep 17 00:00:00 2001 From: Yue Wang Date: Sun, 29 Aug 2021 09:11:10 +0800 Subject: [PATCH] Feat(cue): support access components artifacts in cue template context (#2161) Signed-off-by: roywang Co-authored-by: roywang --- pkg/appfile/appfile.go | 32 +++++- pkg/appfile/appfile_test.go | 179 +++++++++++++++++++++++++++++++++ pkg/cue/process/handle.go | 20 ++++ pkg/cue/process/handle_test.go | 32 ++++++ 4 files changed, 261 insertions(+), 2 deletions(-) diff --git a/pkg/appfile/appfile.go b/pkg/appfile/appfile.go index c69e532f5..479e0ce98 100644 --- a/pkg/appfile/appfile.go +++ b/pkg/appfile/appfile.go @@ -194,6 +194,7 @@ type Appfile struct { Policies []*Workload WorkflowSteps []v1beta1.WorkflowStep Components []common.ApplicationComponent + Artifacts []*types.ComponentManifest } // GenerateWorkflowAndPolicy generates workflow steps and policies from an appFile @@ -210,7 +211,7 @@ func (af *Appfile) GenerateWorkflowAndPolicy(ctx context.Context, m discoverymap func (af *Appfile) generateUnstructureds(workloads []*Workload) ([]*unstructured.Unstructured, error) { var uns []*unstructured.Unstructured for _, wl := range workloads { - un, err := generateUnstructuredFromCUEModule(wl, af.Name, af.RevisionName, af.Namespace, af.Components) + un, err := generateUnstructuredFromCUEModule(wl, af.Name, af.RevisionName, af.Namespace, af.Components, af.Artifacts) if err != nil { return nil, err } @@ -254,23 +255,50 @@ func (af *Appfile) generateSteps(ctx context.Context, dm discoverymapper.Discove return tasks, nil } -func generateUnstructuredFromCUEModule(wl *Workload, appName, revision, ns string, components []common.ApplicationComponent) (*unstructured.Unstructured, error) { +func generateUnstructuredFromCUEModule(wl *Workload, appName, revision, ns string, components []common.ApplicationComponent, artifacts []*types.ComponentManifest) (*unstructured.Unstructured, error) { pCtx := process.NewPolicyContext(ns, wl.Name, appName, revision, components) + pCtx.PushData(process.ContextDataArtifacts, prepareArtifactsData(artifacts)) if err := wl.EvalContext(pCtx); err != nil { return nil, errors.Wrapf(err, "evaluate base template app=%s in namespace=%s", appName, ns) } return makeWorkloadWithContext(pCtx, wl, ns, appName) } +// artifacts contains resouces in unstructured shape of all components +// it allows to access values of workloads and traits in CUE template, i.g., +// `if context.artifacts..ready` to determine whether it's ready to access +// `context.artifacts..workload` to access a workload +// `context.artifacts..traits..` to access a trait +func prepareArtifactsData(comps []*types.ComponentManifest) map[string]interface{} { + artifacts := unstructured.Unstructured{Object: make(map[string]interface{})} + for _, pComp := range comps { + if pComp.InsertConfigNotReady { + _ = unstructured.SetNestedField(artifacts.Object, false, pComp.Name, "ready") + continue + } + _ = unstructured.SetNestedField(artifacts.Object, true, pComp.Name, "ready") + _ = unstructured.SetNestedField(artifacts.Object, pComp.StandardWorkload.Object, pComp.Name, "workload") + for _, t := range pComp.Traits { + _ = unstructured.SetNestedField(artifacts.Object, t.Object, pComp.Name, + "traits", + t.GetLabels()[oam.TraitTypeLabel], + t.GetLabels()[oam.TraitResource]) + } + } + return artifacts.Object +} + // GenerateComponentManifests converts an appFile to a slice of ComponentManifest func (af *Appfile) GenerateComponentManifests() ([]*types.ComponentManifest, error) { compManifests := make([]*types.ComponentManifest, len(af.Workloads)) + af.Artifacts = make([]*types.ComponentManifest, len(af.Workloads)) for i, wl := range af.Workloads { cm, err := af.GenerateComponentManifest(wl) if err != nil { return nil, err } compManifests[i] = cm + af.Artifacts[i] = cm } return compManifests, nil } diff --git a/pkg/appfile/appfile_test.go b/pkg/appfile/appfile_test.go index a635fee82..39af31971 100644 --- a/pkg/appfile/appfile_test.go +++ b/pkg/appfile/appfile_test.go @@ -419,6 +419,107 @@ wait: op.#ConditionalWait & { }) }) +var _ = Describe("Test Policy", func() { + It("test generate Policies", func() { + testAppfile := &Appfile{ + Name: "test-app", + Namespace: "default", + Workloads: []*Workload{ + { + Name: "test-comp", + Type: "worker", + CapabilityCategory: oamtypes.KubeCategory, + engine: definition.NewWorkloadAbstractEngine("test-comp", pd), + FullTemplate: &Template{ + Kube: &common.Kube{ + Template: func() runtime.RawExtension { + yamlStr := `apiVersion: apps/v1 +kind: Deployment +spec: + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + ports: + - containerPort: 80 ` + b, _ := yaml.YAMLToJSON([]byte(yamlStr)) + return runtime.RawExtension{Raw: b} + }(), + }, + }, + }, + }, + Policies: []*Workload{ + { + Name: "test-policy", + Type: "test-policy", + Params: map[string]interface{}{ + "boundComponents": []string{"test-comp"}, + }, + FullTemplate: &Template{TemplateStr: ` output: { + apiVersion: "core.oam.dev/v1alpha2" + kind: "HealthScope" + spec: { + for k, v in parameter.boundComponents + if context.artifacts[v].ready { + compName: v + workload: { + apiVersion: context.artifacts[v].workload.apiVersion + kind: context.artifacts[v].workload.kind + name: v + } + }, + } + } + parameter: { + boundComponents: [...string] + }`}, + engine: definition.NewWorkloadAbstractEngine("test-policy", pd), + }, + }, + } + _, err := testAppfile.GenerateComponentManifests() + Expect(err).Should(BeNil()) + gotPolicies, _, err := testAppfile.GenerateWorkflowAndPolicy(context.Background(), dm, k8sClient, pd, nil) + Expect(err).Should(BeNil()) + Expect(len(gotPolicies)).ShouldNot(Equal(0)) + + expectPolicy := unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "compName": "test-comp", + "workload": map[string]interface{}{ + "name": "test-comp", + "apiVersion": "apps/v1", + "kind": "Deployment", + }, + }, + "metadata": map[string]interface{}{ + "name": "test-policy", + "namespace": "default", + "labels": map[string]interface{}{ + "app.oam.dev/name": "test-app", + "app.oam.dev/component": "test-policy", + "app.oam.dev/appRevision": "", + "workload.oam.dev/type": "test-policy", + }, + }, + "apiVersion": "core.oam.dev/v1alpha2", + "kind": "HealthScope", + }, + } + Expect(len(gotPolicies)).ShouldNot(Equal(0)) + gotPolicy := gotPolicies[0] + Expect(cmp.Diff(gotPolicy.Object, expectPolicy.Object)).Should(BeEmpty()) + }) +}) + var _ = Describe("Test Terraform schematic appfile", func() { It("workload capability is Terraform", func() { var ( @@ -1366,3 +1467,81 @@ parameter: {} assert.DeepEqual(t, tc.expectConfigMapData, tc.workload.UserConfigs) } } + +func TestPrepareArtifactsData(t *testing.T) { + compManifests := []*oamtypes.ComponentManifest{ + &oamtypes.ComponentManifest{ + Name: "readyComp", + Namespace: "ns", + RevisionName: "readyComp-v1", + StandardWorkload: &unstructured.Unstructured{Object: map[string]interface{}{ + "fake": "workload", + }}, + Traits: func() []*unstructured.Unstructured { + ingressYAML := `apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + labels: + trait.oam.dev/resource: ingress + trait.oam.dev/type: ingress + namespace: default +spec: + rules: + - host: testsvc.example.com` + ingress := &unstructured.Unstructured{} + _ = yaml.Unmarshal([]byte(ingressYAML), ingress) + svcYAML := `apiVersion: v1 +kind: Service +metadata: + labels: + trait.oam.dev/resource: service + trait.oam.dev/type: ingress + namespace: default +spec: + clusterIP: 10.96.185.119 + selector: + app.oam.dev/component: express-server + type: ClusterIP` + svc := &unstructured.Unstructured{} + _ = yaml.Unmarshal([]byte(svcYAML), svc) + return []*unstructured.Unstructured{ingress, svc} + }(), + InsertConfigNotReady: false, + }, + } + + gotArtifacts := prepareArtifactsData(compManifests) + gotWorkload, _, err := unstructured.NestedMap(gotArtifacts, "readyComp", "workload") + assert.NilError(t, err) + diff := cmp.Diff(gotWorkload, map[string]interface{}{"fake": string("workload")}) + assert.Equal(t, diff, "") + + _, gotIngress, err := unstructured.NestedMap(gotArtifacts, "readyComp", "traits", "ingress", "ingress") + assert.NilError(t, err) + if !gotIngress { + t.Fatalf("cannot get ingress trait") + } + _, gotSvc, err := unstructured.NestedMap(gotArtifacts, "readyComp", "traits", "ingress", "service") + assert.NilError(t, err) + if !gotSvc { + t.Fatalf("cannot get service trait") + } + + compManifests = []*oamtypes.ComponentManifest{ + &oamtypes.ComponentManifest{ + Name: "notReadyComp", + Namespace: "ns", + RevisionName: "notReadyComp-v1", + InsertConfigNotReady: true, + }, + } + + gotArtifacts = prepareArtifactsData(compManifests) + gotReady, _, err := unstructured.NestedBool(gotArtifacts, "notReadyComp", "ready") + assert.NilError(t, err) + assert.Equal(t, gotReady, false) + + _, foundWorkload, err := unstructured.NestedMap(gotArtifacts, "notReadyComp", "workload") + assert.NilError(t, err) + assert.Equal(t, foundWorkload, false) +} diff --git a/pkg/cue/process/handle.go b/pkg/cue/process/handle.go index 98e7352bd..377fed596 100644 --- a/pkg/cue/process/handle.go +++ b/pkg/cue/process/handle.go @@ -57,6 +57,8 @@ const ( // ComponentRevisionPlaceHolder is the component revision name placeHolder, this field will be replace with real value // after component be created ComponentRevisionPlaceHolder = "KUBEVELA_COMPONENT_REVISION_PLACEHOLDER" + // ContextDataArtifacts is used to store unstructured resources of components + ContextDataArtifacts = "artifacts" ) // Context defines Rendering Context Interface @@ -70,6 +72,7 @@ type Context interface { SetConfigs(configs []map[string]string) InsertSecrets(outputSecretName string, requiredSecrets []RequiredSecrets) SetParameters(params map[string]interface{}) + PushData(key string, data interface{}) } // Auxiliary are objects rendered by definition template. @@ -108,6 +111,8 @@ type templateContext struct { auxiliaryHooks []AuxiliaryHook components []common.ApplicationComponent + + data map[string]interface{} } // RequiredSecrets is used to store all secret names which are generated by cloud resource components and required by current component @@ -240,6 +245,12 @@ func (ctx *templateContext) BaseContextFile() string { if ctx.outputSecretName != "" { buff += fmt.Sprintf("%s:\"%s\"", OutputSecretName, ctx.outputSecretName) } + + if ctx.data != nil { + d, _ := json.Marshal(ctx.data) + buff += fmt.Sprintf("\n %s", structMarshal(string(d))) + } + return fmt.Sprintf("context: %s", structMarshal(buff)) } @@ -286,6 +297,15 @@ func (ctx *templateContext) InsertSecrets(outputSecretName string, requiredSecre } } +// PushData appends arbitrary extension data to context +func (ctx *templateContext) PushData(key string, data interface{}) { + if ctx.data == nil { + ctx.data = map[string]interface{}{key: data} + return + } + ctx.data[key] = data +} + func structMarshal(v string) string { skip := false v = strings.TrimFunc(v, func(r rune) bool { diff --git a/pkg/cue/process/handle_test.go b/pkg/cue/process/handle_test.go index a48556240..64b508f7e 100644 --- a/pkg/cue/process/handle_test.go +++ b/pkg/cue/process/handle_test.go @@ -75,12 +75,36 @@ image: "myserver" }, "parameter3": []string{"item1", "item2"}, } + targetData := map[string]interface{}{ + "int": 10, + "string": "mytxt", + "bool": false, + "map": map[string]interface{}{ + "key": "value", + }, + "slice": []string{ + "str1", "str2", "str3", + }, + } + targetArbitraryData := map[string]interface{}{ + "int": 10, + "string": "mytxt", + "bool": false, + "map": map[string]interface{}{ + "key": "value", + }, + "slice": []string{ + "str1", "str2", "str3", + }, + } ctx := NewContext("myns", "mycomp", "myapp", "myapp-v1") ctx.InsertSecrets("db-conn", targetRequiredSecrets) ctx.SetBase(base) ctx.AppendAuxiliaries(svcAux) ctx.SetParameters(targetParams) + ctx.PushData(ContextDataArtifacts, targetData) + ctx.PushData("arbitraryData", targetArbitraryData) ctxInst, err := r.Compile("-", ctx.ExtendedContextFile()) if err != nil { @@ -123,4 +147,12 @@ image: "myserver" params, err := ctxInst.Lookup("context", ParametersFieldName).MarshalJSON() assert.Equal(t, nil, err) assert.Equal(t, "{\"parameter1\":\"string\",\"parameter2\":{\"key1\":\"value1\",\"key2\":\"value2\"},\"parameter3\":[\"item1\",\"item2\"]}", string(params)) + + artifacts, err := ctxInst.Lookup("context", ContextDataArtifacts).MarshalJSON() + assert.Equal(t, nil, err) + assert.Equal(t, "{\"bool\":false,\"string\":\"mytxt\",\"int\":10,\"map\":{\"key\":\"value\"},\"slice\":[\"str1\",\"str2\",\"str3\"]}", string(artifacts)) + + arbitraryData, err := ctxInst.Lookup("context", "arbitraryData").MarshalJSON() + assert.Equal(t, nil, err) + assert.Equal(t, "{\"bool\":false,\"string\":\"mytxt\",\"int\":10,\"map\":{\"key\":\"value\"},\"slice\":[\"str1\",\"str2\",\"str3\"]}", string(arbitraryData)) }