Feat: add the feature that views in addon are applied independently.(#3905) (#4154)

* Feat: add the feature that views in addon are applied independently.(#3905)

separate the deployment of views in addon and deploy them separately instead of nested in application.

see the issue for details

Refs #3905

Signed-off-by: HanMengnan <1448189829@qq.com>

* Fix: add test case of readViewFile.

Signed-off-by: HanMengnan <1448189829@qq.com>

* Fix: add the apply method of yaml view.

Signed-off-by: HanMengnan <1448189829@qq.com>

* Fix: add addon suit-test case

Signed-off-by: HanMengnan <1448189829@qq.com>

apis/types/capability.go

@@ -80,6 +80,8 @@ const (
 	OpenapiV3JSONSchema string = "openapi-v3-json-schema"
 	// UISchema is the key to store ui custom schema
 	UISchema string = "ui-schema"
+	// VelaQLConfigmapKey is the key to store velaql view
+	VelaQLConfigmapKey string = "template"
 )
 
 // CapabilityCategory defines the category of a capability

pkg/addon/addon.go

@@ -94,6 +94,9 @@ const (
 	// DefSchemaName is the addon definition schemas dir name
 	DefSchemaName string = "schemas"
 
+	// ViewDirName is the addon views dir name
+	ViewDirName string = "views"
+
 	// AddonParameterDataKey is the key of parameter in addon args secrets
 	AddonParameterDataKey string = "addonParameterDataKey"
 
@@ -191,7 +194,7 @@ type Pattern struct {
 }
 
 // Patterns is the file pattern that the addon should be in
-var Patterns = []Pattern{{Value: ReadmeFileName}, {Value: MetadataFileName}, {Value: TemplateFileName}, {Value: ParameterFileName}, {IsDir: true, Value: ResourcesDirName}, {IsDir: true, Value: DefinitionsDirName}, {IsDir: true, Value: DefSchemaName}}
+var Patterns = []Pattern{{Value: ReadmeFileName}, {Value: MetadataFileName}, {Value: TemplateFileName}, {Value: ParameterFileName}, {IsDir: true, Value: ResourcesDirName}, {IsDir: true, Value: DefinitionsDirName}, {IsDir: true, Value: DefSchemaName}, {IsDir: true, Value: ViewDirName}}
 
 // GetPatternFromItem will check if the file path has a valid pattern, return empty string if it's invalid.
 // AsyncReader is needed to calculate relative path
@@ -306,6 +309,7 @@ func GetInstallPackageFromReader(r AsyncReader, meta *SourceMeta, uiData *UIData
 		TemplateFileName: readTemplate,
 		ResourcesDirName: readResFile,
 		DefSchemaName:    readDefSchemaFile,
+		ViewDirName:      readViewFile,
 	}
 	ptItems := ClassifyItemByPattern(meta, r)
 
@@ -408,6 +412,23 @@ func readDefFile(a *UIData, reader AsyncReader, readPath string) error {
 	return nil
 }
 
+// readViewFile read single view file
+func readViewFile(a *InstallPackage, reader AsyncReader, readPath string) error {
+	b, err := reader.ReadFile(readPath)
+	if err != nil {
+		return err
+	}
+	filename := path.Base(readPath)
+	switch filepath.Ext(filename) {
+	case ".cue":
+		a.CUEViews = append(a.CUEViews, ElementFile{Data: b, Name: filepath.Base(readPath)})
+	case ".yaml":
+		a.YAMLViews = append(a.YAMLViews, ElementFile{Data: b, Name: filepath.Base(readPath)})
+	default:
+	}
+	return nil
+}
+
 func readMetadata(a *UIData, reader AsyncReader, readPath string) error {
 	b, err := reader.ReadFile(readPath)
 	if err != nil {
@@ -809,6 +830,26 @@ func RenderDefinitionSchema(addon *InstallPackage) ([]*unstructured.Unstructured
 	return schemaConfigmaps, nil
 }
 
+// RenderViews will render views in addons.
+func RenderViews(addon *InstallPackage) ([]*unstructured.Unstructured, error) {
+	views := make([]*unstructured.Unstructured, 0)
+	for _, view := range addon.YAMLViews {
+		obj, err := renderObject(view)
+		if err != nil {
+			return nil, err
+		}
+		views = append(views, obj)
+	}
+	for _, view := range addon.CUEViews {
+		obj, err := renderCUEView(view)
+		if err != nil {
+			return nil, err
+		}
+		views = append(views, obj)
+	}
+	return views, nil
+}
+
 func allocateDomainForAddon(ctx context.Context, k8sClient client.Client) ([]ObservabilityEnvironment, error) {
 	secrets, err := multicluster.ListExistingClusterSecrets(ctx, k8sClient)
 	if err != nil {
@@ -953,6 +994,16 @@ func renderSchemaConfigmap(elem ElementFile) (*unstructured.Unstructured, error)
 	return util.Object2Unstructured(cm)
 }
 
+func renderCUEView(elem ElementFile) (*unstructured.Unstructured, error) {
+	cm := v1.ConfigMap{
+		TypeMeta:   metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"},
+		ObjectMeta: metav1.ObjectMeta{Namespace: types.DefaultKubeVelaNS, Name: strings.Split(elem.Name, ".")[0]},
+		Data: map[string]string{
+			types.VelaQLConfigmapKey: elem.Data,
+		}}
+	return util.Object2Unstructured(cm)
+}
+
 // renderCUETemplate will return a component from cue template
 func renderCUETemplate(elem ElementFile, parameters string, args map[string]interface{}, metadata Meta) (*common2.ApplicationComponent, error) {
 	bt, err := json.Marshal(args)
@@ -1248,6 +1299,11 @@ func (h *Installer) dispatchAddonResource(addon *InstallPackage) error {
 		return errors.Wrap(err, "render addon definitions' schema fail")
 	}
 
+	views, err := RenderViews(addon)
+	if err != nil {
+		return errors.Wrap(err, "render addon views fail")
+	}
+
 	if err := passDefInAppAnnotation(defs, app); err != nil {
 		return errors.Wrapf(err, "cannot pass definition to addon app's annotation")
 	}
@@ -1272,6 +1328,14 @@ func (h *Installer) dispatchAddonResource(addon *InstallPackage) error {
 		}
 	}
 
+	for _, view := range views {
+		addOwner(view, app)
+		err = h.apply.Apply(h.ctx, view, apply.DisableUpdateAnnotation())
+		if err != nil {
+			return err
+		}
+	}
+
 	if h.args != nil && len(h.args) > 0 {
 		sec := RenderArgsSecret(addon, h.args)
 		addOwner(sec, app)

pkg/addon/addon_suite_test.go

@@ -378,6 +378,23 @@ var _ = Describe("test enable addon in local dir", func() {
 	})
 })
 
+var _ = Describe("test enable addon which applies the views independently", func() {
+	BeforeEach(func() {
+		app := v1beta1.Application{ObjectMeta: metav1.ObjectMeta{Namespace: "vela-system", Name: "addon-test-view"}}
+		Expect(k8sClient.Delete(ctx, &app)).Should(SatisfyAny(BeNil(), util.NotFoundMatcher{}))
+	})
+
+	It("test enable addon which applies the views independently", func() {
+		ctx := context.Background()
+		err := EnableAddonByLocalDir(ctx, "test-view", "./testdata/test-view", k8sClient, dc, apply.NewAPIApplicator(k8sClient), cfg, map[string]interface{}{"example": "test"})
+		Expect(err).Should(BeNil())
+		app := v1beta1.Application{}
+		Expect(k8sClient.Get(ctx, types2.NamespacedName{Namespace: "vela-system", Name: "addon-test-view"}, &app)).Should(BeNil())
+		configMap := v1.ConfigMap{}
+		Expect(k8sClient.Get(ctx, types2.NamespacedName{Namespace: "vela-system", Name: "pod-view"}, &configMap)).Should(BeNil())
+	})
+})
+
 const (
 	appYaml = `apiVersion: core.oam.dev/v1beta1
 kind: Application

pkg/addon/addon_test.go

@@ -288,6 +288,30 @@ func TestRenderDefinitions(t *testing.T) {
 	assert.Nil(t, app.Spec.Workflow)
 }
 
+func TestRenderViews(t *testing.T) {
+	addonDeployToRuntime := viewAddon
+	addonDeployToRuntime.Meta.DeployTo = &DeployTo{
+		DisableControlPlane: false,
+		RuntimeCluster:      false,
+	}
+	views, err := RenderViews(&addonDeployToRuntime)
+	assert.NoError(t, err)
+	assert.Equal(t, len(views), 2)
+
+	view := views[0]
+	assert.Equal(t, view.GetKind(), "ConfigMap")
+	assert.Equal(t, view.GetAPIVersion(), "v1")
+	assert.Equal(t, view.GetNamespace(), types.DefaultKubeVelaNS)
+	assert.Equal(t, view.GetName(), "cloud-resource-view")
+
+	view = views[1]
+	assert.Equal(t, view.GetKind(), "ConfigMap")
+	assert.Equal(t, view.GetAPIVersion(), "v1")
+	assert.Equal(t, view.GetNamespace(), types.DefaultKubeVelaNS)
+	assert.Equal(t, view.GetName(), "pod-view")
+
+}
+
 func TestRenderK8sObjects(t *testing.T) {
 	addonMultiYaml := multiYamlAddon
 	addonMultiYaml.Meta.DeployTo = &DeployTo{
@@ -475,6 +499,24 @@ var multiYamlAddon = InstallPackage{
 	},
 }
 
+var viewAddon = InstallPackage{
+	Meta: Meta{
+		Name: "test-render-view-addon",
+	},
+	YAMLViews: []ElementFile{
+		{
+			Data: testYAMLView,
+			Name: "cloud-resource-view",
+		},
+	},
+	CUEViews: []ElementFile{
+		{
+			Data: testCUEView,
+			Name: "pod-view",
+		},
+	},
+}
+
 var testCueDef = `annotations: {
 	type: "trait"
 	annotations: {}
@@ -553,6 +595,125 @@ spec:
         - containerPort: 80
 `
 
+var testYAMLView = `
+apiVersion: "v1"
+kind: "ConfigMap"
+metadata:
+  name: "cloud-resource-view"
+  namespace: "vela-system"
+data:
+  template: |
+    import (
+    "vela/ql"
+    )
+    
+    parameter: {
+      appName: string
+        appNs:   string
+    }
+    resources: ql.#ListResourcesInApp & {
+      app: {
+        name:      parameter.appName
+          namespace: parameter.appNs
+          filter: {
+            "apiVersion": "terraform.core.oam.dev/v1beta1"
+              "kind":       "Configuration"
+          }
+          withStatus: true
+      }
+    }
+    status: {
+      if resources.err == _|_ {
+        "cloud-resources": [ for i, resource in resources.list {
+          resource.object
+        }]
+      }
+      if resources.err != _|_ {
+        error: resources.err
+      }
+    }
+
+
+`
+var testCUEView = `
+import (
+	"vela/ql"
+)
+
+parameter: {
+	name:      string
+	namespace: string
+	cluster:   *"" | string
+}
+pod: ql.#Read & {
+	value: {
+		apiVersion: "v1"
+		kind:       "Pod"
+		metadata: {
+			name:      parameter.name
+			namespace: parameter.namespace
+		}
+	}
+	cluster: parameter.cluster
+}
+eventList: ql.#SearchEvents & {
+	value: {
+		apiVersion: "v1"
+		kind:       "Pod"
+		metadata:   pod.value.metadata
+	}
+	cluster: parameter.cluster
+}
+podMetrics: ql.#Read & {
+	cluster: parameter.cluster
+	value: {
+		apiVersion: "metrics.k8s.io/v1beta1"
+		kind:       "PodMetrics"
+		metadata: {
+			name:      parameter.name
+			namespace: parameter.namespace
+		}
+	}
+}
+status: {
+	if pod.err == _|_ {
+		containers: [ for container in pod.value.spec.containers {
+			name:  container.name
+			image: container.image
+			resources: {
+				if container.resources.limits != _|_ {
+					limits: container.resources.limits
+				}
+				if container.resources.requests != _|_ {
+					requests: container.resources.requests
+				}
+				if podMetrics.err == _|_ {
+					usage: {for containerUsage in podMetrics.value.containers {
+						if containerUsage.name == container.name {
+							cpu:    containerUsage.usage.cpu
+							memory: containerUsage.usage.memory
+						}
+					}}
+				}
+			}
+			if pod.value.status.containerStatuses != _|_ {
+				status: {for containerStatus in pod.value.status.containerStatuses if containerStatus.name == container.name {
+					state:        containerStatus.state
+					restartCount: containerStatus.restartCount
+				}}
+			}
+		}]
+		if eventList.err == _|_ {
+			events: eventList.list
+		}
+	}
+	if pod.err != _|_ {
+		error: pod.err
+	}
+}
+
+`
+
 func TestRenderApp4Observability(t *testing.T) {
 	k8sClient := fake.NewClientBuilder().Build()
 	testcases := []struct {
@@ -639,6 +800,13 @@ func TestGetPatternFromItem(t *testing.T) {
 	gitItemName := "parameter.cue"
 	gitItemType := FileType
 	gitItemPath := "addons/terraform/resources/parameter.cue"
+
+	viewOSSR := localReader{
+		dir:  "./testdata/test-view",
+		name: "test-view",
+	}
+	viewPath := filepath.Join("./testdata/test-view/views/pod-view.cue", "pod-view.cue")
+
 	testCases := []struct {
 		caseName    string
 		item        Item
@@ -664,6 +832,17 @@ func TestGetPatternFromItem(t *testing.T) {
 			meetPattern: "resources/parameter.cue",
 			r:           gitR,
 		},
+		{
+			caseName: "views case",
+			item: OSSItem{
+				tp:   FileType,
+				path: viewPath,
+				name: "pod-view.cue",
+			},
+			root:        "test-view",
+			meetPattern: "views",
+			r:           viewOSSR,
+		},
 	}
 	for _, tc := range testCases {
 		res := GetPatternFromItem(tc.item, tc.r, tc.root)
@@ -917,6 +1096,36 @@ func TestReadDefFile(t *testing.T) {
 	assert.True(t, len(uiData.Definitions) == 1)
 }
 
+// Test readDefFile only accept .cue
+func TestReadViewFile(t *testing.T) {
+
+	// setup test data
+	testAddonName := "test-view"
+	testAddonDir := fmt.Sprintf("./testdata/%s", testAddonName)
+	reader := localReader{dir: testAddonDir, name: testAddonName}
+	metas, err := reader.ListAddonMeta()
+	testAddonMeta := metas[testAddonName]
+	assert.NoError(t, err)
+
+	// run test
+	var addon = &InstallPackage{}
+	ptItems := ClassifyItemByPattern(&testAddonMeta, reader)
+	items := ptItems[ViewDirName]
+
+	for _, it := range items {
+		err := readViewFile(addon, reader, reader.RelativePath(it))
+		if err != nil {
+			assert.NoError(t, err)
+		}
+	}
+	notExistErr := readViewFile(addon, reader, "not-exist.cue")
+	assert.Error(t, notExistErr)
+
+	// verify
+	assert.True(t, len(addon.CUEViews) == 1)
+	assert.True(t, len(addon.YAMLViews) == 1)
+}
+
 func TestRenderCUETemplate(t *testing.T) {
 	fileDate, err := os.ReadFile("./testdata/example/resources/configmap.cue")
 	assert.NoError(t, err)

pkg/addon/testdata/test-view/metadata.yaml

@@ -0,0 +1,15 @@
+name: test-view
+version: 1.0.0
+description: test
+icon: https://www.terraform.io/assets/images/logo-text-8c3ba8a6.svg
+url: https://terraform.io/
+
+tags: []
+
+deployTo:
+  controlPlane: true
+  runtimeCluster: false
+
+dependencies: []
+
+invisible: false

pkg/addon/testdata/test-view/views/cloud-resource-view.yaml

@@ -0,0 +1,37 @@
+apiVersion: "v1"
+kind: "ConfigMap"
+metadata:
+  name: "cloud-resource-view"
+  namespace: "vela-system"
+data:
+  template: |
+    import (
+    "vela/ql"
+    )
+    
+    parameter: {
+      appName: string
+        appNs:   string
+    }
+    resources: ql.#ListResourcesInApp & {
+      app: {
+        name:      parameter.appName
+          namespace: parameter.appNs
+          filter: {
+            "apiVersion": "terraform.core.oam.dev/v1beta1"
+              "kind":       "Configuration"
+          }
+          withStatus: true
+      }
+    }
+    status: {
+      if resources.err == _|_ {
+        "cloud-resources": [ for i, resource in resources.list {
+          resource.object
+        }]
+      }
+      if resources.err != _|_ {
+        error: resources.err
+      }
+    }
+

pkg/addon/testdata/test-view/views/pod-view.cue

@@ -0,0 +1,75 @@
+import (
+	"vela/ql"
+)
+
+parameter: {
+	name:      string
+	namespace: string
+	cluster:   *"" | string
+}
+pod: ql.#Read & {
+	value: {
+		apiVersion: "v1"
+		kind:       "Pod"
+		metadata: {
+			name:      parameter.name
+			namespace: parameter.namespace
+		}
+	}
+	cluster: parameter.cluster
+}
+eventList: ql.#SearchEvents & {
+	value: {
+		apiVersion: "v1"
+		kind:       "Pod"
+		metadata:   pod.value.metadata
+	}
+	cluster: parameter.cluster
+}
+podMetrics: ql.#Read & {
+	cluster: parameter.cluster
+	value: {
+		apiVersion: "metrics.k8s.io/v1beta1"
+		kind:       "PodMetrics"
+		metadata: {
+			name:      parameter.name
+			namespace: parameter.namespace
+		}
+	}
+}
+status: {
+	if pod.err == _|_ {
+		containers: [ for container in pod.value.spec.containers {
+			name:  container.name
+			image: container.image
+			resources: {
+				if container.resources.limits != _|_ {
+					limits: container.resources.limits
+				}
+				if container.resources.requests != _|_ {
+					requests: container.resources.requests
+				}
+				if podMetrics.err == _|_ {
+					usage: {for containerUsage in podMetrics.value.containers {
+						if containerUsage.name == container.name {
+							cpu:    containerUsage.usage.cpu
+							memory: containerUsage.usage.memory
+						}
+					}}
+				}
+			}
+			if pod.value.status.containerStatuses != _|_ {
+				status: {for containerStatus in pod.value.status.containerStatuses if containerStatus.name == container.name {
+					state:        containerStatus.state
+					restartCount: containerStatus.restartCount
+				}}
+			}
+		}]
+		if eventList.err == _|_ {
+			events: eventList.list
+		}
+	}
+	if pod.err != _|_ {
+		error: pod.err
+	}
+}

pkg/addon/type.go

@@ -48,6 +48,9 @@ type InstallPackage struct {
 	// Definitions and CUEDefinitions are converted as OAM X-Definitions, they will only in control plane cluster
 	Definitions    []ElementFile `json:"definitions"`
 	CUEDefinitions []ElementFile `json:"CUEDefinitions"`
+	// YAMLViews and CUEViews are the instances of velaql, they will only in control plane cluster
+	YAMLViews []ElementFile `json:"YAMLViews"`
+	CUEViews  []ElementFile `json:"CUEViews"`
 	// DefSchemas are UI schemas read by VelaUX, it will only be installed in control plane clusters
 	DefSchemas []ElementFile `json:"defSchemas,omitempty"`