[Backport release-1.4] Feat: enhance velq ql and support cue file (#4266)

* Feat: enhance velq ql and support cue file

Signed-off-by: Jianbo Sun <jianbo.sjb@alibaba-inc.com>
(cherry picked from commit 70ed417326)

* add statement

Signed-off-by: Jianbo Sun <jianbo.sjb@alibaba-inc.com>
(cherry picked from commit 25260eb9ef)

Co-authored-by: Jianbo Sun <jianbo.sjb@alibaba-inc.com>

pkg/cue/model/keyword.go

@@ -51,7 +51,7 @@ const (
 	ContextComponents = "components"
 	// ContextComponentType is the component type of current trait binding with
 	ContextComponentType = "componentType"
-	// ComponentRevisionPlaceHolder is the component revision name placeHolder, this field will be replace with real value
+	// ComponentRevisionPlaceHolder is the component revision name placeHolder, this field will be replaced with real value
 	// after component be created
 	ComponentRevisionPlaceHolder = "KUBEVELA_COMPONENT_REVISION_PLACEHOLDER"
 	// ContextDataArtifacts is used to store unstructured resources of components

pkg/velaql/parse.go

@@ -17,11 +17,15 @@
 package velaql
 
 import (
+	"io/ioutil"
+	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
 
 	"github.com/pkg/errors"
+
+	"github.com/oam-dev/kubevela/pkg/cue/model/value"
 )
 
 // QueryView contains query data
@@ -40,6 +44,8 @@ const (
 	KeyWordView = "view"
 	// KeyWordParameter represent parameter keyword
 	KeyWordParameter = "parameter"
+	// KeyWordTemplate represents template keyword
+	KeyWordTemplate = "template"
 	// KeyWordExport represent export keyword
 	KeyWordExport = "export"
 	// DefaultExportValue is the default Export value
@@ -91,6 +97,36 @@ func ParseVelaQL(ql string) (QueryView, error) {
 	return qv, nil
 }
 
+// ParseVelaQLFromPath will parse a velaQL file path to QueryView
+func ParseVelaQLFromPath(velaQLViewPath string) (*QueryView, error) {
+	body, err := ioutil.ReadFile(filepath.Clean(velaQLViewPath))
+	if err != nil {
+		return nil, errors.Errorf("read view file from %s: %v", velaQLViewPath, err)
+	}
+
+	val, err := value.NewValue(string(body), nil, "")
+	if err != nil {
+		return nil, errors.Errorf("new value for view: %v", err)
+	}
+
+	var expStr string
+	exp, err := val.LookupValue(KeyWordExport)
+	if err == nil {
+		expStr, err = exp.String()
+		if err != nil {
+			expStr = DefaultExportValue
+		}
+	} else {
+		expStr = DefaultExportValue
+	}
+
+	return &QueryView{
+		View:      string(body),
+		Parameter: nil,
+		Export:    strings.Trim(strings.TrimSpace(expStr), `"`),
+	}, nil
+}
+
 // ParseParameter parse parameter to map[string]interface{}
 func ParseParameter(parameter string) (map[string]interface{}, error) {
 	parameter = strings.TrimLeft(parameter, "{")

pkg/velaql/view.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strings"
 
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -37,6 +38,7 @@ import (
 	"github.com/oam-dev/kubevela/pkg/utils"
 	"github.com/oam-dev/kubevela/pkg/utils/apply"
 	"github.com/oam-dev/kubevela/pkg/workflow/tasks"
+	"github.com/oam-dev/kubevela/pkg/workflow/tasks/template"
 	wfTypes "github.com/oam-dev/kubevela/pkg/workflow/types"
 )
 
@@ -73,7 +75,7 @@ func (handler *ViewHandler) QueryView(ctx context.Context, qv QueryView) (*value
 	outputsTemplate := fmt.Sprintf(OutputsTemplate, qv.Export, qv.Export)
 	queryKey := QueryParameterKey{}
 	if err := json.Unmarshal([]byte(outputsTemplate), &queryKey); err != nil {
-		return nil, err
+		return nil, errors.Errorf("unmarhsal query template: %v", err)
 	}
 
 	handler.viewTask = v1beta1.WorkflowStep{
@@ -84,7 +86,12 @@ func (handler *ViewHandler) QueryView(ctx context.Context, qv QueryView) (*value
 	}
 
 	pCtx := process.NewContext(process.ContextData{})
-	taskDiscover := tasks.NewViewTaskDiscover(handler.pd, handler.cli, handler.cfg, handler.dispatch, handler.delete, handler.namespace, 3, pCtx)
+	loader := template.NewViewTemplateLoader(handler.cli, handler.namespace)
+	if len(strings.Split(qv.View, "\n")) > 2 {
+		loader = &template.EchoLoader{}
+	}
+
+	taskDiscover := tasks.NewViewTaskDiscover(handler.pd, handler.cli, handler.cfg, handler.dispatch, handler.delete, handler.namespace, 3, pCtx, loader)
 	genTask, err := taskDiscover.GetTaskGenerator(ctx, handler.viewTask.Type)
 	if err != nil {
 		return nil, err
@@ -97,14 +104,14 @@ func (handler *ViewHandler) QueryView(ctx context.Context, qv QueryView) (*value
 
 	viewCtx, err := NewViewContext()
 	if err != nil {
-		return nil, err
+		return nil, errors.Errorf("new view context: %v", err)
 	}
 	status, _, err := runner.Run(viewCtx, &wfTypes.TaskRunOptions{})
 	if err != nil {
-		return nil, err
+		return nil, errors.Errorf("run query view: %v", err)
 	}
 	if string(status.Phase) != ViewTaskPhaseSucceeded {
-		return nil, errors.Errorf("failed to query the view %s", status.Message)
+		return nil, errors.Errorf("failed to query the view: %s", status.Message)
 	}
 	return viewCtx.GetVar(qv.Export)
 }

pkg/workflow/tasks/discover.go

@@ -272,7 +272,7 @@ func (tr *stepGroupTaskRunner) Run(ctx wfContext.Context, options *types.TaskRun
 }
 
 // NewViewTaskDiscover will create a client for load task generator.
-func NewViewTaskDiscover(pd *packages.PackageDiscover, cli client.Client, cfg *rest.Config, apply kube.Dispatcher, delete kube.Deleter, viewNs string, logLevel int, pCtx process.Context) types.TaskDiscover {
+func NewViewTaskDiscover(pd *packages.PackageDiscover, cli client.Client, cfg *rest.Config, apply kube.Dispatcher, delete kube.Deleter, viewNs string, logLevel int, pCtx process.Context, loader template.Loader) types.TaskDiscover {
 	handlerProviders := providers.NewProviders()
 
 	// install builtin provider
@@ -282,10 +282,9 @@ func NewViewTaskDiscover(pd *packages.PackageDiscover, cli client.Client, cfg *r
 	http.Install(handlerProviders, cli, viewNs)
 	email.Install(handlerProviders)
 
-	templateLoader := template.NewViewTemplateLoader(cli, viewNs)
 	return &taskDiscover{
-		remoteTaskDiscover: custom.NewTaskLoader(templateLoader.LoadTaskTemplate, pd, handlerProviders, logLevel, pCtx),
-		templateLoader:     templateLoader,
+		remoteTaskDiscover: custom.NewTaskLoader(loader.LoadTaskTemplate, pd, handlerProviders, logLevel, pCtx),
+		templateLoader:     loader,
 	}
 }
 

pkg/workflow/tasks/template/load.go

@@ -119,3 +119,12 @@ func NewViewTemplateLoader(client client.Client, namespace string) Loader {
 		namespace: namespace,
 	}
 }
+
+// EchoLoader will load data from input as it is.
+type EchoLoader struct {
+}
+
+// LoadTaskTemplate gets the echo content exactly what it is .
+func (ll *EchoLoader) LoadTaskTemplate(_ context.Context, content string) (string, error) {
+	return content, nil
+}

references/cli/velaql.go

@@ -42,39 +42,92 @@ type Filter struct {
 
 // NewQlCommand creates `ql` command for executing velaQL
 func NewQlCommand(c common.Args, order string, ioStreams util.IOStreams) *cobra.Command {
+	var cueFile, querySts string
 	ctx := context.Background()
 	cmd := &cobra.Command{
-		Use:     "ql",
-		Short:   "Show result of executing velaQL.",
-		Long:    "Show result of executing velaQL.",
-		Example: `vela ql "view{parameter=value1,parameter=value2}"`,
+		Use:   "ql",
+		Short: "Show result of executing velaQL.",
+		Long:  "Show result of executing velaQL.",
+		Example: `Users can query with a query statement:
+		vela ql --query "<inner-view-name>{<param1>=<value1>,<param2>=<value2>}"
+They can also query by a ql file:
+		vela ql --file=./ql.cue
+
+Example content of ql.cue:
+---
+import (
+	"vela/ql"
+)
+configmap: ql.#Read & {
+   value: {
+      kind: "ConfigMap"
+      apiVersion: "v1"
+      metadata: {
+        name: "mycm"
+      }
+   }
+}
+status: configmap.value.data.key
+
+export: "status"
+---
+`,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			argsLength := len(args)
-			if argsLength == 0 {
-				return fmt.Errorf("please specify an VelaQL statement")
+			if cueFile == "" && querySts == "" && len(args) == 0 {
+				return fmt.Errorf("please specify at least on VelaQL statement or velaql file path")
 			}
-			velaQL := args[0]
 			newClient, err := c.GetClient()
 			if err != nil {
 				return err
 			}
-			return printVelaQLResult(ctx, newClient, c, velaQL, cmd)
+
+			if cueFile != "" {
+				return queryFromView(ctx, newClient, c, cueFile, cmd)
+			}
+			if querySts == "" {
+				// for compatibility
+				querySts = args[0]
+			}
+			return queryFromStatement(ctx, newClient, c, querySts, cmd)
 		},
 		Annotations: map[string]string{
 			types.TagCommandOrder: order,
 			types.TagCommandType:  types.TypeApp,
 		},
 	}
+	cmd.Flags().StringVarP(&cueFile, "file", "f", "", "The CUE file path for VelaQL.")
+	cmd.Flags().StringVarP(&querySts, "query", "q", "", "The query statement for VelaQL.")
 	cmd.SetOut(ioStreams.Out)
 	return cmd
 }
 
-// printVelaQLResult show velaQL result
-func printVelaQLResult(ctx context.Context, client client.Client, velaC common.Args, velaQL string, cmd *cobra.Command) error {
-	queryValue, err := QueryValue(ctx, client, velaC, velaQL)
+// queryFromStatement print velaQL result from query statement with inner query view
+func queryFromStatement(ctx context.Context, client client.Client, velaC common.Args, velaQLStatement string, cmd *cobra.Command) error {
+	queryView, err := velaql.ParseVelaQL(velaQLStatement)
 	if err != nil {
 		return err
 	}
+	queryValue, err := QueryValue(ctx, client, velaC, &queryView)
+	if err != nil {
+		return err
+	}
+	return print(queryValue, cmd)
+}
+
+// queryFromView print velaQL result from query view
+func queryFromView(ctx context.Context, client client.Client, velaC common.Args, velaQLViewPath string, cmd *cobra.Command) error {
+	queryView, err := velaql.ParseVelaQLFromPath(velaQLViewPath)
+	if err != nil {
+		return err
+	}
+	queryValue, err := QueryValue(ctx, client, velaC, queryView)
+	if err != nil {
+		return err
+	}
+	return print(queryValue, cmd)
+}
+
+func print(queryValue *value.Value, cmd *cobra.Command) error {
 	response, err := queryValue.CueValue().MarshalJSON()
 	if err != nil {
 		return err
@@ -91,11 +144,11 @@ func printVelaQLResult(ctx context.Context, client client.Client, velaC common.A
 // MakeVelaQL build velaQL
 func MakeVelaQL(view string, params map[string]string, action string) string {
 	var paramString string
-	for key, value := range params {
+	for k, v := range params {
 		if paramString != "" {
-			paramString = fmt.Sprintf("%s, %s=%s", paramString, key, value)
+			paramString = fmt.Sprintf("%s, %s=%s", paramString, k, v)
 		} else {
-			paramString = fmt.Sprintf("%s=%s", key, value)
+			paramString = fmt.Sprintf("%s=%s", k, v)
 		}
 	}
 	return fmt.Sprintf("%s{%s}.%s", view, paramString, action)
@@ -116,7 +169,11 @@ func GetServiceEndpoints(ctx context.Context, client client.Client, appName stri
 	}
 
 	velaQL := MakeVelaQL("service-endpoints-view", params, "status")
-	queryValue, err := QueryValue(ctx, client, velaC, velaQL)
+	queryView, err := velaql.ParseVelaQL(velaQL)
+	if err != nil {
+		return nil, err
+	}
+	queryValue, err := QueryValue(ctx, client, velaC, &queryView)
 	if err != nil {
 		return nil, err
 	}
@@ -134,7 +191,7 @@ func GetServiceEndpoints(ctx context.Context, client client.Client, appName stri
 }
 
 // QueryValue get queryValue from velaQL
-func QueryValue(ctx context.Context, client client.Client, velaC common.Args, velaQL string) (*value.Value, error) {
+func QueryValue(ctx context.Context, client client.Client, velaC common.Args, queryView *velaql.QueryView) (*value.Value, error) {
 	dm, err := velaC.GetDiscoveryMapper()
 	if err != nil {
 		return nil, err
@@ -143,15 +200,11 @@ func QueryValue(ctx context.Context, client client.Client, velaC common.Args, ve
 	if err != nil {
 		return nil, err
 	}
-	queryView, err := velaql.ParseVelaQL(velaQL)
-	if err != nil {
-		return nil, err
-	}
 	config, err := velaC.GetConfig()
 	if err != nil {
 		return nil, err
 	}
-	queryValue, err := velaql.NewViewHandler(client, config, dm, pd).QueryView(ctx, queryView)
+	queryValue, err := velaql.NewViewHandler(client, config, dm, pd).QueryView(ctx, *queryView)
 	if err != nil {
 		return nil, err
 	}

references/cli/velaql_test.go

@@ -17,10 +17,14 @@ limitations under the License.
 package cli
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io/ioutil"
+	"os"
+	"strconv"
 	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -39,6 +43,43 @@ import (
 	common2 "github.com/oam-dev/kubevela/pkg/utils/common"
 )
 
+var _ = Describe("Test velaQL from file", func() {
+	It("Test Query pod data", func() {
+		cm := &corev1.ConfigMap{Data: map[string]string{"key": "my-value"}}
+		cm.Name = "mycm"
+		cm.Namespace = "default"
+		Expect(k8sClient.Create(context.TODO(), cm)).Should(BeNil())
+		view := `import (
+	"vela/ql"
+)
+configmap: ql.#Read & {
+   value: {
+      kind: "ConfigMap"
+      apiVersion: "v1"
+      metadata: {
+        name: "mycm"
+      }
+   }
+}
+status: configmap.value.data.key
+
+export: "status"
+`
+		name := "vela-test-" + strconv.FormatInt(time.Now().UnixNano(), 10) + ".cue"
+		Expect(os.WriteFile(name, []byte(view), 0644)).Should(BeNil())
+		defer os.Remove(name)
+
+		arg := common2.Args{}
+		arg.SetConfig(cfg)
+		arg.SetClient(k8sClient)
+		cmd := NewCommand()
+		var buff = bytes.NewBufferString("")
+		cmd.SetOut(buff)
+		Expect(queryFromView(context.TODO(), k8sClient, arg, name, cmd)).Should(BeNil())
+		Expect(buff.String()).Should(ContainSubstring(`my-value`))
+	})
+})
+
 var _ = Describe("Test velaQL", func() {
 	var appName = "test-velaql"
 	var namespace = "default"