Fix: fail to query the application logs with the special characters (#4308)

Signed-off-by: barnettZQG <barnett.zqg@gmail.com>
(cherry picked from commit 0101396a42)

Co-authored-by: barnettZQG <barnett.zqg@gmail.com>

.github/workflows/release.yml

@@ -123,6 +123,11 @@ jobs:
       - name: sync the latest version file
         if: ${{ !contains(env.VELA_VERSION,'alpha') && !contains(env.VELA_VERSION,'beta') }}
         run: |
+          LATEST_VERSION=$(curl -fsSl https://static.kubevela.net/binary/vela/latest_version)
+          verlte() {
+            [  "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ]
+          }
+          verlte ${{ env.VELA_VERSION }} $LATEST_VERSION && echo "${{ env.VELA_VERSION }} <= $LATEST_VERSION, skip update" && exit 0
           echo ${{ env.VELA_VERSION }} > ./latest_version
           ./ossutil --config-file .ossutilconfig cp -u ./latest_version oss://$BUCKET/binary/vela/latest_version
 

charts/vela-core/templates/defwithtemplate/service-account.yaml

@@ -14,10 +14,114 @@ spec:
   schematic:
     cue:
       template: |
+        #Privileges: {
+        	// +usage=Specify the verbs to be allowed for the resource
+        	verbs: [...string]
+        	// +usage=Specify the apiGroups of the resource
+        	apiGroups?: [...string]
+        	// +usage=Specify the resources to be allowed
+        	resources?: [...string]
+        	// +usage=Specify the resourceNames to be allowed
+        	resourceNames?: [...string]
+        	// +usage=Specify the resource url to be allowed
+        	nonResourceURLs?: [...string]
+        	// +usage=Specify the scope of the privileges, default to be namespace scope
+        	scope: *"namespace" | "cluster"
+        }
         parameter: {
         	// +usage=Specify the name of ServiceAccount
         	name: string
+        	// +usage=Specify whether to create new ServiceAccount or not
+        	create: *false | bool
+        	// +usage=Specify the privileges of the ServiceAccount, if not empty, RoleBindings(ClusterRoleBindings) will be created
+        	privileges?: [...#Privileges]
         }
         // +patchStrategy=retainKeys
         patch: spec: template: spec: serviceAccountName: parameter.name
+        _clusterPrivileges: [ for p in parameter.privileges if p.scope == "cluster" {p}]
+        _namespacePrivileges: [ for p in parameter.privileges if p.scope == "namespace" {p}]
+        outputs: {
+        	if parameter.create {
+        		"service-account": {
+        			apiVersion: "v1"
+        			kind:       "ServiceAccount"
+        			metadata: name: parameter.name
+        		}
+        	}
+        	if parameter.privileges != _|_ {
+        		if len(_clusterPrivileges) > 0 {
+        			"cluster-role": {
+        				apiVersion: "rbac.authorization.k8s.io/v1"
+        				kind:       "ClusterRole"
+        				metadata: name: "\(context.namespace):\(parameter.name)"
+        				rules: [ for p in _clusterPrivileges {
+        					verbs: p.verbs
+        					if p.apiGroups != _|_ {
+        						apiGroups: p.apiGroups
+        					}
+        					if p.resources != _|_ {
+        						resources: p.resources
+        					}
+        					if p.resourceNames != _|_ {
+        						resources: p.resourceNames
+        					}
+        					if p.nonResourceURLs != _|_ {
+        						nonResourceURLs: p.nonResourceURLs
+        					}
+        				}]
+        			}
+        			"cluster-role-binding": {
+        				apiVersion: "rbac.authorization.k8s.io/v1"
+        				kind:       "ClusterRoleBinding"
+        				metadata: name: "\(context.namespace):\(parameter.name)"
+        				roleRef: {
+        					apiGroup: "rbac.authorization.k8s.io"
+        					kind:     "ClusterRole"
+        					name:     "\(context.namespace):\(parameter.name)"
+        				}
+        				subjects: [{
+        					kind:      "ServiceAccount"
+        					name:      parameter.name
+        					namespace: "\(context.namespace)"
+        				}]
+        			}
+        		}
+        		if len(_namespacePrivileges) > 0 {
+        			role: {
+        				apiVersion: "rbac.authorization.k8s.io/v1"
+        				kind:       "Role"
+        				metadata: name: parameter.name
+        				rules: [ for p in _namespacePrivileges {
+        					verbs: p.verbs
+        					if p.apiGroups != _|_ {
+        						apiGroups: p.apiGroups
+        					}
+        					if p.resources != _|_ {
+        						resources: p.resources
+        					}
+        					if p.resourceNames != _|_ {
+        						resources: p.resourceNames
+        					}
+        					if p.nonResourceURLs != _|_ {
+        						nonResourceURLs: p.nonResourceURLs
+        					}
+        				}]
+        			}
+        			"role-binding": {
+        				apiVersion: "rbac.authorization.k8s.io/v1"
+        				kind:       "RoleBinding"
+        				metadata: name: parameter.name
+        				roleRef: {
+        					apiGroup: "rbac.authorization.k8s.io"
+        					kind:     "Role"
+        					name:     parameter.name
+        				}
+        				subjects: [{
+        					kind: "ServiceAccount"
+        					name: parameter.name
+        				}]
+        			}
+        		}
+        	}
+        }
 

charts/vela-minimal/templates/defwithtemplate/service-account.yaml

@@ -14,10 +14,114 @@ spec:
   schematic:
     cue:
       template: |
+        #Privileges: {
+        	// +usage=Specify the verbs to be allowed for the resource
+        	verbs: [...string]
+        	// +usage=Specify the apiGroups of the resource
+        	apiGroups?: [...string]
+        	// +usage=Specify the resources to be allowed
+        	resources?: [...string]
+        	// +usage=Specify the resourceNames to be allowed
+        	resourceNames?: [...string]
+        	// +usage=Specify the resource url to be allowed
+        	nonResourceURLs?: [...string]
+        	// +usage=Specify the scope of the privileges, default to be namespace scope
+        	scope: *"namespace" | "cluster"
+        }
         parameter: {
         	// +usage=Specify the name of ServiceAccount
         	name: string
+        	// +usage=Specify whether to create new ServiceAccount or not
+        	create: *false | bool
+        	// +usage=Specify the privileges of the ServiceAccount, if not empty, RoleBindings(ClusterRoleBindings) will be created
+        	privileges?: [...#Privileges]
         }
         // +patchStrategy=retainKeys
         patch: spec: template: spec: serviceAccountName: parameter.name
+        _clusterPrivileges: [ for p in parameter.privileges if p.scope == "cluster" {p}]
+        _namespacePrivileges: [ for p in parameter.privileges if p.scope == "namespace" {p}]
+        outputs: {
+        	if parameter.create {
+        		"service-account": {
+        			apiVersion: "v1"
+        			kind:       "ServiceAccount"
+        			metadata: name: parameter.name
+        		}
+        	}
+        	if parameter.privileges != _|_ {
+        		if len(_clusterPrivileges) > 0 {
+        			"cluster-role": {
+        				apiVersion: "rbac.authorization.k8s.io/v1"
+        				kind:       "ClusterRole"
+        				metadata: name: "\(context.namespace):\(parameter.name)"
+        				rules: [ for p in _clusterPrivileges {
+        					verbs: p.verbs
+        					if p.apiGroups != _|_ {
+        						apiGroups: p.apiGroups
+        					}
+        					if p.resources != _|_ {
+        						resources: p.resources
+        					}
+        					if p.resourceNames != _|_ {
+        						resources: p.resourceNames
+        					}
+        					if p.nonResourceURLs != _|_ {
+        						nonResourceURLs: p.nonResourceURLs
+        					}
+        				}]
+        			}
+        			"cluster-role-binding": {
+        				apiVersion: "rbac.authorization.k8s.io/v1"
+        				kind:       "ClusterRoleBinding"
+        				metadata: name: "\(context.namespace):\(parameter.name)"
+        				roleRef: {
+        					apiGroup: "rbac.authorization.k8s.io"
+        					kind:     "ClusterRole"
+        					name:     "\(context.namespace):\(parameter.name)"
+        				}
+        				subjects: [{
+        					kind:      "ServiceAccount"
+        					name:      parameter.name
+        					namespace: "\(context.namespace)"
+        				}]
+        			}
+        		}
+        		if len(_namespacePrivileges) > 0 {
+        			role: {
+        				apiVersion: "rbac.authorization.k8s.io/v1"
+        				kind:       "Role"
+        				metadata: name: parameter.name
+        				rules: [ for p in _namespacePrivileges {
+        					verbs: p.verbs
+        					if p.apiGroups != _|_ {
+        						apiGroups: p.apiGroups
+        					}
+        					if p.resources != _|_ {
+        						resources: p.resources
+        					}
+        					if p.resourceNames != _|_ {
+        						resources: p.resourceNames
+        					}
+        					if p.nonResourceURLs != _|_ {
+        						nonResourceURLs: p.nonResourceURLs
+        					}
+        				}]
+        			}
+        			"role-binding": {
+        				apiVersion: "rbac.authorization.k8s.io/v1"
+        				kind:       "RoleBinding"
+        				metadata: name: parameter.name
+        				roleRef: {
+        					apiGroup: "rbac.authorization.k8s.io"
+        					kind:     "Role"
+        					name:     parameter.name
+        				}
+        				subjects: [{
+        					kind: "ServiceAccount"
+        					name: parameter.name
+        				}]
+        			}
+        		}
+        	}
+        }
 

pkg/apiserver/domain/service/velaql.go

@@ -18,6 +18,8 @@ package service
 
 import (
 	"context"
+	"encoding/base64"
+	"strings"
 
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -79,5 +81,9 @@ func (v *velaQLServiceImpl) QueryView(ctx context.Context, velaQL string) (*apis
 		log.Logger.Errorf("decode the velaQL response to json failure %s", err.Error())
 		return nil, bcode.ErrParseQuery2Json
 	}
+	if strings.Contains(velaQL, "collect-logs") {
+		enc, _ := base64.StdEncoding.DecodeString(resp["logs"].(string))
+		resp["logs"] = string(enc)
+	}
 	return &resp, err
 }

pkg/apiserver/domain/service/webhook.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/emicklei/go-restful/v3"
@@ -426,6 +427,11 @@ func (j *jfrogHandlerImpl) handle(ctx context.Context, webhookTrigger *model.App
 		return nil, err
 	}
 	image := fmt.Sprintf("%s/%s:%s", jfrogReq.Data.RepoKey, jfrogReq.Data.ImageName, jfrogReq.Data.Tag)
+	pathArray := strings.Split(jfrogReq.Data.Path, "/")
+	if len(pathArray) > 2 {
+		image = fmt.Sprintf("%s/%s:%s", jfrogReq.Data.RepoKey, strings.Join(pathArray[:len(pathArray)-2], "/"), jfrogReq.Data.Tag)
+	}
+
 	if jfrogReq.Data.URL != "" {
 		image = fmt.Sprintf("%s/%s", jfrogReq.Data.URL, image)
 	}
@@ -441,7 +447,7 @@ func (j *jfrogHandlerImpl) handle(ctx context.Context, webhookTrigger *model.App
 		TriggerType:  apisv1.TriggerTypeWebhook,
 		Force:        true,
 		ImageInfo: &model.ImageInfo{
-			Type: model.PayloadTypeHarbor,
+			Type: model.PayloadTypeJFrog,
 			Resource: &model.ImageResource{
 				Digest: jfrogReq.Data.Digest,
 				Tag:    jfrogReq.Data.Tag,

pkg/auth/userinfo.go

@@ -47,6 +47,11 @@ func ContextWithUserInfo(ctx context.Context, app *v1beta1.Application) context.
 	return request.WithUser(ctx, GetUserInfoInAnnotation(&app.ObjectMeta))
 }
 
+// ContextClearUserInfo clear user info in context
+func ContextClearUserInfo(ctx context.Context) context.Context {
+	return request.WithUser(ctx, nil)
+}
+
 // SetUserInfoInAnnotation set username and group from userInfo into annotations
 // it will clear the existing service account annotation in avoid of permission leak
 func SetUserInfoInAnnotation(obj *metav1.ObjectMeta, userInfo authv1.UserInfo) {

pkg/builtin/http/http.go

@@ -51,7 +51,7 @@ func (c *HTTPCmd) Run(meta *registry.Meta) (res interface{}, err error) {
 	var (
 		r      io.Reader
 		client = &http.Client{
-			Transport: &http.Transport{},
+			Transport: http.DefaultTransport,
 			Timeout:   time.Second * 3,
 		}
 	)

pkg/resourcekeeper/dispatch.go

@@ -105,6 +105,7 @@ func (h *resourceKeeper) record(ctx context.Context, manifests []*unstructured.U
 	}
 
 	cfg := newDispatchConfig(options...)
+	ctx = auth.ContextClearUserInfo(ctx)
 	if len(rootManifests) != 0 {
 		rt, err := h.getRootRT(ctx)
 		if err != nil {

pkg/stdlib/pkgs/query.cue

@@ -86,9 +86,9 @@
 		limitBytes:   *null | int
 	}
 	outputs?: {
-		logs: string
-		err?: string
-		info: {
+		logs?: string
+		err?:  string
+		info?: {
 			fromDate: string
 			toDate:   string
 		}

pkg/velaql/parse.go

@@ -17,8 +17,6 @@
 package velaql
 
 import (
-	"io/ioutil"
-	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
@@ -26,6 +24,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/oam-dev/kubevela/pkg/cue/model/value"
+	"github.com/oam-dev/kubevela/references/common"
 )
 
 // QueryView contains query data
@@ -99,7 +98,7 @@ func ParseVelaQL(ql string) (QueryView, error) {
 
 // ParseVelaQLFromPath will parse a velaQL file path to QueryView
 func ParseVelaQLFromPath(velaQLViewPath string) (*QueryView, error) {
-	body, err := ioutil.ReadFile(filepath.Clean(velaQLViewPath))
+	body, err := common.ReadRemoteOrLocalPath(velaQLViewPath)
 	if err != nil {
 		return nil, errors.Errorf("read view file from %s: %v", velaQLViewPath, err)
 	}

pkg/velaql/providers/query/handler.go

@@ -18,12 +18,12 @@ package query
 
 import (
 	"bufio"
+	"bytes"
 	"context"
+	"encoding/base64"
 	"fmt"
 	"io"
-	"regexp"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -397,14 +397,6 @@ func (h *provider) GeneratorServiceEndpoints(wfctx wfContext.Context, v *value.V
 	return fillQueryResult(v, serviceEndpoints, "list")
 }
 
-var (
-	terminatedContainerNotFoundRegex = regexp.MustCompile("previous terminated container .+ in pod .+ not found")
-)
-
-func isTerminatedContainerNotFound(err error) bool {
-	return err != nil && terminatedContainerNotFoundRegex.MatchString(err.Error())
-}
-
 func (h *provider) CollectLogsInPod(ctx wfContext.Context, v *value.Value, act types.Action) error {
 	cluster, err := v.GetString("cluster")
 	if err != nil {
@@ -429,27 +421,29 @@ func (h *provider) CollectLogsInPod(ctx wfContext.Context, v *value.Value, act t
 	cliCtx := multicluster.ContextWithClusterName(context.Background(), cluster)
 	clientSet, err := kubernetes.NewForConfig(h.cfg)
 	if err != nil {
-		return errors.Wrapf(err, "failed to create kubernetes clientset")
+		return errors.Wrapf(err, "failed to create kubernetes client")
 	}
+	var defaultOutputs = make(map[string]interface{})
+	var errMsg string
 	podInst, err := clientSet.CoreV1().Pods(namespace).Get(cliCtx, pod, v1.GetOptions{})
 	if err != nil {
-		return errors.Wrapf(err, "failed to get pod")
+		errMsg = fmt.Sprintf("failed to get pod:%s", err.Error())
 	}
 	req := clientSet.CoreV1().Pods(namespace).GetLogs(pod, opts)
 	readCloser, err := req.Stream(cliCtx)
-	if err != nil && !isTerminatedContainerNotFound(err) {
-		return errors.Wrapf(err, "failed to get stream logs")
+	if err != nil {
+		errMsg = fmt.Sprintf("failed to get stream logs %s", err.Error())
 	}
-	r := bufio.NewReader(readCloser)
-	var b strings.Builder
-	var readErr error
-	if err == nil {
+	if readCloser != nil && podInst != nil {
+		r := bufio.NewReader(readCloser)
+		buffer := bytes.NewBuffer(nil)
+		var readErr error
 		defer func() {
 			_ = readCloser.Close()
 		}()
 		for {
 			s, err := r.ReadString('\n')
-			b.WriteString(s)
+			buffer.WriteString(s)
 			if err != nil {
 				if !errors.Is(err, io.EOF) {
 					readErr = err
@@ -457,30 +451,34 @@ func (h *provider) CollectLogsInPod(ctx wfContext.Context, v *value.Value, act t
 				break
 			}
 		}
-	} else {
-		readErr = err
+		toDate := v1.Now()
+		var fromDate v1.Time
+		// nolint
+		if opts.SinceTime != nil {
+			fromDate = *opts.SinceTime
+		} else if opts.SinceSeconds != nil {
+			fromDate = v1.NewTime(toDate.Add(time.Duration(-(*opts.SinceSeconds) * int64(time.Second))))
+		} else {
+			fromDate = podInst.CreationTimestamp
+		}
+		// the cue string can not support the special characters
+		logs := base64.StdEncoding.EncodeToString(buffer.Bytes())
+		defaultOutputs = map[string]interface{}{
+			"logs": logs,
+			"info": map[string]interface{}{
+				"fromDate": fromDate,
+				"toDate":   toDate,
+			},
+		}
+		if readErr != nil {
+			errMsg = readErr.Error()
+		}
 	}
-	toDate := v1.Now()
-	var fromDate v1.Time
-	// nolint
-	if opts.SinceTime != nil {
-		fromDate = *opts.SinceTime
-	} else if opts.SinceSeconds != nil {
-		fromDate = v1.NewTime(toDate.Add(time.Duration(-(*opts.SinceSeconds) * int64(time.Second))))
-	} else {
-		fromDate = podInst.CreationTimestamp
+	if errMsg != "" {
+		klog.Warningf(errMsg)
+		defaultOutputs["err"] = errMsg
 	}
-	o := map[string]interface{}{
-		"logs": b.String(),
-		"info": map[string]interface{}{
-			"fromDate": fromDate,
-			"toDate":   toDate,
-		},
-	}
-	if readErr != nil {
-		o["err"] = readErr.Error()
-	}
-	return v.FillObject(o, "outputs")
+	return v.FillObject(defaultOutputs, "outputs")
 }
 
 // Install register handlers to provider discover.

references/cli/velaql.go

@@ -50,12 +50,12 @@ func NewQlCommand(c common.Args, order string, ioStreams util.IOStreams) *cobra.
 		Short: "Show result of executing velaQL.",
 		Long: `Show result of executing velaQL, use it like:
 		vela ql --query "<inner-view-name>{<param1>=<value1>,<param2>=<value2>}
-		vela ql --file=./ql.cue
+		vela ql --file ./ql.cue
 `,
 		Example: `Users can query with a query statement:
 		vela ql --query "<inner-view-name>{<param1>=<value1>,<param2>=<value2>}"
 They can also query by a ql file:
-		vela ql --file=./ql.cue
+		vela ql --file ./ql.cue
 
 Example content of ql.cue:
 ---
@@ -99,7 +99,7 @@ export: "status"
 			types.TagCommandType:  types.TypeApp,
 		},
 	}
-	cmd.Flags().StringVarP(&cueFile, "file", "f", "", "The CUE file path for VelaQL.")
+	cmd.Flags().StringVarP(&cueFile, "file", "f", "", "The CUE file path for VelaQL, it could be a remote url.")
 	cmd.Flags().StringVarP(&querySts, "query", "q", "", "The query statement for VelaQL.")
 	cmd.SetOut(ioStreams.Out)
 	return cmd

vela-templates/definitions/internal/trait/service-account.cue

@@ -9,10 +9,115 @@
 	}
 }
 template: {
+	#Privileges: {
+		// +usage=Specify the verbs to be allowed for the resource
+		verbs: [...string]
+		// +usage=Specify the apiGroups of the resource
+		apiGroups?: [...string]
+		// +usage=Specify the resources to be allowed
+		resources?: [...string]
+		// +usage=Specify the resourceNames to be allowed
+		resourceNames?: [...string]
+		// +usage=Specify the resource url to be allowed
+		nonResourceURLs?: [...string]
+		// +usage=Specify the scope of the privileges, default to be namespace scope
+		scope: *"namespace" | "cluster"
+	}
 	parameter: {
 		// +usage=Specify the name of ServiceAccount
 		name: string
+		// +usage=Specify whether to create new ServiceAccount or not
+		create: *false | bool
+		// +usage=Specify the privileges of the ServiceAccount, if not empty, RoleBindings(ClusterRoleBindings) will be created
+		privileges?: [...#Privileges]
 	}
 	// +patchStrategy=retainKeys
 	patch: spec: template: spec: serviceAccountName: parameter.name
+
+	_clusterPrivileges: [ for p in parameter.privileges if p.scope == "cluster" {p}]
+	_namespacePrivileges: [ for p in parameter.privileges if p.scope == "namespace" {p}]
+	outputs: {
+		if parameter.create {
+			"service-account": {
+				apiVersion: "v1"
+				kind:       "ServiceAccount"
+				metadata: name: parameter.name
+			}
+		}
+		if parameter.privileges != _|_ {
+			if len(_clusterPrivileges) > 0 {
+				"cluster-role": {
+					apiVersion: "rbac.authorization.k8s.io/v1"
+					kind:       "ClusterRole"
+					metadata: name: "\(context.namespace):\(parameter.name)"
+					rules: [ for p in _clusterPrivileges {
+						verbs: p.verbs
+						if p.apiGroups != _|_ {
+							apiGroups: p.apiGroups
+						}
+						if p.resources != _|_ {
+							resources: p.resources
+						}
+						if p.resourceNames != _|_ {
+							resources: p.resourceNames
+						}
+						if p.nonResourceURLs != _|_ {
+							nonResourceURLs: p.nonResourceURLs
+						}
+					}]
+				}
+				"cluster-role-binding": {
+					apiVersion: "rbac.authorization.k8s.io/v1"
+					kind:       "ClusterRoleBinding"
+					metadata: name: "\(context.namespace):\(parameter.name)"
+					roleRef: {
+						apiGroup: "rbac.authorization.k8s.io"
+						kind:     "ClusterRole"
+						name:     "\(context.namespace):\(parameter.name)"
+					}
+					subjects: [{
+						kind:      "ServiceAccount"
+						name:      parameter.name
+						namespace: "\(context.namespace)"
+					}]
+				}
+			}
+			if len(_namespacePrivileges) > 0 {
+				"role": {
+					apiVersion: "rbac.authorization.k8s.io/v1"
+					kind:       "Role"
+					metadata: name: parameter.name
+					rules: [ for p in _namespacePrivileges {
+						verbs: p.verbs
+						if p.apiGroups != _|_ {
+							apiGroups: p.apiGroups
+						}
+						if p.resources != _|_ {
+							resources: p.resources
+						}
+						if p.resourceNames != _|_ {
+							resources: p.resourceNames
+						}
+						if p.nonResourceURLs != _|_ {
+							nonResourceURLs: p.nonResourceURLs
+						}
+					}]
+				}
+				"role-binding": {
+					apiVersion: "rbac.authorization.k8s.io/v1"
+					kind:       "RoleBinding"
+					metadata: name: parameter.name
+					roleRef: {
+						apiGroup: "rbac.authorization.k8s.io"
+						kind:     "Role"
+						name:     parameter.name
+					}
+					subjects: [{
+						kind: "ServiceAccount"
+						name: parameter.name
+					}]
+				}
+			}
+		}
+	}
 }