update local caps before listing workloads&traits

refine output of `vela system update`

add e2e-test

Signed-off-by: roy wang <seiwy2010@gmail.com>

api/types/capability.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 
 	"cuelang.org/go/cue"
+	"github.com/google/go-cmp/cmp"
 	"github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/runtime"
 )
@@ -137,3 +138,68 @@ func SetFlagBy(flags *pflag.FlagSet, v Parameter) {
 		flags.Float64P(v.Name, v.Short, vv, v.Usage)
 	}
 }
+
+var CapabilityCmpOptions = []cmp.Option{
+	cmp.Comparer(func(a, b Parameter) bool {
+		if a.Name != b.Name || a.Short != b.Short || a.Required != b.Required ||
+			a.Usage != b.Usage || a.Type != b.Type {
+			return false
+		}
+		switch a.Type {
+		case cue.IntKind:
+			var va, vb int64
+			switch vala := a.Default.(type) {
+			case int64:
+				va = vala
+			case json.Number:
+				va, _ = vala.Int64()
+			case int:
+				va = int64(vala)
+			case float64:
+				va = int64(vala)
+			}
+			switch valb := b.Default.(type) {
+			case int64:
+				vb = valb
+			case json.Number:
+				vb, _ = valb.Int64()
+			case int:
+				vb = int64(valb)
+			case float64:
+				vb = int64(valb)
+			}
+			return va == vb
+		case cue.StringKind:
+			return a.Default.(string) == b.Default.(string)
+		case cue.BoolKind:
+			return a.Default.(bool) == b.Default.(bool)
+		case cue.NumberKind, cue.FloatKind:
+			var va, vb float64
+			switch vala := a.Default.(type) {
+			case int64:
+				va = float64(vala)
+			case json.Number:
+				va, _ = vala.Float64()
+			case int:
+				va = float64(vala)
+			case float64:
+				va = float64(vala)
+			}
+			switch valb := b.Default.(type) {
+			case int64:
+				vb = float64(valb)
+			case json.Number:
+				vb, _ = valb.Float64()
+			case int:
+				vb = float64(valb)
+			case float64:
+				vb = float64(valb)
+			}
+			return va == vb
+		}
+		return true
+	})}
+
+func EqualCapability(a, b Capability) bool {
+	return cmp.Equal(a, b, CapabilityCmpOptions...)
+}

e2e/commonContext.go

@@ -51,10 +51,7 @@ var (
 			ginkgo.It("Sync commands from your Kubernetes cluster and locally cached them", func() {
 				output, err := Exec("vela system update")
 				gomega.Expect(err).NotTo(gomega.HaveOccurred())
-				gomega.Expect(output).To(gomega.ContainSubstring("syncing workload definitions from cluster..."))
-				gomega.Expect(output).To(gomega.ContainSubstring("sync"))
-				gomega.Expect(output).To(gomega.ContainSubstring("successfully"))
-				gomega.Expect(output).To(gomega.ContainSubstring("remove"))
+				gomega.Expect(output).To(gomega.ContainSubstring("Synchronizing capabilities from cluster"))
 			})
 		})
 	}
@@ -152,6 +149,27 @@ var (
 		})
 	}
 
+	WorkloadCapabilityListContext = func() bool {
+		return ginkgo.Context("list workload capabilities", func() {
+			ginkgo.It("should sync capabilities from cluster before listing workload capabilities", func() {
+				output, err := Exec("vela workloads")
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+				gomega.Expect(output).To(gomega.ContainSubstring("Sync capabilities successfully"))
+				gomega.Expect(output).To(gomega.ContainSubstring("Listing workload capabilities"))
+			})
+		})
+	}
+
+	TraitCapabilityListContext = func() bool {
+		return ginkgo.Context("list traits capabilities", func() {
+			ginkgo.It("should sync capabilities from cluster before listing trait capabilities", func() {
+				output, err := Exec("vela traits")
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+				gomega.Expect(output).To(gomega.ContainSubstring("Sync capabilities successfully"))
+				gomega.Expect(output).To(gomega.ContainSubstring("Listing trait capabilities"))
+			})
+		})
+	}
 	// TraitManualScalerAttachContext used for test trait attach success
 	TraitManualScalerAttachContext = func(context string, traitAlias string, applicationName string) bool {
 		return ginkgo.Context(context, func() {

e2e/trait/trait_test.go

@@ -18,7 +18,7 @@ var (
 )
 
 var _ = ginkgo.Describe("Trait", func() {
-	e2e.RefreshContext("refresh")
+	e2e.TraitCapabilityListContext()
 	e2e.EnvInitContext("env init", envName)
 	e2e.EnvSetContext("env set", envName)
 	e2e.WorkloadRunContext("deploy", fmt.Sprintf("vela svc deploy -t webservice %s -p 80 --image nginx:1.9.4", applicationName))

e2e/workload/workload_test.go

@@ -15,7 +15,7 @@ var (
 )
 
 var _ = ginkgo.Describe("Workload", func() {
-	e2e.RefreshContext("refresh")
+	e2e.WorkloadCapabilityListContext()
 	e2e.EnvInitContext("env init", envName)
 	e2e.EnvSetContext("env set", envName)
 	e2e.WorkloadRunContext("deploy", fmt.Sprintf("vela svc deploy -t webservice %s -p 80 --image nginx:1.9.4", applicationName))

pkg/commands/cli.go

@@ -85,8 +85,8 @@ func NewCommand() *cobra.Command {
 		SystemCommandGroup(commandArgs, ioStream),
 		NewCompletionCommand(),
 
-		NewTraitsCommand(ioStream),
-		NewWorkloadsCommand(ioStream),
+		NewTraitsCommand(commandArgs, ioStream),
+		NewWorkloadsCommand(commandArgs, ioStream),
 
 		NewDashboardCommand(commandArgs, ioStream, fake.FrontendSource),
 

pkg/commands/refresh.go

@@ -3,6 +3,8 @@ package commands
 import (
 	"context"
 
+	"github.com/fatih/color"
+	"github.com/gosuri/uitable"
 	"github.com/oam-dev/kubevela/api/types"
 	cmdutil "github.com/oam-dev/kubevela/pkg/commands/util"
 	"github.com/oam-dev/kubevela/pkg/plugins"
@@ -12,6 +14,15 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+type refreshStatus string
+
+const (
+	added     refreshStatus = "Added"
+	updated   refreshStatus = "Updated"
+	unchanged refreshStatus = "Unchanged"
+	deleted   refreshStatus = "Deleted"
+)
+
 func NewRefreshCommand(c types.Args, ioStreams cmdutil.IOStreams) *cobra.Command {
 	ctx := context.Background()
 	cmd := &cobra.Command{
@@ -36,30 +47,132 @@ func NewRefreshCommand(c types.Args, ioStreams cmdutil.IOStreams) *cobra.Command
 }
 
 func RefreshDefinitions(ctx context.Context, c client.Client, ioStreams cmdutil.IOStreams) error {
+	ioStreams.Infof("Synchronizing capabilities from cluster%s...\n", emojiWait)
 	dir, _ := system.GetCapabilityDir()
 
+	oldCaps, err := plugins.LoadAllInstalledCapability()
+	if err != nil {
+		return err
+	}
 	syncedTemplates := []types.Capability{}
-	ioStreams.Info("syncing workload definitions from cluster...")
-	templates, err := plugins.GetWorkloadsFromCluster(ctx, types.DefaultOAMNS, c, dir, nil)
+	//TODO show errors of handling templates
+	templates, _, err := plugins.GetWorkloadsFromCluster(ctx, types.DefaultOAMNS, c, dir, nil)
 	if err != nil {
 		return err
 	}
 	syncedTemplates = append(syncedTemplates, templates...)
-	ioStreams.Infof("get %d workload definition(s) from cluster, syncing...\n", len(templates))
-	successNum := plugins.SinkTemp2Local(templates, dir)
-	ioStreams.Infof("sync %d workload definition(s) successfully\n", successNum)
+	plugins.SinkTemp2Local(templates, dir)
 
-	ioStreams.Info("syncing trait definitions from cluster...")
-	templates, err = plugins.GetTraitsFromCluster(ctx, types.DefaultOAMNS, c, dir, nil)
+	//TODO show errors of handling templates
+	templates, _, err = plugins.GetTraitsFromCluster(ctx, types.DefaultOAMNS, c, dir, nil)
 	if err != nil {
 		return err
 	}
 	syncedTemplates = append(syncedTemplates, templates...)
-	ioStreams.Infof("get %d trait definition(s) from cluster, syncing...\n", len(templates))
-	successNum = plugins.SinkTemp2Local(templates, dir)
-	ioStreams.Infof("sync %d trait definition(s) successfully\n", successNum)
+	plugins.SinkTemp2Local(templates, dir)
+	plugins.RemoveLegacyTemps(syncedTemplates, dir)
 
-	legacyNum := plugins.RemoveLegacyTemps(syncedTemplates, dir)
-	ioStreams.Infof("remove %d legacy capability definition(s) successfully\n", legacyNum)
+	printRefreshReport(syncedTemplates, oldCaps, ioStreams)
 	return nil
 }
+
+func printRefreshReport(newCaps, oldCaps []types.Capability, io cmdutil.IOStreams) {
+	report := refreshResultReport(newCaps, oldCaps)
+	table := uitable.New()
+	table.AddRow("NAME", "TYPE", "DESCRIPTION")
+
+	if len(report[added]) == 0 && len(report[updated]) == 0 && len(report[deleted]) == 0 {
+		// no change occurs, just show all existing caps
+		// always show workload at first
+		for _, cap := range report[unchanged] {
+			if cap.Type == types.TypeWorkload {
+				table.AddRow(cap.Name, cap.Type, cap.Description)
+			}
+		}
+		for _, cap := range report[unchanged] {
+			if cap.Type == types.TypeTrait {
+				table.AddRow(cap.Name, cap.Type, cap.Description)
+			}
+		}
+		io.Infof("Sync capabilities successfully %s(no changes)\n", emojiSucceed)
+		io.Info(table.String())
+		return
+	}
+
+	io.Infof("Sync capabilities successfully %sAdd(%s) Update(%s) Delete(%s)\n",
+		emojiSucceed,
+		green.Sprint(len(report[added])),
+		yellow.Sprint(len(report[updated])),
+		red.Sprint(len(report[deleted])))
+	// show added/updated/deleted cpas
+	addStsRow(added, report, table)
+	addStsRow(updated, report, table)
+	addStsRow(deleted, report, table)
+	io.Info(table.String())
+}
+
+func addStsRow(sts refreshStatus, report map[refreshStatus][]types.Capability, t *uitable.Table) {
+	caps := report[sts]
+	if len(caps) == 0 {
+		return
+	}
+	var stsIcon string
+	var stsColor *color.Color
+	switch sts {
+	case added:
+		stsIcon = "+"
+		stsColor = green
+	case updated:
+		stsIcon = "*"
+		stsColor = yellow
+	case deleted:
+		stsIcon = "-"
+		stsColor = red
+	}
+	for _, cap := range caps {
+		t.AddRow(
+			// color.New(color.Bold).Sprint(stsColor.Sprint(stsIcon)),
+			stsColor.Sprintf("%s%s", stsIcon, cap.Name),
+			stsColor.Sprint(cap.Type),
+			stsColor.Sprint(cap.Description))
+	}
+}
+
+func refreshResultReport(newCaps, oldCaps []types.Capability) map[refreshStatus][]types.Capability {
+	report := map[refreshStatus][]types.Capability{
+		added:     make([]types.Capability, 0),
+		updated:   make([]types.Capability, 0),
+		unchanged: make([]types.Capability, 0),
+		deleted:   make([]types.Capability, 0),
+	}
+	for _, newCap := range newCaps {
+		found := false
+		for _, oldCap := range oldCaps {
+			if newCap.Name == oldCap.Name {
+				found = true
+				break
+			}
+		}
+		if !found {
+			report[added] = append(report[added], newCap)
+		}
+	}
+	for _, oldCap := range oldCaps {
+		found := false
+		for _, newCap := range newCaps {
+			if oldCap.Name == newCap.Name {
+				found = true
+				if types.EqualCapability(oldCap, newCap) {
+					report[unchanged] = append(report[unchanged], newCap)
+				} else {
+					report[updated] = append(report[updated], newCap)
+				}
+				break
+			}
+		}
+		if !found {
+			report[deleted] = append(report[deleted], oldCap)
+		}
+	}
+	return report
+}

pkg/commands/status.go

@@ -90,6 +90,7 @@ var (
 	// nolint
 	emojiTimeout   = emoji.Sprint(":heavy_exclamation_mark:")
 	emojiLightBulb = emoji.Sprint(":light_bulb:")
+	emojiWait      = emoji.Sprint(":hourglass:")
 )
 
 const (

pkg/commands/traits.go

@@ -1,17 +1,22 @@
 package commands
 
 import (
+	"context"
 	"strings"
 
+	"github.com/oam-dev/kubevela/api/types"
 	"github.com/oam-dev/kubevela/pkg/oam"
 
 	"github.com/gosuri/uitable"
 	cmdutil "github.com/oam-dev/kubevela/pkg/commands/util"
 	"github.com/spf13/cobra"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-func NewTraitsCommand(ioStreams cmdutil.IOStreams) *cobra.Command {
+func NewTraitsCommand(c types.Args, ioStreams cmdutil.IOStreams) *cobra.Command {
 	var workloadName string
+	var syncCluster bool
+	ctx := context.Background()
 	cmd := &cobra.Command{
 		Use:                   "traits [--apply-to WORKLOADNAME]",
 		DisableFlagsInUseLine: true,
@@ -19,12 +24,23 @@ func NewTraitsCommand(ioStreams cmdutil.IOStreams) *cobra.Command {
 		Long:                  "List traits",
 		Example:               `vela traits`,
 		RunE: func(cmd *cobra.Command, args []string) error {
+			if syncCluster {
+				newClient, err := client.New(c.Config, client.Options{Scheme: c.Schema})
+				if err != nil {
+					return err
+				}
+				if err := RefreshDefinitions(ctx, newClient, ioStreams); err != nil {
+					return err
+				}
+				ioStreams.Info("\nListing trait capabilities ...\n")
+			}
 			return printTraitList(&workloadName, ioStreams)
 		},
 	}
 
 	cmd.SetOut(ioStreams.Out)
 	cmd.Flags().StringVar(&workloadName, "apply-to", "", "Workload name")
+	cmd.Flags().BoolVarP(&syncCluster, "sync", "s", true, "Synchronize capabilities from cluster into local")
 	return cmd
 }
 

pkg/commands/workloads.go

@@ -1,15 +1,20 @@
 package commands
 
 import (
+	"context"
+
 	"github.com/oam-dev/kubevela/api/types"
 	cmdutil "github.com/oam-dev/kubevela/pkg/commands/util"
 	"github.com/oam-dev/kubevela/pkg/plugins"
 
 	"github.com/gosuri/uitable"
 	"github.com/spf13/cobra"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-func NewWorkloadsCommand(ioStreams cmdutil.IOStreams) *cobra.Command {
+func NewWorkloadsCommand(c types.Args, ioStreams cmdutil.IOStreams) *cobra.Command {
+	var syncCluster bool
+	ctx := context.Background()
 	cmd := &cobra.Command{
 		Use:                   "workloads",
 		DisableFlagsInUseLine: true,
@@ -17,6 +22,16 @@ func NewWorkloadsCommand(ioStreams cmdutil.IOStreams) *cobra.Command {
 		Long:                  "List workloads",
 		Example:               `vela workloads`,
 		RunE: func(cmd *cobra.Command, args []string) error {
+			if syncCluster {
+				newClient, err := client.New(c.Config, client.Options{Scheme: c.Schema})
+				if err != nil {
+					return err
+				}
+				if err := RefreshDefinitions(ctx, newClient, ioStreams); err != nil {
+					return err
+				}
+				ioStreams.Info("\nListing workload capabilities ...\n")
+			}
 			workloads, err := plugins.LoadInstalledCapabilityWithType(types.TypeWorkload)
 			if err != nil {
 				return err
@@ -25,6 +40,7 @@ func NewWorkloadsCommand(ioStreams cmdutil.IOStreams) *cobra.Command {
 		},
 	}
 	cmd.SetOut(ioStreams.Out)
+	cmd.Flags().BoolVarP(&syncCluster, "sync", "s", true, "Synchronize capabilities from cluster into local")
 	return cmd
 }
 

pkg/plugins/cluster.go

@@ -2,7 +2,6 @@ package plugins
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -12,6 +11,7 @@ import (
 	cmdutil "github.com/oam-dev/kubevela/pkg/commands/util"
 	"github.com/oam-dev/kubevela/pkg/cue"
 	"github.com/oam-dev/kubevela/pkg/utils/system"
+	"github.com/pkg/errors"
 
 	corev1alpha2 "github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2"
 	"k8s.io/apimachinery/pkg/labels"
@@ -22,11 +22,11 @@ import (
 const DescriptionUndefined = "description not defined"
 
 func GetCapabilitiesFromCluster(ctx context.Context, namespace string, c client.Client, syncDir string, selector labels.Selector) ([]types.Capability, error) {
-	workloads, err := GetWorkloadsFromCluster(ctx, namespace, c, syncDir, selector)
+	workloads, _, err := GetWorkloadsFromCluster(ctx, namespace, c, syncDir, selector)
 	if err != nil {
 		return nil, err
 	}
-	traits, err := GetTraitsFromCluster(ctx, namespace, c, syncDir, selector)
+	traits, _, err := GetTraitsFromCluster(ctx, namespace, c, syncDir, selector)
 	if err != nil {
 		return nil, err
 	}
@@ -34,21 +34,20 @@ func GetCapabilitiesFromCluster(ctx context.Context, namespace string, c client.
 	return workloads, nil
 }
 
-func GetWorkloadsFromCluster(ctx context.Context, namespace string, c client.Client, syncDir string, selector labels.Selector) ([]types.Capability, error) {
+func GetWorkloadsFromCluster(ctx context.Context, namespace string, c client.Client, syncDir string, selector labels.Selector) ([]types.Capability, []error, error) {
 	var templates []types.Capability
 	var workloadDefs corev1alpha2.WorkloadDefinitionList
 	err := c.List(ctx, &workloadDefs, &client.ListOptions{Namespace: namespace, LabelSelector: selector})
 	if err != nil {
-		return nil, fmt.Errorf("list WorkloadDefinition err: %s", err)
+		return nil, nil, fmt.Errorf("list WorkloadDefinition err: %s", err)
 	}
 
+	templateErrors := []error{}
 	for _, wd := range workloadDefs.Items {
 		tmp, err := HandleDefinition(wd.Name, syncDir, wd.Spec.Reference.Name, wd.Annotations, wd.Spec.Extension, types.TypeWorkload, nil)
 		if err != nil {
-			fmt.Printf("[WARN] hanlde workload template `%s` failed with error: %v\n", wd.Name, err)
+			templateErrors = append(templateErrors, errors.Wrapf(err, "handle workload template `%s` failed", wd.Name))
 			continue
-		} else {
-			fmt.Printf("imported workload `%s`\n", wd.Name)
 		}
 		if apiVerion, kind := cmdutil.GetAPIVersionKindFromWorkload(wd); apiVerion != "" && kind != "" {
 			tmp.CrdInfo = &types.CrdInfo{
@@ -58,24 +57,23 @@ func GetWorkloadsFromCluster(ctx context.Context, namespace string, c client.Cli
 		}
 		templates = append(templates, tmp)
 	}
-	return templates, nil
+	return templates, templateErrors, nil
 }
 
-func GetTraitsFromCluster(ctx context.Context, namespace string, c client.Client, syncDir string, selector labels.Selector) ([]types.Capability, error) {
+func GetTraitsFromCluster(ctx context.Context, namespace string, c client.Client, syncDir string, selector labels.Selector) ([]types.Capability, []error, error) {
 	var templates []types.Capability
 	var traitDefs corev1alpha2.TraitDefinitionList
 	err := c.List(ctx, &traitDefs, &client.ListOptions{Namespace: namespace, LabelSelector: selector})
 	if err != nil {
-		return nil, fmt.Errorf("list TraitDefinition err: %s", err)
+		return nil, nil, fmt.Errorf("list TraitDefinition err: %s", err)
 	}
 
+	templateErrors := []error{}
 	for _, td := range traitDefs.Items {
 		tmp, err := HandleDefinition(td.Name, syncDir, td.Spec.Reference.Name, td.Annotations, td.Spec.Extension, types.TypeTrait, td.Spec.AppliesToWorkloads)
 		if err != nil {
-			fmt.Printf("[WARN] hanlde trait template `%s` failed with error: %v\n", td.Name, err)
+			templateErrors = append(templateErrors, errors.Wrapf(err, "handle trait template `%s` failed", td.Name))
 			continue
-		} else {
-			fmt.Printf("imported trait `%s`\n", td.Name)
 		}
 		if apiVerion, kind := cmdutil.GetAPIVersionKindFromTrait(td); apiVerion != "" && kind != "" {
 			tmp.CrdInfo = &types.CrdInfo{
@@ -85,7 +83,7 @@ func GetTraitsFromCluster(ctx context.Context, namespace string, c client.Client
 		}
 		templates = append(templates, tmp)
 	}
-	return templates, nil
+	return templates, templateErrors, nil
 }
 
 func HandleDefinition(name, syncDir, crdName string, annotation map[string]string, extention *runtime.RawExtension, tp types.CapType, applyTo []string) (types.Capability, error) {

pkg/plugins/cluster_test.go

@@ -89,7 +89,7 @@ var _ = Describe("DefinitionFiles", func() {
 	// Notice!!  DefinitionPath Object is Cluster Scope object
 	// which means objects created in other DefinitionNamespace will also affect here.
 	It("gettrait", func() {
-		traitDefs, err := GetTraitsFromCluster(context.Background(), DefinitionNamespace, k8sClient, definitionDir, selector)
+		traitDefs, _, err := GetTraitsFromCluster(context.Background(), DefinitionNamespace, k8sClient, definitionDir, selector)
 		Expect(err).Should(BeNil())
 		logf.Log.Info(fmt.Sprintf("Getting trait definitions %v", traitDefs))
 		for i := range traitDefs {
@@ -105,7 +105,7 @@ var _ = Describe("DefinitionFiles", func() {
 	// Notice!!  DefinitionPath Object is Cluster Scope object
 	// which means objects created in other DefinitionNamespace will also affect here.
 	It("getworkload", func() {
-		workloadDefs, err := GetWorkloadsFromCluster(context.Background(), DefinitionNamespace, k8sClient, definitionDir, selector)
+		workloadDefs, _, err := GetWorkloadsFromCluster(context.Background(), DefinitionNamespace, k8sClient, definitionDir, selector)
 		Expect(err).Should(BeNil())
 		logf.Log.Info(fmt.Sprintf("Getting workload definitions  %v", workloadDefs))
 		for i := range workloadDefs {