Merge pull request #122 from wonderflow/plugin

add addon:update and refactor addon code

cmd/vela/main.go

@@ -94,7 +94,7 @@ func newCommand() *cobra.Command {
 	// Getting Start
 	cmd.EnvCommandGroup(cmds, commandArgs, ioStream)
 	// Others
-	cmd.AddonCommandGroup(cmds, commandArgs, ioStream)
+	cmd.AddonCommandGroup(cmds, ioStream)
 	// System
 	cmd.SystemCommandGroup(cmds, commandArgs, ioStream)
 

pkg/cmd/addon.go

@@ -1,88 +1,65 @@
 package cmd
 
 import (
-	"context"
-	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
-	"net/http"
-	"os"
 	"path/filepath"
-	"strings"
+
+	"github.com/cloud-native-application/rudrx/pkg/utils/system"
+
+	"github.com/cloud-native-application/rudrx/pkg/plugins"
 
 	"github.com/cloud-native-application/rudrx/api/types"
-	corev1alpha2 "github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2"
-	"github.com/ghodss/yaml"
 
 	cmdutil "github.com/cloud-native-application/rudrx/pkg/cmd/util"
 	"github.com/gosuri/uitable"
 	"github.com/spf13/cobra"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-var (
-	addonCenterConfigFile = ".vela/addon_config"
-	defaultAddonCenter    = "local"
-)
-
-//Used to store addon center config in file
-type AddonCenterConfig struct {
-	Name    string `json:"name"`
-	IsLocal bool   `json:"isLocal"`
-}
-type PluginFile struct {
-	Name string `json:"name"`
-	Url  string `json:"download_url"`
-	Sha  string `json:"sha"`
-}
-
-type Plugin struct {
-	Name       string `json:"name"`
-	Type       string `json:"type"`
-	Definition string `json:"definition"`
-	Status     string `json:"status"`
-	ApplesTo   string `json:"applies_to"`
-}
-
-func AddonCommandGroup(parentCmd *cobra.Command, c types.Args, ioStream cmdutil.IOStreams) {
-	parentCmd.AddCommand(NewAddonConfigCommand(ioStream),
-		NewAddonListCommand(c, ioStream),
+func AddonCommandGroup(parentCmd *cobra.Command, ioStream cmdutil.IOStreams) {
+	parentCmd.AddCommand(
+		NewAddonConfigCommand(ioStream),
+		NewAddonListCommand(ioStream),
+		NewAddonUpdateCommand(ioStream),
 	)
 }
 
 func NewAddonConfigCommand(ioStreams cmdutil.IOStreams) *cobra.Command {
 	cmd := &cobra.Command{
-		Use:     "addon:config",
+		Use:     "addon:config <reponame> <url>",
 		Short:   "Set the addon center, default is local (built-in ones)",
 		Long:    "Set the addon center, default is local (built-in ones)",
-		Example: `vela addon:config <REPOSITORY>`,
-		Run: func(cmd *cobra.Command, args []string) {
+		Example: `vela addon:config myhub https://github.com/oam-dev/catalog/repository`,
+		RunE: func(cmd *cobra.Command, args []string) error {
 			argsLength := len(args)
-			switch {
-			case argsLength == 0:
-				ioStreams.Errorf("Please set addon center, `local` or an URL.")
-			case argsLength == 1:
-				addonCenter := args[0]
-				config := AddonCenterConfig{
-					Name:    addonCenter,
-					IsLocal: addonCenter == defaultAddonCenter,
-				}
-				var data []byte
-				var err error
-				var homeDir string
-				if data, err = json.Marshal(config); err != nil {
-					ioStreams.Errorf(fmt.Sprintf("Failed to configure Addon center: %s", addonCenter))
-				}
-				if homeDir, err = os.UserHomeDir(); err != nil {
-					ioStreams.Errorf(fmt.Sprintf("Failed to configure Addon center: %s", addonCenter))
-				}
-				if err = ioutil.WriteFile(filepath.Join(homeDir, addonCenterConfigFile), data, 0644); err != nil {
-					ioStreams.Errorf(fmt.Sprintf("Failed to configure Addon center: %s", addonCenter))
-				}
-				ioStreams.Info(fmt.Sprintf("Successfully configured Addon center: %s", addonCenter))
-			case argsLength > 1:
-				ioStreams.Errorf("Unnecessary arguments are specified, please try again")
+			if argsLength < 2 {
+				return errors.New("please set addon repo with <RepoName> and <URL>")
 			}
+			repos, err := plugins.LoadRepos()
+			if err != nil {
+				return err
+			}
+			config := plugins.RepoConfig{
+				Name:    args[0],
+				Address: ConvertURL(args[1]),
+			}
+			var updated bool
+			for idx, r := range repos {
+				if r.Name == config.Name {
+					repos[idx] = config
+					updated = true
+					break
+				}
+			}
+			if !updated {
+				repos = append(repos, config)
+			}
+			if err = plugins.StoreRepos(repos); err != nil {
+				return err
+			}
+			ioStreams.Info(fmt.Sprintf("Successfully configured Addon repo: %s", args[0]))
+			return nil
 		},
 		Annotations: map[string]string{
 			types.TagCommandType: types.TypeOthers,
@@ -91,25 +68,39 @@ func NewAddonConfigCommand(ioStreams cmdutil.IOStreams) *cobra.Command {
 	return cmd
 }
 
-func NewAddonListCommand(c types.Args, ioStreams cmdutil.IOStreams) *cobra.Command {
-	ctx := context.Background()
+func NewAddonUpdateCommand(ioStreams cmdutil.IOStreams) *cobra.Command {
 	cmd := &cobra.Command{
-		Use:     "addon:ls",
-		Short:   "List addons",
-		Long:    "List addons of workloads and traits",
-		Example: `vela addon:ls`,
+		Use:     "addon:update <repoName>",
+		Short:   "Update addon repositories, default for all repo",
+		Long:    "Update addon repositories, default for all repo",
+		Example: `vela addon:update myrepo`,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			env, err := GetEnv()
+			repos, err := plugins.LoadRepos()
 			if err != nil {
 				return err
 			}
-			newClient, err := client.New(c.Config, client.Options{Scheme: c.Schema})
-			if err != nil {
-				return err
+			var specified string
+			if len(args) > 0 {
+				specified = args[0]
 			}
-			err = retrievePlugins(ctx, newClient, ioStreams, env.Namespace)
-			if err != nil {
-				return err
+			find := false
+			if specified != "" {
+				for idx, r := range repos {
+					if r.Name == specified {
+						repos = []plugins.RepoConfig{repos[idx]}
+						find = true
+						break
+					}
+				}
+				if !find {
+					return fmt.Errorf("%s repo not exist", specified)
+				}
+			}
+			for _, d := range repos {
+				err = SyncRemoteAddon(d)
+				if err != nil {
+					return err
+				}
 			}
 			return nil
 		},
@@ -120,129 +111,72 @@ func NewAddonListCommand(c types.Args, ioStreams cmdutil.IOStreams) *cobra.Comma
 	return cmd
 }
 
-func retrievePlugins(ctx context.Context, c client.Client, ioStreams cmdutil.IOStreams, namespace string) error {
-	var pluginList []Plugin
-	var config AddonCenterConfig
-	var data []byte
-	var err error
-	var homeDir string
-	if homeDir, err = os.UserHomeDir(); err != nil {
-		ioStreams.Errorf("Failed to retrieve addon center configuration, please run `vela addon:config` first")
-	}
-	if data, err = ioutil.ReadFile(filepath.Join(homeDir, addonCenterConfigFile)); err != nil {
-		ioStreams.Errorf("Failed to retrieve addon center configuration, please run `vela addon:config` first")
-	}
-	if err := json.Unmarshal(data, &config); err != nil {
-		ioStreams.Errorf("Failed to retrieve addon center configuration, please run `vela addon:config` first")
-	}
-
-	if config.IsLocal {
-		//TODO(zzxwill) merge `vela traits` and `vela workloads`
-		return nil
-	} else {
-		resp, err := http.Get(config.Name)
-		if err != nil {
-			return err
-		}
-		defer resp.Body.Close()
-		result, _ := ioutil.ReadAll(resp.Body)
-		var manifests []PluginFile
-		var traitManifestPrefix, workloadManifestPrefix = "TraitDefinition", "WorkloadDefinition"
-		err = json.Unmarshal(result, &manifests)
-		if err != nil {
-			return err
-		}
-		var manifestResp *http.Response
-		for _, d := range manifests {
-			var template types.Template
-			var workloadDefinition corev1alpha2.WorkloadDefinition
-			var traitDefinition corev1alpha2.TraitDefinition
-			if manifestResp, err = http.Get(d.Url); err != nil {
+func NewAddonListCommand(ioStreams cmdutil.IOStreams) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "addon:ls <repoName>",
+		Short:   "List addons",
+		Long:    "List addons of workloads and traits",
+		Example: `vela addon:ls`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			var repoName string
+			if len(args) > 0 {
+				repoName = args[0]
+			}
+			dir, err := system.GetRepoDir()
+			if err != nil {
 				return err
 			}
-			defer manifestResp.Body.Close()
-
-			if strings.Contains(strings.ToLower(d.Name), strings.ToLower(workloadManifestPrefix)) {
-				result, err := ioutil.ReadAll(manifestResp.Body)
-				if err != nil {
-					return err
-				}
-				err = yaml.Unmarshal(result, &workloadDefinition)
-				if err != nil {
-					return err
-				}
-				var definitionName string
-				if workloadDefinition.Spec.Extension != nil {
-					template, err = types.ConvertTemplateJson2Object(workloadDefinition.Spec.Extension)
-					if err != nil {
-						return err
-					}
-					definitionName = template.Name
-				} else {
-					definitionName = workloadDefinition.Name
-				}
-
-				if err != nil {
-					return err
-				}
-				//Check whether the definition is applied
-				var status = "uninstalled"
-				if _, err = cmdutil.GetWorkloadDefinitionByName(ctx, c, namespace, workloadDefinition.Name); err == nil {
-					status = "installed"
-				}
-				pluginList = append(pluginList, Plugin{
-					Name:       definitionName,
-					Type:       "workload",
-					Definition: workloadDefinition.Spec.Reference.Name,
-					Status:     status,
-					ApplesTo:   "-",
-				})
-			} else if strings.Contains(strings.ToLower(d.Name), strings.ToLower(traitManifestPrefix)) {
-				result, err := ioutil.ReadAll(manifestResp.Body)
-				if err != nil {
-					return err
-				}
-				err = yaml.Unmarshal(result, &traitDefinition)
-				if err != nil {
-					return err
-				}
-				var definitionName string
-				if traitDefinition.Spec.Extension != nil {
-					template, err = types.ConvertTemplateJson2Object(traitDefinition.Spec.Extension)
-					if err != nil {
-						return err
-					}
-					definitionName = template.Name
-				} else {
-					definitionName = traitDefinition.Name
-				}
-
-				//Check whether the definition is applied
-				var status = "uninstalled"
-				if _, err = cmdutil.GetTraitDefinitionByName(ctx, c, namespace, traitDefinition.Name); err == nil {
-					status = "installed"
-				}
-				pluginList = append(pluginList, Plugin{
-					Name:       definitionName,
-					Type:       "trait",
-					Definition: traitDefinition.Spec.Reference.Name,
-					Status:     status,
-					ApplesTo:   strings.Join(traitDefinition.Spec.AppliesToWorkloads, ","),
-				})
-			} else {
-				ioStreams.Errorf(fmt.Sprintf("Those manifests in addon repository should start with %s or %s",
-					workloadManifestPrefix, traitManifestPrefix))
-				os.Exit(1)
+			if repoName != "" {
+				return ListRepoAddons(filepath.Join(dir, repoName), ioStreams)
 			}
-		}
-
-		table := uitable.New()
-		table.MaxColWidth = 60
-		table.AddRow("NAME", "TYPE", "DEFINITION", "STATUS", "APPLIES-TO")
-		for _, p := range pluginList {
-			table.AddRow(p.Name, p.Type, p.Definition, p.Status, p.ApplesTo)
-		}
-		ioStreams.Info(table.String())
+			dirs, err := ioutil.ReadDir(dir)
+			if err != nil {
+				return err
+			}
+			for _, dd := range dirs {
+				if !dd.IsDir() {
+					continue
+				}
+				if err = ListRepoAddons(filepath.Join(dir, dd.Name()), ioStreams); err != nil {
+					return err
+				}
+			}
+			return nil
+		},
+		Annotations: map[string]string{
+			types.TagCommandType: types.TypeOthers,
+		},
 	}
+	return cmd
+}
+
+func ListRepoAddons(repoDir string, ioStreams cmdutil.IOStreams) error {
+	templates, err := plugins.LoadTempFromLocal(repoDir)
+	if err != nil {
+		return err
+	}
+	table := uitable.New()
+	table.AddRow("NAME", "TYPE", "DEFINITION", "STATUS", "APPLIES-TO")
+
+	var status string
+	//TODO(wonderflow): check status whether install or not
+	status = "uninstalled"
+	for _, p := range templates {
+		table.AddRow(p.Name, p.Type, p.Type, status, p.AppliesTo)
+	}
+	ioStreams.Info(table.String())
 	return nil
 }
+
+func ConvertURL(address string) string {
+	//TODO(wonderflow) convert github address here
+	return address
+}
+
+func SyncRemoteAddon(d plugins.RepoConfig) error {
+	addons, err := plugins.GetReposFromRemote(d)
+	if err != nil {
+		return err
+	}
+	return plugins.SyncRemoteAddons(d, addons)
+}

pkg/cmd/refresh.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"context"
-	"os"
 	"path/filepath"
 
 	"github.com/cloud-native-application/rudrx/api/types"
@@ -36,12 +35,6 @@ func NewRefreshCommand(c types.Args, ioStreams cmdutil.IOStreams) *cobra.Command
 	return cmd
 }
 
-func StatOrCreate(dir string) {
-	if _, err := os.Stat(dir); os.IsNotExist(err) {
-		os.MkdirAll(dir, 0755)
-	}
-}
-
 func RefreshDefinitions(ctx context.Context, c client.Client, ioStreams cmdutil.IOStreams) error {
 	dir, _ := system.GetDefinitionDir()
 
@@ -51,7 +44,7 @@ func RefreshDefinitions(ctx context.Context, c client.Client, ioStreams cmdutil.
 		return err
 	}
 	workloadDir := filepath.Join(dir, "workloads")
-	StatOrCreate(workloadDir)
+	system.StatAndCreate(workloadDir)
 	ioStreams.Infof("get %d workload definitions from cluster, syncing to %s...", len(templates), workloadDir)
 	successNum := plugins.SinkTemp2Local(templates, workloadDir)
 	ioStreams.Infof("%d workload definitions successfully synced\n", successNum)
@@ -62,7 +55,7 @@ func RefreshDefinitions(ctx context.Context, c client.Client, ioStreams cmdutil.
 		return err
 	}
 	traitDir := filepath.Join(dir, "traits")
-	StatOrCreate(traitDir)
+	system.StatAndCreate(traitDir)
 	ioStreams.Infof("get %d trait definitions from cluster, syncing to %s...", len(templates), traitDir)
 	successNum = plugins.SinkTemp2Local(templates, traitDir)
 	ioStreams.Infof("%d trait definitions successfully synced\n", successNum)

pkg/plugins/cluster.go

@@ -41,14 +41,11 @@ func GetWorkloadsFromCluster(ctx context.Context, namespace string, c client.Cli
 	}
 
 	for _, wd := range workloadDefs.Items {
-		var tmp types.Template
-		tmp, err := HandleTemplate(wd.Spec.Extension, wd.Name, syncDir)
+		tmp, err := HandleDefinition(wd.Name, syncDir, wd.Spec.Reference.Name, wd.Spec.Extension, types.TypeWorkload, nil)
 		if err != nil {
 			fmt.Printf("[WARN]handle template %s: %v\n", wd.Name, err)
 			continue
 		}
-		tmp.Type = types.TypeWorkload
-		tmp.CrdName = wd.Spec.Reference.Name
 		templates = append(templates, tmp)
 	}
 	return templates, nil
@@ -63,20 +60,30 @@ func GetTraitsFromCluster(ctx context.Context, namespace string, c client.Client
 	}
 
 	for _, td := range traitDefs.Items {
-		var tmp types.Template
-		tmp, err := HandleTemplate(td.Spec.Extension, td.Name, syncDir)
+		tmp, err := HandleDefinition(td.Name, syncDir, td.Spec.Reference.Name, td.Spec.Extension, types.TypeTrait, td.Spec.AppliesToWorkloads)
 		if err != nil {
 			fmt.Printf("[WARN]handle template %s: %v\n", td.Name, err)
 			continue
 		}
-		tmp.Type = types.TypeTrait
-		tmp.AppliesTo = td.Spec.AppliesToWorkloads
-		tmp.CrdName = td.Spec.Reference.Name
 		templates = append(templates, tmp)
 	}
 	return templates, nil
 }
 
+func HandleDefinition(name, syncDir, crdName string, extention *runtime.RawExtension, tp types.DefinitionType, applyTo []string) (types.Template, error) {
+	var tmp types.Template
+	tmp, err := HandleTemplate(extention, name, syncDir)
+	if err != nil {
+		return types.Template{}, err
+	}
+	tmp.Type = tp
+	if tp == types.TypeTrait {
+		tmp.AppliesTo = applyTo
+	}
+	tmp.CrdName = crdName
+	return tmp, nil
+}
+
 func HandleTemplate(in *runtime.RawExtension, name, syncDir string) (types.Template, error) {
 	tmp, err := types.ConvertTemplateJson2Object(in)
 	if err != nil {

pkg/plugins/repository.go

@@ -0,0 +1,153 @@
+package plugins
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/cloud-native-application/rudrx/api/types"
+	"github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2"
+
+	"github.com/cloud-native-application/rudrx/pkg/utils/system"
+	"github.com/ghodss/yaml"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+//Used to store addon center config in file
+type RepoConfig struct {
+	Name    string `json:"repoName"`
+	Address string `json:"repoAddress"`
+}
+
+var (
+	RepoConfigFile = ".vela/addon_config"
+	DefaultRepo    = "local"
+)
+
+type RemoteAddon struct {
+	// Name MUST be xxx.yaml
+	Name string `json:"name"`
+	Url  string `json:"download_url"`
+	Sha  string `json:"sha"`
+	// Type MUST be file
+	Type string `json:"type"`
+}
+
+type RemoteAddons []RemoteAddon
+
+type Plugin struct {
+	Name       string `json:"name"`
+	Type       string `json:"type"`
+	Definition string `json:"definition"`
+	Status     string `json:"status"`
+	ApplesTo   string `json:"applies_to"`
+}
+
+//TODO(wonderflow): we can make default(built-in) repo configurable, then we should make default inside the answer
+func LoadRepos() ([]RepoConfig, error) {
+	config, err := system.GetRepoConfig()
+	if err != nil {
+		return nil, err
+	}
+	data, err := ioutil.ReadFile(config)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return []RepoConfig{}, nil
+		}
+		return nil, err
+	}
+	var repos []RepoConfig
+	if err = yaml.Unmarshal(data, &repos); err != nil {
+		return nil, err
+	}
+	return repos, nil
+}
+
+func StoreRepos(repos []RepoConfig) error {
+	config, err := system.GetRepoConfig()
+	if err != nil {
+		return err
+	}
+	data, err := yaml.Marshal(repos)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(config, data, 0644)
+}
+
+func GetReposFromRemote(r RepoConfig) (RemoteAddons, error) {
+	resp, err := http.Get(r.Address)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	var repos RemoteAddons
+	if err = json.Unmarshal(data, &repos); err != nil {
+		return nil, err
+	}
+	return repos, nil
+}
+
+func GetDefinitionFromURL(address, syncDir string) (types.Template, error) {
+	resp, err := http.Get(address)
+	if err != nil {
+		return types.Template{}, err
+	}
+	defer resp.Body.Close()
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return types.Template{}, err
+	}
+	var obj = unstructured.Unstructured{Object: make(map[string]interface{})}
+	err = yaml.Unmarshal(data, &obj.Object)
+	if err != nil {
+		return types.Template{}, err
+	}
+	switch obj.GetKind() {
+	case "WorkloadDefinition":
+		var rd v1alpha2.WorkloadDefinition
+		err = yaml.Unmarshal(data, &rd)
+		if err != nil {
+			return types.Template{}, err
+		}
+		return HandleDefinition(rd.Name, syncDir, rd.Spec.Reference.Name, rd.Spec.Extension, types.TypeWorkload, nil)
+	case "TraitDefinition":
+		var td v1alpha2.TraitDefinition
+		err = yaml.Unmarshal(data, &td)
+		if err != nil {
+			return types.Template{}, err
+		}
+		return HandleDefinition(td.Name, syncDir, td.Spec.Reference.Name, td.Spec.Extension, types.TypeTrait, td.Spec.AppliesToWorkloads)
+	case "ScopeDefinition":
+		//TODO(wonderflow): support scope definition here.
+	}
+	return types.Template{}, fmt.Errorf("unknown definition Type %s", obj.GetKind())
+}
+
+//TODO(wonderflow): currently we only sync by create, we also need to delete which not exist remotely.
+func SyncRemoteAddons(r RepoConfig, addons RemoteAddons) error {
+	dir, err := system.GetRepoDir()
+	if err != nil {
+		return err
+	}
+	repoDir := filepath.Join(dir, r.Name)
+	system.StatAndCreate(repoDir)
+	var tmps []types.Template
+	for _, addon := range addons {
+		tmp, err := GetDefinitionFromURL(addon.Url, repoDir)
+		if err != nil {
+			return err
+		}
+		tmps = append(tmps, tmp)
+	}
+	success := SinkTemp2Local(tmps, repoDir)
+	fmt.Printf("successfully sync %d remote addons\n", success)
+	return nil
+}

pkg/utils/system/system.go

@@ -23,6 +23,23 @@ func GetVelaHomeDir() (string, error) {
 	return filepath.Join(home, defaultVelaHome), nil
 }
 
+func GetRepoDir() (string, error) {
+	home, err := GetVelaHomeDir()
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(home, ".repo"), nil
+}
+
+func GetRepoConfig() (string, error) {
+	home, err := GetRepoDir()
+	if err != nil {
+		return "", err
+	}
+	StatAndCreate(home)
+	return filepath.Join(home, "config.yaml"), nil
+}
+
 func GetApplicationDir() (string, error) {
 	home, err := GetVelaHomeDir()
 	if err != nil {
@@ -76,9 +93,7 @@ func InitDefaultEnv() error {
 	if err != nil {
 		return err
 	}
-	if err = os.MkdirAll(envDir, 0755); err != nil {
-		return err
-	}
+	StatAndCreate(envDir)
 	data, _ := json.Marshal(&types.EnvMeta{Namespace: types.DefaultEnvName})
 	if err = ioutil.WriteFile(filepath.Join(envDir, types.DefaultEnvName), data, 0644); err != nil {
 		return err
@@ -92,3 +107,9 @@ func InitDefaultEnv() error {
 	}
 	return nil
 }
+
+func StatAndCreate(dir string) {
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		os.MkdirAll(dir, 0755)
+	}
+}