diff --git a/cmd/vela/main.go b/cmd/vela/main.go index 26059c143..df8ba1c91 100644 --- a/cmd/vela/main.go +++ b/cmd/vela/main.go @@ -94,7 +94,7 @@ func newCommand() *cobra.Command { // Getting Start cmd.EnvCommandGroup(cmds, commandArgs, ioStream) // Others - cmd.AddonCommandGroup(cmds, commandArgs, ioStream) + cmd.AddonCommandGroup(cmds, ioStream) // System cmd.SystemCommandGroup(cmds, commandArgs, ioStream) diff --git a/pkg/cmd/addon.go b/pkg/cmd/addon.go index 7c8af10be..7b3f5817b 100644 --- a/pkg/cmd/addon.go +++ b/pkg/cmd/addon.go @@ -1,88 +1,65 @@ package cmd import ( - "context" - "encoding/json" + "errors" "fmt" "io/ioutil" - "net/http" - "os" "path/filepath" - "strings" + + "github.com/cloud-native-application/rudrx/pkg/utils/system" + + "github.com/cloud-native-application/rudrx/pkg/plugins" "github.com/cloud-native-application/rudrx/api/types" - corev1alpha2 "github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2" - "github.com/ghodss/yaml" cmdutil "github.com/cloud-native-application/rudrx/pkg/cmd/util" "github.com/gosuri/uitable" "github.com/spf13/cobra" - "sigs.k8s.io/controller-runtime/pkg/client" ) -var ( - addonCenterConfigFile = ".vela/addon_config" - defaultAddonCenter = "local" -) - -//Used to store addon center config in file -type AddonCenterConfig struct { - Name string `json:"name"` - IsLocal bool `json:"isLocal"` -} -type PluginFile struct { - Name string `json:"name"` - Url string `json:"download_url"` - Sha string `json:"sha"` -} - -type Plugin struct { - Name string `json:"name"` - Type string `json:"type"` - Definition string `json:"definition"` - Status string `json:"status"` - ApplesTo string `json:"applies_to"` -} - -func AddonCommandGroup(parentCmd *cobra.Command, c types.Args, ioStream cmdutil.IOStreams) { - parentCmd.AddCommand(NewAddonConfigCommand(ioStream), - NewAddonListCommand(c, ioStream), +func AddonCommandGroup(parentCmd *cobra.Command, ioStream cmdutil.IOStreams) { + parentCmd.AddCommand( + NewAddonConfigCommand(ioStream), + NewAddonListCommand(ioStream), + NewAddonUpdateCommand(ioStream), ) } func NewAddonConfigCommand(ioStreams cmdutil.IOStreams) *cobra.Command { cmd := &cobra.Command{ - Use: "addon:config", + Use: "addon:config ", Short: "Set the addon center, default is local (built-in ones)", Long: "Set the addon center, default is local (built-in ones)", - Example: `vela addon:config `, - Run: func(cmd *cobra.Command, args []string) { + Example: `vela addon:config myhub https://github.com/oam-dev/catalog/repository`, + RunE: func(cmd *cobra.Command, args []string) error { argsLength := len(args) - switch { - case argsLength == 0: - ioStreams.Errorf("Please set addon center, `local` or an URL.") - case argsLength == 1: - addonCenter := args[0] - config := AddonCenterConfig{ - Name: addonCenter, - IsLocal: addonCenter == defaultAddonCenter, - } - var data []byte - var err error - var homeDir string - if data, err = json.Marshal(config); err != nil { - ioStreams.Errorf(fmt.Sprintf("Failed to configure Addon center: %s", addonCenter)) - } - if homeDir, err = os.UserHomeDir(); err != nil { - ioStreams.Errorf(fmt.Sprintf("Failed to configure Addon center: %s", addonCenter)) - } - if err = ioutil.WriteFile(filepath.Join(homeDir, addonCenterConfigFile), data, 0644); err != nil { - ioStreams.Errorf(fmt.Sprintf("Failed to configure Addon center: %s", addonCenter)) - } - ioStreams.Info(fmt.Sprintf("Successfully configured Addon center: %s", addonCenter)) - case argsLength > 1: - ioStreams.Errorf("Unnecessary arguments are specified, please try again") + if argsLength < 2 { + return errors.New("please set addon repo with and ") } + repos, err := plugins.LoadRepos() + if err != nil { + return err + } + config := plugins.RepoConfig{ + Name: args[0], + Address: ConvertURL(args[1]), + } + var updated bool + for idx, r := range repos { + if r.Name == config.Name { + repos[idx] = config + updated = true + break + } + } + if !updated { + repos = append(repos, config) + } + if err = plugins.StoreRepos(repos); err != nil { + return err + } + ioStreams.Info(fmt.Sprintf("Successfully configured Addon repo: %s", args[0])) + return nil }, Annotations: map[string]string{ types.TagCommandType: types.TypeOthers, @@ -91,25 +68,39 @@ func NewAddonConfigCommand(ioStreams cmdutil.IOStreams) *cobra.Command { return cmd } -func NewAddonListCommand(c types.Args, ioStreams cmdutil.IOStreams) *cobra.Command { - ctx := context.Background() +func NewAddonUpdateCommand(ioStreams cmdutil.IOStreams) *cobra.Command { cmd := &cobra.Command{ - Use: "addon:ls", - Short: "List addons", - Long: "List addons of workloads and traits", - Example: `vela addon:ls`, + Use: "addon:update ", + Short: "Update addon repositories, default for all repo", + Long: "Update addon repositories, default for all repo", + Example: `vela addon:update myrepo`, RunE: func(cmd *cobra.Command, args []string) error { - env, err := GetEnv() + repos, err := plugins.LoadRepos() if err != nil { return err } - newClient, err := client.New(c.Config, client.Options{Scheme: c.Schema}) - if err != nil { - return err + var specified string + if len(args) > 0 { + specified = args[0] } - err = retrievePlugins(ctx, newClient, ioStreams, env.Namespace) - if err != nil { - return err + find := false + if specified != "" { + for idx, r := range repos { + if r.Name == specified { + repos = []plugins.RepoConfig{repos[idx]} + find = true + break + } + } + if !find { + return fmt.Errorf("%s repo not exist", specified) + } + } + for _, d := range repos { + err = SyncRemoteAddon(d) + if err != nil { + return err + } } return nil }, @@ -120,129 +111,72 @@ func NewAddonListCommand(c types.Args, ioStreams cmdutil.IOStreams) *cobra.Comma return cmd } -func retrievePlugins(ctx context.Context, c client.Client, ioStreams cmdutil.IOStreams, namespace string) error { - var pluginList []Plugin - var config AddonCenterConfig - var data []byte - var err error - var homeDir string - if homeDir, err = os.UserHomeDir(); err != nil { - ioStreams.Errorf("Failed to retrieve addon center configuration, please run `vela addon:config` first") - } - if data, err = ioutil.ReadFile(filepath.Join(homeDir, addonCenterConfigFile)); err != nil { - ioStreams.Errorf("Failed to retrieve addon center configuration, please run `vela addon:config` first") - } - if err := json.Unmarshal(data, &config); err != nil { - ioStreams.Errorf("Failed to retrieve addon center configuration, please run `vela addon:config` first") - } - - if config.IsLocal { - //TODO(zzxwill) merge `vela traits` and `vela workloads` - return nil - } else { - resp, err := http.Get(config.Name) - if err != nil { - return err - } - defer resp.Body.Close() - result, _ := ioutil.ReadAll(resp.Body) - var manifests []PluginFile - var traitManifestPrefix, workloadManifestPrefix = "TraitDefinition", "WorkloadDefinition" - err = json.Unmarshal(result, &manifests) - if err != nil { - return err - } - var manifestResp *http.Response - for _, d := range manifests { - var template types.Template - var workloadDefinition corev1alpha2.WorkloadDefinition - var traitDefinition corev1alpha2.TraitDefinition - if manifestResp, err = http.Get(d.Url); err != nil { +func NewAddonListCommand(ioStreams cmdutil.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "addon:ls ", + Short: "List addons", + Long: "List addons of workloads and traits", + Example: `vela addon:ls`, + RunE: func(cmd *cobra.Command, args []string) error { + var repoName string + if len(args) > 0 { + repoName = args[0] + } + dir, err := system.GetRepoDir() + if err != nil { return err } - defer manifestResp.Body.Close() - - if strings.Contains(strings.ToLower(d.Name), strings.ToLower(workloadManifestPrefix)) { - result, err := ioutil.ReadAll(manifestResp.Body) - if err != nil { - return err - } - err = yaml.Unmarshal(result, &workloadDefinition) - if err != nil { - return err - } - var definitionName string - if workloadDefinition.Spec.Extension != nil { - template, err = types.ConvertTemplateJson2Object(workloadDefinition.Spec.Extension) - if err != nil { - return err - } - definitionName = template.Name - } else { - definitionName = workloadDefinition.Name - } - - if err != nil { - return err - } - //Check whether the definition is applied - var status = "uninstalled" - if _, err = cmdutil.GetWorkloadDefinitionByName(ctx, c, namespace, workloadDefinition.Name); err == nil { - status = "installed" - } - pluginList = append(pluginList, Plugin{ - Name: definitionName, - Type: "workload", - Definition: workloadDefinition.Spec.Reference.Name, - Status: status, - ApplesTo: "-", - }) - } else if strings.Contains(strings.ToLower(d.Name), strings.ToLower(traitManifestPrefix)) { - result, err := ioutil.ReadAll(manifestResp.Body) - if err != nil { - return err - } - err = yaml.Unmarshal(result, &traitDefinition) - if err != nil { - return err - } - var definitionName string - if traitDefinition.Spec.Extension != nil { - template, err = types.ConvertTemplateJson2Object(traitDefinition.Spec.Extension) - if err != nil { - return err - } - definitionName = template.Name - } else { - definitionName = traitDefinition.Name - } - - //Check whether the definition is applied - var status = "uninstalled" - if _, err = cmdutil.GetTraitDefinitionByName(ctx, c, namespace, traitDefinition.Name); err == nil { - status = "installed" - } - pluginList = append(pluginList, Plugin{ - Name: definitionName, - Type: "trait", - Definition: traitDefinition.Spec.Reference.Name, - Status: status, - ApplesTo: strings.Join(traitDefinition.Spec.AppliesToWorkloads, ","), - }) - } else { - ioStreams.Errorf(fmt.Sprintf("Those manifests in addon repository should start with %s or %s", - workloadManifestPrefix, traitManifestPrefix)) - os.Exit(1) + if repoName != "" { + return ListRepoAddons(filepath.Join(dir, repoName), ioStreams) } - } - - table := uitable.New() - table.MaxColWidth = 60 - table.AddRow("NAME", "TYPE", "DEFINITION", "STATUS", "APPLIES-TO") - for _, p := range pluginList { - table.AddRow(p.Name, p.Type, p.Definition, p.Status, p.ApplesTo) - } - ioStreams.Info(table.String()) + dirs, err := ioutil.ReadDir(dir) + if err != nil { + return err + } + for _, dd := range dirs { + if !dd.IsDir() { + continue + } + if err = ListRepoAddons(filepath.Join(dir, dd.Name()), ioStreams); err != nil { + return err + } + } + return nil + }, + Annotations: map[string]string{ + types.TagCommandType: types.TypeOthers, + }, } + return cmd +} + +func ListRepoAddons(repoDir string, ioStreams cmdutil.IOStreams) error { + templates, err := plugins.LoadTempFromLocal(repoDir) + if err != nil { + return err + } + table := uitable.New() + table.AddRow("NAME", "TYPE", "DEFINITION", "STATUS", "APPLIES-TO") + + var status string + //TODO(wonderflow): check status whether install or not + status = "uninstalled" + for _, p := range templates { + table.AddRow(p.Name, p.Type, p.Type, status, p.AppliesTo) + } + ioStreams.Info(table.String()) return nil } + +func ConvertURL(address string) string { + //TODO(wonderflow) convert github address here + return address +} + +func SyncRemoteAddon(d plugins.RepoConfig) error { + addons, err := plugins.GetReposFromRemote(d) + if err != nil { + return err + } + return plugins.SyncRemoteAddons(d, addons) +} diff --git a/pkg/cmd/refresh.go b/pkg/cmd/refresh.go index f1514d324..143baa6e2 100644 --- a/pkg/cmd/refresh.go +++ b/pkg/cmd/refresh.go @@ -2,7 +2,6 @@ package cmd import ( "context" - "os" "path/filepath" "github.com/cloud-native-application/rudrx/api/types" @@ -36,12 +35,6 @@ func NewRefreshCommand(c types.Args, ioStreams cmdutil.IOStreams) *cobra.Command return cmd } -func StatOrCreate(dir string) { - if _, err := os.Stat(dir); os.IsNotExist(err) { - os.MkdirAll(dir, 0755) - } -} - func RefreshDefinitions(ctx context.Context, c client.Client, ioStreams cmdutil.IOStreams) error { dir, _ := system.GetDefinitionDir() @@ -51,7 +44,7 @@ func RefreshDefinitions(ctx context.Context, c client.Client, ioStreams cmdutil. return err } workloadDir := filepath.Join(dir, "workloads") - StatOrCreate(workloadDir) + system.StatAndCreate(workloadDir) ioStreams.Infof("get %d workload definitions from cluster, syncing to %s...", len(templates), workloadDir) successNum := plugins.SinkTemp2Local(templates, workloadDir) ioStreams.Infof("%d workload definitions successfully synced\n", successNum) @@ -62,7 +55,7 @@ func RefreshDefinitions(ctx context.Context, c client.Client, ioStreams cmdutil. return err } traitDir := filepath.Join(dir, "traits") - StatOrCreate(traitDir) + system.StatAndCreate(traitDir) ioStreams.Infof("get %d trait definitions from cluster, syncing to %s...", len(templates), traitDir) successNum = plugins.SinkTemp2Local(templates, traitDir) ioStreams.Infof("%d trait definitions successfully synced\n", successNum) diff --git a/pkg/plugins/cluster.go b/pkg/plugins/cluster.go index 111d0a41e..e78f3d5ee 100644 --- a/pkg/plugins/cluster.go +++ b/pkg/plugins/cluster.go @@ -41,14 +41,11 @@ func GetWorkloadsFromCluster(ctx context.Context, namespace string, c client.Cli } for _, wd := range workloadDefs.Items { - var tmp types.Template - tmp, err := HandleTemplate(wd.Spec.Extension, wd.Name, syncDir) + tmp, err := HandleDefinition(wd.Name, syncDir, wd.Spec.Reference.Name, wd.Spec.Extension, types.TypeWorkload, nil) if err != nil { fmt.Printf("[WARN]handle template %s: %v\n", wd.Name, err) continue } - tmp.Type = types.TypeWorkload - tmp.CrdName = wd.Spec.Reference.Name templates = append(templates, tmp) } return templates, nil @@ -63,20 +60,30 @@ func GetTraitsFromCluster(ctx context.Context, namespace string, c client.Client } for _, td := range traitDefs.Items { - var tmp types.Template - tmp, err := HandleTemplate(td.Spec.Extension, td.Name, syncDir) + tmp, err := HandleDefinition(td.Name, syncDir, td.Spec.Reference.Name, td.Spec.Extension, types.TypeTrait, td.Spec.AppliesToWorkloads) if err != nil { fmt.Printf("[WARN]handle template %s: %v\n", td.Name, err) continue } - tmp.Type = types.TypeTrait - tmp.AppliesTo = td.Spec.AppliesToWorkloads - tmp.CrdName = td.Spec.Reference.Name templates = append(templates, tmp) } return templates, nil } +func HandleDefinition(name, syncDir, crdName string, extention *runtime.RawExtension, tp types.DefinitionType, applyTo []string) (types.Template, error) { + var tmp types.Template + tmp, err := HandleTemplate(extention, name, syncDir) + if err != nil { + return types.Template{}, err + } + tmp.Type = tp + if tp == types.TypeTrait { + tmp.AppliesTo = applyTo + } + tmp.CrdName = crdName + return tmp, nil +} + func HandleTemplate(in *runtime.RawExtension, name, syncDir string) (types.Template, error) { tmp, err := types.ConvertTemplateJson2Object(in) if err != nil { diff --git a/pkg/plugins/repository.go b/pkg/plugins/repository.go new file mode 100644 index 000000000..7123de71d --- /dev/null +++ b/pkg/plugins/repository.go @@ -0,0 +1,153 @@ +package plugins + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + + "github.com/cloud-native-application/rudrx/api/types" + "github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2" + + "github.com/cloud-native-application/rudrx/pkg/utils/system" + "github.com/ghodss/yaml" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +//Used to store addon center config in file +type RepoConfig struct { + Name string `json:"repoName"` + Address string `json:"repoAddress"` +} + +var ( + RepoConfigFile = ".vela/addon_config" + DefaultRepo = "local" +) + +type RemoteAddon struct { + // Name MUST be xxx.yaml + Name string `json:"name"` + Url string `json:"download_url"` + Sha string `json:"sha"` + // Type MUST be file + Type string `json:"type"` +} + +type RemoteAddons []RemoteAddon + +type Plugin struct { + Name string `json:"name"` + Type string `json:"type"` + Definition string `json:"definition"` + Status string `json:"status"` + ApplesTo string `json:"applies_to"` +} + +//TODO(wonderflow): we can make default(built-in) repo configurable, then we should make default inside the answer +func LoadRepos() ([]RepoConfig, error) { + config, err := system.GetRepoConfig() + if err != nil { + return nil, err + } + data, err := ioutil.ReadFile(config) + if err != nil { + if os.IsNotExist(err) { + return []RepoConfig{}, nil + } + return nil, err + } + var repos []RepoConfig + if err = yaml.Unmarshal(data, &repos); err != nil { + return nil, err + } + return repos, nil +} + +func StoreRepos(repos []RepoConfig) error { + config, err := system.GetRepoConfig() + if err != nil { + return err + } + data, err := yaml.Marshal(repos) + if err != nil { + return err + } + return ioutil.WriteFile(config, data, 0644) +} + +func GetReposFromRemote(r RepoConfig) (RemoteAddons, error) { + resp, err := http.Get(r.Address) + if err != nil { + return nil, err + } + defer resp.Body.Close() + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + var repos RemoteAddons + if err = json.Unmarshal(data, &repos); err != nil { + return nil, err + } + return repos, nil +} + +func GetDefinitionFromURL(address, syncDir string) (types.Template, error) { + resp, err := http.Get(address) + if err != nil { + return types.Template{}, err + } + defer resp.Body.Close() + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return types.Template{}, err + } + var obj = unstructured.Unstructured{Object: make(map[string]interface{})} + err = yaml.Unmarshal(data, &obj.Object) + if err != nil { + return types.Template{}, err + } + switch obj.GetKind() { + case "WorkloadDefinition": + var rd v1alpha2.WorkloadDefinition + err = yaml.Unmarshal(data, &rd) + if err != nil { + return types.Template{}, err + } + return HandleDefinition(rd.Name, syncDir, rd.Spec.Reference.Name, rd.Spec.Extension, types.TypeWorkload, nil) + case "TraitDefinition": + var td v1alpha2.TraitDefinition + err = yaml.Unmarshal(data, &td) + if err != nil { + return types.Template{}, err + } + return HandleDefinition(td.Name, syncDir, td.Spec.Reference.Name, td.Spec.Extension, types.TypeTrait, td.Spec.AppliesToWorkloads) + case "ScopeDefinition": + //TODO(wonderflow): support scope definition here. + } + return types.Template{}, fmt.Errorf("unknown definition Type %s", obj.GetKind()) +} + +//TODO(wonderflow): currently we only sync by create, we also need to delete which not exist remotely. +func SyncRemoteAddons(r RepoConfig, addons RemoteAddons) error { + dir, err := system.GetRepoDir() + if err != nil { + return err + } + repoDir := filepath.Join(dir, r.Name) + system.StatAndCreate(repoDir) + var tmps []types.Template + for _, addon := range addons { + tmp, err := GetDefinitionFromURL(addon.Url, repoDir) + if err != nil { + return err + } + tmps = append(tmps, tmp) + } + success := SinkTemp2Local(tmps, repoDir) + fmt.Printf("successfully sync %d remote addons\n", success) + return nil +} diff --git a/pkg/utils/system/system.go b/pkg/utils/system/system.go index ca34f63ec..4d10b1707 100644 --- a/pkg/utils/system/system.go +++ b/pkg/utils/system/system.go @@ -23,6 +23,23 @@ func GetVelaHomeDir() (string, error) { return filepath.Join(home, defaultVelaHome), nil } +func GetRepoDir() (string, error) { + home, err := GetVelaHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, ".repo"), nil +} + +func GetRepoConfig() (string, error) { + home, err := GetRepoDir() + if err != nil { + return "", err + } + StatAndCreate(home) + return filepath.Join(home, "config.yaml"), nil +} + func GetApplicationDir() (string, error) { home, err := GetVelaHomeDir() if err != nil { @@ -76,9 +93,7 @@ func InitDefaultEnv() error { if err != nil { return err } - if err = os.MkdirAll(envDir, 0755); err != nil { - return err - } + StatAndCreate(envDir) data, _ := json.Marshal(&types.EnvMeta{Namespace: types.DefaultEnvName}) if err = ioutil.WriteFile(filepath.Join(envDir, types.DefaultEnvName), data, 0644); err != nil { return err @@ -92,3 +107,9 @@ func InitDefaultEnv() error { } return nil } + +func StatAndCreate(dir string) { + if _, err := os.Stat(dir); os.IsNotExist(err) { + os.MkdirAll(dir, 0755) + } +}