diff --git a/api/types/template_types.go b/api/types/capability.go similarity index 79% rename from api/types/template_types.go rename to api/types/capability.go index 83ae87f0a..9b2aa3949 100644 --- a/api/types/template_types.go +++ b/api/types/capability.go @@ -27,17 +27,36 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) -// Template defines the content of a plugin -type Template struct { +type Source struct { + RepoName string `json:"repoName"` +} + +// Capability defines the content of a capability +type Capability struct { Name string `json:"name"` Type DefinitionType `json:"type"` - Template string `json:"template,omitempty"` + CueTemplate string `json:"template,omitempty"` Parameters []Parameter `json:"parameters,omitempty"` DefinitionPath string `json:"definition"` CrdName string `json:"crdName,omitempty"` //trait only AppliesTo []string `json:"appliesTo,omitempty"` + + // Plugin Source + Source *Source `json:"source,omitempty"` + Install *Installation `json:"install,omitempty"` +} + +type Chart struct { + Repo string `json:"repo"` + URl string `json:"url"` + Name string `json:"name"` + Version string `json:"version"` +} + +type Installation struct { + Helm []Chart `json:"helm"` } type DefinitionType string @@ -45,6 +64,7 @@ type DefinitionType string const ( TypeWorkload DefinitionType = "workload" TypeTrait DefinitionType = "trait" + TypeScope DefinitionType = "scope" ) type Parameter struct { @@ -57,9 +77,9 @@ type Parameter struct { } // ConvertTemplateJson2Object convert spec.extension to object -func ConvertTemplateJson2Object(in *runtime.RawExtension) (Template, error) { - var t Template - var extension Template +func ConvertTemplateJson2Object(in *runtime.RawExtension) (Capability, error) { + var t Capability + var extension Capability if in == nil { return t, fmt.Errorf("extension field is nil") } diff --git a/api/types/types.go b/api/types/types.go index 782f4cbd2..fdc771022 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -7,13 +7,12 @@ import ( ) const ( - DefaultOAMNS = "oam-system" - DefaultOAMReleaseName = "core-runtime" - DefaultOAMChartName = "crossplane-master/oam-kubernetes-runtime" - DefaultOAMRuntimeName = "oam-kubernetes-runtime" - DefaultOAMRepoName = "crossplane-master" - DefaultOAMRepoUrl = "https://charts.crossplane.io/master" - DefaultOAMVersion = ">0.0.0-0" + DefaultOAMNS = "oam-system" + DefaultOAMReleaseName = "core-runtime" + DefaultOAMRuntimeChartName = "oam-kubernetes-runtime" + DefaultOAMRepoName = "crossplane-master" + DefaultOAMRepoUrl = "https://charts.crossplane.io/master" + DefaultOAMVersion = ">0.0.0-0" DefaultEnvName = "default" ) diff --git a/api/types/types_test.go b/api/types/types_test.go index 073ddcc77..f07ea6140 100644 --- a/api/types/types_test.go +++ b/api/types/types_test.go @@ -5,8 +5,8 @@ import ( "fmt" "testing" + "github.com/ghodss/yaml" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v3" ) func TestApplication(t *testing.T) { diff --git a/cmd/vela/main.go b/cmd/vela/main.go index 884e3a4bb..16265334e 100644 --- a/cmd/vela/main.go +++ b/cmd/vela/main.go @@ -94,19 +94,15 @@ func newCommand() *cobra.Command { Schema: scheme, } - if err := system.InitApplicationDir(); err != nil { - fmt.Println("InitApplicationDir err", err) - os.Exit(1) - } - if err := system.InitDefinitionDir(); err != nil { - fmt.Println("InitDefinitionDir err", err) + if err := system.InitDirs(); err != nil { + fmt.Println("InitDir err", err) os.Exit(1) } // Getting Start cmd.EnvCommandGroup(cmds, commandArgs, ioStream) // Others - cmd.AddonCommandGroup(cmds, ioStream) + cmd.CapabilityCommandGroup(cmds, commandArgs, ioStream) // System cmd.SystemCommandGroup(cmds, commandArgs, ioStream) @@ -129,18 +125,18 @@ func newCommand() *cobra.Command { ) // Workloads - if err = cmd.AddWorkloadPlugins(cmds, commandArgs, ioStream); err != nil { - fmt.Println("Add plugins from workloadDefinition err", err) + if err = cmd.AddWorkloadCommands(cmds, commandArgs, ioStream); err != nil { + fmt.Println("Add workload commands from workloadDefinition err", err) os.Exit(1) } // Traits - if err = cmd.AddTraitPlugins(cmds, commandArgs, ioStream); err != nil { - fmt.Println("Add plugins from traitDefinition err", err) + if err = cmd.AddTraitCommands(cmds, commandArgs, ioStream); err != nil { + fmt.Println("Add trait commands from traitDefinition err", err) os.Exit(1) } - if err = cmd.DetachTraitPlugins(cmds, commandArgs, ioStream); err != nil { - fmt.Println("Add plugins from traitDefinition err", err) + if err = cmd.AddTraitDetachCommands(cmds, commandArgs, ioStream); err != nil { + fmt.Println("Add trait detach commands from traitDefinition err", err) os.Exit(1) } // this is for mute klog diff --git a/config/samples/traits/route.yaml b/config/samples/traits/route.yaml new file mode 100644 index 000000000..85908f066 --- /dev/null +++ b/config/samples/traits/route.yaml @@ -0,0 +1,41 @@ +apiVersion: core.oam.dev/v1alpha2 +kind: TraitDefinition +metadata: + name: ingresses.networking.k8s.io + annotations: + "oam.appengine.info/apiVersion": "networking.k8s.io/v1beta1" + "oam.appengine.info/kind": "Ingress" +spec: + revisionEnabled: true + appliesToWorkloads: + - core.oam.dev/v1alpha2.ContainerizedWorkload + - deployments.apps + definitionRef: + name: ingresses.networking.k8s.io + extension: + install: + helm: + - repo: stable + name: nginx-ingress + version: 1.41.2 + template: | + #Template: { + apiVersion: "networking.k8s.io/v1beta1" + kind: "Ingress" + spec: { + rules: [{ + host: route.domain + http: paths: [{ + backend: { + serviceName: route.service + servicePort: route.port + }}] + }] + } + } + route: { + domain: string + port: *80 | int + service: string + } + diff --git a/go.mod b/go.mod index 332a157bd..efeda6b7e 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/crossplane/oam-kubernetes-runtime v0.0.8 github.com/ghodss/yaml v1.0.0 github.com/gin-gonic/gin v1.6.3 + github.com/google/go-github/v32 v32.1.0 github.com/gosuri/uitable v0.0.4 github.com/onsi/ginkgo v1.11.0 github.com/onsi/gomega v1.8.1 @@ -17,7 +18,7 @@ require ( github.com/stretchr/testify v1.6.1 go.uber.org/zap v1.10.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 - gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c + golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 gotest.tools v2.2.0+incompatible helm.sh/helm/v3 v3.2.4 k8s.io/api v0.18.6 diff --git a/go.sum b/go.sum index b301b4fab..903b8af2b 100644 --- a/go.sum +++ b/go.sum @@ -334,6 +334,11 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= +github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= diff --git a/pkg/cmd/addon.go b/pkg/cmd/addon.go deleted file mode 100644 index 7b3f5817b..000000000 --- a/pkg/cmd/addon.go +++ /dev/null @@ -1,182 +0,0 @@ -package cmd - -import ( - "errors" - "fmt" - "io/ioutil" - "path/filepath" - - "github.com/cloud-native-application/rudrx/pkg/utils/system" - - "github.com/cloud-native-application/rudrx/pkg/plugins" - - "github.com/cloud-native-application/rudrx/api/types" - - cmdutil "github.com/cloud-native-application/rudrx/pkg/cmd/util" - "github.com/gosuri/uitable" - "github.com/spf13/cobra" -) - -func AddonCommandGroup(parentCmd *cobra.Command, ioStream cmdutil.IOStreams) { - parentCmd.AddCommand( - NewAddonConfigCommand(ioStream), - NewAddonListCommand(ioStream), - NewAddonUpdateCommand(ioStream), - ) -} - -func NewAddonConfigCommand(ioStreams cmdutil.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "addon:config ", - Short: "Set the addon center, default is local (built-in ones)", - Long: "Set the addon center, default is local (built-in ones)", - Example: `vela addon:config myhub https://github.com/oam-dev/catalog/repository`, - RunE: func(cmd *cobra.Command, args []string) error { - argsLength := len(args) - if argsLength < 2 { - return errors.New("please set addon repo with and ") - } - repos, err := plugins.LoadRepos() - if err != nil { - return err - } - config := plugins.RepoConfig{ - Name: args[0], - Address: ConvertURL(args[1]), - } - var updated bool - for idx, r := range repos { - if r.Name == config.Name { - repos[idx] = config - updated = true - break - } - } - if !updated { - repos = append(repos, config) - } - if err = plugins.StoreRepos(repos); err != nil { - return err - } - ioStreams.Info(fmt.Sprintf("Successfully configured Addon repo: %s", args[0])) - return nil - }, - Annotations: map[string]string{ - types.TagCommandType: types.TypeOthers, - }, - } - return cmd -} - -func NewAddonUpdateCommand(ioStreams cmdutil.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "addon:update ", - Short: "Update addon repositories, default for all repo", - Long: "Update addon repositories, default for all repo", - Example: `vela addon:update myrepo`, - RunE: func(cmd *cobra.Command, args []string) error { - repos, err := plugins.LoadRepos() - if err != nil { - return err - } - var specified string - if len(args) > 0 { - specified = args[0] - } - find := false - if specified != "" { - for idx, r := range repos { - if r.Name == specified { - repos = []plugins.RepoConfig{repos[idx]} - find = true - break - } - } - if !find { - return fmt.Errorf("%s repo not exist", specified) - } - } - for _, d := range repos { - err = SyncRemoteAddon(d) - if err != nil { - return err - } - } - return nil - }, - Annotations: map[string]string{ - types.TagCommandType: types.TypeOthers, - }, - } - return cmd -} - -func NewAddonListCommand(ioStreams cmdutil.IOStreams) *cobra.Command { - cmd := &cobra.Command{ - Use: "addon:ls ", - Short: "List addons", - Long: "List addons of workloads and traits", - Example: `vela addon:ls`, - RunE: func(cmd *cobra.Command, args []string) error { - var repoName string - if len(args) > 0 { - repoName = args[0] - } - dir, err := system.GetRepoDir() - if err != nil { - return err - } - if repoName != "" { - return ListRepoAddons(filepath.Join(dir, repoName), ioStreams) - } - dirs, err := ioutil.ReadDir(dir) - if err != nil { - return err - } - for _, dd := range dirs { - if !dd.IsDir() { - continue - } - if err = ListRepoAddons(filepath.Join(dir, dd.Name()), ioStreams); err != nil { - return err - } - } - return nil - }, - Annotations: map[string]string{ - types.TagCommandType: types.TypeOthers, - }, - } - return cmd -} - -func ListRepoAddons(repoDir string, ioStreams cmdutil.IOStreams) error { - templates, err := plugins.LoadTempFromLocal(repoDir) - if err != nil { - return err - } - table := uitable.New() - table.AddRow("NAME", "TYPE", "DEFINITION", "STATUS", "APPLIES-TO") - - var status string - //TODO(wonderflow): check status whether install or not - status = "uninstalled" - for _, p := range templates { - table.AddRow(p.Name, p.Type, p.Type, status, p.AppliesTo) - } - ioStreams.Info(table.String()) - return nil -} - -func ConvertURL(address string) string { - //TODO(wonderflow) convert github address here - return address -} - -func SyncRemoteAddon(d plugins.RepoConfig) error { - addons, err := plugins.GetReposFromRemote(d) - if err != nil { - return err - } - return plugins.SyncRemoteAddons(d, addons) -} diff --git a/pkg/cmd/capability.go b/pkg/cmd/capability.go new file mode 100644 index 000000000..f6a8a8d21 --- /dev/null +++ b/pkg/cmd/capability.go @@ -0,0 +1,327 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "io/ioutil" + "path/filepath" + "strings" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + + "github.com/ghodss/yaml" + + "github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2" + + "github.com/gosuri/uitable" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/cloud-native-application/rudrx/pkg/utils/system" + + "github.com/cloud-native-application/rudrx/pkg/plugins" + + "github.com/cloud-native-application/rudrx/api/types" + + cmdutil "github.com/cloud-native-application/rudrx/pkg/cmd/util" + "github.com/spf13/cobra" +) + +func CapabilityCommandGroup(parentCmd *cobra.Command, c types.Args, ioStream cmdutil.IOStreams) { + parentCmd.AddCommand( + NewCapCenterConfigCommand(ioStream), + NewCapListCommand(ioStream), + NewCapCenterSyncCommand(ioStream), + NewCapAddCommand(c, ioStream), + ) +} + +func NewCapCenterConfigCommand(ioStreams cmdutil.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "cap:center:config ", + Short: "Configure or add the capability center, default is local (built-in capabilities)", + Long: "Configure or add the capability center, default is local (built-in capabilities)", + Example: `vela cap:center:config mycenter https://github.com/oam-dev/catalog/cap-center`, + RunE: func(cmd *cobra.Command, args []string) error { + argsLength := len(args) + if argsLength < 2 { + return errors.New("please set capability center with and ") + } + repos, err := plugins.LoadRepos() + if err != nil { + return err + } + config := &plugins.CapCenterConfig{ + Name: args[0], + Address: args[1], + Token: cmd.Flag("token").Value.String(), + } + var updated bool + for idx, r := range repos { + if r.Name == config.Name { + repos[idx] = *config + updated = true + break + } + } + if !updated { + repos = append(repos, *config) + } + if err = plugins.StoreRepos(repos); err != nil { + return err + } + ioStreams.Info(fmt.Sprintf("Successfully configured capability center: %s, please use 'vela cap:center:sync %s' to sync capabilities", args[0], args[0])) + return nil + }, + Annotations: map[string]string{ + types.TagCommandType: types.TypeOthers, + }, + } + cmd.PersistentFlags().StringP("token", "t", "", "Github Repo token") + return cmd +} + +func NewCapAddCommand(c types.Args, ioStreams cmdutil.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "cap:add
/", + Short: "Add capability into cluster", + Long: "Add capability into cluster", + Example: `vela cap:add mycenter/route`, + RunE: func(cmd *cobra.Command, args []string) error { + argsLength := len(args) + if argsLength < 1 { + return errors.New("you must specify
/ for capability you want to add") + } + newClient, err := client.New(c.Config, client.Options{Scheme: c.Schema}) + if err != nil { + return err + } + ss := strings.Split(args[0], "/") + if len(ss) < 2 { + return errors.New("invalid format for " + args[0] + ", please follow format
/") + } + repoName := ss[0] + name := ss[1] + return InstallCapability(newClient, repoName, name, ioStreams) + }, + Annotations: map[string]string{ + types.TagCommandType: types.TypeOthers, + }, + } + cmd.PersistentFlags().StringP("token", "t", "", "Github Repo token") + return cmd +} + +func NewCapCenterSyncCommand(ioStreams cmdutil.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "cap:center:sync [centerName]", + Short: "Sync capabilities from remote center, default to sync all centers", + Long: "Sync capabilities from remote center, default to sync all centers", + Example: `vela cap:center:sync mycenter`, + RunE: func(cmd *cobra.Command, args []string) error { + repos, err := plugins.LoadRepos() + if err != nil { + return err + } + var specified string + if len(args) > 0 { + specified = args[0] + } + if len(repos) == 0 { + return fmt.Errorf("no capability center configured") + } + find := false + if specified != "" { + for idx, r := range repos { + if r.Name == specified { + repos = []plugins.CapCenterConfig{repos[idx]} + find = true + break + } + } + if !find { + return fmt.Errorf("%s center not exist", specified) + } + } + ctx := context.Background() + for _, d := range repos { + client, err := plugins.NewCenterClient(ctx, d.Name, d.Address, d.Token) + err = client.SyncCapabilityFromCenter() + if err != nil { + return err + } + } + return nil + }, + Annotations: map[string]string{ + types.TagCommandType: types.TypeOthers, + }, + } + return cmd +} + +func NewCapListCommand(ioStreams cmdutil.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "cap:ls [centerName]", + Short: "List all capabilities in center", + Long: "List all capabilities in center", + Example: `vela cap:ls`, + RunE: func(cmd *cobra.Command, args []string) error { + var repoName string + if len(args) > 0 { + repoName = args[0] + } + dir, err := system.GetCapCenterDir() + if err != nil { + return err + } + table := uitable.New() + table.AddRow("NAME", "TYPE", "DEFINITION", "STATUS", "APPLIES-TO") + if repoName != "" { + if err = ListCenterCapabilities(table, filepath.Join(dir, repoName), ioStreams); err != nil { + return err + } + ioStreams.Info(table.String()) + return nil + } + dirs, err := ioutil.ReadDir(dir) + if err != nil { + return err + } + + for _, dd := range dirs { + if !dd.IsDir() { + continue + } + if err = ListCenterCapabilities(table, filepath.Join(dir, dd.Name()), ioStreams); err != nil { + return err + } + } + ioStreams.Info(table.String()) + return nil + }, + Annotations: map[string]string{ + types.TagCommandType: types.TypeOthers, + }, + } + return cmd +} + +func InstallCapability(client client.Client, centerName, capabilityName string, ioStreams cmdutil.IOStreams) error { + dir, _ := system.GetCapCenterDir() + repoDir := filepath.Join(dir, centerName) + tp, err := GetSyncedCapabilities(centerName, capabilityName) + if err != nil { + return err + } + tp.Source = &types.Source{RepoName: centerName} + defDir, _ := system.GetCapabilityDir() + switch tp.Type { + case types.TypeWorkload: + defDir = filepath.Join(defDir, "workloads") + var wd v1alpha2.WorkloadDefinition + workloadData, err := ioutil.ReadFile(filepath.Join(repoDir, tp.CrdName+".yaml")) + if err != nil { + return nil + } + if err = yaml.Unmarshal(workloadData, &wd); err != nil { + return err + } + wd.Namespace = types.DefaultOAMNS + ioStreams.Info("Installing workload capability " + wd.Name) + if tp.Install != nil { + if err = InstallHelmChart(ioStreams, tp.Install.Helm); err != nil { + return err + } + } + if err = client.Create(context.Background(), &wd); err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + case types.TypeTrait: + defDir = filepath.Join(defDir, "traits") + var td v1alpha2.TraitDefinition + traitdata, err := ioutil.ReadFile(filepath.Join(repoDir, tp.CrdName+".yaml")) + if err != nil { + return nil + } + if err = yaml.Unmarshal(traitdata, &td); err != nil { + return err + } + td.Namespace = types.DefaultOAMNS + ioStreams.Info("Installing trait capability " + td.Name) + if tp.Install != nil { + if err = InstallHelmChart(ioStreams, tp.Install.Helm); err != nil { + return err + } + } + if err = client.Create(context.Background(), &td); err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + case types.TypeScope: + //TODO(wonderflow): support install scope here + } + + success := plugins.SinkTemp2Local([]types.Capability{tp}, defDir) + if success == 1 { + ioStreams.Infof("Successfully installed capability %s from %s\n", capabilityName, centerName) + } + return nil +} + +func InstallHelmChart(ioStreams cmdutil.IOStreams, charts []types.Chart) error { + for _, c := range charts { + if err := HelmInstall(ioStreams, c.Repo, c.URl, c.Name, c.Version, c.Name); err != nil { + return err + } + } + return nil +} + +func GetSyncedCapabilities(repoName, addonName string) (types.Capability, error) { + dir, _ := system.GetCapCenterDir() + repoDir := filepath.Join(dir, repoName) + templates, err := plugins.LoadCapabilityFromLocal(repoDir) + if err != nil { + return types.Capability{}, err + } + for _, t := range templates { + if t.Name == addonName { + return t, nil + } + } + return types.Capability{}, fmt.Errorf("%s/%s not exist, try vela cap:center:sync %s to sync from remote", repoName, addonName, repoName) +} + +func ListCenterCapabilities(table *uitable.Table, repoDir string, ioStreams cmdutil.IOStreams) error { + templates, err := plugins.LoadCapabilityFromLocal(repoDir) + if err != nil { + return err + } + if len(templates) < 1 { + return nil + } + baseDir := filepath.Base(repoDir) + for _, p := range templates { + status := CheckInstallStatus(baseDir, p) + table.AddRow(baseDir+"/"+p.Name, p.Type, p.Type, status, p.AppliesTo) + } + return nil +} + +func CheckInstallStatus(repoName string, tmp types.Capability) string { + var status = "uninstalled" + dir, _ := system.GetCapabilityDir() + switch tmp.Type { + case types.TypeTrait: + dir = filepath.Join(dir, "traits") + case types.TypeWorkload: + dir = filepath.Join(dir, "workloads") + } + installed, _ := plugins.LoadTempFromLocal(dir) + for _, i := range installed { + if i.Source != nil && i.Source.RepoName == repoName && i.Name == tmp.Name && i.CrdName == tmp.CrdName { + return "installed" + } + } + return status +} diff --git a/pkg/cmd/fixtures_test.go b/pkg/cmd/fixtures_test.go index 4e0dd9bb7..8976c0c71 100644 --- a/pkg/cmd/fixtures_test.go +++ b/pkg/cmd/fixtures_test.go @@ -22,7 +22,7 @@ func init() { // used in testing var ( - workloadTemplateExample = &types.Template{ + workloadTemplateExample = &types.Capability{ Parameters: []types.Parameter{ types.Parameter{ @@ -38,7 +38,7 @@ var ( }, } - traitTemplateExample = &types.Template{ + traitTemplateExample = &types.Capability{ Parameters: []types.Parameter{ types.Parameter{ diff --git a/pkg/cmd/refresh.go b/pkg/cmd/refresh.go index 143baa6e2..0face6a27 100644 --- a/pkg/cmd/refresh.go +++ b/pkg/cmd/refresh.go @@ -36,7 +36,7 @@ func NewRefreshCommand(c types.Args, ioStreams cmdutil.IOStreams) *cobra.Command } func RefreshDefinitions(ctx context.Context, c client.Client, ioStreams cmdutil.IOStreams) error { - dir, _ := system.GetDefinitionDir() + dir, _ := system.GetCapabilityDir() ioStreams.Info("syncing workload definitions from cluster...") templates, err := plugins.GetWorkloadsFromCluster(ctx, types.DefaultOAMNS, c, dir, nil) diff --git a/pkg/cmd/system.go b/pkg/cmd/system.go index 61149ca25..11e0dc84f 100644 --- a/pkg/cmd/system.go +++ b/pkg/cmd/system.go @@ -171,13 +171,17 @@ func (i *initCmd) IsOamRuntimeExist() bool { return false } } + return IsHelmReleaseRunning(types.DefaultOAMReleaseName, types.DefaultOAMRuntimeChartName, i.ioStreams) +} + +func IsHelmReleaseRunning(releaseName, chartName string, streams cmdutil.IOStreams) bool { releases, err := GetHelmRelease() if err != nil { - i.ioStreams.Error("get helm release err", err) + streams.Error("get helm release err", err) return false } for _, r := range releases { - if strings.Contains(r.Chart.ChartFullPath(), types.DefaultOAMRuntimeName) { + if strings.Contains(r.Chart.ChartFullPath(), chartName) && r.Name == releaseName { return true } } @@ -185,35 +189,38 @@ func (i *initCmd) IsOamRuntimeExist() bool { } func InstallOamRuntime(ioStreams cmdutil.IOStreams, version string) error { + return HelmInstall(ioStreams, types.DefaultOAMRepoName, types.DefaultOAMRepoUrl, types.DefaultOAMRuntimeChartName, version, types.DefaultOAMReleaseName) +} - if !IsHelmRepositoryExist(types.DefaultOAMRepoName, types.DefaultOAMRepoUrl) { - err := AddHelmRepository(types.DefaultOAMRepoName, types.DefaultOAMRepoUrl, +func HelmInstall(ioStreams cmdutil.IOStreams, repoName, repoUrl, chartName, version, releaseName string) error { + if !IsHelmRepositoryExist(repoName, repoUrl) { + err := AddHelmRepository(repoName, repoUrl, "", "", "", "", "", false, ioStreams.Out) if err != nil { return err } } + if IsHelmReleaseRunning(releaseName, chartName, ioStreams) { + return nil + } - chartClient, err := NewHelmInstall(version, ioStreams) + chartClient, err := NewHelmInstall(version, releaseName, ioStreams) if err != nil { return err } - - chartRequested, err := GetChart(chartClient, types.DefaultOAMChartName) + chartRequested, err := GetChart(chartClient, repoName+"/"+chartName) if err != nil { return err } - release, err := chartClient.Run(chartRequested, nil) if err != nil { return err } - - fmt.Println("Successfully installed oam-kubernetes-runtime release: ", release.Name) + ioStreams.Infof("Successfully installed %s as release name %s\n", chartName, release.Name) return nil } -func NewHelmInstall(version string, ioStreams cmdutil.IOStreams) (*action.Install, error) { +func NewHelmInstall(version, releaseName string, ioStreams cmdutil.IOStreams) (*action.Install, error) { actionConfig := new(action.Configuration) if err := actionConfig.Init( @@ -227,12 +234,8 @@ func NewHelmInstall(version string, ioStreams cmdutil.IOStreams) (*action.Instal client := action.NewInstall(actionConfig) client.Namespace = types.DefaultOAMNS - client.ReleaseName = types.DefaultOAMReleaseName - if len(version) > 0 { - client.Version = version - return client, nil - } - client.Version = types.DefaultOAMVersion + client.ReleaseName = releaseName + client.Version = version return client, nil } @@ -329,7 +332,7 @@ func GetOAMReleaseVersion() (string, error) { } for _, result := range results { - if result.Chart.ChartFullPath() == types.DefaultOAMRuntimeName { + if result.Chart.ChartFullPath() == types.DefaultOAMRuntimeChartName { return result.Chart.AppVersion(), nil } } diff --git a/pkg/cmd/trait_bind.go b/pkg/cmd/trait_bind.go index f50f8dc24..52a0f5a9a 100644 --- a/pkg/cmd/trait_bind.go +++ b/pkg/cmd/trait_bind.go @@ -30,7 +30,7 @@ import ( ) type commandOptions struct { - Template types.Template + Template types.Capability Component corev1alpha2.Component AppConfig corev1alpha2.ApplicationConfiguration Client client.Client @@ -44,8 +44,8 @@ func NewCommandOptions(ioStreams cmdutil.IOStreams) *commandOptions { return &commandOptions{IOStreams: ioStreams} } -func AddTraitPlugins(parentCmd *cobra.Command, c types.Args, ioStreams cmdutil.IOStreams) error { - dir, _ := system.GetDefinitionDir() +func AddTraitCommands(parentCmd *cobra.Command, c types.Args, ioStreams cmdutil.IOStreams) error { + dir, _ := system.GetCapabilityDir() templates, err := plugins.LoadTempFromLocal(filepath.Join(dir, "traits")) if err != nil { return err @@ -171,8 +171,8 @@ func (o *commandOptions) Complete(cmd *cobra.Command, args []string, ctx context return nil } -func DetachTraitPlugins(parentCmd *cobra.Command, c types.Args, ioStreams cmdutil.IOStreams) error { - dir, _ := system.GetDefinitionDir() +func AddTraitDetachCommands(parentCmd *cobra.Command, c types.Args, ioStreams cmdutil.IOStreams) error { + dir, _ := system.GetCapabilityDir() templates, err := plugins.LoadTempFromLocal(filepath.Join(dir, "traits")) if err != nil { return err diff --git a/pkg/cmd/traits.go b/pkg/cmd/traits.go index c41046c0c..c6a6da958 100644 --- a/pkg/cmd/traits.go +++ b/pkg/cmd/traits.go @@ -23,7 +23,7 @@ func NewTraitsCommand(ioStreams cmdutil.IOStreams) *cobra.Command { Long: "List traits", Example: `vela traits`, RunE: func(cmd *cobra.Command, args []string) error { - dir, _ := system.GetDefinitionDir() + dir, _ := system.GetCapabilityDir() templates, err := plugins.LoadTempFromLocal(filepath.Join(dir, "traits")) if err != nil { return err @@ -41,7 +41,7 @@ func NewTraitsCommand(ioStreams cmdutil.IOStreams) *cobra.Command { return cmd } -func printTraitList(traits, workloads []types.Template, workloadName *string, ioStreams cmdutil.IOStreams) error { +func printTraitList(traits, workloads []types.Capability, workloadName *string, ioStreams cmdutil.IOStreams) error { table := uitable.New() table.MaxColWidth = 60 @@ -71,7 +71,7 @@ func printTraitList(traits, workloads []types.Template, workloadName *string, io return nil } -func ConvertApplyTo(applyTo []string, workloads []types.Template) []string { +func ConvertApplyTo(applyTo []string, workloads []types.Capability) []string { var converted []string for _, v := range applyTo { newName, exist := check(v, workloads) @@ -83,7 +83,7 @@ func ConvertApplyTo(applyTo []string, workloads []types.Template) []string { return converted } -func check(crdname string, workloads []types.Template) (string, bool) { +func check(crdname string, workloads []types.Capability) (string, bool) { for _, v := range workloads { if crdname == v.CrdName { return v.Name, true diff --git a/pkg/cmd/traits_test.go b/pkg/cmd/traits_test.go index e6bd324c9..0e93c22b0 100644 --- a/pkg/cmd/traits_test.go +++ b/pkg/cmd/traits_test.go @@ -13,7 +13,7 @@ import ( ) func Test_printTraitList(t *testing.T) { - traits := []types.Template{ + traits := []types.Capability{ { Name: "route", CrdName: "routes.oam.dev", @@ -25,7 +25,7 @@ func Test_printTraitList(t *testing.T) { AppliesTo: []string{"deployments.apps"}, }, } - workloads := []types.Template{ + workloads := []types.Capability{ { Name: "deployment", CrdName: "deployments.apps", @@ -54,8 +54,8 @@ func Test_printTraitList(t *testing.T) { tb3.AddRow("route", "routes.oam.dev", "clonset") cases := map[string]struct { - traits []types.Template - workloads []types.Template + traits []types.Capability + workloads []types.Capability workloadName string iostream cmdutil.IOStreams ExpectedString string diff --git a/pkg/cmd/util/helpers.go b/pkg/cmd/util/helpers.go index 4c8e8ba37..fe52d1293 100644 --- a/pkg/cmd/util/helpers.go +++ b/pkg/cmd/util/helpers.go @@ -118,7 +118,7 @@ func GetWorkloadNameAliasKind(ctx context.Context, c client.Client, namespace st w, err := GetWorkloadDefinitionByName(ctx, c, namespace, workloadName) if err == nil { // workloadName is complete name - var workloadTemplate types.Template + var workloadTemplate types.Capability workloadTemplate, err := types.ConvertTemplateJson2Object(w.Spec.Extension) if err == nil { name, alias = w.Name, workloadTemplate.Name diff --git a/pkg/cmd/workload_run.go b/pkg/cmd/workload_run.go index 23ab673a4..1a6dc8ac2 100644 --- a/pkg/cmd/workload_run.go +++ b/pkg/cmd/workload_run.go @@ -12,7 +12,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "gopkg.in/yaml.v3" + "github.com/ghodss/yaml" "cuelang.org/go/cue" @@ -35,7 +35,7 @@ import ( const ComponentWorkloadDefLabel = "vela.oam.dev/workloadDef" type runOptions struct { - Template types.Template + Template types.Capability Env *types.EnvMeta workloadName string client client.Client @@ -47,8 +47,8 @@ func newRunOptions(ioStreams cmdutil.IOStreams) *runOptions { return &runOptions{IOStreams: ioStreams} } -func AddWorkloadPlugins(parentCmd *cobra.Command, c types.Args, ioStreams cmdutil.IOStreams) error { - dir, _ := system.GetDefinitionDir() +func AddWorkloadCommands(parentCmd *cobra.Command, c types.Args, ioStreams cmdutil.IOStreams) error { + dir, _ := system.GetCapabilityDir() templates, err := plugins.LoadTempFromLocal(filepath.Join(dir, "workloads")) if err != nil { return err diff --git a/pkg/cmd/workloads.go b/pkg/cmd/workloads.go index a64eb460f..059ad787d 100644 --- a/pkg/cmd/workloads.go +++ b/pkg/cmd/workloads.go @@ -21,7 +21,7 @@ func NewWorkloadsCommand(ioStreams cmdutil.IOStreams) *cobra.Command { Long: "List workloads", Example: `vela workloads`, RunE: func(cmd *cobra.Command, args []string) error { - dir, _ := system.GetDefinitionDir() + dir, _ := system.GetCapabilityDir() workloads, err := plugins.LoadTempFromLocal(filepath.Join(dir, "workloads")) if err != nil { return err @@ -33,7 +33,7 @@ func NewWorkloadsCommand(ioStreams cmdutil.IOStreams) *cobra.Command { return cmd } -func printWorkloadList(workloadList []types.Template, ioStreams cmdutil.IOStreams) error { +func printWorkloadList(workloadList []types.Capability, ioStreams cmdutil.IOStreams) error { table := uitable.New() table.MaxColWidth = 60 table.AddRow("NAME", "DEFINITION") diff --git a/pkg/cue/convert.go b/pkg/cue/convert.go index 304db47b3..56e9023d6 100644 --- a/pkg/cue/convert.go +++ b/pkg/cue/convert.go @@ -17,59 +17,29 @@ func Eval(templatePath, workloadType string, value map[string]interface{}) (stri r := cue.Runtime{} template, err := r.Compile(templatePath, nil) if err != nil { - return "", err + return "", fmt.Errorf("compile %s err %v", templatePath, err) } tempValue := template.Value() appValue, err := tempValue.Fill(value, workloadType).Eval().Struct() if err != nil { - return "", err + return "", fmt.Errorf("fill value to template err %v", err) } final, err := appValue.FieldByName(Template, true) if err != nil { - return "", err + return "", fmt.Errorf("get template %s err %v", Template, err) } if err := final.Value.Validate(cue.Concrete(true), cue.Final()); err != nil { return "", err } data, err := json.Marshal(final.Value) if err != nil { - return "", err + return "", fmt.Errorf("marshal final value err %v", err) } - return data, nil } -func Parse(templatePath, workloadType string, value map[string]interface{}) error { - r := cue.Runtime{} - template, err := r.Compile(templatePath, nil) - if err != nil { - return err - } - - tempValue := template.Value() - appValue, err := tempValue.Fill(value, workloadType).Eval().Struct() - if err != nil { - return err - } - - final, err := appValue.FieldByName(Template, true) - if err != nil { - return err - } - if err := final.Value.Validate(cue.Concrete(true), cue.Final()); err != nil { - return err - } - data, err := json.Marshal(final.Value) - if err != nil { - return err - } - println(string(data)) - - return nil -} - func GetParameters(templatePath string) ([]types.Parameter, string, error) { r := cue.Runtime{} template, err := r.Compile(templatePath, nil) diff --git a/pkg/cue/testdata/traits/route.cue b/pkg/cue/testdata/traits/route.cue index 0e40bd5bc..44678366c 100644 --- a/pkg/cue/testdata/traits/route.cue +++ b/pkg/cue/testdata/traits/route.cue @@ -1,11 +1,19 @@ #Template: { - apiVersion: "apps/v1" - kind: "Route" + apiVersion: "networking.k8s.io/v1beta1" + kind: "Ingress" spec: { - domain: route.domain + rules: [{ + host: route.domain + http: paths: [{ + backend: { + serviceName: route.service + servicePort: route.port + }}] + }] } } - route: { - domain: string + domain: string + port: *80 | int + service: string } diff --git a/pkg/plugins/capcenter.go b/pkg/plugins/capcenter.go new file mode 100644 index 000000000..d5710761e --- /dev/null +++ b/pkg/plugins/capcenter.go @@ -0,0 +1,244 @@ +package plugins + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + + "golang.org/x/oauth2" + + "github.com/google/go-github/v32/github" + + "github.com/cloud-native-application/rudrx/api/types" + "github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2" + + "github.com/cloud-native-application/rudrx/pkg/utils/system" + "github.com/ghodss/yaml" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +type GithubContent struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Path string `json:"path"` + Ref string `json:"ref"` +} + +//CapCenterConfig is used to store cap center config in file +type CapCenterConfig struct { + Name string `json:"name"` + Address string `json:"address"` + Token string `json:"token"` +} + +type CenterClient interface { + SyncCapabilityFromCenter() error +} + +func NewCenterClient(ctx context.Context, name, address, token string) (CenterClient, error) { + Type, cfg, err := Parse(address) + if err != nil { + return nil, err + } + switch Type { + case TypeGithub: + return NewGithubCenter(ctx, token, name, cfg) + } + return nil, errors.New("we only support github as repository now") +} + +const TypeGithub = "github" +const TypeUnknown = "unknown" + +func Parse(addr string) (string, *GithubContent, error) { + url, err := url.Parse(addr) + if err != nil { + return "", nil, err + } + l := strings.Split(strings.TrimPrefix(url.Path, "/"), "/") + switch url.Host { + case "github.com": + // We support two valid format: + // 1. https://github.com///tree// + // 2. https://github.com/// + if len(l) < 3 { + return "", nil, errors.New("invalid format " + addr) + } + if l[2] == "tree" { + // https://github.com///tree// + if len(l) < 5 { + return "", nil, errors.New("invalid format " + addr) + } + return TypeGithub, &GithubContent{ + Owner: l[0], + Repo: l[1], + Path: strings.Join(l[4:], "/"), + Ref: l[3], + }, nil + } else { + // https://github.com/// + return TypeGithub, &GithubContent{ + Owner: l[0], + Repo: l[1], + Path: strings.Join(l[2:], "/"), + Ref: "", //use default branch + }, nil + } + case "api.github.com": + if len(l) != 5 { + return "", nil, errors.New("invalid format " + addr) + } + //https://api.github.com/repos///contents/ + return TypeGithub, &GithubContent{ + Owner: l[1], + Repo: l[2], + Path: l[4], + Ref: url.Query().Get("ref"), + }, nil + default: + //TODO(wonderflow): support raw url and oss format in the future + } + return TypeUnknown, nil, nil +} + +type RemoteCapability struct { + // Name MUST be xxx.yaml + Name string `json:"name"` + Url string `json:"download_url"` + Sha string `json:"sha"` + // Type MUST be file + Type string `json:"type"` +} + +type RemoteCapabilities []RemoteCapability + +//TODO(wonderflow): we can make default(built-in) repo configurable, then we should make default inside the answer +func LoadRepos() ([]CapCenterConfig, error) { + config, err := system.GetRepoConfig() + if err != nil { + return nil, err + } + data, err := ioutil.ReadFile(config) + if err != nil { + if os.IsNotExist(err) { + return []CapCenterConfig{}, nil + } + return nil, err + } + var repos []CapCenterConfig + if err = yaml.Unmarshal(data, &repos); err != nil { + return nil, err + } + return repos, nil +} + +func StoreRepos(repos []CapCenterConfig) error { + config, err := system.GetRepoConfig() + if err != nil { + return err + } + data, err := yaml.Marshal(repos) + if err != nil { + return err + } + return ioutil.WriteFile(config, data, 0644) +} + +func ParseAndSyncCapability(data []byte, syncDir string) (types.Capability, error) { + var obj = unstructured.Unstructured{Object: make(map[string]interface{})} + err := yaml.Unmarshal(data, &obj.Object) + if err != nil { + return types.Capability{}, err + } + switch obj.GetKind() { + case "WorkloadDefinition": + var rd v1alpha2.WorkloadDefinition + err = yaml.Unmarshal(data, &rd) + if err != nil { + return types.Capability{}, err + } + return HandleDefinition(rd.Name, syncDir, rd.Spec.Reference.Name, rd.Spec.Extension, types.TypeWorkload, nil) + case "TraitDefinition": + var td v1alpha2.TraitDefinition + err = yaml.Unmarshal(data, &td) + if err != nil { + return types.Capability{}, err + } + return HandleDefinition(td.Name, syncDir, td.Spec.Reference.Name, td.Spec.Extension, types.TypeTrait, td.Spec.AppliesToWorkloads) + case "ScopeDefinition": + //TODO(wonderflow): support scope definition here. + } + return types.Capability{}, fmt.Errorf("unknown definition Type %s", obj.GetKind()) +} + +type GithubCenter struct { + client *github.Client + cfg *GithubContent + centerName string + ctx context.Context +} + +var _ CenterClient = &GithubCenter{} + +func NewGithubCenter(ctx context.Context, token, centerName string, r *GithubContent) (*GithubCenter, error) { + var tc *http.Client + if token != "" { + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + tc = oauth2.NewClient(ctx, ts) + } + return &GithubCenter{client: github.NewClient(tc), cfg: r, centerName: centerName, ctx: ctx}, nil +} + +//TODO(wonderflow): currently we only sync by create, we also need to delete which not exist remotely. +func (g *GithubCenter) SyncCapabilityFromCenter() error { + _, dirs, _, err := g.client.Repositories.GetContents(g.ctx, g.cfg.Owner, g.cfg.Repo, g.cfg.Path, &github.RepositoryContentGetOptions{Ref: g.cfg.Ref}) + if err != nil { + return err + } + dir, err := system.GetCapCenterDir() + if err != nil { + return err + } + repoDir := filepath.Join(dir, g.centerName) + system.StatAndCreate(repoDir) + var success, total int + for _, addon := range dirs { + if *addon.Type != "file" { + continue + } + total++ + fileContent, _, _, err := g.client.Repositories.GetContents(g.ctx, g.cfg.Owner, g.cfg.Repo, *addon.Path, &github.RepositoryContentGetOptions{Ref: g.cfg.Ref}) + if err != nil { + return err + } + var data = []byte(*fileContent.Content) + if *fileContent.Encoding == "base64" { + data, err = base64.StdEncoding.DecodeString(*fileContent.Content) + if err != nil { + return fmt.Errorf("decode github content %s err %v", *fileContent.Path, err) + } + } + tmp, err := ParseAndSyncCapability(data, filepath.Join(dir, ".tmp")) + if err != nil { + fmt.Printf("parse definition of %s err %v\n", *fileContent.Name, err) + continue + } + err = ioutil.WriteFile(filepath.Join(repoDir, tmp.CrdName+".yaml"), data, 0644) + if err != nil { + fmt.Printf("write definition %s to %s err %v\n", tmp.CrdName+".yaml", repoDir, err) + continue + } + success++ + } + fmt.Printf("successfully sync %d/%d from %s remote center\n", success, total, g.centerName) + return nil +} diff --git a/pkg/plugins/capcenter_test.go b/pkg/plugins/capcenter_test.go new file mode 100644 index 000000000..8a7ed5148 --- /dev/null +++ b/pkg/plugins/capcenter_test.go @@ -0,0 +1,51 @@ +package plugins + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseURL(t *testing.T) { + cases := map[string]struct { + url string + exp *GithubContent + expType string + }{ + "api-github": { + url: "https://api.github.com/repos/zzxwill/catalog/contents/repository?ref=plugin", + expType: TypeGithub, + exp: &GithubContent{ + Owner: "zzxwill", + Repo: "catalog", + Path: "repository", + Ref: "plugin", + }, + }, + "github-copy-path": { + url: "https://github.com/zzxwill/catalog/tree/plugin/repository", + expType: TypeGithub, + exp: &GithubContent{ + Owner: "zzxwill", + Repo: "catalog", + Path: "repository", + Ref: "plugin", + }, + }, + "github-manuel-write-path": { + url: "https://github.com/zzxwill/catalog/repository", + expType: TypeGithub, + exp: &GithubContent{ + Owner: "zzxwill", + Repo: "catalog", + Path: "repository", + }, + }, + } + for caseName, c := range cases { + tp, content, err := Parse(c.url) + assert.NoError(t, err, caseName) + assert.Equal(t, c.exp, content, caseName) + assert.Equal(t, c.expType, tp, caseName) + } +} diff --git a/pkg/plugins/cluster.go b/pkg/plugins/cluster.go index ed27739af..e3007b76d 100644 --- a/pkg/plugins/cluster.go +++ b/pkg/plugins/cluster.go @@ -7,6 +7,8 @@ import ( "io/ioutil" "path/filepath" + "github.com/cloud-native-application/rudrx/pkg/utils/system" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" @@ -18,7 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func GetTemplatesFromCluster(ctx context.Context, namespace string, c client.Client, syncDir string, selector labels.Selector) ([]types.Template, error) { +func GetCapabilitiesFromCluster(ctx context.Context, namespace string, c client.Client, syncDir string, selector labels.Selector) ([]types.Capability, error) { workloads, err := GetWorkloadsFromCluster(ctx, namespace, c, syncDir, selector) if err != nil { return nil, err @@ -31,8 +33,8 @@ func GetTemplatesFromCluster(ctx context.Context, namespace string, c client.Cli return workloads, nil } -func GetWorkloadsFromCluster(ctx context.Context, namespace string, c client.Client, syncDir string, selector labels.Selector) ([]types.Template, error) { - var templates []types.Template +func GetWorkloadsFromCluster(ctx context.Context, namespace string, c client.Client, syncDir string, selector labels.Selector) ([]types.Capability, error) { + var templates []types.Capability var workloadDefs corev1alpha2.WorkloadDefinitionList err := c.List(ctx, &workloadDefs, &client.ListOptions{Namespace: namespace, LabelSelector: selector}) if err != nil { @@ -50,8 +52,8 @@ func GetWorkloadsFromCluster(ctx context.Context, namespace string, c client.Cli return templates, nil } -func GetTraitsFromCluster(ctx context.Context, namespace string, c client.Client, syncDir string, selector labels.Selector) ([]types.Template, error) { - var templates []types.Template +func GetTraitsFromCluster(ctx context.Context, namespace string, c client.Client, syncDir string, selector labels.Selector) ([]types.Capability, error) { + var templates []types.Capability var traitDefs corev1alpha2.TraitDefinitionList err := c.List(ctx, &traitDefs, &client.ListOptions{Namespace: namespace, LabelSelector: selector}) if err != nil { @@ -69,11 +71,11 @@ func GetTraitsFromCluster(ctx context.Context, namespace string, c client.Client return templates, nil } -func HandleDefinition(name, syncDir, crdName string, extention *runtime.RawExtension, tp types.DefinitionType, applyTo []string) (types.Template, error) { - var tmp types.Template +func HandleDefinition(name, syncDir, crdName string, extention *runtime.RawExtension, tp types.DefinitionType, applyTo []string) (types.Capability, error) { + var tmp types.Capability tmp, err := HandleTemplate(extention, name, syncDir) if err != nil { - return types.Template{}, err + return types.Capability{}, err } tmp.Type = tp if tp == types.TypeTrait { @@ -83,23 +85,24 @@ func HandleDefinition(name, syncDir, crdName string, extention *runtime.RawExten return tmp, nil } -func HandleTemplate(in *runtime.RawExtension, name, syncDir string) (types.Template, error) { +func HandleTemplate(in *runtime.RawExtension, name, syncDir string) (types.Capability, error) { tmp, err := types.ConvertTemplateJson2Object(in) if err != nil { - return types.Template{}, err + return types.Capability{}, err } - if tmp.Template == "" { - return types.Template{}, errors.New("template not exist in definition") + if tmp.CueTemplate == "" { + return types.Capability{}, errors.New("template not exist in definition") } + system.StatAndCreate(syncDir) filePath := filepath.Join(syncDir, name+".cue") - err = ioutil.WriteFile(filePath, []byte(tmp.Template), 0644) + err = ioutil.WriteFile(filePath, []byte(tmp.CueTemplate), 0644) if err != nil { - return types.Template{}, err + return types.Capability{}, err } tmp.DefinitionPath = filePath tmp.Parameters, tmp.Name, err = cue.GetParameters(filePath) if err != nil { - return types.Template{}, err + return types.Capability{}, err } return tmp, nil } diff --git a/pkg/plugins/cluster_test.go b/pkg/plugins/cluster_test.go index f105ab5f7..105ac552f 100644 --- a/pkg/plugins/cluster_test.go +++ b/pkg/plugins/cluster_test.go @@ -19,7 +19,7 @@ import ( var _ = Describe("DefinitionFiles", func() { - route := types.Template{ + route := types.Capability{ Name: "route", Type: types.TypeTrait, Parameters: []types.Parameter{ @@ -32,7 +32,7 @@ var _ = Describe("DefinitionFiles", func() { }, CrdName: "routes.test", } - deployment := types.Template{ + deployment := types.Capability{ Name: "deployment", Type: types.TypeWorkload, CrdName: "deployments.testapps", @@ -74,10 +74,10 @@ var _ = Describe("DefinitionFiles", func() { Expect(err).Should(BeNil()) logf.Log.Info(fmt.Sprintf("Getting trait definitions %v", traitDefs)) for i := range traitDefs { - traitDefs[i].Template = "" + traitDefs[i].CueTemplate = "" traitDefs[i].DefinitionPath = "" } - Expect(traitDefs).Should(Equal([]types.Template{route})) + Expect(traitDefs).Should(Equal([]types.Capability{route})) }) // Notice!! DefinitionPath Object is Cluster Scope object // which means objects created in other DefinitionNamespace will also affect here. @@ -86,19 +86,19 @@ var _ = Describe("DefinitionFiles", func() { Expect(err).Should(BeNil()) logf.Log.Info(fmt.Sprintf("Getting workload definitions %v", workloadDefs)) for i := range workloadDefs { - workloadDefs[i].Template = "" + workloadDefs[i].CueTemplate = "" workloadDefs[i].DefinitionPath = "" } - Expect(workloadDefs).Should(Equal([]types.Template{deployment})) + Expect(workloadDefs).Should(Equal([]types.Capability{deployment})) }) It("getall", func() { - alldef, err := GetTemplatesFromCluster(context.Background(), DefinitionNamespace, k8sClient, definitionDir, selector) + alldef, err := GetCapabilitiesFromCluster(context.Background(), DefinitionNamespace, k8sClient, definitionDir, selector) Expect(err).Should(BeNil()) logf.Log.Info(fmt.Sprintf("Getting all definitions %v", alldef)) for i := range alldef { - alldef[i].Template = "" + alldef[i].CueTemplate = "" alldef[i].DefinitionPath = "" } - Expect(alldef).Should(Equal([]types.Template{deployment, route})) + Expect(alldef).Should(Equal([]types.Capability{deployment, route})) }) }) diff --git a/pkg/plugins/local.go b/pkg/plugins/local.go index f4a8810cb..2990d0ef8 100644 --- a/pkg/plugins/local.go +++ b/pkg/plugins/local.go @@ -12,12 +12,12 @@ import ( "github.com/cloud-native-application/rudrx/api/types" ) -func GetDefFromLocal(dir string, defType types.DefinitionType) ([]types.Template, error) { +func GetDefFromLocal(dir string, defType types.DefinitionType) ([]types.Capability, error) { temps, err := LoadTempFromLocal(dir) if err != nil { return nil, err } - var defs []types.Template + var defs []types.Capability for _, t := range temps { if t.Type != defType { continue @@ -27,7 +27,7 @@ func GetDefFromLocal(dir string, defType types.DefinitionType) ([]types.Template return defs, nil } -func SinkTemp2Local(templates []types.Template, dir string) int { +func SinkTemp2Local(templates []types.Capability, dir string) int { success := 0 for _, tmp := range templates { data, err := json.Marshal(tmp) @@ -45,12 +45,11 @@ func SinkTemp2Local(templates []types.Template, dir string) int { return success } -func LoadTempFromLocal(dir string) ([]types.Template, error) { - var tmps []types.Template +func LoadCapabilityFromLocal(dir string) ([]types.Capability, error) { + var tmps []types.Capability files, err := ioutil.ReadDir(dir) if err != nil { if os.IsNotExist(err) { - fmt.Println("\"no definition files found, use 'vela refresh' to sync from cluster\"") return nil, nil } return nil, err @@ -67,7 +66,38 @@ func LoadTempFromLocal(dir string) ([]types.Template, error) { fmt.Printf("read file %s err %v\n", f.Name(), err) continue } - var tmp types.Template + tmp, err := ParseAndSyncCapability(data, filepath.Join(dir, ".tmp")) + if err != nil { + fmt.Printf("get definition of %s err %v\n", f.Name(), err) + continue + } + tmps = append(tmps, tmp) + } + return tmps, nil +} + +func LoadTempFromLocal(dir string) ([]types.Capability, error) { + var tmps []types.Capability + files, err := ioutil.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + for _, f := range files { + if f.IsDir() { + continue + } + if strings.HasSuffix(f.Name(), ".cue") { + continue + } + data, err := ioutil.ReadFile(filepath.Join(dir, f.Name())) + if err != nil { + fmt.Printf("read file %s err %v\n", f.Name(), err) + continue + } + var tmp types.Capability decoder := json.NewDecoder(bytes.NewBuffer(data)) decoder.UseNumber() if err = decoder.Decode(&tmp); err != nil { @@ -76,8 +106,5 @@ func LoadTempFromLocal(dir string) ([]types.Template, error) { } tmps = append(tmps, tmp) } - if len(tmps) == 0 { - fmt.Println("\"no definition files found, use 'vela refresh' to sync from cluster\"") - } return tmps, nil } diff --git a/pkg/plugins/local_test.go b/pkg/plugins/local_test.go index f8effec2f..5c3ae2f00 100644 --- a/pkg/plugins/local_test.go +++ b/pkg/plugins/local_test.go @@ -10,7 +10,7 @@ import ( ) func TestLocalSink(t *testing.T) { - deployment := types.Template{ + deployment := types.Capability{ Name: "deployment", Type: types.TypeWorkload, Parameters: []types.Parameter{ @@ -21,7 +21,7 @@ func TestLocalSink(t *testing.T) { }, }, } - statefulset := types.Template{ + statefulset := types.Capability{ Name: "statefulset", Type: types.TypeWorkload, Parameters: []types.Parameter{ @@ -32,7 +32,7 @@ func TestLocalSink(t *testing.T) { }, }, } - route := types.Template{ + route := types.Capability{ Name: "route", Type: types.TypeTrait, Parameters: []types.Parameter{ @@ -46,9 +46,9 @@ func TestLocalSink(t *testing.T) { cases := map[string]struct { dir string - tmps []types.Template + tmps []types.Capability Type types.DefinitionType - expDef []types.Template + expDef []types.Capability err error }{ "Test No Templates": { @@ -57,33 +57,33 @@ func TestLocalSink(t *testing.T) { }, "Test Only Workload": { dir: "vela-test2", - tmps: []types.Template{deployment, statefulset}, + tmps: []types.Capability{deployment, statefulset}, Type: types.TypeWorkload, - expDef: []types.Template{deployment, statefulset}, + expDef: []types.Capability{deployment, statefulset}, }, "Test Only Trait": { dir: "vela-test3", - tmps: []types.Template{route}, + tmps: []types.Capability{route}, Type: types.TypeTrait, - expDef: []types.Template{route}, + expDef: []types.Capability{route}, }, "Test Only Workload But want trait": { dir: "vela-test3", - tmps: []types.Template{deployment, statefulset}, + tmps: []types.Capability{deployment, statefulset}, Type: types.TypeTrait, expDef: nil, }, "Test Both have Workload and trait But want Workload": { dir: "vela-test4", - tmps: []types.Template{deployment, route, statefulset}, + tmps: []types.Capability{deployment, route, statefulset}, Type: types.TypeWorkload, - expDef: []types.Template{deployment, statefulset}, + expDef: []types.Capability{deployment, statefulset}, }, "Test Both have Workload and trait But want Trait": { dir: "vela-test5", - tmps: []types.Template{deployment, route, statefulset}, + tmps: []types.Capability{deployment, route, statefulset}, Type: types.TypeTrait, - expDef: []types.Template{route}, + expDef: []types.Capability{route}, }, } for name, c := range cases { @@ -91,7 +91,7 @@ func TestLocalSink(t *testing.T) { } } -func testInDir(t *testing.T, casename, dir string, tmps, defexp []types.Template, Type types.DefinitionType, err1 error) { +func testInDir(t *testing.T, casename, dir string, tmps, defexp []types.Capability, Type types.DefinitionType, err1 error) { err := os.MkdirAll(dir, 0755) assert.NoError(t, err, casename) defer os.RemoveAll(dir) diff --git a/pkg/plugins/repository.go b/pkg/plugins/repository.go deleted file mode 100644 index 7123de71d..000000000 --- a/pkg/plugins/repository.go +++ /dev/null @@ -1,153 +0,0 @@ -package plugins - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "os" - "path/filepath" - - "github.com/cloud-native-application/rudrx/api/types" - "github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2" - - "github.com/cloud-native-application/rudrx/pkg/utils/system" - "github.com/ghodss/yaml" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" -) - -//Used to store addon center config in file -type RepoConfig struct { - Name string `json:"repoName"` - Address string `json:"repoAddress"` -} - -var ( - RepoConfigFile = ".vela/addon_config" - DefaultRepo = "local" -) - -type RemoteAddon struct { - // Name MUST be xxx.yaml - Name string `json:"name"` - Url string `json:"download_url"` - Sha string `json:"sha"` - // Type MUST be file - Type string `json:"type"` -} - -type RemoteAddons []RemoteAddon - -type Plugin struct { - Name string `json:"name"` - Type string `json:"type"` - Definition string `json:"definition"` - Status string `json:"status"` - ApplesTo string `json:"applies_to"` -} - -//TODO(wonderflow): we can make default(built-in) repo configurable, then we should make default inside the answer -func LoadRepos() ([]RepoConfig, error) { - config, err := system.GetRepoConfig() - if err != nil { - return nil, err - } - data, err := ioutil.ReadFile(config) - if err != nil { - if os.IsNotExist(err) { - return []RepoConfig{}, nil - } - return nil, err - } - var repos []RepoConfig - if err = yaml.Unmarshal(data, &repos); err != nil { - return nil, err - } - return repos, nil -} - -func StoreRepos(repos []RepoConfig) error { - config, err := system.GetRepoConfig() - if err != nil { - return err - } - data, err := yaml.Marshal(repos) - if err != nil { - return err - } - return ioutil.WriteFile(config, data, 0644) -} - -func GetReposFromRemote(r RepoConfig) (RemoteAddons, error) { - resp, err := http.Get(r.Address) - if err != nil { - return nil, err - } - defer resp.Body.Close() - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - var repos RemoteAddons - if err = json.Unmarshal(data, &repos); err != nil { - return nil, err - } - return repos, nil -} - -func GetDefinitionFromURL(address, syncDir string) (types.Template, error) { - resp, err := http.Get(address) - if err != nil { - return types.Template{}, err - } - defer resp.Body.Close() - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - return types.Template{}, err - } - var obj = unstructured.Unstructured{Object: make(map[string]interface{})} - err = yaml.Unmarshal(data, &obj.Object) - if err != nil { - return types.Template{}, err - } - switch obj.GetKind() { - case "WorkloadDefinition": - var rd v1alpha2.WorkloadDefinition - err = yaml.Unmarshal(data, &rd) - if err != nil { - return types.Template{}, err - } - return HandleDefinition(rd.Name, syncDir, rd.Spec.Reference.Name, rd.Spec.Extension, types.TypeWorkload, nil) - case "TraitDefinition": - var td v1alpha2.TraitDefinition - err = yaml.Unmarshal(data, &td) - if err != nil { - return types.Template{}, err - } - return HandleDefinition(td.Name, syncDir, td.Spec.Reference.Name, td.Spec.Extension, types.TypeTrait, td.Spec.AppliesToWorkloads) - case "ScopeDefinition": - //TODO(wonderflow): support scope definition here. - } - return types.Template{}, fmt.Errorf("unknown definition Type %s", obj.GetKind()) -} - -//TODO(wonderflow): currently we only sync by create, we also need to delete which not exist remotely. -func SyncRemoteAddons(r RepoConfig, addons RemoteAddons) error { - dir, err := system.GetRepoDir() - if err != nil { - return err - } - repoDir := filepath.Join(dir, r.Name) - system.StatAndCreate(repoDir) - var tmps []types.Template - for _, addon := range addons { - tmp, err := GetDefinitionFromURL(addon.Url, repoDir) - if err != nil { - return err - } - tmps = append(tmps, tmp) - } - success := SinkTemp2Local(tmps, repoDir) - fmt.Printf("successfully sync %d remote addons\n", success) - return nil -} diff --git a/pkg/plugins/suit_test.go b/pkg/plugins/suit_test.go index 7fe548624..9fb0139fa 100644 --- a/pkg/plugins/suit_test.go +++ b/pkg/plugins/suit_test.go @@ -118,7 +118,7 @@ var _ = BeforeSuite(func(done Done) { }, }, } - definitionDir, err = system.GetDefinitionDir() + definitionDir, err = system.GetCapabilityDir() Expect(err).Should(BeNil()) os.MkdirAll(definitionDir, 0755) Expect(k8sClient.Create(context.Background(), &crd)).Should(SatisfyAny(BeNil(), &util.AlreadyExistMatcher{})) diff --git a/pkg/server/util/errors.go b/pkg/server/util/errors.go index 4296eff9b..5baf52e72 100644 --- a/pkg/server/util/errors.go +++ b/pkg/server/util/errors.go @@ -72,10 +72,10 @@ func ConstructError(ec Code, a ...interface{}) error { // - use setErrorAndAbort to abort the rest of the handlers, mostly called in middleware func SetErrorAndAbort(c *gin.Context, code Code, msg ...interface{}) { // Calling abort so no handlers and middlewares will be executed. - c.AbortWithStatusJSON(code.StatusCode(), gin.H{"error": ConstructError(code, msg...).Error()} ) + c.AbortWithStatusJSON(code.StatusCode(), gin.H{"error": ConstructError(code, msg...).Error()}) } func HandleError(c *gin.Context, code Code, msg ...interface{}) { - c.JSON(code.StatusCode(), gin.H{"error": ConstructError(code, msg...).Error()} ) + c.JSON(code.StatusCode(), gin.H{"error": ConstructError(code, msg...).Error()}) } diff --git a/pkg/server/util/middleware.go b/pkg/server/util/middleware.go index 0650b9e09..84a3153f3 100644 --- a/pkg/server/util/middleware.go +++ b/pkg/server/util/middleware.go @@ -125,4 +125,3 @@ func ValidateHeaders() gin.HandlerFunc { } } } - diff --git a/pkg/utils/system/system.go b/pkg/utils/system/system.go index 4d10b1707..6ec6d5b5f 100644 --- a/pkg/utils/system/system.go +++ b/pkg/utils/system/system.go @@ -23,20 +23,19 @@ func GetVelaHomeDir() (string, error) { return filepath.Join(home, defaultVelaHome), nil } -func GetRepoDir() (string, error) { +func GetCapCenterDir() (string, error) { home, err := GetVelaHomeDir() if err != nil { return "", err } - return filepath.Join(home, ".repo"), nil + return filepath.Join(home, "centers"), nil } func GetRepoConfig() (string, error) { - home, err := GetRepoDir() + home, err := GetCapCenterDir() if err != nil { return "", err } - StatAndCreate(home) return filepath.Join(home, "config.yaml"), nil } @@ -48,12 +47,12 @@ func GetApplicationDir() (string, error) { return filepath.Join(home, "applications"), nil } -func GetDefinitionDir() (string, error) { +func GetCapabilityDir() (string, error) { home, err := GetVelaHomeDir() if err != nil { return "", err } - return filepath.Join(home, "definitions"), nil + return filepath.Join(home, "capabilities"), nil } func GetEnvDir() (string, error) { @@ -72,12 +71,33 @@ func GetCurrentEnvPath() (string, error) { return filepath.Join(homedir, "curenv"), nil } -func InitDefinitionDir() error { - dir, err := GetDefinitionDir() +func InitDirs() error { + if err := InitCapabilityDir(); err != nil { + return err + } + if err := InitApplicationDir(); err != nil { + return err + } + if err := InitCapCenterDir(); err != nil { + return err + } + return nil +} + +func InitCapCenterDir() error { + home, err := GetCapCenterDir() if err != nil { return err } - return os.MkdirAll(dir, 0755) + return StatAndCreate(filepath.Join(home, ".tmp")) +} + +func InitCapabilityDir() error { + dir, err := GetCapabilityDir() + if err != nil { + return err + } + return StatAndCreate(dir) } func InitApplicationDir() error { @@ -85,7 +105,7 @@ func InitApplicationDir() error { if err != nil { return err } - return os.MkdirAll(dir, 0755) + return StatAndCreate(dir) } func InitDefaultEnv() error { @@ -108,8 +128,9 @@ func InitDefaultEnv() error { return nil } -func StatAndCreate(dir string) { +func StatAndCreate(dir string) error { if _, err := os.Stat(dir); os.IsNotExist(err) { - os.MkdirAll(dir, 0755) + return os.MkdirAll(dir, 0755) } + return nil }