diff --git a/api/types/types_test.go b/api/types/types_test.go index 073ddcc77..f07ea6140 100644 --- a/api/types/types_test.go +++ b/api/types/types_test.go @@ -5,8 +5,8 @@ import ( "fmt" "testing" + "github.com/ghodss/yaml" "github.com/stretchr/testify/assert" - "gopkg.in/yaml.v3" ) func TestApplication(t *testing.T) { diff --git a/go.mod b/go.mod index 332a157bd..efeda6b7e 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/crossplane/oam-kubernetes-runtime v0.0.8 github.com/ghodss/yaml v1.0.0 github.com/gin-gonic/gin v1.6.3 + github.com/google/go-github/v32 v32.1.0 github.com/gosuri/uitable v0.0.4 github.com/onsi/ginkgo v1.11.0 github.com/onsi/gomega v1.8.1 @@ -17,7 +18,7 @@ require ( github.com/stretchr/testify v1.6.1 go.uber.org/zap v1.10.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 - gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c + golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 gotest.tools v2.2.0+incompatible helm.sh/helm/v3 v3.2.4 k8s.io/api v0.18.6 diff --git a/go.sum b/go.sum index b301b4fab..903b8af2b 100644 --- a/go.sum +++ b/go.sum @@ -334,6 +334,11 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= +github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= diff --git a/pkg/cmd/addon.go b/pkg/cmd/addon.go index 7b3f5817b..1108a71db 100644 --- a/pkg/cmd/addon.go +++ b/pkg/cmd/addon.go @@ -1,11 +1,14 @@ package cmd import ( + "context" "errors" "fmt" "io/ioutil" "path/filepath" + "github.com/gosuri/uitable" + "github.com/cloud-native-application/rudrx/pkg/utils/system" "github.com/cloud-native-application/rudrx/pkg/plugins" @@ -13,7 +16,6 @@ import ( "github.com/cloud-native-application/rudrx/api/types" cmdutil "github.com/cloud-native-application/rudrx/pkg/cmd/util" - "github.com/gosuri/uitable" "github.com/spf13/cobra" ) @@ -40,31 +42,33 @@ func NewAddonConfigCommand(ioStreams cmdutil.IOStreams) *cobra.Command { if err != nil { return err } - config := plugins.RepoConfig{ + config := &plugins.RepoConfig{ Name: args[0], - Address: ConvertURL(args[1]), + Address: args[1], + Token: cmd.Flag("token").Value.String(), } var updated bool for idx, r := range repos { if r.Name == config.Name { - repos[idx] = config + repos[idx] = *config updated = true break } } if !updated { - repos = append(repos, config) + repos = append(repos, *config) } if err = plugins.StoreRepos(repos); err != nil { return err } - ioStreams.Info(fmt.Sprintf("Successfully configured Addon repo: %s", args[0])) + ioStreams.Info(fmt.Sprintf("Successfully configured Addon repo: %s, please use 'vela addon:update %s' to sync addons", args[0], args[0])) return nil }, Annotations: map[string]string{ types.TagCommandType: types.TypeOthers, }, } + cmd.PersistentFlags().StringP("token", "t", "", "Github Repo token") return cmd } @@ -96,8 +100,10 @@ func NewAddonUpdateCommand(ioStreams cmdutil.IOStreams) *cobra.Command { return fmt.Errorf("%s repo not exist", specified) } } + ctx := context.Background() for _, d := range repos { - err = SyncRemoteAddon(d) + client, err := plugins.NewAddClient(ctx, d.Name, d.Address, d.Token) + err = client.SyncRemoteAddons() if err != nil { return err } @@ -126,8 +132,10 @@ func NewAddonListCommand(ioStreams cmdutil.IOStreams) *cobra.Command { if err != nil { return err } + table := uitable.New() + table.AddRow("NAME", "TYPE", "DEFINITION", "STATUS", "APPLIES-TO") if repoName != "" { - return ListRepoAddons(filepath.Join(dir, repoName), ioStreams) + return ListRepoAddons(table, filepath.Join(dir, repoName), ioStreams) } dirs, err := ioutil.ReadDir(dir) if err != nil { @@ -137,7 +145,7 @@ func NewAddonListCommand(ioStreams cmdutil.IOStreams) *cobra.Command { if !dd.IsDir() { continue } - if err = ListRepoAddons(filepath.Join(dir, dd.Name()), ioStreams); err != nil { + if err = ListRepoAddons(table, filepath.Join(dir, dd.Name()), ioStreams); err != nil { return err } } @@ -150,33 +158,21 @@ func NewAddonListCommand(ioStreams cmdutil.IOStreams) *cobra.Command { return cmd } -func ListRepoAddons(repoDir string, ioStreams cmdutil.IOStreams) error { +func ListRepoAddons(table *uitable.Table, repoDir string, ioStreams cmdutil.IOStreams) error { templates, err := plugins.LoadTempFromLocal(repoDir) if err != nil { return err } - table := uitable.New() - table.AddRow("NAME", "TYPE", "DEFINITION", "STATUS", "APPLIES-TO") - + if len(templates) < 1 { + return nil + } + baseDir := filepath.Base(repoDir) var status string //TODO(wonderflow): check status whether install or not status = "uninstalled" for _, p := range templates { - table.AddRow(p.Name, p.Type, p.Type, status, p.AppliesTo) + table.AddRow(baseDir+"/"+p.Name, p.Type, p.Type, status, p.AppliesTo) } ioStreams.Info(table.String()) return nil } - -func ConvertURL(address string) string { - //TODO(wonderflow) convert github address here - return address -} - -func SyncRemoteAddon(d plugins.RepoConfig) error { - addons, err := plugins.GetReposFromRemote(d) - if err != nil { - return err - } - return plugins.SyncRemoteAddons(d, addons) -} diff --git a/pkg/cmd/workload_run.go b/pkg/cmd/workload_run.go index 23ab673a4..c749cea7e 100644 --- a/pkg/cmd/workload_run.go +++ b/pkg/cmd/workload_run.go @@ -12,7 +12,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "gopkg.in/yaml.v3" + "github.com/ghodss/yaml" "cuelang.org/go/cue" diff --git a/pkg/plugins/local.go b/pkg/plugins/local.go index f4a8810cb..9db0ea026 100644 --- a/pkg/plugins/local.go +++ b/pkg/plugins/local.go @@ -50,7 +50,6 @@ func LoadTempFromLocal(dir string) ([]types.Template, error) { files, err := ioutil.ReadDir(dir) if err != nil { if os.IsNotExist(err) { - fmt.Println("\"no definition files found, use 'vela refresh' to sync from cluster\"") return nil, nil } return nil, err @@ -76,8 +75,5 @@ func LoadTempFromLocal(dir string) ([]types.Template, error) { } tmps = append(tmps, tmp) } - if len(tmps) == 0 { - fmt.Println("\"no definition files found, use 'vela refresh' to sync from cluster\"") - } return tmps, nil } diff --git a/pkg/plugins/repository.go b/pkg/plugins/repository.go index 7123de71d..dc3b37f21 100644 --- a/pkg/plugins/repository.go +++ b/pkg/plugins/repository.go @@ -1,12 +1,20 @@ package plugins import ( - "encoding/json" + "context" + "encoding/base64" + "errors" "fmt" "io/ioutil" "net/http" + "net/url" "os" "path/filepath" + "strings" + + "golang.org/x/oauth2" + + "github.com/google/go-github/v32/github" "github.com/cloud-native-application/rudrx/api/types" "github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2" @@ -16,16 +24,89 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) +type GithubContent struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Path string `json:"path"` + Ref string `json:"ref"` +} + //Used to store addon center config in file type RepoConfig struct { Name string `json:"repoName"` Address string `json:"repoAddress"` + Token string `json:"token"` } -var ( - RepoConfigFile = ".vela/addon_config" - DefaultRepo = "local" -) +type AddonClient interface { + SyncRemoteAddons() error +} + +func NewAddClient(ctx context.Context, name, address, token string) (AddonClient, error) { + Type, cfg, err := Parse(address) + if err != nil { + return nil, err + } + switch Type { + case TypeGithub: + return NewGithubAddon(ctx, token, name, cfg) + } + return nil, errors.New("we only support github as repository now") +} + +const TypeGithub = "github" +const TypeUnknown = "unknown" + +func Parse(addr string) (string, *GithubContent, error) { + url, err := url.Parse(addr) + if err != nil { + return "", nil, err + } + l := strings.Split(strings.TrimPrefix(url.Path, "/"), "/") + switch url.Host { + case "github.com": + // We support two valid format: + // 1. https://github.com///tree// + // 2. https://github.com/// + if len(l) < 3 { + return "", nil, errors.New("invalid format " + addr) + } + if l[2] == "tree" { + // https://github.com///tree// + if len(l) < 5 { + return "", nil, errors.New("invalid format " + addr) + } + return TypeGithub, &GithubContent{ + Owner: l[0], + Repo: l[1], + Path: strings.Join(l[4:], "/"), + Ref: l[3], + }, nil + } else { + // https://github.com/// + return TypeGithub, &GithubContent{ + Owner: l[0], + Repo: l[1], + Path: strings.Join(l[2:], "/"), + Ref: "", //use default branch + }, nil + } + case "api.github.com": + if len(l) != 5 { + return "", nil, errors.New("invalid format " + addr) + } + //https://api.github.com/repos///contents/ + return TypeGithub, &GithubContent{ + Owner: l[1], + Repo: l[2], + Path: l[4], + Ref: url.Query().Get("ref"), + }, nil + default: + //TODO(wonderflow): support raw url and oss format in the future + } + return TypeUnknown, nil, nil +} type RemoteAddon struct { // Name MUST be xxx.yaml @@ -78,35 +159,9 @@ func StoreRepos(repos []RepoConfig) error { return ioutil.WriteFile(config, data, 0644) } -func GetReposFromRemote(r RepoConfig) (RemoteAddons, error) { - resp, err := http.Get(r.Address) - if err != nil { - return nil, err - } - defer resp.Body.Close() - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - var repos RemoteAddons - if err = json.Unmarshal(data, &repos); err != nil { - return nil, err - } - return repos, nil -} - -func GetDefinitionFromURL(address, syncDir string) (types.Template, error) { - resp, err := http.Get(address) - if err != nil { - return types.Template{}, err - } - defer resp.Body.Close() - data, err := ioutil.ReadAll(resp.Body) - if err != nil { - return types.Template{}, err - } +func GetDefinitionFromURL(data []byte, syncDir string) (types.Template, error) { var obj = unstructured.Unstructured{Object: make(map[string]interface{})} - err = yaml.Unmarshal(data, &obj.Object) + err := yaml.Unmarshal(data, &obj.Object) if err != nil { return types.Template{}, err } @@ -131,20 +186,59 @@ func GetDefinitionFromURL(address, syncDir string) (types.Template, error) { return types.Template{}, fmt.Errorf("unknown definition Type %s", obj.GetKind()) } +type GithubAddon struct { + client *github.Client + cfg *GithubContent + repoName string + ctx context.Context +} + +var _ AddonClient = &GithubAddon{} + +func NewGithubAddon(ctx context.Context, token, repoName string, r *GithubContent) (*GithubAddon, error) { + var tc *http.Client + if token != "" { + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + tc = oauth2.NewClient(ctx, ts) + } + return &GithubAddon{client: github.NewClient(tc), cfg: r, repoName: repoName, ctx: ctx}, nil +} + //TODO(wonderflow): currently we only sync by create, we also need to delete which not exist remotely. -func SyncRemoteAddons(r RepoConfig, addons RemoteAddons) error { +func (g *GithubAddon) SyncRemoteAddons() error { + _, dirs, _, err := g.client.Repositories.GetContents(g.ctx, g.cfg.Owner, g.cfg.Repo, g.cfg.Path, &github.RepositoryContentGetOptions{Ref: g.cfg.Ref}) + if err != nil { + return err + } dir, err := system.GetRepoDir() if err != nil { return err } - repoDir := filepath.Join(dir, r.Name) + repoDir := filepath.Join(dir, g.repoName) system.StatAndCreate(repoDir) var tmps []types.Template - for _, addon := range addons { - tmp, err := GetDefinitionFromURL(addon.Url, repoDir) + for _, addon := range dirs { + if *addon.Type != "file" { + continue + } + fileContent, _, _, err := g.client.Repositories.GetContents(g.ctx, g.cfg.Owner, g.cfg.Repo, *addon.Path, &github.RepositoryContentGetOptions{Ref: g.cfg.Ref}) if err != nil { return err } + var data = []byte(*fileContent.Content) + if *fileContent.Encoding == "base64" { + data, err = base64.StdEncoding.DecodeString(*fileContent.Content) + if err != nil { + return fmt.Errorf("decode github content %s err %v", *fileContent.Path, err) + } + } + tmp, err := GetDefinitionFromURL(data, repoDir) + if err != nil { + fmt.Printf("get definition of %s err %v\n", *addon.Path, err) + continue + } tmps = append(tmps, tmp) } success := SinkTemp2Local(tmps, repoDir) diff --git a/pkg/plugins/repository_test.go b/pkg/plugins/repository_test.go new file mode 100644 index 000000000..8a7ed5148 --- /dev/null +++ b/pkg/plugins/repository_test.go @@ -0,0 +1,51 @@ +package plugins + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseURL(t *testing.T) { + cases := map[string]struct { + url string + exp *GithubContent + expType string + }{ + "api-github": { + url: "https://api.github.com/repos/zzxwill/catalog/contents/repository?ref=plugin", + expType: TypeGithub, + exp: &GithubContent{ + Owner: "zzxwill", + Repo: "catalog", + Path: "repository", + Ref: "plugin", + }, + }, + "github-copy-path": { + url: "https://github.com/zzxwill/catalog/tree/plugin/repository", + expType: TypeGithub, + exp: &GithubContent{ + Owner: "zzxwill", + Repo: "catalog", + Path: "repository", + Ref: "plugin", + }, + }, + "github-manuel-write-path": { + url: "https://github.com/zzxwill/catalog/repository", + expType: TypeGithub, + exp: &GithubContent{ + Owner: "zzxwill", + Repo: "catalog", + Path: "repository", + }, + }, + } + for caseName, c := range cases { + tp, content, err := Parse(c.url) + assert.NoError(t, err, caseName) + assert.Equal(t, c.exp, content, caseName) + assert.Equal(t, c.expType, tp, caseName) + } +}