diff --git a/pkg/addon/addon.go b/pkg/addon/addon.go index 884a1a129..de54f6c4b 100644 --- a/pkg/addon/addon.go +++ b/pkg/addon/addon.go @@ -21,6 +21,9 @@ import ( "context" "encoding/json" "fmt" + "io/ioutil" + "net/http" + "net/url" "path" "path/filepath" "strings" @@ -86,6 +89,9 @@ const ( // AddonParameterDataKey is the key of parameter in addon args secrets AddonParameterDataKey string = "addonParameterDataKey" + + // DefaultGiteeURL is the addon repository of gitee api + DefaultGiteeURL string = "https://gitee.com/api/v5/" ) // ParameterFileName is the addon resources/parameter.cue file name @@ -422,15 +428,75 @@ func createGitHelper(content *utils.Content, token string) *gitHelper { } } +func createGiteeHelper(content *utils.Content, token string) *giteeHelper { + var ts oauth2.TokenSource + if token != "" { + ts = oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) + } + tc := oauth2.NewClient(context.Background(), ts) + tc.Timeout = time.Second * 20 + cli := NewGiteeClient(tc, nil) + return &giteeHelper{ + Client: cli, + Meta: content, + } +} + // readRepo will read relative path (relative to Meta.Path) func (h *gitHelper) readRepo(relativePath string) (*github.RepositoryContent, []*github.RepositoryContent, error) { - file, items, _, err := h.Client.Repositories.GetContents(context.Background(), h.Meta.Owner, h.Meta.Repo, path.Join(h.Meta.Path, relativePath), nil) + file, items, _, err := h.Client.Repositories.GetContents(context.Background(), h.Meta.GithubContent.Owner, h.Meta.GithubContent.Repo, path.Join(h.Meta.GithubContent.Path, relativePath), nil) if err != nil { return nil, nil, WrapErrRateLimit(err) } return file, items, nil } +// readRepo will read relative path (relative to Meta.Path) +func (h *giteeHelper) readRepo(relativePath string) (*github.RepositoryContent, []*github.RepositoryContent, error) { + file, items, err := h.Client.GetGiteeContents(context.Background(), h.Meta.GiteeContent.Owner, h.Meta.GiteeContent.Repo, path.Join(h.Meta.GiteeContent.Path, relativePath), h.Meta.GiteeContent.Ref) + if err != nil { + return nil, nil, WrapErrRateLimit(err) + } + return file, items, nil +} + +// GetGiteeContents can return either the metadata and content of a single file +func (c *Client) GetGiteeContents(ctx context.Context, owner, repo, path, ref string) (fileContent *github.RepositoryContent, directoryContent []*github.RepositoryContent, err error) { + escapedPath := (&url.URL{Path: path}).String() + u := fmt.Sprintf(c.BaseURL.String()+"repos/%s/%s/contents/%s", owner, repo, escapedPath) + if ref != "" { + u = fmt.Sprintf(u+"?ref=%s", ref) + } + + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, nil, err + } + response, err := c.Client.Do(req.WithContext(ctx)) + if err != nil { + return nil, nil, err + } + //nolint:errcheck + defer response.Body.Close() + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, nil, err + } + return unmarshalToContent(body) +} + +func unmarshalToContent(content []byte) (fileContent *github.RepositoryContent, directoryContent []*github.RepositoryContent, err error) { + fileUnmarshalError := json.Unmarshal(content, &fileContent) + if fileUnmarshalError == nil { + return fileContent, nil, nil + } + directoryUnmarshalError := json.Unmarshal(content, &directoryContent) + if directoryUnmarshalError == nil { + return nil, directoryContent, nil + } + return nil, nil, fmt.Errorf("unmarshalling failed for both file and directory content: %s and %w", fileUnmarshalError, directoryUnmarshalError) +} + func genAddonAPISchema(addonRes *UIData) error { param, err := utils2.PrepareParameterCue(addonRes.Name, addonRes.Parameters) if err != nil { diff --git a/pkg/addon/addon_test.go b/pkg/addon/addon_test.go index 8412fc4fb..6f0d2ba91 100644 --- a/pkg/addon/addon_test.go +++ b/pkg/addon/addon_test.go @@ -748,6 +748,68 @@ func TestCheckAddonVersionMeetRequired(t *testing.T) { assert.NoError(t, checkAddonVersionMeetRequired(ctx, &SystemRequirements{VelaVersion: ">=1.2.4"}, k8sClient, nil)) } +var testUnmarshalToContent1 = ` +{ + "type": "file", + "encoding": "", + "size": 651, + "name": "metadata.yaml", + "path": "example/metadata.yaml", + "content": "name: example\r\nversion: 1.0.0\r\ndescription: Extended workload to do continuous and progressive delivery\r\nicon: https://raw.githubusercontent.com/fluxcd/flux/master/docs/_files/weave-flux.png\r\nurl: https://fluxcd.io\r\n\r\ntags:\r\n - extended_workload\r\n - gitops\r\n - only_example\r\n\r\ndeployTo:\r\n control_plane: true\r\n runtime_cluster: false\r\n\r\ndependencies: []\r\n#- name: addon_name\r\n\r\n# set invisible means this won't be list and will be enabled when depended on\r\n# for example, terraform-alibaba depends on terraform which is invisible,\r\n# when terraform-alibaba is enabled, terraform will be enabled automatically\r\n# default: false\r\ninvisible: false\r\n" +}` +var testUnmarshalToContent2 = ` +[ + { + "type": "dir", + "name": "example", + "path": "example" + }, + { + "type": "dir", + "name": "local", + "path": "local" + }, + { + "type": "dir", + "name": "terraform", + "path": "terraform" + }, + { + "type": "dir", + "name": "terraform-alibaba", + "path": "terraform-alibaba" + }, + { + "type": "dir", + "name": "test-error-addon", + "path": "test-error-addon" + } +]` +var testUnmarshalToContent3 = ` +[ + { + "type": "dir", + "name": "example", + }, + { + "type": "dir", + "name": "local", + "path": "local" + } +]` +var testUnmarshalToContent4 = `` + +func TestUnmarshalToContent(t *testing.T) { + _, _, err1 := unmarshalToContent([]byte(testUnmarshalToContent1)) + assert.NoError(t, err1) + _, _, err2 := unmarshalToContent([]byte(testUnmarshalToContent2)) + assert.NoError(t, err2) + _, _, err3 := unmarshalToContent([]byte(testUnmarshalToContent3)) + assert.Error(t, err3, "unmarshalling failed for both file and directory content: invalid character '}' looking for beginnin") + _, _, err4 := unmarshalToContent([]byte(testUnmarshalToContent4)) + assert.Error(t, err4, "unmarshalling failed for both file and directory content: unexpected end of JSON input and unexpecte") +} + // Test readResFile, only accept .cue and .yaml/.yml func TestReadResFile(t *testing.T) { @@ -787,6 +849,7 @@ func TestReadDefFile(t *testing.T) { var uiData = &UIData{} ptItems := ClassifyItemByPattern(&testAddonMeta, reader) items := ptItems[DefinitionsDirName] + for _, it := range items { err := readDefFile(uiData, reader, reader.RelativePath(it)) if err != nil { diff --git a/pkg/addon/reader_gitee.go b/pkg/addon/reader_gitee.go new file mode 100644 index 000000000..b5d115d12 --- /dev/null +++ b/pkg/addon/reader_gitee.go @@ -0,0 +1,122 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addon + +import ( + "fmt" + "net/http" + "net/url" + "path" + "strings" + + "github.com/pkg/errors" + + "github.com/oam-dev/kubevela/pkg/utils" +) + +var _ AsyncReader = &giteeReader{} + +// giteeHelper helps get addon's file by git +type giteeHelper struct { + Client *Client + Meta *utils.Content +} + +// Client manages communication with the Gitee API +type Client struct { + Client *http.Client + BaseURL *url.URL +} + +type giteeReader struct { + h *giteeHelper +} + +// NewGiteeClient returns a new Gitee API client +func NewGiteeClient(httpClient *http.Client, baseURL *url.URL) *Client { + if httpClient == nil { + httpClient = &http.Client{} + } + if baseURL == nil { + baseURL, _ = baseURL.Parse(DefaultGiteeURL) + } + return &Client{httpClient, baseURL} +} + +// ListAddonMeta relative path to repoURL/basePath +func (g *giteeReader) ListAddonMeta() (map[string]SourceMeta, error) { + subItems := make(map[string]SourceMeta) + _, items, err := g.h.readRepo("") + if err != nil { + return nil, err + } + for _, item := range items { + // single addon + if item.GetType() != DirType { + continue + } + addonName := path.Base(item.GetPath()) + addonMeta, err := g.listAddonMeta(g.RelativePath(item)) + if err != nil { + return nil, errors.Wrapf(err, "fail to get addon meta of %s", addonName) + } + subItems[addonName] = SourceMeta{Name: addonName, Items: addonMeta} + } + return subItems, nil +} + +func (g *giteeReader) listAddonMeta(dirPath string) ([]Item, error) { + _, items, err := g.h.readRepo(dirPath) + if err != nil { + return nil, err + } + res := make([]Item, 0) + for _, item := range items { + switch item.GetType() { + case FileType: + res = append(res, item) + case DirType: + subItems, err := g.listAddonMeta(g.RelativePath(item)) + if err != nil { + return nil, err + } + res = append(res, subItems...) + } + } + return res, nil +} + +// ReadFile read file content from github +func (g *giteeReader) ReadFile(relativePath string) (content string, err error) { + file, _, err := g.h.readRepo(relativePath) + if err != nil { + return + } + if file == nil { + return "", fmt.Errorf("path %s is not a file", relativePath) + } + return file.GetContent() +} + +func (g *giteeReader) RelativePath(item Item) string { + absPath := strings.Split(item.GetPath(), "/") + if g.h.Meta.GiteeContent.Path == "" { + return path.Join(absPath...) + } + base := strings.Split(g.h.Meta.GiteeContent.Path, "/") + return path.Join(absPath[len(base):]...) +} diff --git a/pkg/addon/reader_gitee_test.go b/pkg/addon/reader_gitee_test.go new file mode 100644 index 000000000..eb49adc6a --- /dev/null +++ b/pkg/addon/reader_gitee_test.go @@ -0,0 +1,100 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addon + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "path" + "strings" + "testing" + + "github.com/google/go-github/v32/github" + "github.com/stretchr/testify/assert" + + "github.com/oam-dev/kubevela/pkg/utils" +) + +func giteeSetup() (client *Client, mux *http.ServeMux, teardown func()) { + // mux is the HTTP request multiplexer used with the test server. + mux = http.NewServeMux() + + apiHandler := http.NewServeMux() + apiHandler.Handle(baseURLPath+"/", http.StripPrefix(baseURLPath, mux)) + + // server is a test HTTP server used to provide mock API responses. + server := httptest.NewServer(apiHandler) + + // client is the GitHub client being tested and is + // configured to use test server. + URL, _ := url.Parse(server.URL + baseURLPath + "/") + httpClient := &http.Client{} + client = NewGiteeClient(httpClient, URL) + + return client, mux, server.Close +} + +func TestGiteeReader(t *testing.T) { + client, mux, teardown := giteeSetup() + giteePattern := "/repos/o/r/contents/" + mux.HandleFunc(giteePattern, func(rw http.ResponseWriter, req *http.Request) { + queryPath := strings.TrimPrefix(req.URL.Path, giteePattern) + localPath := path.Join(testdataPrefix, queryPath) + file, err := testdata.ReadFile(localPath) + // test if it's a file + if err == nil { + content := &github.RepositoryContent{Type: String("file"), Name: String(path.Base(queryPath)), Size: Int(len(file)), Encoding: String(""), Path: String(queryPath), Content: String(string(file))} + res, _ := json.Marshal(content) + rw.Write(res) + return + } + + // otherwise, it could be directory + dir, err := testdata.ReadDir(localPath) + if err == nil { + contents := make([]*github.RepositoryContent, 0) + for _, item := range dir { + tp := "file" + if item.IsDir() { + tp = "dir" + } + contents = append(contents, &github.RepositoryContent{Type: String(tp), Name: String(item.Name()), Path: String(path.Join(queryPath, item.Name()))}) + } + dRes, _ := json.Marshal(contents) + rw.Write(dRes) + return + } + + rw.Write([]byte("invalid gitee query")) + }) + defer teardown() + + gith := &giteeHelper{ + Client: client, + Meta: &utils.Content{GiteeContent: utils.GiteeContent{ + Owner: "o", + Repo: "r", + }}, + } + var r AsyncReader = &giteeReader{gith} + _, err := r.ReadFile("example/metadata.yaml") + assert.NoError(t, err) + + testReaderFunc(t, r) +} diff --git a/pkg/addon/reader_github.go b/pkg/addon/reader_github.go index 0f0813481..a96136c4e 100644 --- a/pkg/addon/reader_github.go +++ b/pkg/addon/reader_github.go @@ -96,9 +96,9 @@ func (g *gitReader) ReadFile(relativePath string) (content string, err error) { func (g *gitReader) RelativePath(item Item) string { absPath := strings.Split(item.GetPath(), "/") - if g.h.Meta.Path == "" { + if g.h.Meta.GithubContent.Path == "" { return path.Join(absPath...) } - base := strings.Split(g.h.Meta.Path, "/") + base := strings.Split(g.h.Meta.GithubContent.Path, "/") return path.Join(absPath[len(base):]...) } diff --git a/pkg/addon/registry.go b/pkg/addon/registry.go index 9f835bb99..5456ca308 100644 --- a/pkg/addon/registry.go +++ b/pkg/addon/registry.go @@ -37,8 +37,9 @@ const registriesKey = "registries" type Registry struct { Name string `json:"name"` - Git *GitAddonSource `json:"git,omitempty"` - OSS *OSSAddonSource `json:"oss,omitempty"` + Git *GitAddonSource `json:"git,omitempty"` + OSS *OSSAddonSource `json:"oss,omitempty"` + Gitee *GiteeAddonSource `json:"gitee,omitempty"` } // RegistryDataStore CRUD addon registry data in configmap diff --git a/pkg/addon/source.go b/pkg/addon/source.go index 9af8509ee..a2872d69b 100644 --- a/pkg/addon/source.go +++ b/pkg/addon/source.go @@ -56,6 +56,13 @@ type GitAddonSource struct { Token string `json:"token,omitempty"` } +// GiteeAddonSource defines the information about the Gitee as addon source +type GiteeAddonSource struct { + URL string `json:"url,omitempty" validate:"required"` + Path string `json:"path,omitempty"` + Token string `json:"token,omitempty"` +} + // Item is a partial interface for github.RepositoryContent type Item interface { // GetType return "dir" or "file" @@ -110,8 +117,9 @@ func pathWithParent(subPath, parent string) string { type ReaderType string const ( - gitType ReaderType = "git" - ossType ReaderType = "oss" + gitType ReaderType = "git" + ossType ReaderType = "oss" + giteeType ReaderType = "gitee" ) // NewAsyncReader create AsyncReader from @@ -154,6 +162,21 @@ func NewAsyncReader(baseURL, bucket, subPath, token string, rdType ReaderType) ( path: subPath, client: resty.New(), }, nil + case giteeType: + baseURL = strings.TrimSuffix(baseURL, ".git") + u, err := url.Parse(baseURL) + if err != nil { + return nil, errors.New("addon registry invalid") + } + u.Path = path.Join(u.Path, subPath) + _, content, err := utils.Parse(u.String()) + if err != nil { + return nil, err + } + gitee := createGiteeHelper(content, token) + return &giteeReader{ + h: gitee, + }, nil } return nil, fmt.Errorf("invalid addon registry type '%s'", rdType) } @@ -168,6 +191,10 @@ func (r *Registry) BuildReader() (AsyncReader, error) { g := r.Git return NewAsyncReader(g.URL, "", g.Path, g.Token, gitType) } + if r.Gitee != nil { + g := r.Gitee + return NewAsyncReader(g.URL, "", g.Path, g.Token, giteeType) + } return nil, errors.New("registry don't have enough info to build a reader") } diff --git a/pkg/apiserver/rest/apis/v1/types.go b/pkg/apiserver/rest/apis/v1/types.go index c1ed36b16..565479fa0 100644 --- a/pkg/apiserver/rest/apis/v1/types.go +++ b/pkg/apiserver/rest/apis/v1/types.go @@ -69,22 +69,25 @@ type NameAlias struct { // CreateAddonRegistryRequest defines the format for addon registry create request type CreateAddonRegistryRequest struct { - Name string `json:"name" validate:"checkname"` - Git *addon.GitAddonSource `json:"git,omitempty" ` - Oss *addon.OSSAddonSource `json:"oss,omitempty"` + Name string `json:"name" validate:"checkname"` + Git *addon.GitAddonSource `json:"git,omitempty" ` + Oss *addon.OSSAddonSource `json:"oss,omitempty"` + Gitee *addon.GiteeAddonSource `json:"gitee,omitempty" ` } // UpdateAddonRegistryRequest defines the format for addon registry update request type UpdateAddonRegistryRequest struct { - Git *addon.GitAddonSource `json:"git,omitempty"` - Oss *addon.OSSAddonSource `json:"oss,omitempty"` + Git *addon.GitAddonSource `json:"git,omitempty"` + Oss *addon.OSSAddonSource `json:"oss,omitempty"` + Gitee *addon.GiteeAddonSource `json:"gitee,omitempty" ` } // AddonRegistry defines the format for a single addon registry type AddonRegistry struct { - Name string `json:"name" validate:"required"` - Git *addon.GitAddonSource `json:"git,omitempty"` - OSS *addon.OSSAddonSource `json:"oss,omitempty"` + Name string `json:"name" validate:"required"` + Git *addon.GitAddonSource `json:"git,omitempty"` + OSS *addon.OSSAddonSource `json:"oss,omitempty"` + Gitee *addon.GiteeAddonSource `json:"gitee,omitempty" ` } // ListAddonRegistryResponse list addon registry diff --git a/pkg/apiserver/rest/usecase/addon.go b/pkg/apiserver/rest/usecase/addon.go index e6aacdbc8..a7be5d52c 100644 --- a/pkg/apiserver/rest/usecase/addon.go +++ b/pkg/apiserver/rest/usecase/addon.go @@ -296,9 +296,10 @@ func (u *defaultAddonHandler) CreateAddonRegistry(ctx context.Context, req apis. } return &apis.AddonRegistry{ - Name: r.Name, - Git: r.Git, - OSS: r.OSS, + Name: r.Name, + Git: r.Git, + OSS: r.OSS, + Gitee: r.Gitee, }, nil } @@ -450,9 +451,10 @@ func (u *defaultAddonHandler) UpdateAddon(ctx context.Context, name string, args func addonRegistryModelFromCreateAddonRegistryRequest(req apis.CreateAddonRegistryRequest) pkgaddon.Registry { return pkgaddon.Registry{ - Name: req.Name, - Git: req.Git, - OSS: req.Oss, + Name: req.Name, + Git: req.Git, + OSS: req.Oss, + Gitee: req.Gitee, } } diff --git a/pkg/utils/parse.go b/pkg/utils/parse.go index a89f1ce7a..b1492c8e3 100644 --- a/pkg/utils/parse.go +++ b/pkg/utils/parse.go @@ -33,6 +33,9 @@ const TypeOss = "oss" // TypeGithub represents github const TypeGithub = "github" +// TypeGitee represents gitee +const TypeGitee = "gitee" + // TypeUnknown represents parse failed const TypeUnknown = "unknown" @@ -40,6 +43,7 @@ const TypeUnknown = "unknown" type Content struct { OssContent GithubContent + GiteeContent LocalContent } @@ -62,6 +66,14 @@ type GithubContent struct { Ref string `json:"ref"` } +// GiteeContent for cap center +type GiteeContent struct { + Owner string `json:"gitee_owner"` + Repo string `json:"gitee_repo"` + Path string `json:"gitee_path"` + Ref string `json:"gitee_ref"` +} + // Parse will parse config from address func Parse(addr string) (string, *Content, error) { URL, err := url.Parse(addr) @@ -117,6 +129,38 @@ func Parse(addr string) (string, *Content, error) { }, }, nil + case "gitee.com": + // We support two valid format: + // 1. https://gitee.com///tree// + // 2. https://gitee.com/// + if len(l) < 3 { + return "", nil, errors.New("invalid format " + addr) + } + switch l[2] { + case "tree": + // https://gitee.com///tree// + if len(l) < 5 { + return "", nil, errors.New("invalid format " + addr) + } + return TypeGitee, &Content{ + GiteeContent: GiteeContent{ + Owner: l[0], + Repo: l[1], + Path: strings.Join(l[4:], "/"), + Ref: l[3], + }, + }, nil + default: + // https://gitee.com/// + return TypeGitee, &Content{ + GiteeContent: GiteeContent{ + Owner: l[0], + Repo: l[1], + Path: strings.Join(l[2:], "/"), + Ref: "", // use default branch + }, + }, nil + } default: return "", nil, fmt.Errorf("git type repository only support github for now") }