Feat: support gitee repository in addon registry (#3393)

* Feat: support gitee repository in addon registry
Signed-off-by: fourierr <maxiangboo@163.com>

* Feat: support gitee repository in addon registry
Signed-off-by: maxiangbo <maxiangboo@cmbchina.com>

Signed-off-by: fourierr <maxiangboo@yeah.net>

* Feat: support gitee repository in addon registry Signed-off-by: maxiangbo <maxiangboo@cmbchina.com>

* Feat: support gitee repository in addon registry Signed-off-by: maxiangbo <maxiangboo@cmbchina.com>

Signed-off-by: fourierr <maxiangboo@qq.com>

Co-authored-by: fourierr <maxiangboo@yeah.net>

pkg/addon/addon.go

@@ -21,6 +21,9 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
 	"path"
 	"path/filepath"
 	"strings"
@@ -86,6 +89,9 @@ const (
 
 	// AddonParameterDataKey is the key of parameter in addon args secrets
 	AddonParameterDataKey string = "addonParameterDataKey"
+
+	// DefaultGiteeURL is the addon repository of gitee api
+	DefaultGiteeURL string = "https://gitee.com/api/v5/"
 )
 
 // ParameterFileName is the addon resources/parameter.cue file name
@@ -422,15 +428,75 @@ func createGitHelper(content *utils.Content, token string) *gitHelper {
 	}
 }
 
+func createGiteeHelper(content *utils.Content, token string) *giteeHelper {
+	var ts oauth2.TokenSource
+	if token != "" {
+		ts = oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
+	}
+	tc := oauth2.NewClient(context.Background(), ts)
+	tc.Timeout = time.Second * 20
+	cli := NewGiteeClient(tc, nil)
+	return &giteeHelper{
+		Client: cli,
+		Meta:   content,
+	}
+}
+
 // readRepo will read relative path (relative to Meta.Path)
 func (h *gitHelper) readRepo(relativePath string) (*github.RepositoryContent, []*github.RepositoryContent, error) {
-	file, items, _, err := h.Client.Repositories.GetContents(context.Background(), h.Meta.Owner, h.Meta.Repo, path.Join(h.Meta.Path, relativePath), nil)
+	file, items, _, err := h.Client.Repositories.GetContents(context.Background(), h.Meta.GithubContent.Owner, h.Meta.GithubContent.Repo, path.Join(h.Meta.GithubContent.Path, relativePath), nil)
 	if err != nil {
 		return nil, nil, WrapErrRateLimit(err)
 	}
 	return file, items, nil
 }
 
+// readRepo will read relative path (relative to Meta.Path)
+func (h *giteeHelper) readRepo(relativePath string) (*github.RepositoryContent, []*github.RepositoryContent, error) {
+	file, items, err := h.Client.GetGiteeContents(context.Background(), h.Meta.GiteeContent.Owner, h.Meta.GiteeContent.Repo, path.Join(h.Meta.GiteeContent.Path, relativePath), h.Meta.GiteeContent.Ref)
+	if err != nil {
+		return nil, nil, WrapErrRateLimit(err)
+	}
+	return file, items, nil
+}
+
+// GetGiteeContents can return either the metadata and content of a single file
+func (c *Client) GetGiteeContents(ctx context.Context, owner, repo, path, ref string) (fileContent *github.RepositoryContent, directoryContent []*github.RepositoryContent, err error) {
+	escapedPath := (&url.URL{Path: path}).String()
+	u := fmt.Sprintf(c.BaseURL.String()+"repos/%s/%s/contents/%s", owner, repo, escapedPath)
+	if ref != "" {
+		u = fmt.Sprintf(u+"?ref=%s", ref)
+	}
+
+	req, err := http.NewRequest("GET", u, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	response, err := c.Client.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, nil, err
+	}
+	//nolint:errcheck
+	defer response.Body.Close()
+	body, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return nil, nil, err
+	}
+	return unmarshalToContent(body)
+}
+
+func unmarshalToContent(content []byte) (fileContent *github.RepositoryContent, directoryContent []*github.RepositoryContent, err error) {
+	fileUnmarshalError := json.Unmarshal(content, &fileContent)
+	if fileUnmarshalError == nil {
+		return fileContent, nil, nil
+	}
+	directoryUnmarshalError := json.Unmarshal(content, &directoryContent)
+	if directoryUnmarshalError == nil {
+		return nil, directoryContent, nil
+	}
+	return nil, nil, fmt.Errorf("unmarshalling failed for both file and directory content: %s and %w", fileUnmarshalError, directoryUnmarshalError)
+}
+
 func genAddonAPISchema(addonRes *UIData) error {
 	param, err := utils2.PrepareParameterCue(addonRes.Name, addonRes.Parameters)
 	if err != nil {

pkg/addon/addon_test.go

@@ -748,6 +748,68 @@ func TestCheckAddonVersionMeetRequired(t *testing.T) {
 	assert.NoError(t, checkAddonVersionMeetRequired(ctx, &SystemRequirements{VelaVersion: ">=1.2.4"}, k8sClient, nil))
 }
 
+var testUnmarshalToContent1 = `
+{
+  "type": "file",
+  "encoding": "",
+  "size": 651,
+  "name": "metadata.yaml",
+  "path": "example/metadata.yaml",
+  "content": "name: example\r\nversion: 1.0.0\r\ndescription: Extended workload to do continuous and progressive delivery\r\nicon: https://raw.githubusercontent.com/fluxcd/flux/master/docs/_files/weave-flux.png\r\nurl: https://fluxcd.io\r\n\r\ntags:\r\n  - extended_workload\r\n  - gitops\r\n  - only_example\r\n\r\ndeployTo:\r\n  control_plane: true\r\n  runtime_cluster: false\r\n\r\ndependencies: []\r\n#- name: addon_name\r\n\r\n# set invisible means this won't be list and will be enabled when depended on\r\n# for example, terraform-alibaba depends on terraform which is invisible,\r\n# when terraform-alibaba is enabled, terraform will be enabled automatically\r\n# default: false\r\ninvisible: false\r\n"
+}`
+var testUnmarshalToContent2 = `
+[
+  {
+    "type": "dir",
+    "name": "example",
+    "path": "example"
+  },
+  {
+    "type": "dir",
+    "name": "local",
+    "path": "local"
+  },
+  {
+    "type": "dir",
+    "name": "terraform",
+    "path": "terraform"
+  },
+  {
+    "type": "dir",
+    "name": "terraform-alibaba",
+    "path": "terraform-alibaba"
+  },
+  {
+    "type": "dir",
+    "name": "test-error-addon",
+    "path": "test-error-addon"
+  }
+]`
+var testUnmarshalToContent3 = `
+[
+  {
+    "type": "dir",
+    "name": "example",
+  },
+  {
+    "type": "dir",
+    "name": "local",
+    "path": "local"
+  }
+]`
+var testUnmarshalToContent4 = ``
+
+func TestUnmarshalToContent(t *testing.T) {
+	_, _, err1 := unmarshalToContent([]byte(testUnmarshalToContent1))
+	assert.NoError(t, err1)
+	_, _, err2 := unmarshalToContent([]byte(testUnmarshalToContent2))
+	assert.NoError(t, err2)
+	_, _, err3 := unmarshalToContent([]byte(testUnmarshalToContent3))
+	assert.Error(t, err3, "unmarshalling failed for both file and directory content: invalid character '}' looking for beginnin")
+	_, _, err4 := unmarshalToContent([]byte(testUnmarshalToContent4))
+	assert.Error(t, err4, "unmarshalling failed for both file and directory content: unexpected end of JSON input and unexpecte")
+}
+
 // Test readResFile, only accept .cue and .yaml/.yml
 func TestReadResFile(t *testing.T) {
 
@@ -787,6 +849,7 @@ func TestReadDefFile(t *testing.T) {
 	var uiData = &UIData{}
 	ptItems := ClassifyItemByPattern(&testAddonMeta, reader)
 	items := ptItems[DefinitionsDirName]
+
 	for _, it := range items {
 		err := readDefFile(uiData, reader, reader.RelativePath(it))
 		if err != nil {

pkg/addon/reader_gitee.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2021 The KubeVela Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package addon
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/oam-dev/kubevela/pkg/utils"
+)
+
+var _ AsyncReader = &giteeReader{}
+
+// giteeHelper helps get addon's file by git
+type giteeHelper struct {
+	Client *Client
+	Meta   *utils.Content
+}
+
+// Client manages communication with the Gitee API
+type Client struct {
+	Client  *http.Client
+	BaseURL *url.URL
+}
+
+type giteeReader struct {
+	h *giteeHelper
+}
+
+// NewGiteeClient returns a new Gitee API client
+func NewGiteeClient(httpClient *http.Client, baseURL *url.URL) *Client {
+	if httpClient == nil {
+		httpClient = &http.Client{}
+	}
+	if baseURL == nil {
+		baseURL, _ = baseURL.Parse(DefaultGiteeURL)
+	}
+	return &Client{httpClient, baseURL}
+}
+
+// ListAddonMeta relative path to repoURL/basePath
+func (g *giteeReader) ListAddonMeta() (map[string]SourceMeta, error) {
+	subItems := make(map[string]SourceMeta)
+	_, items, err := g.h.readRepo("")
+	if err != nil {
+		return nil, err
+	}
+	for _, item := range items {
+		// single addon
+		if item.GetType() != DirType {
+			continue
+		}
+		addonName := path.Base(item.GetPath())
+		addonMeta, err := g.listAddonMeta(g.RelativePath(item))
+		if err != nil {
+			return nil, errors.Wrapf(err, "fail to get addon meta of %s", addonName)
+		}
+		subItems[addonName] = SourceMeta{Name: addonName, Items: addonMeta}
+	}
+	return subItems, nil
+}
+
+func (g *giteeReader) listAddonMeta(dirPath string) ([]Item, error) {
+	_, items, err := g.h.readRepo(dirPath)
+	if err != nil {
+		return nil, err
+	}
+	res := make([]Item, 0)
+	for _, item := range items {
+		switch item.GetType() {
+		case FileType:
+			res = append(res, item)
+		case DirType:
+			subItems, err := g.listAddonMeta(g.RelativePath(item))
+			if err != nil {
+				return nil, err
+			}
+			res = append(res, subItems...)
+		}
+	}
+	return res, nil
+}
+
+// ReadFile read file content from github
+func (g *giteeReader) ReadFile(relativePath string) (content string, err error) {
+	file, _, err := g.h.readRepo(relativePath)
+	if err != nil {
+		return
+	}
+	if file == nil {
+		return "", fmt.Errorf("path %s is not a file", relativePath)
+	}
+	return file.GetContent()
+}
+
+func (g *giteeReader) RelativePath(item Item) string {
+	absPath := strings.Split(item.GetPath(), "/")
+	if g.h.Meta.GiteeContent.Path == "" {
+		return path.Join(absPath...)
+	}
+	base := strings.Split(g.h.Meta.GiteeContent.Path, "/")
+	return path.Join(absPath[len(base):]...)
+}

pkg/addon/reader_gitee_test.go

@@ -0,0 +1,100 @@
+/*
+Copyright 2021 The KubeVela Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package addon
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/google/go-github/v32/github"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/oam-dev/kubevela/pkg/utils"
+)
+
+func giteeSetup() (client *Client, mux *http.ServeMux, teardown func()) {
+	// mux is the HTTP request multiplexer used with the test server.
+	mux = http.NewServeMux()
+
+	apiHandler := http.NewServeMux()
+	apiHandler.Handle(baseURLPath+"/", http.StripPrefix(baseURLPath, mux))
+
+	// server is a test HTTP server used to provide mock API responses.
+	server := httptest.NewServer(apiHandler)
+
+	// client is the GitHub client being tested and is
+	// configured to use test server.
+	URL, _ := url.Parse(server.URL + baseURLPath + "/")
+	httpClient := &http.Client{}
+	client = NewGiteeClient(httpClient, URL)
+
+	return client, mux, server.Close
+}
+
+func TestGiteeReader(t *testing.T) {
+	client, mux, teardown := giteeSetup()
+	giteePattern := "/repos/o/r/contents/"
+	mux.HandleFunc(giteePattern, func(rw http.ResponseWriter, req *http.Request) {
+		queryPath := strings.TrimPrefix(req.URL.Path, giteePattern)
+		localPath := path.Join(testdataPrefix, queryPath)
+		file, err := testdata.ReadFile(localPath)
+		// test if it's a file
+		if err == nil {
+			content := &github.RepositoryContent{Type: String("file"), Name: String(path.Base(queryPath)), Size: Int(len(file)), Encoding: String(""), Path: String(queryPath), Content: String(string(file))}
+			res, _ := json.Marshal(content)
+			rw.Write(res)
+			return
+		}
+
+		// otherwise, it could be directory
+		dir, err := testdata.ReadDir(localPath)
+		if err == nil {
+			contents := make([]*github.RepositoryContent, 0)
+			for _, item := range dir {
+				tp := "file"
+				if item.IsDir() {
+					tp = "dir"
+				}
+				contents = append(contents, &github.RepositoryContent{Type: String(tp), Name: String(item.Name()), Path: String(path.Join(queryPath, item.Name()))})
+			}
+			dRes, _ := json.Marshal(contents)
+			rw.Write(dRes)
+			return
+		}
+
+		rw.Write([]byte("invalid gitee query"))
+	})
+	defer teardown()
+
+	gith := &giteeHelper{
+		Client: client,
+		Meta: &utils.Content{GiteeContent: utils.GiteeContent{
+			Owner: "o",
+			Repo:  "r",
+		}},
+	}
+	var r AsyncReader = &giteeReader{gith}
+	_, err := r.ReadFile("example/metadata.yaml")
+	assert.NoError(t, err)
+
+	testReaderFunc(t, r)
+}

pkg/addon/reader_github.go

@@ -96,9 +96,9 @@ func (g *gitReader) ReadFile(relativePath string) (content string, err error) {
 
 func (g *gitReader) RelativePath(item Item) string {
 	absPath := strings.Split(item.GetPath(), "/")
-	if g.h.Meta.Path == "" {
+	if g.h.Meta.GithubContent.Path == "" {
 		return path.Join(absPath...)
 	}
-	base := strings.Split(g.h.Meta.Path, "/")
+	base := strings.Split(g.h.Meta.GithubContent.Path, "/")
 	return path.Join(absPath[len(base):]...)
 }

pkg/addon/registry.go

@@ -37,8 +37,9 @@ const registriesKey = "registries"
 type Registry struct {
 	Name string `json:"name"`
 
-	Git *GitAddonSource `json:"git,omitempty"`
-	OSS *OSSAddonSource `json:"oss,omitempty"`
+	Git   *GitAddonSource   `json:"git,omitempty"`
+	OSS   *OSSAddonSource   `json:"oss,omitempty"`
+	Gitee *GiteeAddonSource `json:"gitee,omitempty"`
 }
 
 // RegistryDataStore CRUD addon registry data in configmap

pkg/addon/source.go

@@ -56,6 +56,13 @@ type GitAddonSource struct {
 	Token string `json:"token,omitempty"`
 }
 
+// GiteeAddonSource defines the information about the Gitee as addon source
+type GiteeAddonSource struct {
+	URL   string `json:"url,omitempty" validate:"required"`
+	Path  string `json:"path,omitempty"`
+	Token string `json:"token,omitempty"`
+}
+
 // Item is a partial interface for github.RepositoryContent
 type Item interface {
 	// GetType return "dir" or "file"
@@ -110,8 +117,9 @@ func pathWithParent(subPath, parent string) string {
 type ReaderType string
 
 const (
-	gitType ReaderType = "git"
-	ossType ReaderType = "oss"
+	gitType   ReaderType = "git"
+	ossType   ReaderType = "oss"
+	giteeType ReaderType = "gitee"
 )
 
 // NewAsyncReader create AsyncReader from
@@ -154,6 +162,21 @@ func NewAsyncReader(baseURL, bucket, subPath, token string, rdType ReaderType) (
 			path:           subPath,
 			client:         resty.New(),
 		}, nil
+	case giteeType:
+		baseURL = strings.TrimSuffix(baseURL, ".git")
+		u, err := url.Parse(baseURL)
+		if err != nil {
+			return nil, errors.New("addon registry invalid")
+		}
+		u.Path = path.Join(u.Path, subPath)
+		_, content, err := utils.Parse(u.String())
+		if err != nil {
+			return nil, err
+		}
+		gitee := createGiteeHelper(content, token)
+		return &giteeReader{
+			h: gitee,
+		}, nil
 	}
 	return nil, fmt.Errorf("invalid addon registry type '%s'", rdType)
 }
@@ -168,6 +191,10 @@ func (r *Registry) BuildReader() (AsyncReader, error) {
 		g := r.Git
 		return NewAsyncReader(g.URL, "", g.Path, g.Token, gitType)
 	}
+	if r.Gitee != nil {
+		g := r.Gitee
+		return NewAsyncReader(g.URL, "", g.Path, g.Token, giteeType)
+	}
 	return nil, errors.New("registry don't have enough info to build a reader")
 }
 

pkg/apiserver/rest/apis/v1/types.go

@@ -69,22 +69,25 @@ type NameAlias struct {
 
 // CreateAddonRegistryRequest defines the format for addon registry create request
 type CreateAddonRegistryRequest struct {
-	Name string                `json:"name" validate:"checkname"`
-	Git  *addon.GitAddonSource `json:"git,omitempty" `
-	Oss  *addon.OSSAddonSource `json:"oss,omitempty"`
+	Name  string                  `json:"name" validate:"checkname"`
+	Git   *addon.GitAddonSource   `json:"git,omitempty" `
+	Oss   *addon.OSSAddonSource   `json:"oss,omitempty"`
+	Gitee *addon.GiteeAddonSource `json:"gitee,omitempty" `
 }
 
 // UpdateAddonRegistryRequest defines the format for addon registry update request
 type UpdateAddonRegistryRequest struct {
-	Git *addon.GitAddonSource `json:"git,omitempty"`
-	Oss *addon.OSSAddonSource `json:"oss,omitempty"`
+	Git   *addon.GitAddonSource   `json:"git,omitempty"`
+	Oss   *addon.OSSAddonSource   `json:"oss,omitempty"`
+	Gitee *addon.GiteeAddonSource `json:"gitee,omitempty" `
 }
 
 // AddonRegistry defines the format for a single addon registry
 type AddonRegistry struct {
-	Name string                `json:"name" validate:"required"`
-	Git  *addon.GitAddonSource `json:"git,omitempty"`
-	OSS  *addon.OSSAddonSource `json:"oss,omitempty"`
+	Name  string                  `json:"name" validate:"required"`
+	Git   *addon.GitAddonSource   `json:"git,omitempty"`
+	OSS   *addon.OSSAddonSource   `json:"oss,omitempty"`
+	Gitee *addon.GiteeAddonSource `json:"gitee,omitempty" `
 }
 
 // ListAddonRegistryResponse list addon registry

pkg/apiserver/rest/usecase/addon.go

@@ -296,9 +296,10 @@ func (u *defaultAddonHandler) CreateAddonRegistry(ctx context.Context, req apis.
 	}
 
 	return &apis.AddonRegistry{
-		Name: r.Name,
-		Git:  r.Git,
-		OSS:  r.OSS,
+		Name:  r.Name,
+		Git:   r.Git,
+		OSS:   r.OSS,
+		Gitee: r.Gitee,
 	}, nil
 }
 
@@ -450,9 +451,10 @@ func (u *defaultAddonHandler) UpdateAddon(ctx context.Context, name string, args
 
 func addonRegistryModelFromCreateAddonRegistryRequest(req apis.CreateAddonRegistryRequest) pkgaddon.Registry {
 	return pkgaddon.Registry{
-		Name: req.Name,
-		Git:  req.Git,
-		OSS:  req.Oss,
+		Name:  req.Name,
+		Git:   req.Git,
+		OSS:   req.Oss,
+		Gitee: req.Gitee,
 	}
 }
 

pkg/utils/parse.go

@@ -33,6 +33,9 @@ const TypeOss = "oss"
 // TypeGithub represents github
 const TypeGithub = "github"
 
+// TypeGitee represents gitee
+const TypeGitee = "gitee"
+
 // TypeUnknown represents parse failed
 const TypeUnknown = "unknown"
 
@@ -40,6 +43,7 @@ const TypeUnknown = "unknown"
 type Content struct {
 	OssContent
 	GithubContent
+	GiteeContent
 	LocalContent
 }
 
@@ -62,6 +66,14 @@ type GithubContent struct {
 	Ref   string `json:"ref"`
 }
 
+// GiteeContent for cap center
+type GiteeContent struct {
+	Owner string `json:"gitee_owner"`
+	Repo  string `json:"gitee_repo"`
+	Path  string `json:"gitee_path"`
+	Ref   string `json:"gitee_ref"`
+}
+
 // Parse will parse config from address
 func Parse(addr string) (string, *Content, error) {
 	URL, err := url.Parse(addr)
@@ -117,6 +129,38 @@ func Parse(addr string) (string, *Content, error) {
 					},
 				},
 				nil
+		case "gitee.com":
+			// We support two valid format:
+			// 1. https://gitee.com/<owner>/<repo>/tree/<branch>/<path-to-dir>
+			// 2. https://gitee.com/<owner>/<repo>/<path-to-dir>
+			if len(l) < 3 {
+				return "", nil, errors.New("invalid format " + addr)
+			}
+			switch l[2] {
+			case "tree":
+				// https://gitee.com/<owner>/<repo>/tree/<branch>/<path-to-dir>
+				if len(l) < 5 {
+					return "", nil, errors.New("invalid format " + addr)
+				}
+				return TypeGitee, &Content{
+					GiteeContent: GiteeContent{
+						Owner: l[0],
+						Repo:  l[1],
+						Path:  strings.Join(l[4:], "/"),
+						Ref:   l[3],
+					},
+				}, nil
+			default:
+				// https://gitee.com/<owner>/<repo>/<path-to-dir>
+				return TypeGitee, &Content{
+					GiteeContent: GiteeContent{
+						Owner: l[0],
+						Repo:  l[1],
+						Path:  strings.Join(l[2:], "/"),
+						Ref:   "", // use default branch
+					},
+				}, nil
+			}
 		default:
 			return "", nil, fmt.Errorf("git type repository only support github for now")
 		}