Add oss registry support (#1715)

* Add oss registry support

* fix test

* refactor and clean up URL parse logic

* add:design docs

* change default registry to oss

design/api/vela-plugin-registry-reference.md

@@ -0,0 +1,33 @@
+# KubeVela Plugin Registry Reference
+
+## registry interface intro
+
+`Registry` interface definitions have two methods: `GetCap` and `ListCaps`. Here "Cap" can represent trait or component definition
+
+```go
+// Registry define a registry stores trait & component defs
+type Registry interface {
+	GetCap(addonName string) (types.Capability, []byte, error)
+	ListCaps() ([]types.Capability, error)
+}
+```
+
+Now we have implemented:`GithubRegistry`, `LocalRegistry` and `OssRegistry`, which represent three types of registry source. 
+
+To create a `Registry` object you should and must call `NewRegistry()`. 
+
+### Helper type or function
+
+You could use `RegistryFile` to convert `[]byte` (easily got from all kinds of source) to `Capability`. Here is `RegistryFile`'s definition.
+
+```go
+// RegistryFile describes a file item in registry
+type RegistryFile struct {
+	data []byte // file content
+	name string // file's name
+}
+```
+
+## Registry vs Capcenter
+
+How they differ from each other? `Capcenter` is a type for vela CLI. It has some function to sync content from remote and local `~/.vela` directory and apply some `ComponentDefinition` or `TraitDefinition` to the cluster. In the contrast, `Registry` is a type for kubectl vela plugin, which focuses on cluster directly. In one word, `Registry` has no local operations. They share some basic function.

docs/en/end-user/components/more.md

@@ -16,7 +16,7 @@ For example, let's try to list all available components in a registry:
 
 ```shell
 $ kubectl vela comp --discover
-Showing components from registry: https://github.com/oam-dev/catalog/tree/master/registry
+Showing components from registry: https://registry.kubevela.net
 NAME              	REGITSRY	DEFINITION                 	
 cloneset          	default	    clonesets.apps.kruise.io
 kruise-statefulset	default	    statefulsets.apps.kruise.io
@@ -43,7 +43,7 @@ cloneset     CloneSet        Describes long-running, scalable, containerized ser
 
 ```
 
-By default, the two commands will retrieve capabilities from [repo](https://github.com/oam-dev/catalog/tree/master/registry) maintained by KubeVela.
+By default, the two commands will retrieve capabilities from [repo](https://registry.kubevela.net) maintained by KubeVela.
 
 ## 2. Designed by yourself 
 Check below documentations about how to bring your own components to the system in various approaches.

docs/en/end-user/traits/more.md

@@ -17,7 +17,7 @@ For example, let's try to list all available traits in registry:
 
 ```shell
 $ kubectl vela trait --discover
-Showing traits from registry: https://github.com/oam-dev/catalog/tree/master/registry
+Showing traits from registry: https://registry.kubevela.net
 NAME           	REGISTRY	  DEFINITION                    		APPLIES-TO               
 service-account	default  	                              		    [webservice worker]      
 env            	default 		                                    [webservice worker]      
@@ -51,7 +51,7 @@ init-container   ["webservice","worker"]   add an init container with a shared v
 ```
 
 By default, the two commands will retrieve capabilities
-from [repo](https://github.com/oam-dev/catalog/tree/master/registry) maintained by KubeVela.
+from [repo](https://registry.kubevela.net) maintained by KubeVela.
 
 ## 2. Designed by yourself
 

e2e/plugin/plugin_test.go

@@ -171,7 +171,7 @@ var _ = Describe("Test Kubectl Plugin", func() {
 			Expect(output).Should(ContainSubstring("Successfully install component: cloneset"))
 		})
 		It("Test install a sample trait", func() {
-			output, err := e2e.Exec("kubectl-vela trait get init-container")
+			output, err := e2e.Exec("kubectl-vela trait get init-container --url=" + testRegistryPath)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(output).Should(ContainSubstring("Successfully install trait: init-container"))
 		})

e2e/plugin/testdata/dynamic-sa.yaml

@@ -0,0 +1,32 @@
+kind: TraitDefinition
+metadata:
+  annotations:
+    definition.oam.dev/description: "dynamically specify service account"
+  name: service-account
+spec:
+  appliesToWorkloads:
+    - deployments.apps
+  schematic:
+    cue:
+      template: |-
+        processing: {
+          output: {
+              credentials?: string
+          }
+          http: {
+              method: *"GET" | string
+              url:    parameter.serviceURL
+              request: {
+                  header: {
+                      "authorization.token": parameter.uidtoken
+                  }
+              }
+          }
+        }
+        patch: {
+          spec: template: spec: serviceAccountName: processing.output.credentials
+        }
+        parameter: {
+          uidtoken:   string
+          serviceURL: string
+        }

references/cli/traits.go

@@ -178,7 +178,7 @@ func InstallTraitByName(args common2.Args, ioStream cmdutil.IOStreams, traitName
 }
 
 // DefaultRegistry is default capability center of kubectl-vela
-var DefaultRegistry = "https://github.com/oam-dev/catalog/tree/master/registry"
+var DefaultRegistry = "oss://registry.kubevela.net"
 
 const installed = "installed"
 const uninstalled = "uninstalled"

references/plugins/capcenter.go

@@ -40,6 +40,23 @@ import (
 	"github.com/oam-dev/kubevela/pkg/utils/system"
 )
 
+// Content contains different type of content needed when building Registry or GithubCenter
+type Content struct {
+	OssContent
+	GithubContent
+	LocalContent
+}
+
+// LocalContent for local registry
+type LocalContent struct {
+	AbsDir string `json:"abs_dir"`
+}
+
+// OssContent for oss registry
+type OssContent struct {
+	BucketURL string `json:"bucket_url"`
+}
+
 // GithubContent for cap center
 type GithubContent struct {
 	Owner string `json:"owner"`
@@ -68,12 +85,18 @@ func NewCenterClient(ctx context.Context, name, address, token string) (CenterCl
 	}
 	switch Type {
 	case TypeGithub:
-		return NewGithubCenter(ctx, token, name, cfg)
+		return NewGithubCenter(ctx, token, name, &cfg.GithubContent)
 	default:
 	}
 	return nil, errors.New("we only support github as repository now")
 }
 
+// TypeLocal represents github
+const TypeLocal = "local"
+
+// TypeOss represent oss
+const TypeOss = "oss"
+
 // TypeGithub represents github
 const TypeGithub = "github"
 
@@ -81,53 +104,77 @@ const TypeGithub = "github"
 const TypeUnknown = "unknown"
 
 // Parse will parse config from address
-func Parse(addr string) (string, *GithubContent, error) {
-	url, err := url.Parse(addr)
+func Parse(addr string) (string, *Content, error) {
+	URL, err := url.Parse(addr)
 	if err != nil {
 		return "", nil, err
 	}
-	l := strings.Split(strings.TrimPrefix(url.Path, "/"), "/")
-	switch url.Host {
-	case "github.com":
-		// We support two valid format:
-		// 1. https://github.com/<owner>/<repo>/tree/<branch>/<path-to-dir>
-		// 2. https://github.com/<owner>/<repo>/<path-to-dir>
-		if len(l) < 3 {
-			return "", nil, errors.New("invalid format " + addr)
-		}
-		if l[2] == "tree" {
-			// https://github.com/<owner>/<repo>/tree/<branch>/<path-to-dir>
-			if len(l) < 5 {
+	l := strings.Split(strings.TrimPrefix(URL.Path, "/"), "/")
+	switch URL.Scheme {
+	case "http", "https":
+		switch URL.Host {
+		case "github.com":
+			// We support two valid format:
+			// 1. https://github.com/<owner>/<repo>/tree/<branch>/<path-to-dir>
+			// 2. https://github.com/<owner>/<repo>/<path-to-dir>
+			if len(l) < 3 {
 				return "", nil, errors.New("invalid format " + addr)
 			}
-			return TypeGithub, &GithubContent{
-				Owner: l[0],
-				Repo:  l[1],
-				Path:  strings.Join(l[4:], "/"),
-				Ref:   l[3],
-			}, nil
+			if l[2] == "tree" {
+				// https://github.com/<owner>/<repo>/tree/<branch>/<path-to-dir>
+				if len(l) < 5 {
+					return "", nil, errors.New("invalid format " + addr)
+				}
+				return TypeGithub, &Content{
+					GithubContent: GithubContent{
+						Owner: l[0],
+						Repo:  l[1],
+						Path:  strings.Join(l[4:], "/"),
+						Ref:   l[3],
+					},
+				}, nil
+			}
+			// https://github.com/<owner>/<repo>/<path-to-dir>
+			return TypeGithub, &Content{
+					GithubContent: GithubContent{
+						Owner: l[0],
+						Repo:  l[1],
+						Path:  strings.Join(l[2:], "/"),
+						Ref:   "", // use default branch
+					},
+				},
+				nil
+		case "api.github.com":
+			if len(l) != 5 {
+				return "", nil, errors.New("invalid format " + addr)
+			}
+			//https://api.github.com/repos/<owner>/<repo>/contents/<path-to-dir>
+			return TypeGithub, &Content{
+					GithubContent: GithubContent{
+						Owner: l[1],
+						Repo:  l[2],
+						Path:  l[4],
+						Ref:   URL.Query().Get("ref"),
+					},
+				},
+				nil
+		default:
 		}
-		// https://github.com/<owner>/<repo>/<path-to-dir>
-		return TypeGithub, &GithubContent{
-			Owner: l[0],
-			Repo:  l[1],
-			Path:  strings.Join(l[2:], "/"),
-			Ref:   "", // use default branch
+	case "oss":
+		return TypeOss, &Content{
+			OssContent: OssContent{
+				BucketURL: URL.Host,
+			},
 		}, nil
-	case "api.github.com":
-		if len(l) != 5 {
-			return "", nil, errors.New("invalid format " + addr)
-		}
-		//https://api.github.com/repos/<owner>/<repo>/contents/<path-to-dir>
-		return TypeGithub, &GithubContent{
-			Owner: l[1],
-			Repo:  l[2],
-			Path:  l[4],
-			Ref:   url.Query().Get("ref"),
+	case "file":
+		return TypeLocal, &Content{
+			LocalContent: LocalContent{
+				AbsDir: URL.Path,
+			},
 		}, nil
-	default:
-		// TODO(wonderflow): support raw url and oss format in the future
+
 	}
+
 	return TypeUnknown, nil, nil
 }
 

references/plugins/capcenter_test.go

@@ -61,7 +61,7 @@ func TestParseURL(t *testing.T) {
 	for caseName, c := range cases {
 		tp, content, err := Parse(c.url)
 		assert.NoError(t, err, caseName)
-		assert.Equal(t, c.exp, content, caseName)
+		assert.Equal(t, c.exp, &content.GithubContent, caseName)
 		assert.Equal(t, c.expType, tp, caseName)
 	}
 }

references/plugins/registry.go

@@ -19,6 +19,7 @@ package plugins
 import (
 	"context"
 	"encoding/base64"
+	"encoding/xml"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -40,7 +41,7 @@ type Registry interface {
 	ListCaps() ([]types.Capability, error)
 }
 
-// GithubRegistry is Registry's implementation trait github url as resource
+// GithubRegistry is Registry's implementation treat github url as resource
 type GithubRegistry struct {
 	client *github.Client
 	cfg    *GithubContent
@@ -50,12 +51,12 @@ type GithubRegistry struct {
 
 // NewRegistry will create a registry implementation
 func NewRegistry(ctx context.Context, token, registryName string, regURL string) (Registry, error) {
-	if strings.HasPrefix(regURL, "http") {
-		// todo(qiaozp) support oss
-		_, cfg, err := Parse(regURL)
-		if err != nil {
-			return nil, err
-		}
+	tp, cfg, err := Parse(regURL)
+	if err != nil {
+		return nil, err
+	}
+	switch tp {
+	case TypeGithub:
 		var tc *http.Client
 		if token != "" {
 			ts := oauth2.StaticTokenSource(
@@ -63,15 +64,23 @@ func NewRegistry(ctx context.Context, token, registryName string, regURL string)
 			)
 			tc = oauth2.NewClient(ctx, ts)
 		}
-		return GithubRegistry{client: github.NewClient(tc), cfg: cfg, ctx: ctx, name: registryName}, nil
-	} else if strings.HasPrefix(regURL, "file://") {
-		dir := strings.TrimPrefix(regURL, "file://")
-		_, err := os.Stat(dir)
+		return GithubRegistry{client: github.NewClient(tc), cfg: &cfg.GithubContent, ctx: ctx, name: registryName}, nil
+	case TypeOss:
+		var tc http.Client
+		return OssRegistry{
+			Client:    &tc,
+			bucketURL: fmt.Sprintf("https://%s/", cfg.BucketURL),
+		}, nil
+	case TypeLocal:
+		_, err := os.Stat(cfg.AbsDir)
 		if os.IsNotExist(err) {
 			return LocalRegistry{}, err
 		}
-		return LocalRegistry{absPath: dir}, nil
+		return LocalRegistry{absPath: cfg.AbsDir}, nil
+	case TypeUnknown:
+		return nil, fmt.Errorf("not supported url")
 	}
+
 	return nil, fmt.Errorf("not supported url")
 }
 
@@ -149,7 +158,79 @@ func (g *GithubRegistry) getRepoFile() ([]RegistryFile, error) {
 	return items, nil
 }
 
-// LocalRegistry is Registry's implementation trait local url as resource
+// OssRegistry is Registry's implementation treat OSS url as resource
+type OssRegistry struct {
+	*http.Client
+	bucketURL string
+}
+
+// GetCap return capability object and raw data specified by cap name
+func (o OssRegistry) GetCap(addonName string) (types.Capability, []byte, error) {
+	filename := addonName + ".yaml"
+	req, _ := http.NewRequestWithContext(
+		context.Background(),
+		http.MethodGet,
+		o.bucketURL+filename,
+		nil,
+	)
+	resp, err := o.Client.Do(req)
+	if err != nil {
+		return types.Capability{}, nil, err
+	}
+	data, err := ioutil.ReadAll(resp.Body)
+	_ = resp.Body.Close()
+	if err != nil {
+		return types.Capability{}, nil, err
+	}
+	rf := RegistryFile{
+		data: data,
+		name: filename,
+	}
+	capa, err := rf.toAddon()
+	if err != nil {
+		return types.Capability{}, nil, err
+	}
+
+	return capa, data, nil
+}
+
+// ListCaps list all capabilities of registry
+func (o OssRegistry) ListCaps() ([]types.Capability, error) {
+	req, _ := http.NewRequestWithContext(
+		context.Background(),
+		http.MethodGet,
+		o.bucketURL+"?list-type=2",
+		nil,
+	)
+	resp, err := o.Client.Do(req)
+	if err != nil {
+		return []types.Capability{}, err
+	}
+	data, err := ioutil.ReadAll(resp.Body)
+	_ = resp.Body.Close()
+	if err != nil {
+		return []types.Capability{}, err
+	}
+	list := &ListBucketResult{}
+	err = xml.Unmarshal(data, list)
+	if err != nil {
+		return []types.Capability{}, err
+	}
+	capas := make([]types.Capability, 0)
+
+	for _, fileName := range list.File {
+		addonName := strings.Split(fileName, ".")[0]
+		capa, _, err := o.GetCap(addonName)
+		if err != nil {
+			fmt.Printf("Get %s err: %s\n", fileName, err)
+			continue
+		}
+		capas = append(capas, capa)
+	}
+	return capas, nil
+}
+
+// LocalRegistry is Registry's implementation treat local url as resource
 type LocalRegistry struct {
 	absPath string
 }
@@ -213,3 +294,9 @@ type RegistryFile struct {
 	data []byte // file content
 	name string // file's name
 }
+
+// ListBucketResult describe a file list from OSS
+type ListBucketResult struct {
+	File  []string `xml:"Contents>Key"`
+	Count int      `xml:"KeyCount"`
+}

references/plugins/registry_test.go

@@ -25,7 +25,7 @@ import (
 )
 
 func TestRegistry(t *testing.T) {
-	testAddon := "init-container"
+	testAddon := "dynamic-sa"
 	regName := "testReg"
 	localPath, err := filepath.Abs("../../e2e/plugin/testdata")
 	assert.Nil(t, err)
@@ -34,6 +34,10 @@ func TestRegistry(t *testing.T) {
 		url       string
 		expectReg Registry
 	}{
+		"oss registry": {
+			url:       "oss://registry.kubevela.net/",
+			expectReg: OssRegistry{},
+		},
 		"github registry": {
 			url:       "https://github.com/oam-dev/catalog/tree/master/registry",
 			expectReg: GithubRegistry{},