github/kubevela
1
0
Fork 0
mirror of https://github.com/kubevela/kubevela.git synced 2026-08-18 03:56:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix addon add and list

Browse Source
This commit is contained in:
天元
2020-08-12 13:31:30 +08:00
parent 85c0f93bb3
commit 0174305213
8 changed files with 214 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/types/types_test.go
+1 -1
View File
@@ -5,8 +5,8 @@ import (
"fmt"
"testing"
"github.com/ghodss/yaml"
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v3"
)
func TestApplication(t *testing.T) {
go.mod
+2 -1
View File
@@ -8,6 +8,7 @@ require (
github.com/crossplane/oam-kubernetes-runtime v0.0.8
github.com/ghodss/yaml v1.0.0
github.com/gin-gonic/gin v1.6.3
github.com/google/go-github/v32 v32.1.0
github.com/gosuri/uitable v0.0.4
github.com/onsi/ginkgo v1.11.0
github.com/onsi/gomega v1.8.1
@@ -17,7 +18,7 @@ require (
github.com/stretchr/testify v1.6.1
go.uber.org/zap v1.10.0
gopkg.in/natefinch/lumberjack.v2 v2.0.0
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
gotest.tools v2.2.0+incompatible
helm.sh/helm/v3 v3.2.4
k8s.io/api v0.18.6
go.sum
+5
View File
@@ -334,6 +334,11 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
pkg/cmd/addon.go
+23 -27
View File
@@ -1,11 +1,14 @@
package cmd
import (
"context"
"errors"
"fmt"
"io/ioutil"
"path/filepath"
"github.com/gosuri/uitable"
"github.com/cloud-native-application/rudrx/pkg/utils/system"
"github.com/cloud-native-application/rudrx/pkg/plugins"
@@ -13,7 +16,6 @@ import (
"github.com/cloud-native-application/rudrx/api/types"
cmdutil "github.com/cloud-native-application/rudrx/pkg/cmd/util"
"github.com/gosuri/uitable"
"github.com/spf13/cobra"
)
@@ -40,31 +42,33 @@ func NewAddonConfigCommand(ioStreams cmdutil.IOStreams) *cobra.Command {
if err != nil {
return err
}
config := plugins.RepoConfig{
config := &plugins.RepoConfig{
Name: args[0],
Address: ConvertURL(args[1]),
Address: args[1],
Token: cmd.Flag("token").Value.String(),
}
var updated bool
for idx, r := range repos {
if r.Name == config.Name {
repos[idx] = config
repos[idx] = *config
updated = true
break
}
}
if !updated {
repos = append(repos, config)
repos = append(repos, *config)
}
if err = plugins.StoreRepos(repos); err != nil {
return err
}
ioStreams.Info(fmt.Sprintf("Successfully configured Addon repo: %s", args[0]))
ioStreams.Info(fmt.Sprintf("Successfully configured Addon repo: %s, please use 'vela addon:update %s' to sync addons", args[0], args[0]))
return nil
},
Annotations: map[string]string{
types.TagCommandType: types.TypeOthers,
},
}
cmd.PersistentFlags().StringP("token", "t", "", "Github Repo token")
return cmd
}
@@ -96,8 +100,10 @@ func NewAddonUpdateCommand(ioStreams cmdutil.IOStreams) *cobra.Command {
return fmt.Errorf("%s repo not exist", specified)
}
}
ctx := context.Background()
for _, d := range repos {
err = SyncRemoteAddon(d)
client, err := plugins.NewAddClient(ctx, d.Name, d.Address, d.Token)
err = client.SyncRemoteAddons()
if err != nil {
return err
}
@@ -126,8 +132,10 @@ func NewAddonListCommand(ioStreams cmdutil.IOStreams) *cobra.Command {
if err != nil {
return err
}
table := uitable.New()
table.AddRow("NAME", "TYPE", "DEFINITION", "STATUS", "APPLIES-TO")
if repoName != "" {
return ListRepoAddons(filepath.Join(dir, repoName), ioStreams)
return ListRepoAddons(table, filepath.Join(dir, repoName), ioStreams)
}
dirs, err := ioutil.ReadDir(dir)
if err != nil {
@@ -137,7 +145,7 @@ func NewAddonListCommand(ioStreams cmdutil.IOStreams) *cobra.Command {
if !dd.IsDir() {
continue
}
if err = ListRepoAddons(filepath.Join(dir, dd.Name()), ioStreams); err != nil {
if err = ListRepoAddons(table, filepath.Join(dir, dd.Name()), ioStreams); err != nil {
return err
}
}
@@ -150,33 +158,21 @@ func NewAddonListCommand(ioStreams cmdutil.IOStreams) *cobra.Command {
return cmd
}
func ListRepoAddons(repoDir string, ioStreams cmdutil.IOStreams) error {
func ListRepoAddons(table *uitable.Table, repoDir string, ioStreams cmdutil.IOStreams) error {
templates, err := plugins.LoadTempFromLocal(repoDir)
if err != nil {
return err
}
table := uitable.New()
table.AddRow("NAME", "TYPE", "DEFINITION", "STATUS", "APPLIES-TO")
if len(templates) < 1 {
return nil
}
baseDir := filepath.Base(repoDir)
var status string
//TODO(wonderflow): check status whether install or not
status = "uninstalled"
for _, p := range templates {
table.AddRow(p.Name, p.Type, p.Type, status, p.AppliesTo)
table.AddRow(baseDir+"/"+p.Name, p.Type, p.Type, status, p.AppliesTo)
}
ioStreams.Info(table.String())
return nil
}
func ConvertURL(address string) string {
//TODO(wonderflow) convert github address here
return address
}
func SyncRemoteAddon(d plugins.RepoConfig) error {
addons, err := plugins.GetReposFromRemote(d)
if err != nil {
return err
}
return plugins.SyncRemoteAddons(d, addons)
}
pkg/cmd/workload_run.go
+1 -1
View File
@@ -12,7 +12,7 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"gopkg.in/yaml.v3"
"github.com/ghodss/yaml"
"cuelang.org/go/cue"
pkg/plugins/local.go
-4
View File
@@ -50,7 +50,6 @@ func LoadTempFromLocal(dir string) ([]types.Template, error) {
files, err := ioutil.ReadDir(dir)
if err != nil {
if os.IsNotExist(err) {
fmt.Println("\"no definition files found, use 'vela refresh' to sync from cluster\"")
return nil, nil
}
return nil, err
@@ -76,8 +75,5 @@ func LoadTempFromLocal(dir string) ([]types.Template, error) {
}
tmps = append(tmps, tmp)
}
if len(tmps) == 0 {
fmt.Println("\"no definition files found, use 'vela refresh' to sync from cluster\"")
}
return tmps, nil
}
pkg/plugins/repository.go
+131 -37
View File
@@ -1,12 +1,20 @@
package plugins
import (
"encoding/json"
"context"
"encoding/base64"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"golang.org/x/oauth2"
"github.com/google/go-github/v32/github"
"github.com/cloud-native-application/rudrx/api/types"
"github.com/crossplane/oam-kubernetes-runtime/apis/core/v1alpha2"
@@ -16,16 +24,89 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
type GithubContent struct {
Owner string `json:"owner"`
Repo string `json:"repo"`
Path string `json:"path"`
Ref string `json:"ref"`
}
//Used to store addon center config in file
type RepoConfig struct {
Name string `json:"repoName"`
Address string `json:"repoAddress"`
Token string `json:"token"`
}
var (
RepoConfigFile = ".vela/addon_config"
DefaultRepo = "local"
)
type AddonClient interface {
SyncRemoteAddons() error
}
func NewAddClient(ctx context.Context, name, address, token string) (AddonClient, error) {
Type, cfg, err := Parse(address)
if err != nil {
return nil, err
}
switch Type {
case TypeGithub:
return NewGithubAddon(ctx, token, name, cfg)
}
return nil, errors.New("we only support github as repository now")
}
const TypeGithub = "github"
const TypeUnknown = "unknown"
func Parse(addr string) (string, *GithubContent, error) {
url, err := url.Parse(addr)
if err != nil {
return "", nil, err
}
l := strings.Split(strings.TrimPrefix(url.Path, "/"), "/")
switch url.Host {
case "github.com":
// We support two valid format:
// 1. https://github.com/<owner>/<repo>/tree/<branch>/<path-to-dir>
// 2. https://github.com/<owner>/<repo>/<path-to-dir>
if len(l) < 3 {
return "", nil, errors.New("invalid format " + addr)
}
if l[2] == "tree" {
// https://github.com/<owner>/<repo>/tree/<branch>/<path-to-dir>
if len(l) < 5 {
return "", nil, errors.New("invalid format " + addr)
}
return TypeGithub, &GithubContent{
Owner: l[0],
Repo: l[1],
Path: strings.Join(l[4:], "/"),
Ref: l[3],
}, nil
} else {
// https://github.com/<owner>/<repo>/<path-to-dir>
return TypeGithub, &GithubContent{
Owner: l[0],
Repo: l[1],
Path: strings.Join(l[2:], "/"),
Ref: "", //use default branch
}, nil
}
case "api.github.com":
if len(l) != 5 {
return "", nil, errors.New("invalid format " + addr)
}
//https://api.github.com/repos/<owner>/<repo>/contents/<path-to-dir>
return TypeGithub, &GithubContent{
Owner: l[1],
Repo: l[2],
Path: l[4],
Ref: url.Query().Get("ref"),
}, nil
default:
//TODO(wonderflow): support raw url and oss format in the future
}
return TypeUnknown, nil, nil
}
type RemoteAddon struct {
// Name MUST be xxx.yaml
@@ -78,35 +159,9 @@ func StoreRepos(repos []RepoConfig) error {
return ioutil.WriteFile(config, data, 0644)
}
func GetReposFromRemote(r RepoConfig) (RemoteAddons, error) {
resp, err := http.Get(r.Address)
if err != nil {
return nil, err
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
var repos RemoteAddons
if err = json.Unmarshal(data, &repos); err != nil {
return nil, err
}
return repos, nil
}
func GetDefinitionFromURL(address, syncDir string) (types.Template, error) {
resp, err := http.Get(address)
if err != nil {
return types.Template{}, err
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return types.Template{}, err
}
func GetDefinitionFromURL(data []byte, syncDir string) (types.Template, error) {
var obj = unstructured.Unstructured{Object: make(map[string]interface{})}
err = yaml.Unmarshal(data, &obj.Object)
err := yaml.Unmarshal(data, &obj.Object)
if err != nil {
return types.Template{}, err
}
@@ -131,20 +186,59 @@ func GetDefinitionFromURL(address, syncDir string) (types.Template, error) {
return types.Template{}, fmt.Errorf("unknown definition Type %s", obj.GetKind())
}
type GithubAddon struct {
client *github.Client
cfg *GithubContent
repoName string
ctx context.Context
}
var _ AddonClient = &GithubAddon{}
func NewGithubAddon(ctx context.Context, token, repoName string, r *GithubContent) (*GithubAddon, error) {
var tc *http.Client
if token != "" {
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: token},
)
tc = oauth2.NewClient(ctx, ts)
}
return &GithubAddon{client: github.NewClient(tc), cfg: r, repoName: repoName, ctx: ctx}, nil
}
//TODO(wonderflow): currently we only sync by create, we also need to delete which not exist remotely.
func SyncRemoteAddons(r RepoConfig, addons RemoteAddons) error {
func (g *GithubAddon) SyncRemoteAddons() error {
_, dirs, _, err := g.client.Repositories.GetContents(g.ctx, g.cfg.Owner, g.cfg.Repo, g.cfg.Path, &github.RepositoryContentGetOptions{Ref: g.cfg.Ref})
if err != nil {
return err
}
dir, err := system.GetRepoDir()
if err != nil {
return err
}
repoDir := filepath.Join(dir, r.Name)
repoDir := filepath.Join(dir, g.repoName)
system.StatAndCreate(repoDir)
var tmps []types.Template
for _, addon := range addons {
tmp, err := GetDefinitionFromURL(addon.Url, repoDir)
for _, addon := range dirs {
if *addon.Type != "file" {
continue
}
fileContent, _, _, err := g.client.Repositories.GetContents(g.ctx, g.cfg.Owner, g.cfg.Repo, *addon.Path, &github.RepositoryContentGetOptions{Ref: g.cfg.Ref})
if err != nil {
return err
}
var data = []byte(*fileContent.Content)
if *fileContent.Encoding == "base64" {
data, err = base64.StdEncoding.DecodeString(*fileContent.Content)
if err != nil {
return fmt.Errorf("decode github content %s err %v", *fileContent.Path, err)
}
}
tmp, err := GetDefinitionFromURL(data, repoDir)
if err != nil {
fmt.Printf("get definition of %s err %v\n", *addon.Path, err)
continue
}
tmps = append(tmps, tmp)
}
success := SinkTemp2Local(tmps, repoDir)
pkg/plugins/repository_test.go
+51
View File
@@ -0,0 +1,51 @@
package plugins
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestParseURL(t *testing.T) {
cases := map[string]struct {
url string
exp *GithubContent
expType string
}{
"api-github": {
url: "https://api.github.com/repos/zzxwill/catalog/contents/repository?ref=plugin",
expType: TypeGithub,
exp: &GithubContent{
Owner: "zzxwill",
Repo: "catalog",
Path: "repository",
Ref: "plugin",
},
},
"github-copy-path": {
url: "https://github.com/zzxwill/catalog/tree/plugin/repository",
expType: TypeGithub,
exp: &GithubContent{
Owner: "zzxwill",
Repo: "catalog",
Path: "repository",
Ref: "plugin",
},
},
"github-manuel-write-path": {
url: "https://github.com/zzxwill/catalog/repository",
expType: TypeGithub,
exp: &GithubContent{
Owner: "zzxwill",
Repo: "catalog",
Path: "repository",
},
},
}
for caseName, c := range cases {
tp, content, err := Parse(c.url)
assert.NoError(t, err, caseName)
assert.Equal(t, c.exp, content, caseName)
assert.Equal(t, c.expType, tp, caseName)
}
}