Feat: add definitions to addon detail API, fix addon cache, async download files (#2738)

* add definition to addon detail API

* change little

* tmp

* fix cache

* fix import
This commit is contained in:
qiaozp
2021-11-20 12:24:35 +08:00
committed by GitHub
parent 36f5bbc973
commit 54eb662959
4 changed files with 377 additions and 212 deletions
+233 -141
View File
@@ -8,6 +8,7 @@ import (
"path"
"path/filepath"
"strings"
"sync"
"time"
v1 "k8s.io/api/core/v1"
@@ -32,7 +33,6 @@ import (
"github.com/oam-dev/kubevela/pkg/oam"
"github.com/oam-dev/kubevela/pkg/oam/util"
"github.com/oam-dev/kubevela/pkg/utils"
addonutil "github.com/oam-dev/kubevela/pkg/utils/addon"
"github.com/oam-dev/kubevela/pkg/utils/common"
)
@@ -53,6 +53,26 @@ const (
DefinitionsDirName string = "definitions"
)
type ListOptions struct {
GetDetail bool
GetDefinition bool
GetResource bool
GetParameter bool
GetTemplate bool
}
var (
ListLevelOptions = ListOptions{}
GetLevelOptions = ListOptions{GetDetail: true, GetDefinition: true, GetParameter: true}
EnableLevelOptions = ListOptions{GetDetail: true, GetDefinition: true, GetResource: true, GetTemplate: true, GetParameter: true}
)
type AddonErr error
var (
AddonNotExist AddonErr = errors.New("addon not exist")
)
type gitHelper struct {
Client *github.Client
Meta *utils.Content
@@ -65,33 +85,40 @@ type GitAddonSource struct {
Token string `json:"token,omitempty"`
}
// GetAddon get a detailed addon info from GitAddonSource
func GetAddon(name string, git *GitAddonSource) (*types.Addon, error) {
addons, err := ListAddons(true, git)
type AddonReader struct {
addon *types.Addon
h *gitHelper
item *github.RepositoryContent
errChan chan error
}
func (r *AddonReader) SetReadContent(content *github.RepositoryContent) {
r.item = content
}
// GetAddon get a addon info from GitAddonSource, can be used for get or enable
func GetAddon(name string, git *GitAddonSource, opt ListOptions) (*types.Addon, error) {
addon, err := getSingleAddonFromGit(git.URL, git.Path, name, git.Token, opt)
if err != nil {
return nil, err
}
for _, addon := range addons {
if addon.Name == name {
return addon, nil
}
}
return nil, errors.New("addon not exist")
return addon, nil
}
// ListAddons list addons' info from GitAddonSource, if not detailed, result only contains types.AddonMeta
func ListAddons(detailed bool, git *GitAddonSource) ([]*types.Addon, error) {
var gitAddons []*types.Addon
gitAddons, err := getAddonsFromGit(git.URL, git.Path, git.Token, detailed)
// ListAddons list addons' info from GitAddonSource
func ListAddons(git *GitAddonSource, opt ListOptions) ([]*types.Addon, error) {
gitAddons, err := getAddonsFromGit(git.URL, git.Path, git.Token, opt)
if err != nil {
return nil, err
}
return gitAddons, nil
}
func getAddonsFromGit(baseURL, dir, token string, detailed bool) ([]*types.Addon, error) {
func getAddonsFromGit(baseURL, dir, token string, opt ListOptions) ([]*types.Addon, error) {
var addons []*types.Addon
var err error
var wg sync.WaitGroup
errChan := make(chan error, 1)
gith, err := createGitHelper(baseURL, dir, token)
if err != nil {
@@ -106,168 +133,233 @@ func getAddonsFromGit(baseURL, dir, token string, detailed bool) ([]*types.Addon
if subItems.GetType() != "dir" {
continue
}
addonRes := &types.Addon{}
_, files, err := gith.readRepo(subItems.GetPath())
if err != nil {
return nil, err
}
for _, file := range files {
var err error
switch strings.ToLower(file.GetName()) {
case ReadmeFileName:
if !detailed {
break
}
err = readReadme(addonRes, gith, file)
case MetadataFileName:
err = readMetadata(addonRes, gith, file)
addonRes.Name = addonutil.TransAddonName(addonRes.Name)
case DefinitionsDirName:
if !detailed {
break
}
err = readDefinitions(addonRes, gith, file)
case ResourcesDirName:
if !detailed {
break
}
err = readResources(addonRes, gith, file)
case TemplateFileName:
if !detailed {
break
}
err = readTemplate(addonRes, gith, file)
}
wg.Add(1)
go func(item *github.RepositoryContent) {
defer wg.Done()
addonRes, err := getSingleAddonFromGit(baseURL, dir, item.GetName(), token, opt)
if err != nil {
return nil, err
errChan <- err
return
}
}
if detailed && addonRes.Parameters != "" {
err = genAddonAPISchema(addonRes)
if err != nil {
continue
}
}
addons = append(addons, addonRes)
addons = append(addons, addonRes)
}(subItems)
}
wg.Wait()
if len(errChan) != 0 {
return nil, <-errChan
}
return addons, nil
}
func readTemplate(addon *types.Addon, h *gitHelper, file *github.RepositoryContent) error {
content, _, err := h.readRepo(*file.Path)
func getSingleAddonFromGit(baseURL, dir, addonName, token string, opt ListOptions) (*types.Addon, error) {
var wg sync.WaitGroup
gith, err := createGitHelper(baseURL, path.Join(dir, addonName), token)
if err != nil {
return err
return nil, err
}
_, items, err := gith.readRepo(gith.Meta.Path)
reader := AddonReader{
addon: &types.Addon{},
h: gith,
errChan: make(chan error, 1),
}
for _, item := range items {
switch strings.ToLower(item.GetName()) {
case ReadmeFileName:
if !opt.GetDetail {
break
}
reader.SetReadContent(item)
wg.Add(1)
go readReadme(&wg, reader)
case MetadataFileName:
reader.SetReadContent(item)
wg.Add(1)
go readMetadata(&wg, reader)
case DefinitionsDirName:
if !opt.GetDefinition {
break
}
reader.SetReadContent(item)
wg.Add(1)
go readDefinitions(&wg, reader)
case ResourcesDirName:
if !opt.GetResource {
break
}
reader.SetReadContent(item)
wg.Add(1)
go readResources(&wg, reader)
case TemplateFileName:
if !opt.GetTemplate {
break
}
reader.SetReadContent(item)
wg.Add(1)
go readTemplate(&wg, reader)
}
}
wg.Wait()
if opt.GetParameter && reader.addon.Parameters != "" {
err = genAddonAPISchema(reader.addon)
if err != nil {
return nil, err
}
}
return reader.addon, nil
}
func readTemplate(wg *sync.WaitGroup, reader AddonReader) {
defer wg.Done()
content, _, err := reader.h.readRepo(*reader.item.Path)
if err != nil {
reader.errChan <- err
return
}
data, err := content.GetContent()
if err != nil {
return err
reader.errChan <- err
return
}
dec := k8syaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme)
addon.AppTemplate = &v1beta1.Application{}
_, _, err = dec.Decode([]byte(data), nil, addon.AppTemplate)
reader.addon.AppTemplate = &v1beta1.Application{}
_, _, err = dec.Decode([]byte(data), nil, reader.addon.AppTemplate)
if err != nil {
return err
reader.errChan <- err
return
}
return nil
}
func readResources(addon *types.Addon, h *gitHelper, dir *github.RepositoryContent) error {
dirPath := strings.Split(dir.GetPath(), "/")
func readResources(wg *sync.WaitGroup, reader AddonReader) {
defer wg.Done()
dirPath := strings.Split(reader.item.GetPath(), "/")
dirPath, err := cutPathUntil(dirPath, ResourcesDirName)
if err != nil {
return err
reader.errChan <- err
}
_, files, err := h.readRepo(*dir.Path)
_, items, err := reader.h.readRepo(*reader.item.Path)
if err != nil {
return err
reader.errChan <- err
return
}
for _, file := range files {
switch file.GetType() {
for _, item := range items {
switch item.GetType() {
case "file":
content, _, err := h.readRepo(*file.Path)
if err != nil {
return err
}
b, err := content.GetContent()
if err != nil {
return err
}
if file.GetName() == "parameter.cue" {
addon.Parameters = b
break
}
switch filepath.Ext(file.GetName()) {
case ".cue":
addon.CUETemplates = append(addon.CUETemplates, types.AddonElementFile{Data: b, Name: file.GetName(), Path: dirPath})
default:
addon.YAMLTemplates = append(addon.YAMLTemplates, types.AddonElementFile{Data: b, Name: file.GetName(), Path: dirPath})
}
reader.SetReadContent(item)
wg.Add(1)
go readResFile(wg, reader, dirPath)
case "dir":
err = readResources(addon, h, file)
if err != nil {
return err
}
reader.SetReadContent(item)
wg.Add(1)
go readResources(wg, reader)
}
}
return nil
}
func readDefinitions(addon *types.Addon, h *gitHelper, dir *github.RepositoryContent) error {
dirPath := strings.Split(dir.GetPath(), "/")
dirPath, err := cutPathUntil(dirPath, DefinitionsDirName)
// readResFile read single resource file
func readResFile(wg *sync.WaitGroup, reader AddonReader, dirPath []string) {
defer wg.Done()
content, _, err := reader.h.readRepo(*reader.item.Path)
if err != nil {
return err
}
_, files, err := h.readRepo(*dir.Path)
if err != nil {
return err
}
for _, file := range files {
switch file.GetType() {
case "file":
content, _, err := h.readRepo(*file.Path)
if err != nil {
return err
}
b, err := content.GetContent()
if err != nil {
return err
}
addon.Definitions = append(addon.Definitions, types.AddonElementFile{Data: b, Name: file.GetName(), Path: dirPath})
case "dir":
err = readDefinitions(addon, h, file)
if err != nil {
return err
}
}
}
return nil
}
func readMetadata(addon *types.Addon, h *gitHelper, file *github.RepositoryContent) error {
content, _, err := h.readRepo(*file.Path)
if err != nil {
return err
reader.errChan <- err
return
}
b, err := content.GetContent()
if err != nil {
return err
reader.errChan <- err
return
}
if reader.item.GetName() == "parameter.cue" {
reader.addon.Parameters = b
return
}
switch filepath.Ext(reader.item.GetName()) {
case ".cue":
reader.addon.CUETemplates = append(reader.addon.CUETemplates, types.AddonElementFile{Data: b, Name: reader.item.GetName(), Path: dirPath})
default:
reader.addon.YAMLTemplates = append(reader.addon.YAMLTemplates, types.AddonElementFile{Data: b, Name: reader.item.GetName(), Path: dirPath})
}
return yaml.Unmarshal([]byte(b), &addon.AddonMeta)
}
func readReadme(addon *types.Addon, h *gitHelper, file *github.RepositoryContent) error {
content, _, err := h.readRepo(*file.Path)
func readDefinitions(wg *sync.WaitGroup, reader AddonReader) {
defer wg.Done()
dirPath := strings.Split(reader.item.GetPath(), "/")
dirPath, err := cutPathUntil(dirPath, DefinitionsDirName)
if err != nil {
return err
reader.errChan <- err
return
}
addon.Detail, err = content.GetContent()
return err
_, items, err := reader.h.readRepo(*reader.item.Path)
if err != nil {
reader.errChan <- err
return
}
for _, item := range items {
switch item.GetType() {
case "file":
reader.SetReadContent(item)
wg.Add(1)
go readDefFile(wg, reader, dirPath)
case "dir":
reader.SetReadContent(item)
wg.Add(1)
go readDefinitions(wg, reader)
}
}
}
// readDefFile read single definition file
func readDefFile(wg *sync.WaitGroup, reader AddonReader, dirPath []string) {
defer wg.Done()
content, _, err := reader.h.readRepo(*reader.item.Path)
if err != nil {
reader.errChan <- err
return
}
b, err := content.GetContent()
if err != nil {
reader.errChan <- err
return
}
reader.addon.Definitions = append(reader.addon.Definitions, types.AddonElementFile{Data: b, Name: reader.item.GetName(), Path: dirPath})
}
func readMetadata(wg *sync.WaitGroup, reader AddonReader) {
defer wg.Done()
content, _, err := reader.h.readRepo(*reader.item.Path)
if err != nil {
reader.errChan <- err
return
}
b, err := content.GetContent()
if err != nil {
reader.errChan <- err
return
}
err = yaml.Unmarshal([]byte(b), &reader.addon.AddonMeta)
if err != nil {
reader.errChan <- err
return
}
return
}
func readReadme(wg *sync.WaitGroup, reader AddonReader) {
defer wg.Done()
content, _, err := reader.h.readRepo(*reader.item.Path)
if err != nil {
reader.errChan <- err
return
}
reader.addon.Detail, err = content.GetContent()
return
}
func createGitHelper(baseURL, dir, token string) (*gitHelper, error) {
+10 -1
View File
@@ -97,7 +97,16 @@ type DetailAddonResponse struct {
UISchema []*utils.UIParameter `json:"uiSchema"`
// More details about the addon, e.g. README
Detail string `json:"detail,omitempty"`
Detail string `json:"detail,omitempty"`
Definitions []*AddonDefinition `json:"definitions"`
}
// AddonDefinition is definition an addon can provide
type AddonDefinition struct {
Name string `json:"name,omitempty"`
// can be component/trait...definition
DefType string `json:"type,omitempty"`
Description string `json:"description,omitempty"`
}
// AddonStatusResponse defines the format of addon status response
+131 -68
View File
@@ -12,6 +12,7 @@ import (
errors2 "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
k8syaml "k8s.io/apimachinery/pkg/runtime/serializer/yaml"
"sigs.k8s.io/controller-runtime/pkg/client"
common2 "github.com/oam-dev/kubevela/apis/core.oam.dev/common"
@@ -35,7 +36,7 @@ type AddonUsecase interface {
DeleteAddonRegistry(ctx context.Context, name string) error
UpdateAddonRegistry(ctx context.Context, name string, req apis.UpdateAddonRegistryRequest) (*apis.AddonRegistryMeta, error)
ListAddonRegistries(ctx context.Context) ([]*apis.AddonRegistryMeta, error)
ListAddons(ctx context.Context, detailed bool, registry, query string) ([]*apis.DetailAddonResponse, error)
ListAddons(ctx context.Context, registry, query string) ([]*apis.DetailAddonResponse, error)
StatusAddon(ctx context.Context, name string) (*apis.AddonStatusResponse, error)
GetAddon(ctx context.Context, name string, registry string) (*apis.DetailAddonResponse, error)
EnableAddon(ctx context.Context, name string, args apis.EnableAddonRequest) error
@@ -43,13 +44,28 @@ type AddonUsecase interface {
}
// AddonImpl2AddonRes convert types.Addon to the type apiserver need
func AddonImpl2AddonRes(impl *types.Addon) *apis.DetailAddonResponse {
return &apis.DetailAddonResponse{
AddonMeta: impl.AddonMeta,
APISchema: impl.APISchema,
UISchema: impl.UISchema,
Detail: impl.Detail,
func AddonImpl2AddonRes(impl *types.Addon) (*apis.DetailAddonResponse, error) {
var defs []*apis.AddonDefinition
for _, def := range impl.Definitions {
obj := &unstructured.Unstructured{}
dec := k8syaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme)
_, _, err := dec.Decode([]byte(def.Data), nil, obj)
if err != nil {
return nil, errors.New(fmt.Sprintf("convert %s file content to definition fail", def.Name))
}
defs = append(defs, &apis.AddonDefinition{
obj.GetName(),
obj.GetKind(),
obj.GetAnnotations()["definition.oam.dev/description"],
})
}
return &apis.DetailAddonResponse{
AddonMeta: impl.AddonMeta,
APISchema: impl.APISchema,
UISchema: impl.UISchema,
Detail: impl.Detail,
Definitions: defs,
}, nil
}
// NewAddonUsecase returns a addon usecase
@@ -75,16 +91,47 @@ type addonUsecaseImpl struct {
// GetAddon will get addon information
func (u *addonUsecaseImpl) GetAddon(ctx context.Context, name string, registry string) (*apis.DetailAddonResponse, error) {
addonDetails, err := u.ListAddons(ctx, true, registry, "")
var addon *types.Addon
var err error
var exist bool
if registry == "" {
registries, err := u.ListAddonRegistries(ctx)
if err != nil {
return nil, err
}
for _, r := range registries {
if addon, exist = u.tryGetAddonFromCache(r.Name, name); !exist {
addon, err = pkgaddon.GetAddon(name, r.Git, pkgaddon.GetLevelOptions)
}
if err != nil && !errors.Is(err, pkgaddon.AddonNotExist) {
return nil, err
}
if addon != nil {
break
}
}
} else {
if addon, exist = u.tryGetAddonFromCache(registry, name); !exist {
addonRegistry, err := u.GetAddonRegistry(ctx, registry)
if err != nil {
return nil, err
}
addon, err = pkgaddon.GetAddon(name, addonRegistry.Git, pkgaddon.GetLevelOptions)
if err != nil && !errors.Is(err, pkgaddon.AddonNotExist) {
return nil, err
}
}
}
if addon == nil {
return nil, bcode.ErrAddonNotExist
}
a, err := AddonImpl2AddonRes(addon)
if err != nil {
return nil, err
}
for _, a := range addonDetails {
if a.Name == name {
return a, nil
}
}
return nil, bcode.ErrAddonNotExist
return a, nil
}
func (u *addonUsecaseImpl) StatusAddon(ctx context.Context, name string) (*apis.AddonStatusResponse, error) {
@@ -130,67 +177,66 @@ func (u *addonUsecaseImpl) StatusAddon(ctx context.Context, name string) (*apis.
}
}
// getCacheKeyWithListOptions will get right cache key for given method registry and detailed, to split different
func getCacheKeyWithListOptions(registry string, detailed bool, query string) string {
var d string
if detailed {
d = "detailed"
}
return fmt.Sprintf("%s/%s/%s", registry, d, query)
}
func (u *addonUsecaseImpl) ListAddons(ctx context.Context, detailed bool, registry, query string) ([]*apis.DetailAddonResponse, error) {
func (u *addonUsecaseImpl) ListAddons(ctx context.Context, registry, query string) ([]*apis.DetailAddonResponse, error) {
var addons []*types.Addon
var listAddons []*types.Addon
cacheKey := getCacheKeyWithListOptions(registry, detailed, query)
if u.isRegistryCacheUpToDate(cacheKey) {
addons = u.getRegistryCache(cacheKey)
} else {
rs, err := u.ListAddonRegistries(ctx)
if err != nil {
return nil, err
}
rs, err := u.ListAddonRegistries(ctx)
if err != nil {
return nil, err
}
for _, r := range rs {
if registry != "" && r.Name != registry {
continue
}
listAddons, err = pkgaddon.ListAddons(detailed, r.Git)
for _, r := range rs {
if registry != "" && r.Name != registry {
continue
}
if u.isRegistryCacheUpToDate(r.Name) {
listAddons = u.getRegistryCache(r.Name)
} else {
listAddons, err = pkgaddon.ListAddons(r.Git, pkgaddon.GetLevelOptions)
if err != nil {
log.Logger.Errorf("fail to get addons from registry %s", r.Name)
continue
}
addons = mergeAddons(addons, listAddons)
}
if query != "" {
var filtered []*types.Addon
for i, addon := range addons {
if strings.Contains(addon.Name, query) || strings.Contains(addon.Description, query) {
filtered = append(filtered, addons[i])
// if list addons, details will be retrieved later
go func() {
addonDetails, err := pkgaddon.ListAddons(r.Git, pkgaddon.EnableLevelOptions)
if err != nil {
return
}
}
addons = filtered
u.putRegistryCache(r.Name, addonDetails)
}()
}
sort.Slice(addons, func(i, j int) bool {
return addons[i].Name < addons[j].Name
})
if detailed {
for _, addon := range addons {
// render default ui schema
addon.UISchema = renderDefaultUISchema(addon.APISchema)
}
}
u.putRegistryCache(cacheKey, addons)
addons = mergeAddons(addons, listAddons)
}
var addonRes []*apis.DetailAddonResponse
if query != "" {
var filtered []*types.Addon
for i, addon := range addons {
if strings.Contains(addon.Name, query) || strings.Contains(addon.Description, query) {
filtered = append(filtered, addons[i])
}
}
addons = filtered
}
sort.Slice(addons, func(i, j int) bool {
return addons[i].Name < addons[j].Name
})
for _, addon := range addons {
// render default ui schema
addon.UISchema = renderDefaultUISchema(addon.APISchema)
}
var addonReses []*apis.DetailAddonResponse
for _, a := range addons {
addonRes = append(addonRes, AddonImpl2AddonRes(a))
addonRes, err := AddonImpl2AddonRes(a)
if err != nil {
log.Logger.Errorf("err while converting AddonImpl to DetailAddonResponse: %v", err)
continue
}
addonReses = append(addonReses, addonRes)
}
return addonRes, nil
return addonReses, nil
}
func (u *addonUsecaseImpl) DeleteAddonRegistry(ctx context.Context, name string) error {
@@ -262,19 +308,36 @@ func (u *addonUsecaseImpl) ListAddonRegistries(ctx context.Context) ([]*apis.Add
return list, nil
}
func (u *addonUsecaseImpl) tryGetAddonFromCache(registry, addonName string) (*types.Addon, bool) {
if u.isRegistryCacheUpToDate(registry) {
addons := u.getRegistryCache(registry)
for _, a := range addons {
if a.Name == addonName {
return a, true
}
}
}
return nil, false
}
func (u *addonUsecaseImpl) EnableAddon(ctx context.Context, name string, args apis.EnableAddonRequest) error {
var addon *types.Addon
var err error
registries, err := u.ListAddonRegistries(ctx)
if err != nil {
return err
}
for _, r := range registries {
addon, err := pkgaddon.GetAddon(name, r.Git)
if err != nil && errors.Is(err, bcode.ErrAddonNotExist) {
continue
} else if err != nil {
var exist bool
if addon, exist = u.tryGetAddonFromCache(r.Name, name); !exist {
addon, err = pkgaddon.GetAddon(name, r.Git, pkgaddon.EnableLevelOptions)
}
if err != nil && !errors.Is(err, pkgaddon.AddonNotExist) {
return bcode.WrapGithubRateLimitErr(err)
}
if addon == nil {
continue
}
app, defs, err := pkgaddon.RenderApplication(addon, args.Args)
if err != nil {
@@ -322,7 +385,7 @@ func (u *addonUsecaseImpl) getRegistryCache(name string) []*types.Addon {
}
func (u *addonUsecaseImpl) putRegistryCache(name string, addons []*types.Addon) {
u.addonRegistryCache[name] = restutils.NewMemoryCache(addons, time.Minute*3)
u.addonRegistryCache[name] = restutils.NewMemoryCache(addons, time.Minute*10)
}
func (u *addonUsecaseImpl) isRegistryCacheUpToDate(name string) bool {
+3 -2
View File
@@ -65,6 +65,7 @@ func (s *addonWebService) GetWebService() *restful.WebService {
Returns(200, "", apis.DetailAddonResponse{}).
Returns(400, "", bcode.Bcode{}).
Param(ws.PathParameter("name", "addon name to query detail").DataType("string").Required(true)).
Param(ws.QueryParameter("registry", "filter addons from given registry").DataType("string")).
Writes(apis.DetailAddonResponse{}))
// GET status
@@ -99,7 +100,7 @@ func (s *addonWebService) GetWebService() *restful.WebService {
}
func (s *addonWebService) listAddons(req *restful.Request, res *restful.Response) {
detailAddons, err := s.addonUsecase.ListAddons(req.Request.Context(), false, req.QueryParameter("registry"), req.QueryParameter("query"))
detailAddons, err := s.addonUsecase.ListAddons(req.Request.Context(), req.QueryParameter("registry"), req.QueryParameter("query"))
if err != nil {
bcode.ReturnError(req, res, err)
return
@@ -120,7 +121,7 @@ func (s *addonWebService) listAddons(req *restful.Request, res *restful.Response
func (s *addonWebService) detailAddon(req *restful.Request, res *restful.Response) {
name := req.PathParameter("name")
addon, err := s.addonUsecase.GetAddon(req.Request.Context(), name, "")
addon, err := s.addonUsecase.GetAddon(req.Request.Context(), name, req.QueryParameter("registry"))
if err != nil {
bcode.ReturnError(req, res, err)
return