kubevela/pkg/addon/source.go

/*
Copyright 2021 The KubeVela Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package addon

import (
	"fmt"
	"net/url"
	"path"
	"strings"

	"github.com/go-resty/resty/v2"
	"github.com/pkg/errors"
	gitlab "gitlab.com/gitlab-org/api/client-go"

	"github.com/oam-dev/kubevela/pkg/utils"
)

const (
	// EOFError is error returned by xml parse
	EOFError string = "EOF"
	// DirType means a directory
	DirType = "dir"
	// FileType means a file
	FileType = "file"
	// BlobType means a blob
	BlobType = "blob"
	// TreeType means a tree
	TreeType = "tree"

	bucketTmpl        = "%s://%s.%s"
	singleOSSFileTmpl = "%s/%s"
	listOSSFileTmpl   = "%s?max-keys=1000&prefix=%s"
)

// Source is where to get addons, Registry implement this interface
type Source interface {
	GetUIData(meta *SourceMeta, opt ListOptions) (*UIData, error)
	ListUIData(registryAddonMeta map[string]SourceMeta, opt ListOptions) ([]*UIData, error)
	GetInstallPackage(meta *SourceMeta, uiData *UIData) (*InstallPackage, error)
	ListAddonMeta() (map[string]SourceMeta, error)
}

// GitAddonSource defines the information about the Git as addon source
type GitAddonSource struct {
	URL            string `json:"url,omitempty" validate:"required"`
	Path           string `json:"path,omitempty"`
	Token          string `json:"token,omitempty"`
	TokenSecretRef string `json:"tokenSecretRef,omitempty"`
}

// GetToken returns the token of the source
func (g *GitAddonSource) GetToken() string {
	return g.Token
}

// SetToken set the token of the source
func (g *GitAddonSource) SetToken(token string) {
	g.Token = token
	g.TokenSecretRef = ""
}

// SetTokenSecretRef set the token secret ref to the source
func (g *GitAddonSource) SetTokenSecretRef(secretName string) {
	g.Token = ""
	g.TokenSecretRef = secretName
}

// GetTokenSecretRef return the token secret ref of the source
func (g *GitAddonSource) GetTokenSecretRef() string {
	return g.TokenSecretRef
}

// SafeCopy hides field Token
func (g *GitAddonSource) SafeCopy() *GitAddonSource {
	if g == nil {
		return nil
	}
	return &GitAddonSource{
		URL:            g.URL,
		Path:           g.Path,
		TokenSecretRef: g.TokenSecretRef,
	}
}

// GiteeAddonSource defines the information about the Gitee as addon source
type GiteeAddonSource struct {
	URL            string `json:"url,omitempty" validate:"required"`
	Path           string `json:"path,omitempty"`
	Token          string `json:"token,omitempty"`
	TokenSecretRef string `json:"tokenSecretRef,omitempty"`
}

// GetToken return the token of the source
func (g *GiteeAddonSource) GetToken() string {
	return g.Token
}

// SetToken set the token of the source
func (g *GiteeAddonSource) SetToken(token string) {
	g.Token = token
	g.TokenSecretRef = ""
}

// SetTokenSecretRef set the token secret ref to the source
func (g *GiteeAddonSource) SetTokenSecretRef(secretName string) {
	g.Token = ""
	g.TokenSecretRef = secretName
}

// GetTokenSecretRef return the token secret ref of the source
func (g *GiteeAddonSource) GetTokenSecretRef() string {
	return g.TokenSecretRef
}

// SafeCopy hides field Token
func (g *GiteeAddonSource) SafeCopy() *GiteeAddonSource {
	if g == nil {
		return nil
	}
	return &GiteeAddonSource{
		URL:            g.URL,
		Path:           g.Path,
		TokenSecretRef: g.TokenSecretRef,
	}
}

// GitlabAddonSource defines the information about Gitlab as an addon source
type GitlabAddonSource struct {
	URL            string `json:"url,omitempty" validate:"required"`
	Repo           string `json:"repo,omitempty" validate:"required"`
	Path           string `json:"path,omitempty"`
	Token          string `json:"token,omitempty"`
	TokenSecretRef string `json:"tokenSecretRef,omitempty"`
}

// GetToken return the token of the source
func (g *GitlabAddonSource) GetToken() string {
	return g.Token
}

// SetToken set the token of the source
func (g *GitlabAddonSource) SetToken(token string) {
	g.Token = token
	g.TokenSecretRef = ""
}

// SetTokenSecretRef set the token secret ref to the source
func (g *GitlabAddonSource) SetTokenSecretRef(secretName string) {
	g.Token = ""
	g.TokenSecretRef = secretName
}

// GetTokenSecretRef return the token secret ref of the source
func (g *GitlabAddonSource) GetTokenSecretRef() string {
	return g.TokenSecretRef
}

// SafeCopy hides field Token
func (g *GitlabAddonSource) SafeCopy() *GitlabAddonSource {
	if g == nil {
		return nil
	}
	return &GitlabAddonSource{
		URL:  g.URL,
		Repo: g.Repo,
		Path: g.Path,
	}
}

// HelmSource  defines the information about the helm repo addon source
type HelmSource struct {
	URL             string `json:"url,omitempty" validate:"required"`
	InsecureSkipTLS bool   `json:"insecureSkipTLS,omitempty"`
	Username        string `json:"username,omitempty"`
	Password        string `json:"password,omitempty"`
}

// SafeCopier is an interface to copy struct without sensitive fields, such as Token, Username, Password
type SafeCopier interface {
	SafeCopy() interface{}
}

// SafeCopy hides field Username, Password
func (h *HelmSource) SafeCopy() *HelmSource {
	if h == nil {
		return nil
	}
	return &HelmSource{
		URL: h.URL,
	}
}

// Item is a partial interface for github.RepositoryContent
type Item interface {
	// GetType return "dir" or "file"
	GetType() string
	GetPath() string
	GetName() string
}

// SourceMeta record the whole metadata of an addon
type SourceMeta struct {
	Name  string
	Items []Item
}

// ClassifyItemByPattern will filter and classify addon data, data will be classified by pattern it meets
func ClassifyItemByPattern(meta *SourceMeta, r AsyncReader) map[string][]Item {
	var p = make(map[string][]Item)
	for _, it := range meta.Items {
		pt := GetPatternFromItem(it, r, meta.Name)
		if pt == "" {
			continue
		}
		items := p[pt]
		items = append(items, it)
		p[pt] = items
	}
	return p
}

// AsyncReader helps async read files of addon
type AsyncReader interface {
	// ListAddonMeta will return directory tree contain addon metadata only
	ListAddonMeta() (addonCandidates map[string]SourceMeta, err error)

	// ReadFile should accept relative path to github repo/path or OSS bucket, and report the file content
	ReadFile(path string) (content string, err error)

	// RelativePath return a relative path to GitHub repo/path or OSS bucket/path
	RelativePath(item Item) string
}

// pathWithParent joins path with its parent directory, suffix slash is reserved
func pathWithParent(subPath, parent string) string {
	actualPath := path.Join(parent, subPath)
	if strings.HasSuffix(subPath, "/") {
		actualPath += "/"
	}
	return actualPath
}

// ReaderType marks where to read addon files
type ReaderType string

const (
	gitType    ReaderType = "git"
	ossType    ReaderType = "oss"
	giteeType  ReaderType = "gitee"
	gitlabType ReaderType = "gitlab"
)

// NewAsyncReader create AsyncReader from
// 1. GitHub url and directory
// 2. OSS endpoint and bucket
func NewAsyncReader(baseURL, bucket, repo, subPath, token string, rdType ReaderType) (AsyncReader, error) {

	switch rdType {
	case gitType:
		baseURL = strings.TrimSuffix(baseURL, ".git")
		u, err := url.Parse(baseURL)
		if err != nil {
			return nil, errors.New("addon registry invalid")
		}
		u.Path = path.Join(u.Path, subPath)
		_, content, err := utils.Parse(u.String())
		if err != nil {
			return nil, err
		}
		gith := createGitHelper(content, token)
		return &gitReader{
			h: gith,
		}, nil
	case ossType:
		ossURL, err := url.Parse(baseURL)
		if err != nil {
			return nil, err
		}
		var bucketEndPoint string
		if bucket == "" {
			bucketEndPoint = ossURL.String()
		} else {
			if ossURL.Scheme == "" {
				ossURL.Scheme = "https"
			}
			bucketEndPoint = fmt.Sprintf(bucketTmpl, ossURL.Scheme, bucket, ossURL.Host)
		}
		return &ossReader{
			bucketEndPoint: bucketEndPoint,
			path:           subPath,
			client:         resty.New(),
		}, nil
	case giteeType:
		baseURL = strings.TrimSuffix(baseURL, ".git")
		u, err := url.Parse(baseURL)
		if err != nil {
			return nil, errors.New("addon registry invalid")
		}
		u.Path = path.Join(u.Path, subPath)
		_, content, err := utils.Parse(u.String())
		if err != nil {
			return nil, err
		}
		gitee := createGiteeHelper(content, token)
		return &giteeReader{
			h: gitee,
		}, nil
	case gitlabType:
		baseURL = strings.TrimSuffix(baseURL, ".git")
		u, err := url.Parse(baseURL)
		if err != nil {
			return nil, errors.New("addon registry invalid")
		}
		_, content, err := utils.ParseGitlab(u.String(), repo)
		content.GitlabContent.Path = subPath
		if err != nil {
			return nil, err
		}
		gitlabHelper, err := createGitlabHelper(content, token)
		if err != nil {
			return nil, errors.New("addon registry connect fail")
		}

		err = gitlabHelper.getGitlabProject(content)
		if err != nil {
			return nil, err
		}

		return &gitlabReader{
			h: gitlabHelper,
		}, nil
	}
	return nil, fmt.Errorf("invalid addon registry type '%s'", rdType)
}

// getGitlabProject get gitlab project , set project id
func (h *gitlabHelper) getGitlabProject(content *utils.Content) error {
	projectURL := content.GitlabContent.Owner + "/" + content.GitlabContent.Repo
	projects, _, err := h.Client.Projects.GetProject(projectURL, &gitlab.GetProjectOptions{})
	if err != nil {
		return err
	}
	content.GitlabContent.PId = projects.ID

	return nil
}

// BuildReader will build a AsyncReader from registry, AsyncReader are needed to read addon files
func (r *Registry) BuildReader() (AsyncReader, error) {
	if r.OSS != nil {
		o := r.OSS
		return NewAsyncReader(o.Endpoint, o.Bucket, "", o.Path, "", ossType)
	}
	if r.Git != nil {
		g := r.Git
		return NewAsyncReader(g.URL, "", "", g.Path, g.Token, gitType)
	}
	if r.Gitee != nil {
		g := r.Gitee
		return NewAsyncReader(g.URL, "", "", g.Path, g.Token, giteeType)
	}
	if r.Gitlab != nil {
		g := r.Gitlab
		return NewAsyncReader(g.URL, "", g.Repo, g.Path, g.Token, gitlabType)
	}
	return nil, errors.New("registry don't have enough info to build a reader")
}

// GetUIData get UIData of an addon
func (r *Registry) GetUIData(meta *SourceMeta, opt ListOptions) (*UIData, error) {
	reader, err := r.BuildReader()
	if err != nil {
		return nil, err
	}
	addon, err := GetUIDataFromReader(reader, meta, opt)
	if err != nil {
		return nil, err
	}
	if len(addon.GlobalParameters) != 0 {
		addon.Parameters = addon.GlobalParameters
	}
	addon.RegistryName = r.Name
	return addon, nil
}

// ListUIData list UI data from addon registry
func (r *Registry) ListUIData(registryAddonMeta map[string]SourceMeta, opt ListOptions) ([]*UIData, error) {
	reader, err := r.BuildReader()
	if err != nil {
		return nil, err
	}
	return ListAddonUIDataFromReader(reader, registryAddonMeta, r.Name, opt)
}

// GetInstallPackage get install package which is all needed to enable an addon from addon registry
func (r *Registry) GetInstallPackage(meta *SourceMeta, uiData *UIData) (*InstallPackage, error) {
	reader, err := r.BuildReader()
	if err != nil {
		return nil, err
	}
	return GetInstallPackageFromReader(reader, meta, uiData)
}

// ListAddonMeta list addon file meta(path and name) from a registry
func (r *Registry) ListAddonMeta() (map[string]SourceMeta, error) {
	reader, err := r.BuildReader()
	if err != nil {
		return nil, err
	}
	return reader.ListAddonMeta()
}

// ItemInfo contains summary information about an addon
type ItemInfo struct {
	Name              string
	Description       string
	AvailableVersions []string
}

type itemInfoMap map[string]ItemInfo

// ListAddonInfo lists addon info (name, versions, etc.) from a registry
func (r *Registry) ListAddonInfo() (map[string]ItemInfo, error) {
	addonInfoMap := make(map[string]ItemInfo)

	// local registry doesn't support listing addons
	if IsLocalRegistry(*r) {
		return addonInfoMap, nil
	}

	if IsVersionRegistry(*r) {
		versionedRegistry, err := ToVersionedRegistry(*r)
		if err != nil {
			return nil, err
		}
		addonList, err := versionedRegistry.ListAddon()
		if err != nil {
			return nil, err
		}
		for _, a := range addonList {
			addonInfoMap[a.Name] = ItemInfo{
				Name:              a.Name,
				Description:       a.Description,
				AvailableVersions: a.AvailableVersions,
			}
		}
	} else {
		meta, err := r.ListAddonMeta()
		if err != nil {
			return nil, err
		}
		addonList, err := r.ListUIData(meta, ListOptions{})
		if err != nil {
			return nil, err
		}
		for _, a := range addonList {
			addonInfoMap[a.Name] = ItemInfo{
				Name:              a.Name,
				Description:       a.Description,
				AvailableVersions: a.AvailableVersions,
			}
		}
	}

	return addonInfoMap, nil
}