/* Copyright 2021 The KubeVela Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package plugins import ( "context" "encoding/base64" "encoding/xml" "fmt" "io/ioutil" "net/http" "os" "path" "path/filepath" "github.com/oam-dev/kubevela/apis/types" "github.com/google/go-github/v32/github" "github.com/pkg/errors" "golang.org/x/oauth2" "github.com/oam-dev/kubevela/pkg/utils/common" ) // Registry define a registry stores trait & component defs type Registry interface { GetCap(addonName string) (types.Capability, []byte, error) ListCaps() ([]types.Capability, error) } // GithubRegistry is Registry's implementation treat github url as resource type GithubRegistry struct { client *github.Client cfg *GithubContent ctx context.Context centerName string // to be used to cache registry } // NewRegistry will create a registry implementation func NewRegistry(ctx context.Context, token, registryName string, regURL string) (Registry, error) { tp, cfg, err := Parse(regURL) if err != nil { return nil, err } switch tp { case TypeGithub: var tc *http.Client if token != "" { ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: token}, ) tc = oauth2.NewClient(ctx, ts) } return GithubRegistry{client: github.NewClient(tc), cfg: &cfg.GithubContent, ctx: ctx, centerName: registryName}, nil case TypeOss: var tc http.Client return OssRegistry{ Client: &tc, bucketURL: fmt.Sprintf("https://%s/", cfg.BucketURL), }, nil case TypeLocal: _, err := os.Stat(cfg.AbsDir) if os.IsNotExist(err) { return LocalRegistry{}, err } return LocalRegistry{absPath: cfg.AbsDir}, nil case TypeUnknown: return nil, fmt.Errorf("not supported url") } return nil, fmt.Errorf("not supported url") } // ListCaps list all capabilities of registry func (g GithubRegistry) ListCaps() ([]types.Capability, error) { var addons []types.Capability itemContents, err := g.getRepoFile() if err != nil { return []types.Capability{}, err } for _, item := range itemContents { capa, err := item.toAddon() if err != nil { fmt.Printf("parse definition of %s err %v\n", item.name, err) continue } addons = append(addons, capa) } return addons, nil } // GetCap return capability object and raw data specified by cap name func (g GithubRegistry) GetCap(addonName string) (types.Capability, []byte, error) { fileContent, _, _, err := g.client.Repositories.GetContents(context.Background(), g.cfg.Owner, g.cfg.Repo, fmt.Sprintf("%s/%s.yaml", g.cfg.Path, addonName), &github.RepositoryContentGetOptions{Ref: g.cfg.Ref}) if err != nil { return types.Capability{}, []byte{}, err } var data []byte if *fileContent.Encoding == "base64" { data, err = base64.StdEncoding.DecodeString(*fileContent.Content) if err != nil { fmt.Printf("decode github content %s err %s\n", fileContent.GetPath(), err) } } repoFile := RegistryFile{ data: data, name: *fileContent.Name, } addon, err := repoFile.toAddon() if err != nil { return types.Capability{}, []byte{}, err } return addon, data, nil } func (g *GithubRegistry) getRepoFile() ([]RegistryFile, error) { var items []RegistryFile _, dirs, _, err := g.client.Repositories.GetContents(g.ctx, g.cfg.Owner, g.cfg.Repo, g.cfg.Path, &github.RepositoryContentGetOptions{Ref: g.cfg.Ref}) if err != nil { return []RegistryFile{}, err } for _, repoItem := range dirs { if *repoItem.Type != "file" { continue } fileContent, _, _, err := g.client.Repositories.GetContents(g.ctx, g.cfg.Owner, g.cfg.Repo, *repoItem.Path, &github.RepositoryContentGetOptions{Ref: g.cfg.Ref}) if err != nil { fmt.Printf("Getting content URL %s error: %s\n", repoItem.GetURL(), err) continue } var data []byte if *fileContent.Encoding == "base64" { data, err = base64.StdEncoding.DecodeString(*fileContent.Content) if err != nil { fmt.Printf("decode github content %s err %s\n", fileContent.GetPath(), err) continue } } items = append(items, RegistryFile{ data: data, name: *fileContent.Name, }) } return items, nil } // OssRegistry is Registry's implementation treat OSS url as resource type OssRegistry struct { *http.Client bucketURL string centerName string } // GetCap return capability object and raw data specified by cap name func (o OssRegistry) GetCap(addonName string) (types.Capability, []byte, error) { filename := addonName + ".yaml" req, _ := http.NewRequestWithContext( context.Background(), http.MethodGet, o.bucketURL+filename, nil, ) resp, err := o.Client.Do(req) if err != nil { return types.Capability{}, nil, err } data, err := ioutil.ReadAll(resp.Body) _ = resp.Body.Close() if err != nil { return types.Capability{}, nil, err } rf := RegistryFile{ data: data, name: filename, } capa, err := rf.toAddon() if err != nil { return types.Capability{}, nil, err } return capa, data, nil } // ListCaps list all capabilities of registry func (o OssRegistry) ListCaps() ([]types.Capability, error) { rfs, err := o.getRegFiles() if err != nil { return []types.Capability{}, errors.Wrap(err, "Get raw files fail") } capas := make([]types.Capability, 0) for _, rf := range rfs { capa, err := rf.toAddon() if err != nil { fmt.Printf("[WARN] Parse file %s fail: %s\n", rf.name, err.Error()) } capas = append(capas, capa) } return capas, nil } func (o OssRegistry) getRegFiles() ([]RegistryFile, error) { req, _ := http.NewRequestWithContext( context.Background(), http.MethodGet, o.bucketURL+"?list-type=2", nil, ) resp, err := o.Client.Do(req) if err != nil { return []RegistryFile{}, err } data, err := ioutil.ReadAll(resp.Body) _ = resp.Body.Close() if err != nil { return []RegistryFile{}, err } list := &ListBucketResult{} err = xml.Unmarshal(data, list) if err != nil { return []RegistryFile{}, err } rfs := make([]RegistryFile, 0) for _, fileName := range list.File { req, _ := http.NewRequestWithContext( context.Background(), http.MethodGet, o.bucketURL+fileName, nil, ) resp, err := o.Client.Do(req) if err != nil { fmt.Printf("[WARN] %s download fail\n", fileName) continue } data, _ := ioutil.ReadAll(resp.Body) _ = resp.Body.Close() rf := RegistryFile{ data: data, name: fileName, } rfs = append(rfs, rf) } return rfs, nil } // LocalRegistry is Registry's implementation treat local url as resource type LocalRegistry struct { absPath string } // GetCap return capability object and raw data specified by cap name func (l LocalRegistry) GetCap(addonName string) (types.Capability, []byte, error) { fileName := addonName + ".yaml" filePath := fmt.Sprintf("%s/%s", l.absPath, fileName) data, err := os.ReadFile(filePath) if err != nil { return types.Capability{}, []byte{}, err } file := RegistryFile{ data: data, name: fileName, } capa, err := file.toAddon() if err != nil { return types.Capability{}, []byte{}, err } return capa, data, nil } // ListCaps list all capabilities of registry func (l LocalRegistry) ListCaps() ([]types.Capability, error) { glob := filepath.Join(filepath.Clean(l.absPath), "*") files, _ := filepath.Glob(glob) capas := make([]types.Capability, 0) for _, file := range files { // nolint:gosec data, err := ioutil.ReadFile(file) if err != nil { return nil, err } capa, err := RegistryFile{ data: data, name: path.Base(file), }.toAddon() if err != nil { fmt.Printf("parsing file: %s err: %s\n", file, err) continue } capas = append(capas, capa) } return capas, nil } func (item RegistryFile) toAddon() (types.Capability, error) { dm, err := (&common.Args{}).GetDiscoveryMapper() if err != nil { return types.Capability{}, err } capability, err := ParseCapability(dm, item.data) if err != nil { return types.Capability{}, err } return capability, nil } // RegistryFile describes a file item in registry type RegistryFile struct { data []byte // file content name string // file's name } // ListBucketResult describe a file list from OSS type ListBucketResult struct { File []string `xml:"Contents>Key"` Count int `xml:"KeyCount"` }