package plugins import ( "context" "encoding/base64" "errors" "fmt" "io/ioutil" "net/http" "net/url" "os" "path/filepath" "strings" "github.com/ghodss/yaml" "github.com/google/go-github/v32/github" "golang.org/x/oauth2" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/oam-dev/kubevela/apis/core.oam.dev/v1alpha2" "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/utils/system" ) // GithubContent for cap center type GithubContent struct { Owner string `json:"owner"` Repo string `json:"repo"` Path string `json:"path"` Ref string `json:"ref"` } // CapCenterConfig is used to store cap center config in file type CapCenterConfig struct { Name string `json:"name"` Address string `json:"address"` Token string `json:"token"` } // CenterClient defines an interface for cap center client type CenterClient interface { SyncCapabilityFromCenter() error } // NewCenterClient create a client from type func NewCenterClient(ctx context.Context, name, address, token string) (CenterClient, error) { Type, cfg, err := Parse(address) if err != nil { return nil, err } switch Type { case TypeGithub: return NewGithubCenter(ctx, token, name, cfg) default: } return nil, errors.New("we only support github as repository now") } // TypeGithub represents github const TypeGithub = "github" // TypeUnknown represents parse failed const TypeUnknown = "unknown" // Parse will parse config from address func Parse(addr string) (string, *GithubContent, error) { url, err := url.Parse(addr) if err != nil { return "", nil, err } l := strings.Split(strings.TrimPrefix(url.Path, "/"), "/") switch url.Host { case "github.com": // We support two valid format: // 1. https://github.com///tree// // 2. https://github.com/// if len(l) < 3 { return "", nil, errors.New("invalid format " + addr) } if l[2] == "tree" { // https://github.com///tree// if len(l) < 5 { return "", nil, errors.New("invalid format " + addr) } return TypeGithub, &GithubContent{ Owner: l[0], Repo: l[1], Path: strings.Join(l[4:], "/"), Ref: l[3], }, nil } // https://github.com/// return TypeGithub, &GithubContent{ Owner: l[0], Repo: l[1], Path: strings.Join(l[2:], "/"), Ref: "", // use default branch }, nil case "api.github.com": if len(l) != 5 { return "", nil, errors.New("invalid format " + addr) } //https://api.github.com/repos///contents/ return TypeGithub, &GithubContent{ Owner: l[1], Repo: l[2], Path: l[4], Ref: url.Query().Get("ref"), }, nil default: // TODO(wonderflow): support raw url and oss format in the future } return TypeUnknown, nil, nil } // RemoteCapability defines the capability discovered from remote cap center type RemoteCapability struct { // Name MUST be xxx.yaml Name string `json:"name"` URL string `json:"downloadUrl"` Sha string `json:"sha"` // Type MUST be file Type string `json:"type"` } // RemoteCapabilities is slice of cap center type RemoteCapabilities []RemoteCapability // LoadRepos will load all cap center repos // TODO(wonderflow): we can make default(built-in) repo configurable, then we should make default inside the answer func LoadRepos() ([]CapCenterConfig, error) { config, err := system.GetRepoConfig() if err != nil { return nil, err } data, err := ioutil.ReadFile(filepath.Clean(config)) if err != nil { if os.IsNotExist(err) { return []CapCenterConfig{}, nil } return nil, err } var repos []CapCenterConfig if err = yaml.Unmarshal(data, &repos); err != nil { return nil, err } return repos, nil } // StoreRepos will store cap center repo locally func StoreRepos(repos []CapCenterConfig) error { config, err := system.GetRepoConfig() if err != nil { return err } data, err := yaml.Marshal(repos) if err != nil { return err } //nolint:gosec return ioutil.WriteFile(config, data, 0644) } // ParseAndSyncCapability will convert config from remote center to capability func ParseAndSyncCapability(data []byte, syncDir string) (types.Capability, error) { var obj = unstructured.Unstructured{Object: make(map[string]interface{})} err := yaml.Unmarshal(data, &obj.Object) if err != nil { return types.Capability{}, err } switch obj.GetKind() { case "WorkloadDefinition": var rd v1alpha2.WorkloadDefinition err = yaml.Unmarshal(data, &rd) if err != nil { return types.Capability{}, err } return HandleDefinition(rd.Name, syncDir, rd.Spec.Reference.Name, rd.Annotations, rd.Spec.Extension, types.TypeWorkload, nil) case "TraitDefinition": var td v1alpha2.TraitDefinition err = yaml.Unmarshal(data, &td) if err != nil { return types.Capability{}, err } return HandleDefinition(td.Name, syncDir, td.Spec.Reference.Name, td.Annotations, td.Spec.Extension, types.TypeTrait, td.Spec.AppliesToWorkloads) case "ScopeDefinition": // TODO(wonderflow): support scope definition here. } return types.Capability{}, fmt.Errorf("unknown definition Type %s", obj.GetKind()) } // GithubCenter implementation of cap center type GithubCenter struct { client *github.Client cfg *GithubContent centerName string ctx context.Context } var _ CenterClient = &GithubCenter{} // NewGithubCenter will create client by github center implementation func NewGithubCenter(ctx context.Context, token, centerName string, r *GithubContent) (*GithubCenter, error) { var tc *http.Client if token != "" { ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: token}, ) tc = oauth2.NewClient(ctx, ts) } return &GithubCenter{client: github.NewClient(tc), cfg: r, centerName: centerName, ctx: ctx}, nil } // SyncCapabilityFromCenter will sync capability from github cap center // TODO(wonderflow): currently we only sync by create, we also need to delete which not exist remotely. func (g *GithubCenter) SyncCapabilityFromCenter() error { _, dirs, _, err := g.client.Repositories.GetContents(g.ctx, g.cfg.Owner, g.cfg.Repo, g.cfg.Path, &github.RepositoryContentGetOptions{Ref: g.cfg.Ref}) if err != nil { return err } dir, err := system.GetCapCenterDir() if err != nil { return err } repoDir := filepath.Join(dir, g.centerName) _, _ = system.CreateIfNotExist(repoDir) var success, total int for _, addon := range dirs { if *addon.Type != "file" { continue } total++ fileContent, _, _, err := g.client.Repositories.GetContents(g.ctx, g.cfg.Owner, g.cfg.Repo, *addon.Path, &github.RepositoryContentGetOptions{Ref: g.cfg.Ref}) if err != nil { return err } var data = []byte(*fileContent.Content) if *fileContent.Encoding == "base64" { data, err = base64.StdEncoding.DecodeString(*fileContent.Content) if err != nil { return fmt.Errorf("decode github content %s err %w", *fileContent.Path, err) } } tmp, err := ParseAndSyncCapability(data, filepath.Join(dir, ".tmp")) if err != nil { fmt.Printf("parse definition of %s err %v\n", *fileContent.Name, err) continue } //nolint:gosec err = ioutil.WriteFile(filepath.Join(repoDir, tmp.CrdName+".yaml"), data, 0644) if err != nil { fmt.Printf("write definition %s to %s err %v\n", tmp.CrdName+".yaml", repoDir, err) continue } success++ } fmt.Printf("successfully sync %d/%d from %s remote center\n", success, total, g.centerName) return nil }