diff --git a/.github/workflows/apiserver-test.yaml b/.github/workflows/apiserver-test.yaml index 25483be82..66e8e0275 100644 --- a/.github/workflows/apiserver-test.yaml +++ b/.github/workflows/apiserver-test.yaml @@ -93,6 +93,7 @@ jobs: run: | export ALIYUN_ACCESS_KEY_ID=${{ secrets.ALIYUN_ACCESS_KEY_ID }} export ALIYUN_ACCESS_KEY_SECRET=${{ secrets.ALIYUN_ACCESS_KEY_SECRET }} + export GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} make e2e-apiserver-test - name: Stop kubevela, get profile diff --git a/charts/vela-core/templates/addons/fluxcd.yaml b/charts/vela-core/templates/addons/fluxcd.yaml index ed1737aa8..35021e140 100644 --- a/charts/vela-core/templates/addons/fluxcd.yaml +++ b/charts/vela-core/templates/addons/fluxcd.yaml @@ -6020,6 +6020,12 @@ data: - name: apply-resources type: apply-remaining status: {} + detail: "# fluxcd\n\nThis addon is built based [FluxCD](https://fluxcd.io/) \n\n## + install\n\n```shell\nvela addon enable fluxcd\n```\n\n## X-Definitions\n\nEnable + fluxcd addon to use these X-definitions\n\n- [helm](https://kubevela.io/docs/end-user/components/helm) + helps to deploy a helm chart from everywhere:\ngit repo / helm repo / S3 compatible + bucket.\n\n- [kustomize](https://kubevela.io/docs/end-user/components/kustomize) + helps to deploy a kustomize style artifact.\n" kind: ConfigMap metadata: annotations: diff --git a/charts/vela-core/templates/addons/istio.yaml b/charts/vela-core/templates/addons/istio.yaml index f3d96695d..8d809d270 100644 --- a/charts/vela-core/templates/addons/istio.yaml +++ b/charts/vela-core/templates/addons/istio.yaml @@ -245,6 +245,10 @@ data: - name: apply-resources type: apply-remaining status: {} + detail: |- + # istio + + This addon provides istio support for vela rollout. kind: ConfigMap metadata: annotations: diff --git a/charts/vela-core/templates/addons/keda.yaml b/charts/vela-core/templates/addons/keda.yaml index db035c2fb..6936a196f 100644 --- a/charts/vela-core/templates/addons/keda.yaml +++ b/charts/vela-core/templates/addons/keda.yaml @@ -26,6 +26,10 @@ data: - name: apply-resources type: apply-application status: {} + detail: |- + # keda + + keda kind: ConfigMap metadata: annotations: diff --git a/charts/vela-core/templates/addons/kruise.yaml b/charts/vela-core/templates/addons/kruise.yaml index 45fed21c4..2a57c3a77 100644 --- a/charts/vela-core/templates/addons/kruise.yaml +++ b/charts/vela-core/templates/addons/kruise.yaml @@ -170,6 +170,10 @@ data: - name: apply-resources type: apply-application status: {} + detail: |- + # kruise + + This addon provides [open-kruise](https://github.com/openkruise/kruise) workload. kind: ConfigMap metadata: annotations: diff --git a/charts/vela-core/templates/addons/observability.yaml b/charts/vela-core/templates/addons/observability.yaml index 8bf60330e..faaa1f327 100644 --- a/charts/vela-core/templates/addons/observability.yaml +++ b/charts/vela-core/templates/addons/observability.yaml @@ -231,6 +231,10 @@ data: - name: apply-resources type: apply-remaining status: {} + detail: |- + # observability + + This addon expose system and application level metrics for KubeVela. kind: ConfigMap metadata: annotations: diff --git a/charts/vela-core/templates/addons/ocm-cluster-manager.yaml b/charts/vela-core/templates/addons/ocm-cluster-manager.yaml index 0c6f4f05f..9b6d35e24 100644 --- a/charts/vela-core/templates/addons/ocm-cluster-manager.yaml +++ b/charts/vela-core/templates/addons/ocm-cluster-manager.yaml @@ -503,6 +503,10 @@ data: - name: apply-resources type: apply-remaining status: {} + detail: |- + # ocm-cluster-manager + + This addon aims to support multi-cluster application deployment. kind: ConfigMap metadata: annotations: diff --git a/charts/vela-core/templates/addons/prometheus.yaml b/charts/vela-core/templates/addons/prometheus.yaml index 1af8c7791..8c6f9e314 100644 --- a/charts/vela-core/templates/addons/prometheus.yaml +++ b/charts/vela-core/templates/addons/prometheus.yaml @@ -27,6 +27,10 @@ data: - name: apply-resources type: apply-application status: {} + detail: | + # prometheus + + prometheus kind: ConfigMap metadata: annotations: diff --git a/charts/vela-core/templates/addons/terraform-provider-alibaba.yaml b/charts/vela-core/templates/addons/terraform-provider-alibaba.yaml index abeeceeed..7bef5dae5 100644 --- a/charts/vela-core/templates/addons/terraform-provider-alibaba.yaml +++ b/charts/vela-core/templates/addons/terraform-provider-alibaba.yaml @@ -43,6 +43,10 @@ data: region: '[[ index .Args "ALICLOUD_REGION" ]]' type: raw status: {} + detail: |- + # terraform/provider-alibaba + + This addon contains terraform provider for Alibaba Cloud. kind: ConfigMap metadata: annotations: diff --git a/charts/vela-core/templates/addons/terraform-provider-aws.yaml b/charts/vela-core/templates/addons/terraform-provider-aws.yaml index 7ecaf8275..d47bc477d 100644 --- a/charts/vela-core/templates/addons/terraform-provider-aws.yaml +++ b/charts/vela-core/templates/addons/terraform-provider-aws.yaml @@ -43,6 +43,10 @@ data: region: '[[ index .Args "AWS_DEFAULT_REGION" ]]' type: raw status: {} + detail: |- + # terraform/provider-aws + + This addon contains terraform provider for AWS kind: ConfigMap metadata: annotations: diff --git a/charts/vela-core/templates/addons/terraform-provider-azure.yaml b/charts/vela-core/templates/addons/terraform-provider-azure.yaml index a8c60409b..c74593283 100644 --- a/charts/vela-core/templates/addons/terraform-provider-azure.yaml +++ b/charts/vela-core/templates/addons/terraform-provider-azure.yaml @@ -43,6 +43,10 @@ data: provider: azure type: raw status: {} + detail: |- + # terraform/provider-azure + + This addon contains terraform provider for Azure. kind: ConfigMap metadata: annotations: diff --git a/charts/vela-core/templates/addons/terraform.yaml b/charts/vela-core/templates/addons/terraform.yaml index 0d6fbf5ce..6c1e84893 100644 --- a/charts/vela-core/templates/addons/terraform.yaml +++ b/charts/vela-core/templates/addons/terraform.yaml @@ -522,6 +522,8 @@ data: - name: apply-resources type: apply-remaining status: {} + detail: "# Terraform\n\nThis addon contains terraform operation kit, which allows + you to arrange, \ngenerate and use cloud service from different cloud vendor." kind: ConfigMap metadata: annotations: diff --git a/pkg/apiserver/model/addon.go b/pkg/apiserver/model/addon.go new file mode 100644 index 000000000..fffd52e38 --- /dev/null +++ b/pkg/apiserver/model/addon.go @@ -0,0 +1,51 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +// AddonRegistry defines the data model of a AddonRegistry +type AddonRegistry struct { + Model + Name string `json:"name"` + + Git *GitAddonSource `json:"git,omitempty"` +} + +// GitAddonSource defines the information about the Git as addon source +type GitAddonSource struct { + URL string `json:"url,omitempty"` + Path string `json:"path,omitempty"` + Token string `json:"token,omitempty"` +} + +// TableName return custom table name +func (a *AddonRegistry) TableName() string { + return tableNamePrefix + "addon_registry" +} + +// PrimaryKey return custom primary key +func (a *AddonRegistry) PrimaryKey() string { + return a.Name +} + +// Index return custom index +func (a *AddonRegistry) Index() map[string]string { + index := make(map[string]string) + if a.Name != "" { + index["name"] = a.Name + } + return index +} diff --git a/pkg/apiserver/model/catalog.go b/pkg/apiserver/model/catalog.go deleted file mode 100644 index 2e334d1ab..000000000 --- a/pkg/apiserver/model/catalog.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2021 The KubeVela Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package model - -// Catalog defines the data model of a Catalog -type Catalog struct { - Name string `json:"name,omitempty"` - Desc string `json:"desc,omitempty"` - // UpdatedAt is the unix time of the last time when the catalog is updated. - UpdatedAt int64 `json:"updated_at,omitempty"` - // Type of the Catalog, such as "github" for a github repo. - Type string `json:"type,omitempty"` - // URL of the Catalog. - URL string `json:"url,omitempty"` - // Auth token used to sync Catalog. - Token string `json:"token,omitempty"` -} diff --git a/pkg/apiserver/rest/apis/v1/types.go b/pkg/apiserver/rest/apis/v1/types.go index 4457f50a2..047611ae6 100644 --- a/pkg/apiserver/rest/apis/v1/types.go +++ b/pkg/apiserver/rest/apis/v1/types.go @@ -25,8 +25,10 @@ import ( "github.com/oam-dev/kubevela/pkg/cloudprovider" ) -// CtxKeyApplication request context key of application -var CtxKeyApplication = "application" +var ( + // CtxKeyApplication request context key of application + CtxKeyApplication = "application" +) // CtxKeyWorkflow request context key of workflow var CtxKeyWorkflow = "workflow" @@ -37,8 +39,6 @@ type AddonPhase string const ( // AddonPhaseDisabled indicates the addon is disabled AddonPhaseDisabled AddonPhase = "disabled" - // AddonPhaseDisabling indicates the addon is disabling - AddonPhaseDisabling AddonPhase = "disabling" // AddonPhaseEnabled indicates the addon is enabled AddonPhaseEnabled AddonPhase = "enabled" // AddonPhaseEnabling indicates the addon is enabling @@ -48,32 +48,30 @@ const ( // EmptyResponse empty response, it will used for delete api type EmptyResponse struct{} -// CreateAddonRequest defines the format for addon create request -type CreateAddonRequest struct { - Name string `json:"name" validate:"name"` +// CreateAddonRegistryRequest defines the format for addon registry create request +type CreateAddonRegistryRequest struct { + Name string `json:"name" validate:"required"` - Version string `json:"version" validate:"required"` + Git *model.GitAddonSource `json:"git,omitempty"` +} - // Short description about the addon. - Description string `json:"description,omitempty"` +// AddonRegistryMeta defines the format for a single addon registry +type AddonRegistryMeta struct { + Name string `json:"name" validate:"required"` - Icon string `json:"icon"` + Git *model.GitAddonSource `json:"git,omitempty"` +} - Tags []string `json:"tags"` +// EnableAddonRequest defines the format for enable addon request +type EnableAddonRequest struct { - // The detail of the addon. Could be the entire README data. - Detail string `json:"detail,omitempty"` - - // DeployData is the object to deploy to the cluster to enable addon - DeployData string `json:"deploy_data,omitempty" validate:"required_without=deploy_url"` - - // DeployURL is the URL to the data file location in a Git repository - DeployURL string `json:"deploy_url,omitempty" validate:"required_without=deploy_data"` + // Args is the key-value environment variables, e.g. AK/SK credentials. + Args map[string]string `json:"args,omitempty"` } // ListAddonResponse defines the format for addon list response type ListAddonResponse struct { - Addons []AddonMeta `json:"addons"` + Addons []*AddonMeta `json:"addons"` } // AddonMeta defines the format for a single addon @@ -87,26 +85,30 @@ type AddonMeta struct { Icon string `json:"icon"` Tags []string `json:"tags"` - - Phase AddonPhase `json:"phase"` } // DetailAddonResponse defines the format for showing the addon details type DetailAddonResponse struct { AddonMeta + // More details about the addon, e.g. README Detail string `json:"detail,omitempty"` - // DeployData is the object to deploy to the cluster to enable addon + // DeployData is the object to apply to enable addon, e.g. Application DeployData string `json:"deploy_data,omitempty"` - - // DeployURL is the URL to the data file location in a Git repository - DeployURL string `json:"deploy_url,omitempty"` } // AddonStatusResponse defines the format of addon status response type AddonStatusResponse struct { Phase AddonPhase `json:"phase"` + + EnablingProgress *EnablingProgress `json:"enabling_progress,omitempty"` +} + +// EnablingProgress defines the progress of enabling an addon +type EnablingProgress struct { + EnabledComponents int `json:"enabled_components"` + TotalComponents int `json:"total_components"` } // AccessKeyRequest request parameters to access cloud provider diff --git a/pkg/apiserver/rest/rest_server.go b/pkg/apiserver/rest/rest_server.go index 8685f2166..0203fb7b1 100644 --- a/pkg/apiserver/rest/rest_server.go +++ b/pkg/apiserver/rest/rest_server.go @@ -22,7 +22,7 @@ import ( "net/http" restfulspec "github.com/emicklei/go-restful-openapi/v2" - restful "github.com/emicklei/go-restful/v3" + "github.com/emicklei/go-restful/v3" "github.com/go-openapi/spec" "github.com/oam-dev/kubevela/pkg/apiserver/datastore" @@ -68,11 +68,12 @@ func New(cfg Config) (a APIServer, err error) { case "kubeapi": ds, err = kubeapi.New(context.Background(), cfg.Datastore) if err != nil { - return nil, fmt.Errorf("create mongodb datastore instance failure %w", err) + return nil, fmt.Errorf("create kubeapi datastore instance failure %w", err) } default: return nil, fmt.Errorf("not support datastore type %s", cfg.Datastore.Type) } + s := &restServer{ webContainer: restful.NewContainer(), cfg: cfg, diff --git a/pkg/apiserver/rest/usecase/addon.go b/pkg/apiserver/rest/usecase/addon.go new file mode 100644 index 000000000..0b4e38dfa --- /dev/null +++ b/pkg/apiserver/rest/usecase/addon.go @@ -0,0 +1,403 @@ +package usecase + +import ( + "bytes" + "context" + "errors" + "fmt" + "github.com/Masterminds/sprig" + "github.com/oam-dev/kubevela/pkg/apiserver/log" + "golang.org/x/oauth2" + "net/http" + "net/url" + "path" + "sort" + "strings" + "text/template" + + errors2 "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/google/go-github/v32/github" + common2 "github.com/oam-dev/kubevela/apis/core.oam.dev/common" + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + "github.com/oam-dev/kubevela/apis/types" + "github.com/oam-dev/kubevela/pkg/apiserver/clients" + "github.com/oam-dev/kubevela/pkg/apiserver/datastore" + "github.com/oam-dev/kubevela/pkg/apiserver/model" + apis "github.com/oam-dev/kubevela/pkg/apiserver/rest/apis/v1" + restutils "github.com/oam-dev/kubevela/pkg/apiserver/rest/utils" + "github.com/oam-dev/kubevela/pkg/apiserver/rest/utils/bcode" + "github.com/oam-dev/kubevela/pkg/utils" + addonutil "github.com/oam-dev/kubevela/pkg/utils/addon" + "github.com/oam-dev/kubevela/pkg/utils/apply" +) + +const ( + // AddonFileName is the addon file name + AddonFileName string = "addon.yaml" + // AddonReadmeFileName is the addon readme file name + AddonReadmeFileName string = "readme.md" +) + +// AddonUsecase addon usecase +type AddonUsecase interface { + GetAddonRegistryModel(ctx context.Context, name string) (*model.AddonRegistry, error) + CreateAddonRegistry(ctx context.Context, req apis.CreateAddonRegistryRequest) (*apis.AddonRegistryMeta, error) + ListAddons(ctx context.Context, detailed bool) ([]*apis.DetailAddonResponse, error) + StatusAddon(name string) (*apis.AddonStatusResponse, error) + GetAddon(ctx context.Context, name string) (*apis.DetailAddonResponse, error) + EnableAddon(ctx context.Context, name string, args apis.EnableAddonRequest) error + DisableAddon(ctx context.Context, name string) error +} + +// NewAddonUsecase returns a addon usecase +func NewAddonUsecase(ds datastore.DataStore) AddonUsecase { + kubecli, err := clients.GetKubeClient() + if err != nil { + panic(err) + } + return &addonUsecaseImpl{ + ds: ds, + kubeClient: kubecli, + apply: apply.NewAPIApplicator(kubecli), + } +} + +type addonUsecaseImpl struct { + ds datastore.DataStore + kubeClient client.Client + apply apply.Applicator +} + +func (u *addonUsecaseImpl) GetAddon(ctx context.Context, name string) (*apis.DetailAddonResponse, error) { + addons, err := u.ListAddons(ctx, true) + if err != nil { + return nil, err + } + + for _, addon := range addons { + if addon.Name == name { + return addon, nil + } + } + return nil, bcode.ErrAddonNotExist +} + +func (u *addonUsecaseImpl) StatusAddon(name string) (*apis.AddonStatusResponse, error) { + _, err := u.GetAddon(context.TODO(), name) + if err != nil { + return nil, err + } + + var app v1beta1.Application + err = u.kubeClient.Get(context.Background(), client.ObjectKey{ + Namespace: types.DefaultKubeVelaNS, + Name: addonutil.TransAddonName(name), + }, &app) + if err != nil { + if errors2.IsNotFound(err) { + return &apis.AddonStatusResponse{ + Phase: apis.AddonPhaseDisabled, + EnablingProgress: nil, + }, nil + } + return nil, bcode.ErrGetApplicationFail + } + + switch app.Status.Phase { + case common2.ApplicationRunning, common2.ApplicationWorkflowFinished: + return &apis.AddonStatusResponse{ + Phase: apis.AddonPhaseEnabled, + EnablingProgress: nil, + }, nil + default: + return &apis.AddonStatusResponse{ + Phase: apis.AddonPhaseEnabling, + EnablingProgress: nil, + }, nil + } +} + +func (u *addonUsecaseImpl) ListAddons(ctx context.Context, detailed bool) ([]*apis.DetailAddonResponse, error) { + // Backward compatibility with ConfigMap addons. + // We will deprecate ConfigMap and use Git based registry. + addons, err := getAddonsFromConfigMap(detailed) + if err != nil { + return nil, err + } + + rs, err := u.listAddonRegistries(ctx) + if err != nil { + return nil, err + } + for _, r := range rs { + gitAddons, err := getAddonsFromGit(r.Git.URL, r.Git.Path, r.Git.Token, detailed) + if err != nil { + return nil, err + } + addons = mergeAddons(addons, gitAddons) + } + sort.Slice(addons, func(i, j int) bool { + return addons[i].Name < addons[j].Name + }) + return addons, nil +} + +func (u *addonUsecaseImpl) CreateAddonRegistry(ctx context.Context, req apis.CreateAddonRegistryRequest) (*apis.AddonRegistryMeta, error) { + r := addonRegistryModelFromCreateAddonRegistryRequest(req) + + err := u.ds.Add(ctx, r) + if err != nil { + if errors.Is(err, datastore.ErrRecordExist) { + return nil, bcode.ErrAddonRegistryExist + } + return nil, err + } + + return &apis.AddonRegistryMeta{ + Name: r.Name, + Git: r.Git, + }, nil + +} + +func (u *addonUsecaseImpl) GetAddonRegistryModel(ctx context.Context, name string) (*model.AddonRegistry, error) { + var r = model.AddonRegistry{ + Name: name, + } + err := u.ds.Get(ctx, &r) + if err != nil { + return nil, err + } + return &r, nil +} + +func (u *addonUsecaseImpl) listAddonRegistries(ctx context.Context) ([]*apis.AddonRegistryMeta, error) { + var r = model.AddonRegistry{} + entities, err := u.ds.List(ctx, &r, &datastore.ListOptions{}) + if err != nil { + return nil, err + } + var list []*apis.AddonRegistryMeta + for _, entity := range entities { + list = append(list, restutils.ConvertAddonRegistryModel2AddonRegistryMeta(entity.(*model.AddonRegistry))) + } + return list, nil +} + +func (u *addonUsecaseImpl) EnableAddon(ctx context.Context, name string, args apis.EnableAddonRequest) error { + addon, err := u.GetAddon(ctx, name) + if err != nil { + return err + } + err = u.applyAddonData(addon.DeployData, args) + if err != nil { + return err + } + return nil + +} + +func (u *addonUsecaseImpl) DisableAddon(ctx context.Context, name string) error { + addon, err := u.GetAddon(ctx, name) + if err != nil { + return err + } + err = u.deleteAddonData(addon.DeployData) + if err != nil { + return err + } + return nil +} + +func (u *addonUsecaseImpl) applyAddonData(data string, request apis.EnableAddonRequest) error { + app, err := renderAddonApp(data, &request) + if err != nil { + return err + } + applicator := apply.NewAPIApplicator(u.kubeClient) + err = applicator.Apply(context.TODO(), app) + if err != nil { + log.Logger.Errorf("apply application fail: %s", err.Error()) + return bcode.ErrAddonApplyFail + } + return nil +} + +func (u *addonUsecaseImpl) deleteAddonData(data string) error { + app, err := renderAddonApp(data, nil) + if err != nil { + return err + } + err = u.kubeClient.Get(context.Background(), client.ObjectKey{ + Namespace: app.GetNamespace(), + Name: app.GetName(), + }, app) + if err != nil { + return bcode.ErrAddonNotEnabled + } + err = u.kubeClient.Delete(context.Background(), app) + if err != nil { + return bcode.ErrAddonDisableFail + } + return nil + +} + +// renderAddonApp can render string to unstructured, args can be nil +func renderAddonApp(data string, args *apis.EnableAddonRequest) (*unstructured.Unstructured, error) { + if args == nil { + args = &apis.EnableAddonRequest{Args: map[string]string{}} + } + + t, err := template.New("addon-template").Delims("[[", "]]").Funcs(sprig.TxtFuncMap()).Parse(data) + if err != nil { + return nil, bcode.ErrAddonRenderFail + } + buf := bytes.Buffer{} + err = t.Execute(&buf, args) + if err != nil { + return nil, bcode.ErrAddonRenderFail + } + dec := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) + obj := &unstructured.Unstructured{} + _, _, err = dec.Decode(buf.Bytes(), nil, obj) + if err != nil { + return nil, bcode.ErrAddonRenderFail + } + return obj, nil +} + +func addonRegistryModelFromCreateAddonRegistryRequest(req apis.CreateAddonRegistryRequest) *model.AddonRegistry { + return &model.AddonRegistry{ + Name: req.Name, + Git: req.Git, + } +} + +func mergeAddons(a1, a2 []*apis.DetailAddonResponse) []*apis.DetailAddonResponse { + for _, item := range a2 { + if hasAddon(a1, item.Name) { + continue + } + a1 = append(a1, item) + } + return a1 +} + +func hasAddon(addons []*apis.DetailAddonResponse, name string) bool { + for _, addon := range addons { + if addon.Name == name { + return true + } + } + return false +} + +func getAddonsFromGit(baseURL, dir, token string, detailed bool) ([]*apis.DetailAddonResponse, error) { + addons := []*apis.DetailAddonResponse{} + dec := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) + var tc *http.Client + if token != "" { + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + tc = oauth2.NewClient(context.Background(), ts) + } + clt := github.NewClient(tc) + // TODO add error handling + baseURL = strings.TrimSuffix(baseURL, ".git") + u, err := url.Parse(baseURL) + if err != nil { + return nil, err + } + u.Path = path.Join(u.Path, dir) + _, content, err := utils.Parse(u.String()) + if err != nil { + return nil, err + } + _, dirs, _, err := clt.Repositories.GetContents(context.Background(), content.Owner, content.Repo, content.Path, nil) + if err != nil { + return nil, err + } + for _, subItems := range dirs { + if *subItems.Type == "file" { + continue + } + addonRes := apis.DetailAddonResponse{ + AddonMeta: apis.AddonMeta{ + Name: *subItems.Name, + }, + } + var err error + _, files, _, err := clt.Repositories.GetContents(context.Background(), content.Owner, content.Repo, *subItems.Path, nil) + // get addon.yaml and readme.md + for _, file := range files { + switch *file.Name { + case AddonFileName: + addonContent, _, _, err := clt.Repositories.GetContents(context.Background(), content.Owner, content.Repo, *file.Path, nil) + if err != nil { + break + } + addonStr, _ := addonContent.GetContent() + obj := &unstructured.Unstructured{} + _, _, err = dec.Decode([]byte(addonStr), nil, obj) + if err != nil { + break + } + addonRes.AddonMeta.Description = obj.GetAnnotations()[addonutil.DescAnnotation] + addonRes.DeployData = addonStr + case AddonReadmeFileName: + if detailed { + detailContent, _, _, err := clt.Repositories.GetContents(context.Background(), content.Owner, content.Repo, *file.Path, nil) + if err != nil { + break + } + addonRes.Detail, err = detailContent.GetContent() + if err != nil { + break + } + } + default: + continue + } + + } + if err != nil { + continue + } + addons = append(addons, &addonRes) + } + return addons, nil +} + +func getAddonsFromConfigMap(detailed bool) ([]*apis.DetailAddonResponse, error) { + repo, err := addonutil.NewAddonRepo() + if err != nil { + return nil, fmt.Errorf("failed to get configMap addon repo: %w", err) + } + cliAddons := repo.ListAddons() + addons := []*apis.DetailAddonResponse{} + for _, addon := range cliAddons { + d := &apis.DetailAddonResponse{ + AddonMeta: apis.AddonMeta{ + Name: addon.Name, + // TODO add actual Version, Icon, tags + Version: "v1alpha1", + Description: addon.Description, + Icon: "", + Tags: nil, + }, + DeployData: addon.Data, + } + if detailed { + d.Detail = addon.Detail + } + addons = append(addons, d) + } + return addons, nil + +} diff --git a/pkg/apiserver/rest/utils/bcode/addon.go b/pkg/apiserver/rest/utils/bcode/addon.go new file mode 100644 index 000000000..4b1ef71a0 --- /dev/null +++ b/pkg/apiserver/rest/utils/bcode/addon.go @@ -0,0 +1,47 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bcode + +var ( + + // ErrAddonNotExist addon not exist + ErrAddonNotExist = NewBcode(400, 50001, "addon not exist") + + // ErrAddonRegistryExist application is exist + ErrAddonRegistryExist = NewBcode(400, 50002, "addon name already exists") + + // ErrAddonRenderFail fail to render addon application + ErrAddonRenderFail = NewBcode(500, 50010, "addon render fail") + + // ErrAddonApplyFail fail to apply application to cluster + ErrAddonApplyFail = NewBcode(500, 50011, "fail to apply addon application") + + // ErrGetClientFail fail to get k8s client + ErrGetClientFail = NewBcode(500, 50012, "fail to initialize kubernetes client") + + // ErrGetApplicationFail fail to get addon application + ErrGetApplicationFail = NewBcode(500, 50013, "fail to get addon application") + + // ErrGetConfigMapAddonFail fail to get addon info in configmap + ErrGetConfigMapAddonFail = NewBcode(500, 50014, "fail to get addon information in ConfigMap") + + // ErrAddonDisableFail fail to disable addon + ErrAddonDisableFail = NewBcode(500, 50016, "fail to disable addon") + + // ErrAddonNotEnabled means addon can't be disable because it's not enabled + ErrAddonNotEnabled = NewBcode(400, 50017, "addon not enabled") +) diff --git a/pkg/apiserver/rest/utils/convert.go b/pkg/apiserver/rest/utils/convert.go new file mode 100644 index 000000000..9da4c237f --- /dev/null +++ b/pkg/apiserver/rest/utils/convert.go @@ -0,0 +1,14 @@ +package utils + +import ( + "github.com/oam-dev/kubevela/pkg/apiserver/model" + apisv1 "github.com/oam-dev/kubevela/pkg/apiserver/rest/apis/v1" +) + +// ConvertAddonRegistryModel2AddonRegistryMeta will convert from model to AddonRegistryMeta +func ConvertAddonRegistryModel2AddonRegistryMeta(r *model.AddonRegistry) *apisv1.AddonRegistryMeta { + return &apisv1.AddonRegistryMeta{ + Name: r.Name, + Git: r.Git, + } +} diff --git a/pkg/apiserver/rest/webservice/addon.go b/pkg/apiserver/rest/webservice/addon.go index 182266c31..52e18f988 100644 --- a/pkg/apiserver/rest/webservice/addon.go +++ b/pkg/apiserver/rest/webservice/addon.go @@ -19,16 +19,25 @@ package webservice import ( restfulspec "github.com/emicklei/go-restful-openapi/v2" "github.com/emicklei/go-restful/v3" - apis "github.com/oam-dev/kubevela/pkg/apiserver/rest/apis/v1" + "github.com/oam-dev/kubevela/pkg/apiserver/rest/usecase" + "github.com/oam-dev/kubevela/pkg/apiserver/rest/utils/bcode" ) -type addonWebService struct { +// NewAddonWebService returns addon web service +func NewAddonWebService(u usecase.AddonUsecase) WebService { + return &addonWebService{ + addonUsecase: u, + } } -func (c *addonWebService) GetWebService() *restful.WebService { +type addonWebService struct { + addonUsecase usecase.AddonUsecase +} + +func (s *addonWebService) GetWebService() *restful.WebService { ws := new(restful.WebService) - ws.Path("/v1/addons"). + ws.Path(versionPrefix+"/addons"). Consumes(restful.MIME_XML, restful.MIME_JSON). Produces(restful.MIME_JSON, restful.MIME_XML). Doc("api for addon management") @@ -36,53 +45,132 @@ func (c *addonWebService) GetWebService() *restful.WebService { tags := []string{"addon"} // List - ws.Route(ws.GET("/").To(noop). + ws.Route(ws.GET("/").To(s.listAddons). Doc("list all addons"). Metadata(restfulspec.KeyOpenAPITags, tags). - Param(ws.QueryParameter("cluster", "Cluster-based search").DataType("string")). - Writes(apis.ListAddonResponse{}).Do(returns200, returns500)) - - // Create - ws.Route(ws.POST("/").To(noop). - Doc("create an addon"). - Metadata(restfulspec.KeyOpenAPITags, tags). - Reads(apis.CreateAddonRequest{}). - Writes(apis.AddonMeta{})) - - // Delete - ws.Route(ws.DELETE("/{name}").To(noop). - Doc("delete an addon"). - Metadata(restfulspec.KeyOpenAPITags, tags). - Param(ws.PathParameter("name", "identifier of the addon").DataType("string")). - Writes(apis.AddonMeta{})) + Returns(200, "", apis.ListAddonResponse{}). + Returns(400, "", bcode.Bcode{}). + Writes(apis.ListAddonResponse{})) // GET - ws.Route(ws.GET("/{name}").To(noop). + ws.Route(ws.GET("/{name}").To(s.detailAddon). Doc("show details of an addon"). Metadata(restfulspec.KeyOpenAPITags, tags). - Param(ws.PathParameter("name", "identifier of the addon").DataType("string")). + Metadata(restfulspec.KeyOpenAPITags, tags). + Returns(200, "", apis.DetailAddonResponse{}). + Returns(400, "", bcode.Bcode{}). + Param(ws.QueryParameter("name", "addon name to query detail").DataType("string").Required(true)). Writes(apis.DetailAddonResponse{})) // GET status - ws.Route(ws.GET("/{name}/status").To(noop). + ws.Route(ws.GET("/status").To(s.statusAddon). Doc("show status of an addon"). Metadata(restfulspec.KeyOpenAPITags, tags). - Param(ws.PathParameter("name", "identifier of the addon").DataType("string")). + Returns(200, "", apis.AddonStatusResponse{}). + Returns(400, "", bcode.Bcode{}). + Param(ws.QueryParameter("name", "addon name to query status").DataType("string").Required(true)). Writes(apis.AddonStatusResponse{})) - // vela enable addon - ws.Route(ws.POST("/{name}/enable").To(noop). - Doc("enable an addon on a cluster"). + // enable addon + ws.Route(ws.POST("/enable").To(s.enableAddon). + Doc("enable an addon"). Metadata(restfulspec.KeyOpenAPITags, tags). - Param(ws.QueryParameter("cluster", "cluster name").DataType("string")). - Writes(apis.AddonMeta{})) + Returns(200, "", apis.AddonStatusResponse{}). + Returns(400, "", bcode.Bcode{}). + Param(ws.QueryParameter("name", "addon name to enable").DataType("string").Required(true)). + Writes(apis.AddonStatusResponse{})) - // vela disable addon - ws.Route(ws.POST("/{name}/disable").To(noop). - Doc("disable an addon on a cluster"). + // disable addon + ws.Route(ws.POST("/disable").To(s.disableAddon). + Doc("disable an addon"). Metadata(restfulspec.KeyOpenAPITags, tags). - Param(ws.QueryParameter("cluster", "cluster name").DataType("string")). - Writes(apis.AddonMeta{})) + Returns(200, "", apis.AddonStatusResponse{}). + Returns(400, "", bcode.Bcode{}). + Param(ws.QueryParameter("name", "addon name to enable").DataType("string").Required(true)). + Writes(apis.AddonStatusResponse{})) return ws } + +func (s *addonWebService) listAddons(req *restful.Request, res *restful.Response) { + detailAddons, err := s.addonUsecase.ListAddons(req.Request.Context(), false) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + + var addons []*apis.AddonMeta + + for _, d := range detailAddons { + addons = append(addons, &d.AddonMeta) + } + + err = res.WriteEntity(apis.ListAddonResponse{Addons: addons}) + if err != nil { + bcode.ReturnError(req, res, err) + return + } +} + +func (s *addonWebService) detailAddon(req *restful.Request, res *restful.Response) { + name := req.QueryParameter("name") + addon, err := s.addonUsecase.GetAddon(req.Request.Context(), name) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + + err = res.WriteEntity(addon) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + +} + +func (s *addonWebService) enableAddon(req *restful.Request, res *restful.Response) { + var createReq apis.EnableAddonRequest + err := req.ReadEntity(&createReq) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + if err = validate.Struct(&createReq); err != nil { + bcode.ReturnError(req, res, err) + return + } + + name := req.QueryParameter("name") + err = s.addonUsecase.EnableAddon(req.Request.Context(), name, createReq) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + + s.statusAddon(req, res) +} + +func (s *addonWebService) disableAddon(req *restful.Request, res *restful.Response) { + name := req.QueryParameter("name") + err := s.addonUsecase.DisableAddon(req.Request.Context(), name) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + s.statusAddon(req, res) +} + +func (s *addonWebService) statusAddon(req *restful.Request, res *restful.Response) { + name := req.QueryParameter("name") + status, err := s.addonUsecase.StatusAddon(name) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + + err = res.WriteEntity(*status) + if err != nil { + bcode.ReturnError(req, res, err) + return + } +} diff --git a/pkg/apiserver/rest/webservice/addon_registry.go b/pkg/apiserver/rest/webservice/addon_registry.go new file mode 100644 index 000000000..8b45fb84f --- /dev/null +++ b/pkg/apiserver/rest/webservice/addon_registry.go @@ -0,0 +1,109 @@ +/* +Copyright 2021 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webservice + +import ( + restfulspec "github.com/emicklei/go-restful-openapi/v2" + "github.com/emicklei/go-restful/v3" + + "github.com/oam-dev/kubevela/pkg/apiserver/log" + apis "github.com/oam-dev/kubevela/pkg/apiserver/rest/apis/v1" + "github.com/oam-dev/kubevela/pkg/apiserver/rest/usecase" + "github.com/oam-dev/kubevela/pkg/apiserver/rest/utils" + "github.com/oam-dev/kubevela/pkg/apiserver/rest/utils/bcode" +) + +// NewAddonRegistryWebService returns addon registry web service +func NewAddonRegistryWebService(u usecase.AddonUsecase) WebService { + return &addonRegistryWebService{ + addonUsecase: u, + } +} + +type addonRegistryWebService struct { + addonUsecase usecase.AddonUsecase +} + +func (s *addonRegistryWebService) GetWebService() *restful.WebService { + ws := new(restful.WebService) + ws.Path(versionPrefix+"/addon_registries"). + Consumes(restful.MIME_XML, restful.MIME_JSON). + Produces(restful.MIME_JSON, restful.MIME_XML). + Doc("api for addon registry management") + + tags := []string{"addon_registry"} + + // Create + ws.Route(ws.POST("/").To(s.createAddonRegistry). + Doc("create an addon registry"). + Metadata(restfulspec.KeyOpenAPITags, tags). + Reads(apis.CreateAddonRegistryRequest{}). + Returns(200, "", apis.AddonRegistryMeta{}). + Returns(400, "", bcode.Bcode{}). + Writes(apis.AddonRegistryMeta{})) + + // Delete + ws.Route(ws.DELETE("/{name}").To(s.deleteAddonRegistry). + Doc("delete an addon registry"). + Metadata(restfulspec.KeyOpenAPITags, tags). + Param(ws.PathParameter("name", "identifier of the addon registry").DataType("string")). + Returns(200, "", apis.AddonRegistryMeta{}). + Returns(400, "", bcode.Bcode{}). + Writes(apis.AddonRegistryMeta{})) + + return ws +} + +func (s *addonRegistryWebService) createAddonRegistry(req *restful.Request, res *restful.Response) { + // Verify the validity of parameters + var createReq apis.CreateAddonRegistryRequest + if err := req.ReadEntity(&createReq); err != nil { + bcode.ReturnError(req, res, err) + return + } + if err := validate.Struct(&createReq); err != nil { + bcode.ReturnError(req, res, err) + return + } + + // Call the usecase layer code + meta, err := s.addonUsecase.CreateAddonRegistry(req.Request.Context(), createReq) + if err != nil { + log.Logger.Errorf("create addon registry failure %s", err.Error()) + bcode.ReturnError(req, res, err) + return + } + + // Write back response data + if err := res.WriteEntity(meta); err != nil { + bcode.ReturnError(req, res, err) + return + } +} + +func (s *addonRegistryWebService) deleteAddonRegistry(req *restful.Request, res *restful.Response) { + r, err := s.addonUsecase.GetAddonRegistryModel(req.Request.Context(), req.PathParameter("name")) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + + if err := res.WriteEntity(*utils.ConvertAddonRegistryModel2AddonRegistryMeta(r)); err != nil { + bcode.ReturnError(req, res, err) + return + } +} diff --git a/pkg/apiserver/rest/webservice/webservice.go b/pkg/apiserver/rest/webservice/webservice.go index d057ce176..a7e92227f 100644 --- a/pkg/apiserver/rest/webservice/webservice.go +++ b/pkg/apiserver/rest/webservice/webservice.go @@ -65,11 +65,13 @@ func Init(ctx context.Context, ds datastore.DataStore) { namespaceUsecase := usecase.NewNamespaceUsecase() oamApplicationUsecase := usecase.NewOAMApplicationUsecase() definitionUsecase := usecase.NewDefinitionUsecase() + addonUsecase := usecase.NewAddonUsecase(ds) RegistWebService(NewClusterWebService(clusterUsecase)) RegistWebService(NewApplicationWebService(applicationUsecase)) RegistWebService(NewNamespaceWebService(namespaceUsecase)) RegistWebService(NewComponentDefinitionWebservice(definitionUsecase)) - RegistWebService(&addonWebService{}) + RegistWebService(NewAddonWebService(addonUsecase)) + RegistWebService(NewAddonRegistryWebService(addonUsecase)) RegistWebService(NewOAMApplication(oamApplicationUsecase)) RegistWebService(&policyDefinitionWebservice{}) RegistWebService(NewWorkflowWebService(workflowUsecase, applicationUsecase)) diff --git a/pkg/utils/addon/addon.go b/pkg/utils/addon/addon.go new file mode 100644 index 000000000..403ba95ec --- /dev/null +++ b/pkg/utils/addon/addon.go @@ -0,0 +1,281 @@ +package addon + +import ( + "bytes" + "context" + "fmt" + "strings" + "text/template" + "time" + + "github.com/Masterminds/sprig" + "github.com/pkg/errors" + + "github.com/oam-dev/kubevela/pkg/utils/common" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + types2 "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "sigs.k8s.io/controller-runtime/pkg/client" + + common2 "github.com/oam-dev/kubevela/apis/core.oam.dev/common" + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" + "github.com/oam-dev/kubevela/apis/types" + "github.com/oam-dev/kubevela/pkg/oam" + "github.com/oam-dev/kubevela/pkg/oam/discoverymapper" + "github.com/oam-dev/kubevela/pkg/utils/apply" +) + +const ( + // DescAnnotation records the Description of addon + DescAnnotation = "addons.oam.dev/description" +) + +var ( + // StatusUninstalled means addon not installed + StatusUninstalled = "uninstalled" + // StatusInstalled means addon installed + StatusInstalled = "installed" + clt client.Client + clientArgs common.Args +) + +func init() { + clientArgs, _ = common.InitBaseRestConfig() + clt, _ = clientArgs.GetClient() + +} + +func newAddon(data *v1.ConfigMap) *Addon { + description := data.ObjectMeta.Annotations[DescAnnotation] + a := Addon{ + Name: data.Annotations[oam.AnnotationAddonsName], + Description: description, + Detail: data.Data["detail"], + Data: data.Data["application"], + } + return &a +} + +// Repo is a place to store addon info +type Repo interface { + GetAddon(name string) (Addon, error) + ListAddons() []Addon +} + +// NewAddonRepo create new addon repo,now only support ConfigMap +func NewAddonRepo() (Repo, error) { + list := v1.ConfigMapList{} + matchLabels := metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{{ + Key: oam.LabelAddonsName, + Operator: metav1.LabelSelectorOpExists, + }}, + } + selector, err := metav1.LabelSelectorAsSelector(&matchLabels) + if err != nil { + return nil, err + } + err = clt.List(context.Background(), &list, &client.ListOptions{LabelSelector: selector}) + if err != nil { + return nil, errors.Wrap(err, "Get addon list failed") + } + return configMapAddonRepo{maps: list.Items}, nil +} + +type configMapAddonRepo struct { + maps []v1.ConfigMap +} + +// NotFoundErr means addon not found +type NotFoundErr struct { + addonName string +} + +func (e NotFoundErr) Error() string { + return fmt.Sprintf("addon %s not found", e.addonName) +} + +// GetAddon will get addon from ConfigMap +func (c configMapAddonRepo) GetAddon(name string) (Addon, error) { + for i := range c.maps { + if addonName, ok := c.maps[i].Annotations[oam.AnnotationAddonsName]; ok && name == addonName { + return *newAddon(&c.maps[i]), nil + } + } + return Addon{}, NotFoundErr{addonName: name} +} + +// ListAddons will list addons from ConfigMap +func (c configMapAddonRepo) ListAddons() []Addon { + var addons []Addon + for i := range c.maps { + addon := newAddon(&c.maps[i]) + addons = append(addons, *addon) + } + return addons +} + +// Addon consist of a Initializer resource to Enable an addon +type Addon struct { + Name string + Description string + Data string + // Args is map for renderInitializer + Args map[string]string + application *unstructured.Unstructured + gvk *schema.GroupVersionKind + + // Detail is doc for addon + Detail string +} + +// GetGVK will return addon's application's GVK +func (a *Addon) GetGVK() (*schema.GroupVersionKind, error) { + if a.gvk == nil { + if a.application == nil { + _, err := a.RenderApplication() + if err != nil { + return nil, err + } + } + gvk := schema.FromAPIVersionAndKind(a.application.GetAPIVersion(), a.application.GetKind()) + a.gvk = &gvk + } + return a.gvk, nil +} + +// RenderApplication will render addon application +// this will use addon's Data and Args +func (a *Addon) RenderApplication() (*unstructured.Unstructured, error) { + if a.Args == nil { + a.Args = map[string]string{} + } + t, err := template.New("addon-template").Delims("[[", "]]").Funcs(sprig.TxtFuncMap()).Parse(a.Data) + if err != nil { + return nil, errors.Wrap(err, "parsing addon initializer template error") + } + buf := bytes.Buffer{} + err = t.Execute(&buf, a) + if err != nil { + return nil, errors.Wrap(err, "application template render fail") + } + dec := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) + obj := &unstructured.Unstructured{} + _, gvk, err := dec.Decode(buf.Bytes(), nil, obj) + if err != nil { + return nil, err + } + a.application = obj + a.gvk = gvk + return a.application, nil +} + +// Enable will enable an addon by apply application +func (a *Addon) Enable() error { + applicator := apply.NewAPIApplicator(clt) + ctx := context.Background() + obj, err := a.RenderApplication() + if err != nil { + return err + } + err = applicator.Apply(ctx, obj) + if err != nil { + return errors.Wrapf(err, "Error occurs when apply addon application: %s\n", a.Name) + } + err = waitApplicationRunning(a.application) + if err != nil { + return errors.Wrap(err, "Error occurs when waiting addon applicatoin running") + } + return nil +} + +func waitApplicationRunning(obj *unstructured.Unstructured) error { + ctx := context.Background() + period := 20 * time.Second + timeout := 10 * time.Minute + var app v1beta1.Application + return wait.PollImmediate(period, timeout, func() (done bool, err error) { + err = clt.Get(ctx, types2.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()}, &app) + if err != nil { + return false, client.IgnoreNotFound(err) + } + phase := app.Status.Phase + if phase == common2.ApplicationRunning { + return true, nil + } + fmt.Printf("Application %s is in phase:%s...\n", obj.GetName(), phase) + return false, nil + }) +} + +// Disable will delete addon's application +func (a *Addon) Disable() error { + dynamicClient, err := dynamic.NewForConfig(clientArgs.Config) + if err != nil { + return err + } + mapper, err := discoverymapper.New(clientArgs.Config) + if err != nil { + return err + } + obj, err := a.RenderApplication() + if err != nil { + return err + } + gvk, err := a.GetGVK() + if err != nil { + return err + } + mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return err + } + var resourceREST dynamic.ResourceInterface + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + // namespaced resources should specify the namespace + resourceREST = dynamicClient.Resource(mapping.Resource).Namespace(obj.GetNamespace()) + } else { + // for cluster-wide resources + resourceREST = dynamicClient.Resource(mapping.Resource) + } + deletePolicy := metav1.DeletePropagationForeground + deleteOptions := metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + } + fmt.Println("Deleting all resources...") + err = resourceREST.Delete(context.TODO(), obj.GetName(), deleteOptions) + if err != nil { + return err + } + return nil +} + +// GetStatus will return if an Addon is enabled +func (a *Addon) GetStatus() string { + var application v1beta1.Application + err := clt.Get(context.Background(), client.ObjectKey{ + Namespace: types.DefaultKubeVelaNS, + Name: TransAddonName(a.Name), + }, &application) + if err != nil { + return StatusUninstalled + } + return StatusInstalled +} + +// SetArgs will set Args for application render +func (a *Addon) SetArgs(args map[string]string) { + a.Args = args +} + +// TransAddonName will turn addon's name from xxx/yyy to xxx-yyy +func TransAddonName(name string) string { + return strings.ReplaceAll(name, "/", "-") +} diff --git a/pkg/utils/parse.go b/pkg/utils/parse.go new file mode 100644 index 000000000..fb1872957 --- /dev/null +++ b/pkg/utils/parse.go @@ -0,0 +1,120 @@ +package utils + +import ( + "net/url" + "strings" + + "github.com/pkg/errors" +) + +// TypeLocal represents github +const TypeLocal = "local" + +// TypeOss represent oss +const TypeOss = "oss" + +// TypeGithub represents github +const TypeGithub = "github" + +// TypeUnknown represents parse failed +const TypeUnknown = "unknown" + +// Content contains different type of content needed when building Registry +type Content struct { + OssContent + GithubContent + LocalContent +} + +// LocalContent for local registry +type LocalContent struct { + AbsDir string `json:"abs_dir"` +} + +// OssContent for oss registry +type OssContent struct { + BucketURL string `json:"bucket_url"` +} + +// GithubContent for cap center +type GithubContent struct { + Owner string `json:"owner"` + Repo string `json:"repo"` + Path string `json:"path"` + Ref string `json:"ref"` +} + +// Parse will parse config from address +func Parse(addr string) (string, *Content, error) { + URL, err := url.Parse(addr) + if err != nil { + return "", nil, err + } + l := strings.Split(strings.TrimPrefix(URL.Path, "/"), "/") + switch URL.Scheme { + case "http", "https": + switch URL.Host { + case "github.com": + // We support two valid format: + // 1. https://github.com///tree// + // 2. https://github.com/// + if len(l) < 3 { + return "", nil, errors.New("invalid format " + addr) + } + if l[2] == "tree" { + // https://github.com///tree// + if len(l) < 5 { + return "", nil, errors.New("invalid format " + addr) + } + return TypeGithub, &Content{ + GithubContent: GithubContent{ + Owner: l[0], + Repo: l[1], + Path: strings.Join(l[4:], "/"), + Ref: l[3], + }, + }, nil + } + // https://github.com/// + return TypeGithub, &Content{ + GithubContent: GithubContent{ + Owner: l[0], + Repo: l[1], + Path: strings.Join(l[2:], "/"), + Ref: "", // use default branch + }, + }, + nil + case "api.github.com": + if len(l) != 5 { + return "", nil, errors.New("invalid format " + addr) + } + //https://api.github.com/repos///contents/ + return TypeGithub, &Content{ + GithubContent: GithubContent{ + Owner: l[1], + Repo: l[2], + Path: l[4], + Ref: URL.Query().Get("ref"), + }, + }, + nil + default: + } + case "oss": + return TypeOss, &Content{ + OssContent: OssContent{ + BucketURL: URL.Host, + }, + }, nil + case "file": + return TypeLocal, &Content{ + LocalContent: LocalContent{ + AbsDir: URL.Path, + }, + }, nil + + } + + return TypeUnknown, nil, nil +} diff --git a/references/cli/addon.go b/references/cli/addon.go index d5c8f1570..f687eb5b7 100644 --- a/references/cli/addon.go +++ b/references/cli/addon.go @@ -17,53 +17,27 @@ limitations under the License. package cli import ( - "bytes" "context" "fmt" "strings" - "text/template" - "time" - common2 "github.com/oam-dev/kubevela/apis/core.oam.dev/common" - - "github.com/Masterminds/sprig" "github.com/gosuri/uitable" "github.com/pkg/errors" "github.com/spf13/cobra" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer/yaml" - types2 "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/dynamic" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/apis/types" - "github.com/oam-dev/kubevela/pkg/oam" - "github.com/oam-dev/kubevela/pkg/oam/discoverymapper" - "github.com/oam-dev/kubevela/pkg/utils/apply" + addonutil "github.com/oam-dev/kubevela/pkg/utils/addon" "github.com/oam-dev/kubevela/pkg/utils/common" cmdutil "github.com/oam-dev/kubevela/pkg/utils/util" ) -const ( - // DescAnnotation records the description of addon - DescAnnotation = "addons.oam.dev/description" -) - -var statusUninstalled = "uninstalled" -var statusInstalled = "installed" -var clt client.Client -var clientArgs common.Args - var legacyAddonNamespace map[string]string +var clt client.Client func init() { - clientArgs, _ = common.InitBaseRestConfig() + clientArgs, _ := common.InitBaseRestConfig() clt, _ = clientArgs.GetClient() legacyAddonNamespace = map[string]string{ "fluxcd": types.DefaultKubeVelaNS, @@ -182,31 +156,31 @@ func NewAddonDisableCommand(ioStream cmdutil.IOStreams) *cobra.Command { } func listAddons() error { - repo, err := NewAddonRepo() + repo, err := addonutil.NewAddonRepo() if err != nil { return err } - addons := repo.listAddons() + addons := repo.ListAddons() table := uitable.New() table.AddRow("NAME", "DESCRIPTION", "STATUS") for _, addon := range addons { - table.AddRow(addon.name, addon.description, addon.getStatus()) + table.AddRow(addon.Name, addon.Description, addon.GetStatus()) } fmt.Println(table.String()) return nil } func enableAddon(name string, args map[string]string) error { - repo, err := NewAddonRepo() + repo, err := addonutil.NewAddonRepo() if err != nil { return err } - addon, err := repo.getAddon(name) + addon, err := repo.GetAddon(name) if err != nil { return err } - addon.setArgs(args) - err = addon.enable() + addon.SetArgs(args) + err = addon.Enable() return err } @@ -214,25 +188,25 @@ func disableAddon(name string) error { if isLegacyAddonExist(name) { return tryDisableInitializerAddon(name) } - repo, err := NewAddonRepo() + repo, err := addonutil.NewAddonRepo() if err != nil { return err } - addon, err := repo.getAddon(name) + addon, err := repo.GetAddon(name) if err != nil { return errors.Wrap(err, "get addon err") } - if addon.getStatus() == statusUninstalled { - fmt.Printf("Addon %s is not installed\n", addon.name) + if addon.GetStatus() == addonutil.StatusUninstalled { + fmt.Printf("Addon %s is not installed\n", addon.Name) return nil } - return addon.disable() + return addon.Disable() } func isLegacyAddonExist(name string) bool { if namespace, ok := legacyAddonNamespace[name]; ok { - convertedAddonName := TransAddonName(name) + convertedAddonName := addonutil.TransAddonName(name) init := unstructured.Unstructured{ Object: map[string]interface{}{ "apiVersion": "core.oam.dev/v1beta1", @@ -255,7 +229,7 @@ func tryDisableInitializerAddon(addonName string) error { "apiVersion": "core.oam.dev/v1beta1", "kind": "Initializer", "metadata": map[string]interface{}{ - "name": TransAddonName(addonName), + "name": addonutil.TransAddonName(addonName), "namespace": legacyAddonNamespace[addonName], }, }, @@ -263,212 +237,3 @@ func tryDisableInitializerAddon(addonName string) error { return clt.Delete(context.TODO(), &init) } -func newAddon(data *v1.ConfigMap) *Addon { - description := data.ObjectMeta.Annotations[DescAnnotation] - a := Addon{name: data.Annotations[oam.AnnotationAddonsName], description: description, data: data.Data["application"]} - return &a -} - -// AddonRepo is a place to store addon info -type AddonRepo interface { - getAddon(name string) (Addon, error) - listAddons() []Addon -} - -// NewAddonRepo create new addon repo,now only support ConfigMap -func NewAddonRepo() (AddonRepo, error) { - list := v1.ConfigMapList{} - matchLabels := metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{{ - Key: oam.LabelAddonsName, - Operator: metav1.LabelSelectorOpExists, - }}, - } - selector, err := metav1.LabelSelectorAsSelector(&matchLabels) - if err != nil { - return nil, err - } - err = clt.List(context.Background(), &list, &client.ListOptions{LabelSelector: selector}) - if err != nil { - return nil, errors.Wrap(err, "Get addon list failed") - } - return configMapAddonRepo{maps: list.Items}, nil -} - -type configMapAddonRepo struct { - maps []v1.ConfigMap -} - -// AddonNotFoundErr means addon not found -type AddonNotFoundErr struct { - addonName string -} - -func (e AddonNotFoundErr) Error() string { - return fmt.Sprintf("addon %s not found", e.addonName) -} - -func (c configMapAddonRepo) getAddon(name string) (Addon, error) { - for i := range c.maps { - if addonName, ok := c.maps[i].Annotations[oam.AnnotationAddonsName]; ok && name == addonName { - return *newAddon(&c.maps[i]), nil - } - } - return Addon{}, AddonNotFoundErr{addonName: name} -} - -func (c configMapAddonRepo) listAddons() []Addon { - var addons []Addon - for i := range c.maps { - addon := newAddon(&c.maps[i]) - addons = append(addons, *addon) - } - return addons -} - -// Addon consist of a Initializer resource to enable an addon -type Addon struct { - name string - description string - data string - // Args is map for renderInitializer - Args map[string]string - application *unstructured.Unstructured - gvk *schema.GroupVersionKind -} - -func (a *Addon) getGVK() (*schema.GroupVersionKind, error) { - if a.gvk == nil { - if a.application == nil { - _, err := a.renderApplication() - if err != nil { - return nil, err - } - } - gvk := schema.FromAPIVersionAndKind(a.application.GetAPIVersion(), a.application.GetKind()) - a.gvk = &gvk - } - return a.gvk, nil -} - -func (a *Addon) renderApplication() (*unstructured.Unstructured, error) { - if a.Args == nil { - a.Args = map[string]string{} - } - t, err := template.New("addon-template").Delims("[[", "]]").Funcs(sprig.TxtFuncMap()).Parse(a.data) - if err != nil { - return nil, errors.Wrap(err, "parsing addon initializer template error") - } - buf := bytes.Buffer{} - err = t.Execute(&buf, a) - if err != nil { - return nil, errors.Wrap(err, "application template render fail") - } - dec := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) - obj := &unstructured.Unstructured{} - _, gvk, err := dec.Decode(buf.Bytes(), nil, obj) - if err != nil { - return nil, err - } - a.application = obj - a.gvk = gvk - return a.application, nil -} - -func (a *Addon) enable() error { - applicator := apply.NewAPIApplicator(clt) - ctx := context.Background() - obj, err := a.renderApplication() - if err != nil { - return err - } - err = applicator.Apply(ctx, obj) - if err != nil { - return errors.Wrapf(err, "Error occurs when apply addon application: %s\n", a.name) - } - err = waitApplicationRunning(a.application) - if err != nil { - return errors.Wrap(err, "Error occurs when waiting addon applicatoin running") - } - return nil -} - -func waitApplicationRunning(obj *unstructured.Unstructured) error { - ctx := context.Background() - period := 20 * time.Second - timeout := 10 * time.Minute - var app v1beta1.Application - return wait.PollImmediate(period, timeout, func() (done bool, err error) { - err = clt.Get(ctx, types2.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()}, &app) - if err != nil { - return false, client.IgnoreNotFound(err) - } - phase := app.Status.Phase - if phase == common2.ApplicationRunning { - return true, nil - } - fmt.Printf("Application %s is in phase:%s...\n", obj.GetName(), phase) - return false, nil - }) -} -func (a *Addon) disable() error { - dynamicClient, err := dynamic.NewForConfig(clientArgs.Config) - if err != nil { - return err - } - mapper, err := discoverymapper.New(clientArgs.Config) - if err != nil { - return err - } - obj, err := a.renderApplication() - if err != nil { - return err - } - gvk, err := a.getGVK() - if err != nil { - return err - } - mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) - if err != nil { - return err - } - var resourceREST dynamic.ResourceInterface - if mapping.Scope.Name() == meta.RESTScopeNameNamespace { - // namespaced resources should specify the namespace - resourceREST = dynamicClient.Resource(mapping.Resource).Namespace(obj.GetNamespace()) - } else { - // for cluster-wide resources - resourceREST = dynamicClient.Resource(mapping.Resource) - } - deletePolicy := metav1.DeletePropagationForeground - deleteOptions := metav1.DeleteOptions{ - PropagationPolicy: &deletePolicy, - } - fmt.Println("Deleting all resources...") - err = resourceREST.Delete(context.TODO(), obj.GetName(), deleteOptions) - if err != nil { - return err - } - return nil -} - -func (a *Addon) getStatus() string { - var application v1beta1.Application - err := clt.Get(context.Background(), client.ObjectKey{ - Namespace: types.DefaultKubeVelaNS, - Name: TransAddonName(a.name), - }, &application) - if err != nil { - return statusUninstalled - } - return statusInstalled -} - -func (a *Addon) setArgs(args map[string]string) { - a.Args = args -} - -// TransAddonName will turn addon's name from xxx/yyy to xxx-yyy -func TransAddonName(name string) string { - return strings.ReplaceAll(name, "/", "-") -} diff --git a/references/plugins/capcenter.go b/references/plugins/capcenter.go index 01c4038ac..55afcc306 100644 --- a/references/plugins/capcenter.go +++ b/references/plugins/capcenter.go @@ -20,17 +20,18 @@ import ( "context" "errors" "fmt" + "net/http" - "net/url" "os" "path/filepath" - "strings" "github.com/google/go-github/v32/github" "golang.org/x/oauth2" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/yaml" + "github.com/oam-dev/kubevela/pkg/utils" + "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/apis/types" "github.com/oam-dev/kubevela/pkg/oam/discoverymapper" @@ -38,31 +39,6 @@ import ( "github.com/oam-dev/kubevela/pkg/utils/system" ) -// Content contains different type of content needed when building Registry -type Content struct { - OssContent - GithubContent - LocalContent -} - -// LocalContent for local registry -type LocalContent struct { - AbsDir string `json:"abs_dir"` -} - -// OssContent for oss registry -type OssContent struct { - BucketURL string `json:"bucket_url"` -} - -// GithubContent for cap center -type GithubContent struct { - Owner string `json:"owner"` - Repo string `json:"repo"` - Path string `json:"path"` - Ref string `json:"ref"` -} - // CapCenterConfig is used to store cap center config in file type CapCenterConfig struct { Name string `json:"name"` @@ -77,107 +53,20 @@ type CenterClient interface { // NewCenterClient create a client from type func NewCenterClient(ctx context.Context, name, address, token string) (CenterClient, error) { - Type, cfg, err := Parse(address) + Type, cfg, err := utils.Parse(address) if err != nil { return nil, err } switch Type { - case TypeGithub: + case utils.TypeGithub: return NewGithubCenter(ctx, token, name, &cfg.GithubContent) - case TypeOss: + case utils.TypeOss: return NewOssCenter(fmt.Sprintf("https://%s/", cfg.BucketURL), name), nil default: } return nil, errors.New("we only support github as repository now") } -// TypeLocal represents github -const TypeLocal = "local" - -// TypeOss represent oss -const TypeOss = "oss" - -// TypeGithub represents github -const TypeGithub = "github" - -// TypeUnknown represents parse failed -const TypeUnknown = "unknown" - -// Parse will parse config from address -func Parse(addr string) (string, *Content, error) { - URL, err := url.Parse(addr) - if err != nil { - return "", nil, err - } - l := strings.Split(strings.TrimPrefix(URL.Path, "/"), "/") - switch URL.Scheme { - case "http", "https": - switch URL.Host { - case "github.com": - // We support two valid format: - // 1. https://github.com///tree// - // 2. https://github.com/// - if len(l) < 3 { - return "", nil, errors.New("invalid format " + addr) - } - if l[2] == "tree" { - // https://github.com///tree// - if len(l) < 5 { - return "", nil, errors.New("invalid format " + addr) - } - return TypeGithub, &Content{ - GithubContent: GithubContent{ - Owner: l[0], - Repo: l[1], - Path: strings.Join(l[4:], "/"), - Ref: l[3], - }, - }, nil - } - // https://github.com/// - return TypeGithub, &Content{ - GithubContent: GithubContent{ - Owner: l[0], - Repo: l[1], - Path: strings.Join(l[2:], "/"), - Ref: "", // use default branch - }, - }, - nil - case "api.github.com": - if len(l) != 5 { - return "", nil, errors.New("invalid format " + addr) - } - //https://api.github.com/repos///contents/ - return TypeGithub, &Content{ - GithubContent: GithubContent{ - Owner: l[1], - Repo: l[2], - Path: l[4], - Ref: URL.Query().Get("ref"), - }, - }, - nil - default: - } - case "oss": - return TypeOss, &Content{ - OssContent: OssContent{ - BucketURL: URL.Host, - }, - }, nil - case "file": - return TypeLocal, &Content{ - LocalContent: LocalContent{ - AbsDir: URL.Path, - }, - }, nil - - } - - return TypeUnknown, nil, nil -} - // LoadRepos will load all cap center repos func LoadRepos() ([]CapCenterConfig, error) { defaultRepo := CapCenterConfig{ @@ -259,7 +148,7 @@ func ParseCapability(mapper discoverymapper.DiscoveryMapper, data []byte) (types } // NewGithubCenter will create client by github center implementation -func NewGithubCenter(ctx context.Context, token, centerName string, r *GithubContent) (*GithubRegistry, error) { +func NewGithubCenter(ctx context.Context, token, centerName string, r *utils.GithubContent) (*GithubRegistry, error) { var tc *http.Client if token != "" { ts := oauth2.StaticTokenSource( diff --git a/references/plugins/capcenter_test.go b/references/plugins/capcenter_test.go index 3a53047a5..38875995e 100644 --- a/references/plugins/capcenter_test.go +++ b/references/plugins/capcenter_test.go @@ -19,19 +19,21 @@ package plugins import ( "testing" + "github.com/oam-dev/kubevela/pkg/utils" + "github.com/stretchr/testify/assert" ) func TestParseURL(t *testing.T) { cases := map[string]struct { url string - exp *GithubContent + exp *utils.GithubContent expType string }{ "api-github": { url: "https://api.github.com/repos/zzxwill/catalog/contents/repository?ref=plugin", - expType: TypeGithub, - exp: &GithubContent{ + expType: utils.TypeGithub, + exp: &utils.GithubContent{ Owner: "zzxwill", Repo: "catalog", Path: "repository", @@ -40,8 +42,8 @@ func TestParseURL(t *testing.T) { }, "github-copy-path": { url: "https://github.com/zzxwill/catalog/tree/plugin/repository", - expType: TypeGithub, - exp: &GithubContent{ + expType: utils.TypeGithub, + exp: &utils.GithubContent{ Owner: "zzxwill", Repo: "catalog", Path: "repository", @@ -50,8 +52,8 @@ func TestParseURL(t *testing.T) { }, "github-manuel-write-path": { url: "https://github.com/zzxwill/catalog/repository", - expType: TypeGithub, - exp: &GithubContent{ + expType: utils.TypeGithub, + exp: &utils.GithubContent{ Owner: "zzxwill", Repo: "catalog", Path: "repository", @@ -59,7 +61,7 @@ func TestParseURL(t *testing.T) { }, } for caseName, c := range cases { - tp, content, err := Parse(c.url) + tp, content, err := utils.Parse(c.url) assert.NoError(t, err, caseName) assert.Equal(t, c.exp, &content.GithubContent, caseName) assert.Equal(t, c.expType, tp, caseName) diff --git a/references/plugins/registry.go b/references/plugins/registry.go index f60eefcfa..5c24e8ccc 100644 --- a/references/plugins/registry.go +++ b/references/plugins/registry.go @@ -27,6 +27,8 @@ import ( "path" "path/filepath" + "github.com/oam-dev/kubevela/pkg/utils" + "github.com/oam-dev/kubevela/apis/types" "github.com/google/go-github/v32/github" @@ -45,19 +47,19 @@ type Registry interface { // GithubRegistry is Registry's implementation treat github url as resource type GithubRegistry struct { client *github.Client - cfg *GithubContent + cfg *utils.GithubContent ctx context.Context centerName string // to be used to cache registry } // NewRegistry will create a registry implementation func NewRegistry(ctx context.Context, token, registryName string, regURL string) (Registry, error) { - tp, cfg, err := Parse(regURL) + tp, cfg, err := utils.Parse(regURL) if err != nil { return nil, err } switch tp { - case TypeGithub: + case utils.TypeGithub: var tc *http.Client if token != "" { ts := oauth2.StaticTokenSource( @@ -66,19 +68,19 @@ func NewRegistry(ctx context.Context, token, registryName string, regURL string) tc = oauth2.NewClient(ctx, ts) } return GithubRegistry{client: github.NewClient(tc), cfg: &cfg.GithubContent, ctx: ctx, centerName: registryName}, nil - case TypeOss: + case utils.TypeOss: var tc http.Client return OssRegistry{ Client: &tc, bucketURL: fmt.Sprintf("https://%s/", cfg.BucketURL), }, nil - case TypeLocal: + case utils.TypeLocal: _, err := os.Stat(cfg.AbsDir) if os.IsNotExist(err) { return LocalRegistry{}, err } return LocalRegistry{absPath: cfg.AbsDir}, nil - case TypeUnknown: + case utils.TypeUnknown: return nil, fmt.Errorf("not supported url") } diff --git a/test/e2e-apiserver-test/addon_test.go b/test/e2e-apiserver-test/addon_test.go new file mode 100644 index 000000000..d31549847 --- /dev/null +++ b/test/e2e-apiserver-test/addon_test.go @@ -0,0 +1,125 @@ +package e2e_apiserver + +import ( + "bytes" + "encoding/json" + "net/http" + "os" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/oam-dev/kubevela/pkg/apiserver/model" + apis "github.com/oam-dev/kubevela/pkg/apiserver/rest/apis/v1" +) + +const baseURL = "http://127.0.0.1:8000" + +func post(path string, body interface{}) *http.Response { + b, err := json.Marshal(body) + Expect(err).Should(BeNil()) + + res, err := http.Post(baseURL+path, "application/json", bytes.NewBuffer(b)) + Expect(err).Should(BeNil()) + return res +} + +func get(path string) *http.Response { + res, err := http.Get(baseURL + path) + Expect(err).Should(BeNil()) + return res +} + +var _ = Describe("Test addon rest api", func() { + It("should add a registry and list addons from it and delete the registry", func() { + defer GinkgoRecover() + + By("add registry") + createReq := apis.CreateAddonRegistryRequest{ + Name: "test-addon-registry-1", + Git: &model.GitAddonSource{ + URL: "https://github.com/oam-dev/catalog", + Path: "addon/", + Token: os.Getenv("GITHUB_TOKEN"), + }, + } + createRes := post("/api/v1/addon_registries", createReq) + Expect(createRes).ShouldNot(BeNil()) + Expect(createRes.StatusCode).Should(Equal(200)) + Expect(createRes.Body).ShouldNot(BeNil()) + + defer createRes.Body.Close() + + var rmeta apis.AddonRegistryMeta + err := json.NewDecoder(createRes.Body).Decode(&rmeta) + Expect(err).Should(BeNil()) + Expect(rmeta.Name).Should(Equal(createReq.Name)) + Expect(rmeta.Git).Should(Equal(createReq.Git)) + + By("list addons") + listRes := get("/api/v1/addons/") + defer listRes.Body.Close() + + var lres apis.ListAddonResponse + err = json.NewDecoder(listRes.Body).Decode(&lres) + Expect(err).Should(BeNil()) + Expect(lres.Addons).ShouldNot(BeZero()) + firstAddon := lres.Addons[0] + Expect(firstAddon.Name).Should(Equal("fluxcd")) + + By("delete registry") + deleteReq, err := http.NewRequest(http.MethodDelete, baseURL+"/api/v1/addon_registries/"+createReq.Name, nil) + Expect(err).Should(BeNil()) + deleteRes, err := http.DefaultClient.Do(deleteReq) + Expect(err).Should(BeNil()) + Expect(deleteRes).ShouldNot(BeNil()) + Expect(deleteRes.StatusCode).Should(Equal(200)) + }) + + It("should enable and disable an addon", func() { + defer GinkgoRecover() + req := apis.EnableAddonRequest{ + Args: map[string]string{}, + } + testAddon := "fluxcd" + res := post("/api/v1/addons/enable?name="+testAddon, req) + Expect(res).ShouldNot(BeNil()) + Expect(res.StatusCode).Should(Equal(200)) + Expect(res.Body).ShouldNot(BeNil()) + + defer res.Body.Close() + + var statusRes apis.AddonStatusResponse + err := json.NewDecoder(res.Body).Decode(&statusRes) + + Expect(err).Should(BeNil()) + Expect(statusRes.Phase).Should(Equal(apis.AddonPhaseEnabling)) + + // Wait for addon enabled + + period := 20 * time.Second + timeout := 5 * time.Minute + err = wait.PollImmediate(period, timeout, func() (done bool, err error) { + res = get("/api/v1/addons/status?name=" + testAddon) + err = json.NewDecoder(res.Body).Decode(&statusRes) + Expect(err).Should(BeNil()) + if statusRes.Phase == apis.AddonPhaseEnabled { + return true, nil + } + return false, nil + }) + Expect(err).Should(BeNil()) + + res = post("/api/v1/addons/disable?name="+testAddon, req) + Expect(res).ShouldNot(BeNil()) + Expect(res.StatusCode).Should(Equal(200)) + Expect(res.Body).ShouldNot(BeNil()) + + err = json.NewDecoder(res.Body).Decode(&statusRes) + Expect(err).Should(BeNil()) + + }) +}) diff --git a/test/e2e-apiserver-test/suite_test.go b/test/e2e-apiserver-test/suite_test.go index 4014e9af8..02a879bb7 100644 --- a/test/e2e-apiserver-test/suite_test.go +++ b/test/e2e-apiserver-test/suite_test.go @@ -18,6 +18,8 @@ package e2e_apiserver_test import ( "context" + "errors" + "net/http" "testing" "time" @@ -61,6 +63,17 @@ var _ = BeforeSuite(func() { err = server.Run(ctx) Expect(err).ShouldNot(HaveOccurred()) }() + By("wait for api server to start") + Eventually( + func() error { + res, err := http.Get("http://127.0.0.1:8000/api/v1/namespaces") + if err != nil { + return err + } + if res.StatusCode == http.StatusOK { + return nil + } + return errors.New("rest service not ready") + }, time.Second*5, time.Millisecond*200).Should(BeNil()) By("api server started") - time.Sleep(time.Second * 2) }) diff --git a/vela-templates/addons/fluxcd/readme.md b/vela-templates/addons/fluxcd/readme.md new file mode 100644 index 000000000..d0ea0da55 --- /dev/null +++ b/vela-templates/addons/fluxcd/readme.md @@ -0,0 +1,18 @@ +# fluxcd + +This addon is built based [FluxCD](https://fluxcd.io/) + +## install + +```shell +vela addon enable fluxcd +``` + +## X-Definitions + +Enable fluxcd addon to use these X-definitions + +- [helm](https://kubevela.io/docs/end-user/components/helm) helps to deploy a helm chart from everywhere: +git repo / helm repo / S3 compatible bucket. + +- [kustomize](https://kubevela.io/docs/end-user/components/kustomize) helps to deploy a kustomize style artifact. diff --git a/vela-templates/addons/istio/readme.md b/vela-templates/addons/istio/readme.md new file mode 100644 index 000000000..de9bdab97 --- /dev/null +++ b/vela-templates/addons/istio/readme.md @@ -0,0 +1,3 @@ +# istio + +This addon provides istio support for vela rollout. \ No newline at end of file diff --git a/vela-templates/addons/keda/readme.md b/vela-templates/addons/keda/readme.md new file mode 100644 index 000000000..8cb7a9f0f --- /dev/null +++ b/vela-templates/addons/keda/readme.md @@ -0,0 +1,3 @@ +# keda + +keda \ No newline at end of file diff --git a/vela-templates/addons/kruise/readme.md b/vela-templates/addons/kruise/readme.md new file mode 100644 index 000000000..775584c2c --- /dev/null +++ b/vela-templates/addons/kruise/readme.md @@ -0,0 +1,3 @@ +# kruise + +This addon provides [open-kruise](https://github.com/openkruise/kruise) workload. \ No newline at end of file diff --git a/vela-templates/addons/observability/readme.md b/vela-templates/addons/observability/readme.md new file mode 100644 index 000000000..de09d6dd8 --- /dev/null +++ b/vela-templates/addons/observability/readme.md @@ -0,0 +1,3 @@ +# observability + +This addon expose system and application level metrics for KubeVela. \ No newline at end of file diff --git a/vela-templates/addons/ocm-cluster-manager/readme.md b/vela-templates/addons/ocm-cluster-manager/readme.md new file mode 100644 index 000000000..f9b214285 --- /dev/null +++ b/vela-templates/addons/ocm-cluster-manager/readme.md @@ -0,0 +1,3 @@ +# ocm-cluster-manager + +This addon aims to support multi-cluster application deployment. \ No newline at end of file diff --git a/vela-templates/addons/prometheus/readme.md b/vela-templates/addons/prometheus/readme.md new file mode 100644 index 000000000..4917cfd76 --- /dev/null +++ b/vela-templates/addons/prometheus/readme.md @@ -0,0 +1,3 @@ +# prometheus + +prometheus diff --git a/vela-templates/addons/terraform-provider-alibaba/readme.md b/vela-templates/addons/terraform-provider-alibaba/readme.md new file mode 100644 index 000000000..b092c1804 --- /dev/null +++ b/vela-templates/addons/terraform-provider-alibaba/readme.md @@ -0,0 +1,3 @@ +# terraform/provider-alibaba + +This addon contains terraform provider for Alibaba Cloud. \ No newline at end of file diff --git a/vela-templates/addons/terraform-provider-aws/readme.md b/vela-templates/addons/terraform-provider-aws/readme.md new file mode 100644 index 000000000..82ddc7e7c --- /dev/null +++ b/vela-templates/addons/terraform-provider-aws/readme.md @@ -0,0 +1,3 @@ +# terraform/provider-aws + +This addon contains terraform provider for AWS \ No newline at end of file diff --git a/vela-templates/addons/terraform-provider-azure/readme.md b/vela-templates/addons/terraform-provider-azure/readme.md new file mode 100644 index 000000000..dfc36ff1f --- /dev/null +++ b/vela-templates/addons/terraform-provider-azure/readme.md @@ -0,0 +1,3 @@ +# terraform/provider-azure + +This addon contains terraform provider for Azure. \ No newline at end of file diff --git a/vela-templates/addons/terraform/readme.md b/vela-templates/addons/terraform/readme.md new file mode 100644 index 000000000..55016b4ea --- /dev/null +++ b/vela-templates/addons/terraform/readme.md @@ -0,0 +1,4 @@ +# Terraform + +This addon contains terraform operation kit, which allows you to arrange, +generate and use cloud service from different cloud vendor. \ No newline at end of file diff --git a/vela-templates/gen_addons.go b/vela-templates/gen_addons.go index 3643b2033..4fd68ea32 100644 --- a/vela-templates/gen_addons.go +++ b/vela-templates/gen_addons.go @@ -29,6 +29,8 @@ import ( "strings" "text/template" + addonutil "github.com/oam-dev/kubevela/pkg/utils/addon" + "github.com/Masterminds/sprig" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" @@ -39,20 +41,22 @@ import ( "github.com/oam-dev/kubevela/apis/core.oam.dev/v1beta1" "github.com/oam-dev/kubevela/pkg/oam/util" - "github.com/oam-dev/kubevela/references/cli" ) const ( + // DetailFileName is readme for each addon + DetailFileName = "readme.md" + // TemplateName represents the Application template file of addons TemplateName = "template.yaml" // ApplicationFileDir is where we store generated application & component definition ApplicationFileDir = "auto-gen" - // ComponentDefDir is where we store correspond componentDefinition for addon - ComponentDefDir = "definitions" + // DefinitionDir is where we store correspond X-Definition for addon + DefinitionDir = "definitions" - // ResourceDir is where we store correspond componentDefinition for addon + // ResourceDir is where we store correspond resources for addon ResourceDir = "resource" // DescAnnotation records the description of addon @@ -66,6 +70,12 @@ const ( // NameAnnotation marked the addon's name if exist, or application's name NameAnnotation = "addons.oam.dev/name" + + // ApplicationKey is the key to store application in ConfigMap + ApplicationKey = "application" + + // DetailKey is the key to store detail information in ConfigMap + DetailKey = "detail" ) // DefaultEnableAddons is default enabled addons @@ -77,11 +87,11 @@ type velaFile struct { Content string } -// AddonInfo records addon's metadata -type AddonInfo struct { +// AddonGenerateInfo records addon's metadata used in addon generation +type AddonGenerateInfo struct { ResourceFiles []velaFile DefinitionFiles []velaFile - HasDefs bool + DetailFile velaFile Name string StoreName string Description string @@ -134,13 +144,14 @@ func newWalkFn(files *[]velaFile) filepath.WalkFunc { } } -func getAddonInfo(addon string, addonsPath string) (*AddonInfo, error) { +func getAddonInfo(addon string, addonsPath string) (*AddonGenerateInfo, error) { addonRoot := filepath.Clean(addonsPath + "/" + addon) resourceRoot := filepath.Clean(addonRoot + "/" + ResourceDir) - defRoot := filepath.Clean(addonRoot + "/" + ComponentDefDir) + defRoot := filepath.Clean(addonRoot + "/" + DefinitionDir) + detailFile := filepath.Clean(addonRoot + "/" + DetailFileName) resourcesFiles := make([]velaFile, 0) defFiles := make([]velaFile, 0) - addInfo := &AddonInfo{ + addInfo := &AddonGenerateInfo{ TemplatePath: filepath.Join(addonRoot, TemplateName), } // raw resources directory @@ -155,9 +166,20 @@ func getAddonInfo(addon string, addonsPath string) (*AddonInfo, error) { if err := filepath.Walk(defRoot, newWalkFn(&defFiles)); err != nil { return nil, err } - addInfo.HasDefs = true addInfo.DefinitionFiles = defFiles } + + if pathExist(detailFile) { + content, err := os.ReadFile(detailFile) + if err != nil { + return nil, errors.Wrapf(err, "read %s detail file fail", addon) + } + addInfo.DetailFile = velaFile{ + RelativePath: detailFile, + Name: filepath.Base(detailFile), + Content: string(content), + } + } return addInfo, nil } @@ -180,7 +202,7 @@ func WriteToFile(filename string, data string) error { return file.Sync() } -func generateApplication(addon *AddonInfo) (*v1beta1.Application, error) { +func generateApplication(addon *AddonGenerateInfo) (*v1beta1.Application, error) { templatePath := strings.Split(addon.TemplatePath, "/") templateName := templatePath[len(templatePath)-1] t, err := template.New(templateName).Funcs(sprig.TxtFuncMap()).ParseFiles(addon.TemplatePath) @@ -202,12 +224,12 @@ func generateApplication(addon *AddonInfo) (*v1beta1.Application, error) { return app, err } -func setConfigMapLabels(addonInfo *AddonInfo) map[string]string { +func setConfigMapLabels(addonInfo *AddonGenerateInfo) map[string]string { return map[string]string{ MarkLabel: addonInfo.StoreName, } } -func setConfigMapAnnotations(addonInfo *AddonInfo) map[string]string { +func setConfigMapAnnotations(addonInfo *AddonGenerateInfo) map[string]string { return map[string]string{ NameAnnotation: addonInfo.Name, DescAnnotation: addonInfo.Description, @@ -229,7 +251,7 @@ func removeUselessInplace(s *string) { } // storeConfigMap store configMap in helm chart -func storeConfigMap(addonInfo *AddonInfo, application *v1beta1.Application, storePath string) error { +func storeConfigMap(addonInfo *AddonGenerateInfo, application *v1beta1.Application, storePath string) error { configMap := &corev1.ConfigMap{ TypeMeta: v1.TypeMeta{ APIVersion: "v1", @@ -247,7 +269,8 @@ func storeConfigMap(addonInfo *AddonInfo, application *v1beta1.Application, stor if err != nil { return err } - data["application"] = string(initContent) + data[ApplicationKey] = string(initContent) + data[DetailKey] = addonInfo.DetailFile.Content configMap.Data = data content, err := yaml.Marshal(configMap) if err != nil { @@ -329,7 +352,7 @@ func main() { } } -func setAddonName(addInfo *AddonInfo, app *v1beta1.Application) { +func setAddonName(addInfo *AddonGenerateInfo, app *v1beta1.Application) { var name string if val, ok := app.Annotations[NameAnnotation]; ok { name = val @@ -337,5 +360,5 @@ func setAddonName(addInfo *AddonInfo, app *v1beta1.Application) { name = app.Name } addInfo.Name = name - addInfo.StoreName = cli.TransAddonName(name) + addInfo.StoreName = addonutil.TransAddonName(name) }