Feat: add helm repo list endpoint (#3564)

* add helm repo list

Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com>

* fix commit

Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com>

* fix comments

Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com>

* build swagger

Signed-off-by: 楚岳 <wangyike.wyk@alibaba-inc.com>

docs/apidoc/swagger.json

@@ -4304,6 +4304,49 @@
 				}
 			}
 		},
+		"/api/v1/repository/chart_repos": {
+			"get": {
+				"consumes": [
+					"application/xml",
+					"application/json"
+				],
+				"produces": [
+					"application/json",
+					"application/xml"
+				],
+				"tags": [
+					"repository",
+					"helm"
+				],
+				"summary": "list chart repo",
+				"operationId": "listRepo",
+				"parameters": [
+					{
+						"type": "string",
+						"description": "the config project",
+						"name": "project",
+						"in": "query"
+					}
+				],
+				"responses": {
+					"200": {
+						"description": "OK",
+						"schema": {
+							"type": "array",
+							"items": {
+								"type": "string"
+							}
+						}
+					},
+					"400": {
+						"description": "Bad Request",
+						"schema": {
+							"$ref": "#/definitions/bcode.Bcode"
+						}
+					}
+				}
+			}
+		},
 		"/api/v1/repository/charts": {
 			"get": {
 				"consumes": [
@@ -4326,6 +4369,12 @@
 						"description": "helm repository url",
 						"name": "repoUrl",
 						"in": "query"
+					},
+					{
+						"type": "string",
+						"description": "secret of the repo",
+						"name": "secretName",
+						"in": "query"
 					}
 				],
 				"responses": {
@@ -4369,6 +4418,12 @@
 						"description": "helm repository url",
 						"name": "repoUrl",
 						"in": "query"
+					},
+					{
+						"type": "string",
+						"description": "secret of the repo",
+						"name": "secretName",
+						"in": "query"
 					}
 				],
 				"responses": {
@@ -4409,6 +4464,12 @@
 						"description": "helm repository url",
 						"name": "repoUrl",
 						"in": "query"
+					},
+					{
+						"type": "string",
+						"description": "secret of the repo",
+						"name": "secretName",
+						"in": "query"
 					}
 				],
 				"responses": {
@@ -7148,10 +7209,10 @@
 		},
 		"v1.ApplicationDeployResponse": {
 			"required": [
-				"createTime",
 				"version",
-				"note",
 				"status",
+				"note",
+				"createTime",
 				"envName",
 				"triggerType"
 			],
@@ -8244,10 +8305,10 @@
 		"v1.DetailAddonResponse": {
 			"required": [
 				"name",
-				"icon",
-				"invisible",
-				"version",
 				"description",
+				"icon",
+				"version",
+				"invisible",
 				"schema",
 				"uiSchema",
 				"definitions",
@@ -8328,12 +8389,12 @@
 		"v1.DetailApplicationResponse": {
 			"required": [
 				"alias",
-				"description",
-				"createTime",
-				"name",
 				"project",
-				"updateTime",
+				"description",
 				"icon",
+				"name",
+				"createTime",
+				"updateTime",
 				"policies",
 				"envBindings",
 				"applicationType",
@@ -8394,20 +8455,20 @@
 		},
 		"v1.DetailClusterResponse": {
 			"required": [
-				"updateTime",
-				"kubeConfig",
-				"name",
-				"icon",
-				"reason",
-				"provider",
+				"status",
 				"apiServerURL",
 				"dashboardURL",
-				"createTime",
+				"kubeConfigSecret",
+				"icon",
+				"labels",
+				"kubeConfig",
+				"updateTime",
+				"name",
 				"alias",
 				"description",
-				"labels",
-				"status",
-				"kubeConfigSecret",
+				"reason",
+				"createTime",
+				"provider",
 				"resourceInfo"
 			],
 			"properties": {
@@ -8465,14 +8526,14 @@
 		},
 		"v1.DetailComponentResponse": {
 			"required": [
+				"createTime",
 				"updateTime",
 				"appPrimaryKey",
+				"type",
+				"main",
+				"name",
 				"creator",
 				"alias",
-				"name",
-				"main",
-				"createTime",
-				"type",
 				"definition"
 			],
 			"properties": {
@@ -8574,13 +8635,13 @@
 		},
 		"v1.DetailPolicyResponse": {
 			"required": [
+				"type",
 				"description",
 				"creator",
 				"properties",
 				"createTime",
 				"updateTime",
-				"name",
-				"type"
+				"name"
 			],
 			"properties": {
 				"createTime": {
@@ -8610,17 +8671,17 @@
 		},
 		"v1.DetailRevisionResponse": {
 			"required": [
-				"envName",
-				"reason",
-				"deployUser",
 				"updateTime",
-				"status",
-				"note",
 				"triggerType",
-				"workflowName",
+				"version",
+				"deployUser",
 				"createTime",
+				"status",
+				"workflowName",
+				"envName",
 				"appPrimaryKey",
-				"version"
+				"reason",
+				"note"
 			],
 			"properties": {
 				"appPrimaryKey": {
@@ -8674,9 +8735,9 @@
 		},
 		"v1.DetailTargetResponse": {
 			"required": [
-				"name",
-				"createTime",
 				"project",
+				"createTime",
+				"name",
 				"updateTime"
 			],
 			"properties": {
@@ -8717,11 +8778,11 @@
 		},
 		"v1.DetailUserResponse": {
 			"required": [
-				"name",
-				"email",
 				"disabled",
 				"createTime",
 				"lastLoginTime",
+				"name",
+				"email",
 				"projects",
 				"roles"
 			],
@@ -8762,12 +8823,12 @@
 		},
 		"v1.DetailWorkflowRecordResponse": {
 			"required": [
-				"workflowName",
-				"workflowAlias",
-				"applicationRevision",
 				"status",
 				"name",
 				"namespace",
+				"workflowName",
+				"workflowAlias",
+				"applicationRevision",
 				"deployTime",
 				"deployUser",
 				"note",
@@ -8819,14 +8880,14 @@
 		},
 		"v1.DetailWorkflowResponse": {
 			"required": [
-				"description",
-				"enable",
+				"envName",
 				"createTime",
 				"updateTime",
-				"name",
-				"alias",
+				"enable",
 				"default",
-				"envName"
+				"description",
+				"name",
+				"alias"
 			],
 			"properties": {
 				"alias": {
@@ -9021,8 +9082,8 @@
 		},
 		"v1.EnvBindingTarget": {
 			"required": [
-				"name",
-				"alias"
+				"alias",
+				"name"
 			],
 			"properties": {
 				"alias": {
@@ -9405,11 +9466,11 @@
 		},
 		"v1.LoginUserInfoResponse": {
 			"required": [
-				"name",
-				"email",
 				"disabled",
 				"createTime",
 				"lastLoginTime",
+				"name",
+				"email",
 				"projects",
 				"platformPermissions",
 				"projectPermissions"
@@ -9733,11 +9794,11 @@
 		},
 		"v1.SystemInfoResponse": {
 			"required": [
-				"createTime",
-				"updateTime",
 				"installID",
 				"enableCollection",
 				"loginType",
+				"createTime",
+				"updateTime",
 				"systemVersion"
 			],
 			"properties": {

pkg/apiserver/rest/apis/v1/types.go

@@ -1277,3 +1277,14 @@ type LoginUserInfoResponse struct {
 	PlatformPermissions []PermissionBase            `json:"platformPermissions"`
 	ProjectPermissions  map[string][]PermissionBase `json:"projectPermissions"`
 }
+
+// ChartRepoResponse the response body of  chart repo
+type ChartRepoResponse struct {
+	URL        string `json:"url"`
+	SecretName string `json:"secretName"`
+}
+
+// ChartRepoResponseList the response body of list chart repo
+type ChartRepoResponseList struct {
+	ChartRepoResponse []*ChartRepoResponse `json:"repos"`
+}

pkg/apiserver/rest/usecase/helm.go

@@ -20,32 +20,48 @@ import (
 	"context"
 	"strconv"
 
-	"helm.sh/helm/v3/pkg/repo"
-
+	"github.com/oam-dev/kubevela/apis/types"
+	"github.com/oam-dev/kubevela/pkg/apiserver/clients"
 	"github.com/oam-dev/kubevela/pkg/apiserver/log"
+	v1 "github.com/oam-dev/kubevela/pkg/apiserver/rest/apis/v1"
 	"github.com/oam-dev/kubevela/pkg/apiserver/rest/utils/bcode"
+	"github.com/oam-dev/kubevela/pkg/oam"
 	"github.com/oam-dev/kubevela/pkg/utils/helm"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"helm.sh/helm/v3/pkg/repo"
 )
 
 // NewHelmUsecase return a helmHandler
 func NewHelmUsecase() HelmHandler {
+	c, err := clients.GetKubeClient()
+	if err != nil {
+		log.Logger.Fatalf("get kube client failure %s", err.Error())
+	}
 	return defaultHelmHandler{
-		helper: helm.NewHelperWithCache(),
+		helper:    helm.NewHelperWithCache(),
+		k8sClient: c,
 	}
 }
 
 // HelmHandler responsible handle helm related interface
 type HelmHandler interface {
-	ListChartNames(ctx context.Context, url string, skipCache bool) ([]string, error)
-	ListChartVersions(ctx context.Context, url string, chartName string, skipCache bool) (repo.ChartVersions, error)
-	GetChartValues(ctx context.Context, url string, chartName string, version string, skipCache bool) (map[string]interface{}, error)
+	ListChartNames(ctx context.Context, url string, secretName string, skipCache bool) ([]string, error)
+	ListChartVersions(ctx context.Context, url string, chartName string, secretName string, skipCache bool) (repo.ChartVersions, error)
+	GetChartValues(ctx context.Context, url string, chartName string, version string, secretName string, skipCache bool) (map[string]interface{}, error)
+	ListChartRepo(ctx context.Context, projectName string) (*v1.ChartRepoResponseList, error)
 }
 
 type defaultHelmHandler struct {
-	helper *helm.Helper
+	helper    *helm.Helper
+	k8sClient client.Client
 }
 
-func (d defaultHelmHandler) ListChartNames(ctx context.Context, url string, skipCache bool) ([]string, error) {
+func (d defaultHelmHandler) ListChartNames(ctx context.Context, url string, secretName string, skipCache bool) ([]string, error) {
+	// TODO(wangyikewxgm): support authority helm repo
 	charts, err := d.helper.ListChartsFromRepo(url, skipCache)
 	if err != nil {
 		log.Logger.Errorf("cannot fetch charts repo: %s, error: %s", url, err.Error())
@@ -54,7 +70,7 @@ func (d defaultHelmHandler) ListChartNames(ctx context.Context, url string, skip
 	return charts, nil
 }
 
-func (d defaultHelmHandler) ListChartVersions(ctx context.Context, url string, chartName string, skipCache bool) (repo.ChartVersions, error) {
+func (d defaultHelmHandler) ListChartVersions(ctx context.Context, url string, chartName string, secretName string, skipCache bool) (repo.ChartVersions, error) {
 	chartVersions, err := d.helper.ListVersions(url, chartName, skipCache)
 	if err != nil {
 		log.Logger.Errorf("cannot fetch chart versions repo: %s, chart: %s error: %s", url, chartName, err.Error())
@@ -67,7 +83,7 @@ func (d defaultHelmHandler) ListChartVersions(ctx context.Context, url string, c
 	return chartVersions, nil
 }
 
-func (d defaultHelmHandler) GetChartValues(ctx context.Context, url string, chartName string, version string, skipCache bool) (map[string]interface{}, error) {
+func (d defaultHelmHandler) GetChartValues(ctx context.Context, url string, chartName string, version string, secretName string, skipCache bool) (map[string]interface{}, error) {
 	v, err := d.helper.GetValuesFromChart(url, chartName, version, skipCache)
 	if err != nil {
 		log.Logger.Errorf("cannot fetch chart values repo: %s, chart: %s, version: %s, error: %s", url, chartName, version, err.Error())
@@ -78,6 +94,50 @@ func (d defaultHelmHandler) GetChartValues(ctx context.Context, url string, char
 	return res, nil
 }
 
+func (d defaultHelmHandler) ListChartRepo(ctx context.Context, projectName string) (*v1.ChartRepoResponseList, error) {
+	var res []*v1.ChartRepoResponse
+	var err error
+
+	if len(projectName) != 0 {
+		projectSecrets := corev1.SecretList{}
+		opts := []client.ListOption{
+			client.MatchingLabels{oam.LabelConfigType: "helm-repository", oam.LabelProject: projectName},
+			client.InNamespace(types.DefaultKubeVelaNS),
+		}
+		err = d.k8sClient.List(ctx, &projectSecrets, opts...)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, item := range projectSecrets.Items {
+			res = append(res, &v1.ChartRepoResponse{URL: string(item.Data["url"]), SecretName: item.Name})
+		}
+	}
+
+	globalSecrets := corev1.SecretList{}
+	selector := metav1.LabelSelector{
+		MatchLabels: map[string]string{oam.LabelConfigType: "helm-repository"},
+		MatchExpressions: []metav1.LabelSelectorRequirement{
+			{Key: oam.LabelProject, Operator: metav1.LabelSelectorOpDoesNotExist},
+		},
+	}
+
+	ls, _ := metav1.LabelSelectorAsSelector(&selector)
+	err = d.k8sClient.List(ctx, &globalSecrets, &client.ListOptions{
+		LabelSelector: ls,
+		Namespace:     types.DefaultKubeVelaNS,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, item := range globalSecrets.Items {
+		res = append(res, &v1.ChartRepoResponse{URL: string(item.Data["url"]), SecretName: item.Name})
+	}
+
+	return &v1.ChartRepoResponseList{ChartRepoResponse: res}, nil
+}
+
 // this func will flatten a nested map, the key will flatten with separator "." and the value's type will be keep
 // src is the map you want to flatten the output will be set in dest map
 // eg : src is  {a:{b:{c:true}}} , the dest is {a.b.c:true}

pkg/apiserver/rest/usecase/helm_test.go

@@ -17,9 +17,19 @@ limitations under the License.
 package usecase
 
 import (
+	"context"
 	"encoding/json"
 	"testing"
 
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/oam-dev/kubevela/pkg/oam/util"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
+
 	"github.com/stretchr/testify/assert"
 )
 
@@ -37,6 +47,63 @@ func TestFlattenKeyFunc(t *testing.T) {
 	assert.Equal(t, dstMap, res)
 }
 
+var _ = Describe("Test helm repo list", func() {
+	ctx := context.Background()
+	var pSec, gSec v1.Secret
+
+	BeforeEach(func() {
+		pSec = v1.Secret{}
+		gSec = v1.Secret{}
+		Expect(k8sClient.Create(ctx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "vela-system"}})).Should(SatisfyAny(BeNil(), util.AlreadyExistMatcher{}))
+		Expect(yaml.Unmarshal([]byte(projectSecret), &pSec)).Should(BeNil())
+		Expect(yaml.Unmarshal([]byte(globalSecret), &gSec)).Should(BeNil())
+		Expect(k8sClient.Create(ctx, &pSec)).Should(BeNil())
+		Expect(k8sClient.Create(ctx, &gSec)).Should(BeNil())
+	})
+
+	AfterEach(func() {
+		Expect(k8sClient.Delete(ctx, &gSec)).Should(BeNil())
+		Expect(k8sClient.Delete(ctx, &pSec)).Should(BeNil())
+	})
+
+	It("Test list with project ", func() {
+		u := NewHelmUsecase()
+		list, err := u.ListChartRepo(ctx, "my-project")
+		Expect(err).Should(BeNil())
+		Expect(len(list.ChartRepoResponse)).Should(BeEquivalentTo(2))
+		found := 0
+		for _, response := range list.ChartRepoResponse {
+			if response.SecretName == "project-helm-repo" {
+				Expect(response.URL).Should(BeEquivalentTo("https://kedacore.github.io/charts"))
+				found++
+			}
+			if response.SecretName == "global-helm-repo" {
+				Expect(response.URL).Should(BeEquivalentTo("https://charts.bitnami.com/bitnami"))
+				found++
+			}
+		}
+		Expect(found).Should(BeEquivalentTo(2))
+	})
+
+	It("Test list func with not exist project", func() {
+		u := NewHelmUsecase()
+		list, err := u.ListChartRepo(ctx, "not-exist-project")
+		Expect(err).Should(BeNil())
+		Expect(len(list.ChartRepoResponse)).Should(BeEquivalentTo(1))
+		Expect(list.ChartRepoResponse[0].URL).Should(BeEquivalentTo("https://charts.bitnami.com/bitnami"))
+		Expect(list.ChartRepoResponse[0].SecretName).Should(BeEquivalentTo("global-helm-repo"))
+	})
+
+	It("Test list func without project", func() {
+		u := NewHelmUsecase()
+		list, err := u.ListChartRepo(ctx, "")
+		Expect(err).Should(BeNil())
+		Expect(len(list.ChartRepoResponse)).Should(BeEquivalentTo(1))
+		Expect(list.ChartRepoResponse[0].URL).Should(BeEquivalentTo("https://charts.bitnami.com/bitnami"))
+		Expect(list.ChartRepoResponse[0].SecretName).Should(BeEquivalentTo("global-helm-repo"))
+	})
+})
+
 var (
 	src = `{
     "OAMSpecVer":"v0.2",
@@ -175,4 +242,29 @@ var (
     "webhookService.port": 11443,
     "webhookService.type": "ClusterIP"
 }`
+	globalSecret = `
+apiVersion: v1
+stringData:
+  url: https://charts.bitnami.com/bitnami
+kind: Secret
+metadata:
+  labels:
+    config.oam.dev/type: helm-repository
+  name: global-helm-repo
+  namespace: vela-system
+type: Opaque
+`
+	projectSecret = `
+apiVersion: v1
+kind: Secret
+metadata:
+  name: project-helm-repo
+  namespace: vela-system
+  labels:
+    config.oam.dev/type: helm-repository
+    core.oam.dev/project: my-project
+stringData:
+  url: https://kedacore.github.io/charts
+type: Opaque
+`
 )

pkg/apiserver/rest/webservice/helm.go

@@ -46,11 +46,21 @@ func (h helmWebService) GetWebService() *restful.WebService {
 
 	tags := []string{"repository", "helm"}
 
+	// List charts
+	ws.Route(ws.GET("/chart_repos").To(h.listRepo).
+		Doc("list chart repo").
+		Metadata(restfulspec.KeyOpenAPITags, tags).
+		Param(ws.QueryParameter("project", "the config project").DataType("string")).
+		Returns(200, "OK", []string{}).
+		Returns(400, "Bad Request", bcode.Bcode{}).
+		Writes([]string{}))
+
 	// List charts
 	ws.Route(ws.GET("/charts").To(h.listCharts).
 		Doc("list charts").
 		Metadata(restfulspec.KeyOpenAPITags, tags).
 		Param(ws.QueryParameter("repoUrl", "helm repository url").DataType("string")).
+		Param(ws.QueryParameter("secretName", "secret of the repo").DataType("string")).
 		Returns(200, "OK", []string{}).
 		Returns(400, "Bad Request", bcode.Bcode{}).
 		Writes([]string{}))
@@ -60,6 +70,7 @@ func (h helmWebService) GetWebService() *restful.WebService {
 		Doc("list versions").
 		Metadata(restfulspec.KeyOpenAPITags, tags).
 		Param(ws.QueryParameter("repoUrl", "helm repository url").DataType("string")).
+		Param(ws.QueryParameter("secretName", "secret of the repo").DataType("string")).
 		Returns(200, "OK", v1.ChartVersionListResponse{}).
 		Returns(400, "Bad Request", bcode.Bcode{}).
 		Writes([]string{}))
@@ -69,6 +80,7 @@ func (h helmWebService) GetWebService() *restful.WebService {
 		Doc("get chart value").
 		Metadata(restfulspec.KeyOpenAPITags, tags).
 		Param(ws.QueryParameter("repoUrl", "helm repository url").DataType("string")).
+		Param(ws.QueryParameter("secretName", "secret of the repo").DataType("string")).
 		Returns(200, "OK", map[string]interface{}{}).
 		Returns(400, "Bad Request", bcode.Bcode{}).
 		Writes([]string{}))
@@ -79,12 +91,13 @@ func (h helmWebService) GetWebService() *restful.WebService {
 
 func (h helmWebService) listCharts(req *restful.Request, res *restful.Response) {
 	url := req.QueryParameter("repoUrl")
+	secName := req.QueryParameter("secretName")
 	skipCache, err := isSkipCache(req)
 	if err != nil {
 		bcode.ReturnError(req, res, bcode.ErrSkipCacheParameter)
 		return
 	}
-	charts, err := h.usecase.ListChartNames(context.Background(), url, skipCache)
+	charts, err := h.usecase.ListChartNames(context.Background(), url, secName, skipCache)
 	if err != nil {
 		bcode.ReturnError(req, res, err)
 		return
@@ -99,13 +112,14 @@ func (h helmWebService) listCharts(req *restful.Request, res *restful.Response)
 func (h helmWebService) listVersions(req *restful.Request, res *restful.Response) {
 	url := req.QueryParameter("repoUrl")
 	chartName := req.PathParameter("chart")
+	secName := req.QueryParameter("secretName")
 	skipCache, err := isSkipCache(req)
 	if err != nil {
 		bcode.ReturnError(req, res, bcode.ErrSkipCacheParameter)
 		return
 	}
 
-	versions, err := h.usecase.ListChartVersions(context.Background(), url, chartName, skipCache)
+	versions, err := h.usecase.ListChartVersions(context.Background(), url, chartName, secName, skipCache)
 	if err != nil {
 		bcode.ReturnError(req, res, err)
 		return
@@ -119,6 +133,7 @@ func (h helmWebService) listVersions(req *restful.Request, res *restful.Response
 
 func (h helmWebService) chartValues(req *restful.Request, res *restful.Response) {
 	url := req.QueryParameter("repoUrl")
+	secName := req.QueryParameter("secretName")
 	chartName := req.PathParameter("chart")
 	version := req.PathParameter("version")
 	skipCache, err := isSkipCache(req)
@@ -127,7 +142,7 @@ func (h helmWebService) chartValues(req *restful.Request, res *restful.Response)
 		return
 	}
 
-	versions, err := h.usecase.GetChartValues(context.Background(), url, chartName, version, skipCache)
+	versions, err := h.usecase.GetChartValues(context.Background(), url, chartName, version, secName, skipCache)
 	if err != nil {
 		bcode.ReturnError(req, res, err)
 		return
@@ -139,6 +154,20 @@ func (h helmWebService) chartValues(req *restful.Request, res *restful.Response)
 	}
 }
 
+func (h helmWebService) listRepo(req *restful.Request, res *restful.Response) {
+	project := req.QueryParameter("project")
+	repos, err := h.usecase.ListChartRepo(context.Background(), project)
+	if err != nil {
+		bcode.ReturnError(req, res, err)
+		return
+	}
+	err = res.WriteEntity(repos)
+	if err != nil {
+		bcode.ReturnError(req, res, err)
+		return
+	}
+}
+
 func isSkipCache(req *restful.Request) (bool, error) {
 	skipStr := req.QueryParameter("skipCache")
 	skipCache := false

pkg/oam/labels.go

@@ -90,6 +90,12 @@ const (
 	// LabelRuntimeNamespaceUsage mark the usage of the namespace in runtime cluster.
 	// A control plane cluster can also be used as runtime cluster
 	LabelRuntimeNamespaceUsage = "usage.oam.dev/runtime"
+
+	// LabelConfigType means the config type
+	LabelConfigType = "config.oam.dev/type"
+
+	// LabelProject recorde the project the resource belong to
+	LabelProject = "core.oam.dev/project"
 )
 
 const (