diff --git a/pkg/apiserver/domain/service/helm.go b/pkg/apiserver/domain/service/helm.go index 2320aa512..1761a4759 100644 --- a/pkg/apiserver/domain/service/helm.go +++ b/pkg/apiserver/domain/service/helm.go @@ -45,8 +45,9 @@ func NewHelmService() HelmService { type HelmService interface { ListChartNames(ctx context.Context, url string, secretName string, skipCache bool) ([]string, error) ListChartVersions(ctx context.Context, url string, chartName string, secretName string, skipCache bool) (repo.ChartVersions, error) - GetChartValues(ctx context.Context, url string, chartName string, version string, secretName string, skipCache bool) (map[string]interface{}, error) + ListChartValuesFiles(ctx context.Context, url string, chartName string, version string, secretName string, repoType string, skipCache bool) (map[string]string, error) ListChartRepo(ctx context.Context, projectName string) (*v1.ChartRepoResponseList, error) + GetChartValues(ctx context.Context, repoURL string, chartName string, version string, secretName string, repoType string, skipCache bool) (map[string]interface{}, error) } type defaultHelmImpl struct { @@ -99,7 +100,7 @@ func (d defaultHelmImpl) ListChartVersions(ctx context.Context, repoURL string, return chartVersions, nil } -func (d defaultHelmImpl) GetChartValues(ctx context.Context, repoURL string, chartName string, version string, secretName string, skipCache bool) (map[string]interface{}, error) { +func (d defaultHelmImpl) ListChartValuesFiles(ctx context.Context, repoURL string, chartName string, version string, secretName string, repoType string, skipCache bool) (map[string]string, error) { if !utils.IsValidURL(repoURL) { return nil, bcode.ErrRepoInvalidURL } @@ -111,13 +112,33 @@ func (d defaultHelmImpl) GetChartValues(ctx context.Context, repoURL string, cha return nil, bcode.ErrRepoBasicAuth } } - v, err := d.helper.GetValuesFromChart(repoURL, chartName, version, skipCache, opts) + v, err := d.helper.GetValuesFromChart(repoURL, chartName, version, skipCache, repoType, opts) if err != nil { klog.Errorf("cannot fetch chart values repo: %s, chart: %s, version: %s, error: %s", utils.Sanitize(repoURL), utils.Sanitize(chartName), utils.Sanitize(version), err.Error()) return nil, bcode.ErrGetChartValues } - res := make(map[string]interface{}, len(v)) - flattenKey("", v, res) + return v.Data, nil +} + +func (d defaultHelmImpl) GetChartValues(ctx context.Context, repoURL string, chartName string, version string, secretName string, repoType string, skipCache bool) (map[string]interface{}, error) { + if !utils.IsValidURL(repoURL) { + return nil, bcode.ErrRepoInvalidURL + } + var opts *common.HTTPOption + var err error + if len(secretName) != 0 { + opts, err = helm.SetHTTPOption(ctx, d.K8sClient, types2.NamespacedName{Namespace: types.DefaultKubeVelaNS, Name: secretName}) + if err != nil { + return nil, bcode.ErrRepoBasicAuth + } + } + v, err := d.helper.GetValuesFromChart(repoURL, chartName, version, skipCache, repoType, opts) + if err != nil { + klog.Errorf("cannot fetch chart values repo: %s, chart: %s, version: %s, error: %s", utils.Sanitize(repoURL), utils.Sanitize(chartName), utils.Sanitize(version), err.Error()) + return nil, bcode.ErrGetChartValues + } + res := make(map[string]interface{}, len(v.Values)) + flattenKey("", v.Values, res) return res, nil } diff --git a/pkg/apiserver/domain/service/helm_test.go b/pkg/apiserver/domain/service/helm_test.go index 1538d882d..7afecc920 100644 --- a/pkg/apiserver/domain/service/helm_test.go +++ b/pkg/apiserver/domain/service/helm_test.go @@ -213,7 +213,7 @@ var _ = Describe("test helm usecasae", func() { Expect(len(versions)).Should(BeEquivalentTo(1)) Expect(versions[0].Version).Should(BeEquivalentTo("8.8.23")) - values, err := u.GetChartValues(ctx, mockServer.URL, "mysql", "8.8.23", "repo-secret", false) + values, err := u.ListChartValuesFiles(ctx, mockServer.URL, "mysql", "8.8.23", "repo-secret", "helm", false) Expect(err).Should(BeNil()) Expect(values).ShouldNot(BeNil()) Expect(len(values)).ShouldNot(BeEquivalentTo(0)) @@ -228,7 +228,7 @@ var _ = Describe("test helm usecasae", func() { _, err = u.ListChartVersions(ctx, "http://127.0.0.1:8080", "mysql", "repo-secret-notExist", false) Expect(err).ShouldNot(BeNil()) - _, err = u.GetChartValues(ctx, "http://127.0.0.1:8080", "mysql", "8.8.23", "repo-secret-notExist", false) + _, err = u.ListChartValuesFiles(ctx, "http://127.0.0.1:8080", "mysql", "8.8.23", "repo-secret-notExist", "helm", false) Expect(err).ShouldNot(BeNil()) }) }) diff --git a/pkg/apiserver/interfaces/api/repository.go b/pkg/apiserver/interfaces/api/repository.go index d79474239..0f0e42734 100644 --- a/pkg/apiserver/interfaces/api/repository.go +++ b/pkg/apiserver/interfaces/api/repository.go @@ -69,9 +69,19 @@ func (h repository) GetWebServiceRoute() *restful.WebService { Writes([]string{})) // List available chart versions - ws.Route(ws.GET("/charts/{chart}/versions").To(h.listVersions). + ws.Route(ws.GET("/chart/versions").To(h.listVersionsFromQuery). Doc("list versions"). Metadata(restfulspec.KeyOpenAPITags, tags). + Param(ws.QueryParameter("chart", "helm chart").DataType("string").Required(true)). + Param(ws.QueryParameter("repoUrl", "helm repository url").DataType("string").Required(true)). + Param(ws.QueryParameter("secretName", "secret of the repo").DataType("string")). + Returns(200, "OK", v1.ChartVersionListResponse{}). + Returns(400, "Bad Request", bcode.Bcode{}). + Writes([]string{})) + + ws.Route(ws.GET("/charts/{chart}/versions").To(h.listChartVersions). + Doc("list versions").Deprecate(). + Metadata(restfulspec.KeyOpenAPITags, tags). Param(ws.QueryParameter("repoUrl", "helm repository url").DataType("string")). Param(ws.QueryParameter("secretName", "secret of the repo").DataType("string")). Returns(200, "OK", v1.ChartVersionListResponse{}). @@ -79,14 +89,26 @@ func (h repository) GetWebServiceRoute() *restful.WebService { Writes([]string{})) // List available chart versions - ws.Route(ws.GET("/charts/{chart}/versions/{version}/values").To(h.chartValues). + ws.Route(ws.GET("/chart/values").To(h.chartValues). Doc("get chart value"). Metadata(restfulspec.KeyOpenAPITags, tags). + Param(ws.QueryParameter("chart", "helm chart").DataType("string").Required(true)). + Param(ws.QueryParameter("version", "helm chart version").DataType("string").Required(true)). + Param(ws.QueryParameter("repoUrl", "helm repository url").DataType("string").Required(true)). + Param(ws.QueryParameter("repoType", "helm repository type").DataType("string").Required(true)). + Param(ws.QueryParameter("secretName", "secret of the repo").DataType("string")). + Returns(200, "OK", ""). + Returns(400, "Bad Request", bcode.Bcode{}). + Writes(map[string]string{})) + + ws.Route(ws.GET("/charts/{chart}/versions/{version}/values").To(h.getChartValues). + Doc("get chart value").Deprecate(). + Metadata(restfulspec.KeyOpenAPITags, tags). Param(ws.QueryParameter("repoUrl", "helm repository url").DataType("string")). Param(ws.QueryParameter("secretName", "secret of the repo").DataType("string")). Returns(200, "OK", map[string]interface{}{}). Returns(400, "Bad Request", bcode.Bcode{}). - Writes([]string{})) + Writes(map[string]interface{}{})) ws.Route(ws.GET("/image/repos").To(h.getImageRepos). Doc("get the oci repos"). @@ -132,9 +154,9 @@ func (h repository) listCharts(req *restful.Request, res *restful.Response) { } } -func (h repository) listVersions(req *restful.Request, res *restful.Response) { +func (h repository) listVersionsFromQuery(req *restful.Request, res *restful.Response) { url := req.QueryParameter("repoUrl") - chartName := req.PathParameter("chart") + chartName := req.QueryParameter("chart") secName := req.QueryParameter("secretName") skipCache, err := isSkipCache(req) if err != nil { @@ -154,7 +176,7 @@ func (h repository) listVersions(req *restful.Request, res *restful.Response) { } } -func (h repository) chartValues(req *restful.Request, res *restful.Response) { +func (h repository) getChartValues(req *restful.Request, res *restful.Response) { url := req.QueryParameter("repoUrl") secName := req.QueryParameter("secretName") chartName := req.PathParameter("chart") @@ -165,12 +187,57 @@ func (h repository) chartValues(req *restful.Request, res *restful.Response) { return } - versions, err := h.HelmService.GetChartValues(req.Request.Context(), url, chartName, version, secName, skipCache) + values, err := h.HelmService.GetChartValues(req.Request.Context(), url, chartName, version, secName, "helm", skipCache) if err != nil { bcode.ReturnError(req, res, err) return } - err = res.WriteEntity(versions) + err = res.WriteEntity(values) + if err != nil { + bcode.ReturnError(req, res, err) + return + } +} + +func (h repository) listChartVersions(req *restful.Request, res *restful.Response) { + url := req.QueryParameter("repoUrl") + chartName := req.PathParameter("chart") + secName := req.QueryParameter("secretName") + skipCache, err := isSkipCache(req) + if err != nil { + bcode.ReturnError(req, res, bcode.ErrSkipCacheParameter) + return + } + versions, err := h.HelmService.ListChartVersions(req.Request.Context(), url, chartName, secName, skipCache) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + err = res.WriteEntity(v1.ChartVersionListResponse{Versions: versions}) + if err != nil { + bcode.ReturnError(req, res, err) + return + } +} + +func (h repository) chartValues(req *restful.Request, res *restful.Response) { + url := req.QueryParameter("repoUrl") + secName := req.QueryParameter("secretName") + chartName := req.QueryParameter("chart") + version := req.QueryParameter("version") + repoType := req.QueryParameter("repoType") + skipCache, err := isSkipCache(req) + if err != nil { + bcode.ReturnError(req, res, bcode.ErrSkipCacheParameter) + return + } + + values, err := h.HelmService.ListChartValuesFiles(req.Request.Context(), url, chartName, version, secName, repoType, skipCache) + if err != nil { + bcode.ReturnError(req, res, err) + return + } + err = res.WriteEntity(values) if err != nil { bcode.ReturnError(req, res, err) return diff --git a/pkg/utils/helm/helm_helper.go b/pkg/utils/helm/helm_helper.go index aa155b293..cd167a8fe 100644 --- a/pkg/utils/helm/helm_helper.go +++ b/pkg/utils/helm/helm_helper.go @@ -24,6 +24,7 @@ import ( "os" "path" "path/filepath" + "regexp" "strings" "time" @@ -32,6 +33,9 @@ import ( "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/pkg/cli" + "helm.sh/helm/v3/pkg/downloader" + "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/kube" "helm.sh/helm/v3/pkg/release" relutil "helm.sh/helm/v3/pkg/releaseutil" @@ -56,6 +60,12 @@ const ( valuesPatten = "repoUrl: %s, chart: %s, version: %s" ) +// ChartValues contain all values files in chart and default chart values +type ChartValues struct { + Data map[string]string + Values map[string]interface{} +} + // Helper provides helper functions for common Helm operations type Helper struct { cache *utils2.MemoryCacheStore @@ -309,12 +319,22 @@ func (h *Helper) ListChartsFromRepo(repoURL string, skipCache bool, opts *common } // GetValuesFromChart will extract the parameter from a helm chart -func (h *Helper) GetValuesFromChart(repoURL string, chartName string, version string, skipCache bool, opts *common.HTTPOption) (map[string]interface{}, error) { +func (h *Helper) GetValuesFromChart(repoURL string, chartName string, version string, skipCache bool, repoType string, opts *common.HTTPOption) (*ChartValues, error) { if h.cache != nil && !skipCache { if v := h.cache.Get(fmt.Sprintf(valuesPatten, repoURL, chartName, version)); v != nil { - return v.(map[string]interface{}), nil + return v.(*ChartValues), nil } } + if repoType == "oci" { + v, err := fetchChartValuesFromOciRepo(repoURL, chartName, version, opts) + if err != nil { + return nil, err + } + if h.cache != nil { + h.cache.Put(fmt.Sprintf(valuesPatten, repoURL, chartName, version), v, 20*time.Minute) + } + return v, nil + } i, err := h.GetIndexInfo(repoURL, skipCache, opts) if err != nil { return nil, err @@ -334,10 +354,17 @@ func (h *Helper) GetValuesFromChart(repoURL string, chartName string, version st if err != nil { continue } - if h.cache != nil { - h.cache.Put(fmt.Sprintf(valuesPatten, repoURL, chartName, version), c.Values, calculateCacheTimeFromIndex(len(i.Entries))) + v := &ChartValues{ + Data: loadValuesYamlFile(c), + Values: c.Values, } - return c.Values, nil + if err != nil { + return nil, err + } + if h.cache != nil { + h.cache.Put(fmt.Sprintf(valuesPatten, repoURL, chartName, version), v, calculateCacheTimeFromIndex(len(i.Entries))) + } + return v, nil } return nil, fmt.Errorf("cannot load chart from chart repo") } @@ -351,3 +378,49 @@ func calculateCacheTimeFromIndex(length int) time.Duration { } return cacheTime } + +// nolint +func fetchChartValuesFromOciRepo(repoURL string, chartName string, version string, opts *common.HTTPOption) (*ChartValues, error) { + d := downloader.ChartDownloader{ + Verify: downloader.VerifyNever, + Getters: getter.All(cli.New()), + } + + if opts != nil { + d.Options = append(d.Options, getter.WithInsecureSkipVerifyTLS(opts.InsecureSkipTLS), + getter.WithTLSClientConfig(opts.CertFile, opts.KeyFile, opts.CaFile), + getter.WithBasicAuth(opts.Username, opts.Password)) + } + + var err error + dest, err := os.MkdirTemp("", "helm-") + if err != nil { + return nil, errors.Wrap(err, "failed to fetch values file") + } + defer os.RemoveAll(dest) + + chartRef := fmt.Sprintf("%s/%s", repoURL, chartName) + saved, _, err := d.DownloadTo(chartRef, version, dest) + if err != nil { + return nil, err + } + c, err := loader.Load(saved) + if err != nil { + return nil, errors.Wrap(err, "failed to fetch values file") + } + return &ChartValues{ + Data: loadValuesYamlFile(c), + Values: c.Values, + }, nil +} + +func loadValuesYamlFile(chart *chart.Chart) map[string]string { + result := map[string]string{} + re := regexp.MustCompile(`.*yaml$`) + for _, f := range chart.Raw { + if re.MatchString(f.Name) && !strings.Contains(f.Name, "/") && f.Name != "Chart.yaml" { + result[f.Name] = string(f.Data) + } + } + return result +} diff --git a/pkg/utils/helm/helm_helper_test.go b/pkg/utils/helm/helm_helper_test.go index 6c16e84d0..7a388abe0 100644 --- a/pkg/utils/helm/helm_helper_test.go +++ b/pkg/utils/helm/helm_helper_test.go @@ -114,9 +114,9 @@ var _ = Describe("Test helm helper", func() { It("Test getValues from chart", func() { helper := NewHelper() - values, err := helper.GetValuesFromChart("./testdata", "autoscalertrait", "0.2.0", true, nil) + values, err := helper.GetValuesFromChart("./testdata", "autoscalertrait", "0.2.0", true, "helm", nil) Expect(err).Should(BeNil()) - Expect(values).ShouldNot(BeEmpty()) + Expect(values).ShouldNot(BeNil()) }) }) diff --git a/test/e2e-apiserver-test/repository_test.go b/test/e2e-apiserver-test/repository_test.go new file mode 100644 index 000000000..05d7087d9 --- /dev/null +++ b/test/e2e-apiserver-test/repository_test.go @@ -0,0 +1,42 @@ +/* +Copyright 2022 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_apiserver_test + +import ( + "io" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Helm rest api test", func() { + + Describe("helm repo api test", func() { + It("test fetching chart values in OCI registry", func() { + resp := getWithQuery("/repository/chart/values", map[string]string{ + "repoUrl": "oci://ghcr.io", + "chart": "stefanprodan/charts/podinfo", + "repoType": "oci", + "version": "6.1.0", + }) + defer resp.Body.Close() + values, err := io.ReadAll(resp.Body) + Expect(err).Should(BeNil()) + Expect(len(values)).ShouldNot(BeEquivalentTo(0)) + }) + }) +}) diff --git a/test/e2e-apiserver-test/suite_test.go b/test/e2e-apiserver-test/suite_test.go index 5d59ea95f..68602d988 100644 --- a/test/e2e-apiserver-test/suite_test.go +++ b/test/e2e-apiserver-test/suite_test.go @@ -191,6 +191,26 @@ func get(path string) *http.Response { return response } +func getWithQuery(path string, params map[string]string) *http.Response { + client := &http.Client{} + if !strings.HasPrefix(path, "/v1") { + path = baseURL + path + } else { + path = baseDomain + path + } + req, err := http.NewRequest(http.MethodGet, path, nil) + Expect(err).Should(BeNil()) + req.Header.Add("Authorization", token) + query := req.URL.Query() + for k, v := range params { + query.Set(k, v) + } + req.URL.RawQuery = query.Encode() + response, err := client.Do(req) + Expect(err).Should(BeNil()) + return response +} + func delete(path string) *http.Response { client := &http.Client{} if !strings.HasPrefix(path, "/v1") {