From c007ad8df04ed445d5aad050cd966cdeb15173b3 Mon Sep 17 00:00:00 2001 From: JohnJan Date: Wed, 12 Apr 2023 13:58:12 +0800 Subject: [PATCH] Feat: support config validation depends on CueX (#5796) * Feat: support config validation depends on CueX Signed-off-by: wuzhongjian * Feat: support config validation depends on CueX Signed-off-by: wuzhongjian * Feat: support config validation depends on CueX Signed-off-by: wuzhongjian * Feat: support config validation depends on CueX Signed-off-by: wuzhongjian --------- Signed-off-by: wuzhongjian --- go.mod | 2 + go.sum | 5 + pkg/config/factory.go | 52 ++++++- pkg/cue/cuex/compiler.go | 32 +++++ pkg/cue/cuex/providers/config/config.cue | 59 ++++++++ pkg/cue/cuex/providers/config/config.go | 125 +++++++++++++++++ pkg/cue/cuex/providers/config/config_test.go | 114 +++++++++++++++ pkg/utils/helm/helm_helper.go | 27 ++++ pkg/utils/helm/helm_helper_test.go | 22 +++ pkg/utils/helm/repo_index.go | 99 +++++++++++++ pkg/utils/helm/repo_index_test.go | 52 +++++++ pkg/utils/helm/types.go | 41 ++++++ pkg/utils/registries/options.go | 86 ++++++++++++ pkg/utils/registries/registry_helper.go | 41 ++++++ pkg/utils/registries/secret_authenticator.go | 131 ++++++++++++++++++ .../registries/secret_authenticator_test.go | 110 +++++++++++++++ pkg/utils/registries/types.go | 45 ++++++ 17 files changed, 1037 insertions(+), 6 deletions(-) create mode 100644 pkg/cue/cuex/compiler.go create mode 100644 pkg/cue/cuex/providers/config/config.cue create mode 100644 pkg/cue/cuex/providers/config/config.go create mode 100644 pkg/cue/cuex/providers/config/config_test.go create mode 100644 pkg/utils/helm/repo_index.go create mode 100644 pkg/utils/helm/repo_index_test.go create mode 100644 pkg/utils/helm/types.go create mode 100644 pkg/utils/registries/options.go create mode 100644 pkg/utils/registries/registry_helper.go create mode 100644 pkg/utils/registries/secret_authenticator.go create mode 100644 pkg/utils/registries/secret_authenticator_test.go create mode 100644 pkg/utils/registries/types.go diff --git a/go.mod b/go.mod index e24455929..4a17d33d6 100644 --- a/go.mod +++ b/go.mod @@ -112,6 +112,7 @@ require ( require ( github.com/dave/jennifer v1.6.0 github.com/ettle/strcase v0.1.1 + github.com/google/go-containerregistry v0.13.0 github.com/oam-dev/cluster-gateway v1.9.0-alpha.1 golang.org/x/tools v0.7.0 ) @@ -226,6 +227,7 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/locker v1.0.1 // indirect diff --git a/go.sum b/go.sum index 96f78e9c3..2b1c07473 100644 --- a/go.sum +++ b/go.sum @@ -287,6 +287,7 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= github.com/containerd/containerd v1.6.18 h1:qZbsLvmyu+Vlty0/Ex5xc0z2YtKpIsb5n45mAMI+2Ns= github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw= +github.com/containerd/stargz-snapshotter/estargz v0.12.1 h1:+7nYmHJb0tEkcRaAW+MHqoKaJYZmkikupxCqVtmPuY0= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -703,6 +704,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.13.0 h1:y1C7Z3e149OJbOPDBxLYR8ITPz8dTKqQwjErKVHJC8k= +github.com/google/go-containerregistry v0.13.0/go.mod h1:J9FQ+eSS4a1aC2GNZxvNpbWhgp0487v+cgiilB4FqDo= github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -1058,6 +1061,7 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v0.0.0-20161203194507-b8bc1bf76747/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= @@ -1447,6 +1451,7 @@ github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= diff --git a/pkg/config/factory.go b/pkg/config/factory.go index a4736efe8..ccfb8a926 100644 --- a/pkg/config/factory.go +++ b/pkg/config/factory.go @@ -24,6 +24,11 @@ import ( "strings" "time" + velacuex "github.com/oam-dev/kubevela/pkg/cue/cuex" + + cuelang "cuelang.org/go/cue" + "github.com/kubevela/pkg/cue/cuex" + "github.com/getkin/kin-openapi/openapi3" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -69,6 +74,15 @@ const SaveTemplateKey = "template" // TemplateConfigMapNamePrefix the prefix of the configmap name. const TemplateConfigMapNamePrefix = "config-template-" +// TemplateValidation define the key name for the config-template validation +const TemplateValidation = SaveTemplateKey + ".validation" + +// TemplateOutput define the key name for the config-template output +const TemplateOutput = SaveTemplateKey + ".output" + +// TemplateParameter define the key name for the config-template parameter +const TemplateParameter = SaveTemplateKey + ".parameter" + // ErrSensitiveConfig means this config can not be read directly. var ErrSensitiveConfig = errors.New("the config is sensitive") @@ -182,6 +196,17 @@ type CreateDistributionSpec struct { Targets []*ClusterTarget } +// Validation the response of the validation +type Validation struct { + Result bool `json:"result"` + Message string `json:"message"` +} + +// Error return the error message +func (e *Validation) Error() string { + return fmt.Sprintf("failed to validate config: %s", e.Message) +} + // Factory handle the config type Factory interface { ParseTemplate(defaultName string, content []byte) (*Template, error) @@ -446,13 +471,28 @@ func (k *kubeConfigFactory) ParseConfig(ctx context.Context, Name: meta.Name, Namespace: meta.Namespace, } - // Render the output secret - output, err := template.Template.RunAndOutput(contextValue, meta.Properties) - if err != nil && !cue.IsFieldNotExist(err) { - return nil, err + // Compile the config template + contextOption := cuex.WithExtraData("context", contextValue) + parameterOption := cuex.WithExtraData(TemplateParameter, meta.Properties) + val, err := velacuex.KubeVelaDefaultCompiler.Get().CompileStringWithOptions(ctx, string(template.Template), contextOption, parameterOption) + if err != nil { + return nil, fmt.Errorf("failed to compile config template: %w", err) } - if output != nil { - if err := output.UnmarshalTo(&secret); err != nil { + // Render the validation response and check validation result + valid := val.LookupPath(cuelang.ParsePath(TemplateValidation)) + validation := Validation{} + if valid.Exists() { + if err := valid.Decode(&validation); err != nil { + return nil, fmt.Errorf("the validation format must be validation") + } + } + if len(validation.Message) > 0 { + return nil, &validation + } + // Render the output secret + output := val.LookupPath(cuelang.ParsePath(TemplateOutput)) + if output.Exists() { + if err := output.Decode(&secret); err != nil { return nil, fmt.Errorf("the output format must be secret") } } diff --git a/pkg/cue/cuex/compiler.go b/pkg/cue/cuex/compiler.go new file mode 100644 index 000000000..998d4c00f --- /dev/null +++ b/pkg/cue/cuex/compiler.go @@ -0,0 +1,32 @@ +/* +Copyright 2023 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cuex + +import ( + "github.com/kubevela/pkg/cue/cuex" + "github.com/kubevela/pkg/util/singleton" + + "github.com/oam-dev/kubevela/pkg/cue/cuex/providers/config" +) + +// KubeVelaDefaultCompiler compiler for cuex to compile +var KubeVelaDefaultCompiler = singleton.NewSingleton[*cuex.Compiler](func() *cuex.Compiler { + compiler := cuex.NewCompilerWithInternalPackages( + config.Package, + ) + return compiler +}) diff --git a/pkg/cue/cuex/providers/config/config.cue b/pkg/cue/cuex/providers/config/config.cue new file mode 100644 index 000000000..7005320ad --- /dev/null +++ b/pkg/cue/cuex/providers/config/config.cue @@ -0,0 +1,59 @@ +package config + +#ImageRegistry: { + #do: "image-registry" + #provider: "config" + + // +usage=The params of this action + $params: { + // +usage=Image registry FQDN, such as: index.docker.io + registry: *"index.docker.io" | string + // +usage=Authenticate the image registry + auth?: { + // +usage=Private Image registry username + username: string + // +usage=Private Image registry password + password: string + // +usage=Private Image registry email + email?: string + } + // +usage=For the registry server that uses the self-signed certificate + insecure?: bool + // +usage=For the registry server that uses the HTTP protocol + useHTTP?: bool + } + // +usage=The result of this action, will be filled with the validation response after the action is executed + $returns: { + // +usage=The result of the response + result: bool + // +usage=The message of the response + message: string + ... + } + ... +} + +#HelmRepository: { + #do: "helm-repository" + #provider: "config" + + $params: { + // +usage=The public url of the helm chart repository. + url: string + // +usage=The username of basic auth repo. + username?: string + // +usage=The password of basic auth repo. + password?: string + // +usage=The ca certificate of helm repository. Please encode this data with base64. + caFile?: string + } + // +usage=The result of this action, will be filled with the validation response after the action is executed + $returns: { + // +usage=The result of the response + result: bool + // +usage=The message of the response + message: string + ... + } + ... +} diff --git a/pkg/cue/cuex/providers/config/config.go b/pkg/cue/cuex/providers/config/config.go new file mode 100644 index 000000000..b6ff21cc8 --- /dev/null +++ b/pkg/cue/cuex/providers/config/config.go @@ -0,0 +1,125 @@ +/* +Copyright 2023 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "context" + _ "embed" + + "github.com/oam-dev/kubevela/pkg/utils/helm" + "github.com/oam-dev/kubevela/pkg/utils/registries" + + "github.com/kubevela/pkg/cue/cuex/providers" + cuexruntime "github.com/kubevela/pkg/cue/cuex/runtime" + "github.com/kubevela/pkg/util/runtime" + "k8s.io/klog/v2" +) + +// ImageRegistryVars is the vars for image registry validation +type ImageRegistryVars struct { + Registry string `json:"registry"` + Auth struct { + Username string `json:"username"` + Password string `json:"password"` + Email string `json:"email"` + } `json:"auth"` + Insecure bool `json:"insecure"` + UseHTTP bool `json:"useHTTP"` +} + +// HelmRepositoryVars is the vars for helm repository validation +type HelmRepositoryVars struct { + URL string `json:"url"` + Username string `json:"username"` + Password string `json:"password"` + CAFile string `json:"caFile"` +} + +// ResponseVars is the returns for resource +type ResponseVars struct { + Result bool `json:"result"` + Message string `json:"message"` +} + +// ImageRegistryParams is the params for image registry +type ImageRegistryParams providers.Params[ImageRegistryVars] + +// HelmRepositoryParams is the params for helm repository +type HelmRepositoryParams providers.Params[HelmRepositoryVars] + +// ValidationReturns returned struct for http response +type ValidationReturns providers.Returns[ResponseVars] + +// ImageRegistry . +func ImageRegistry(ctx context.Context, validationParams *ImageRegistryParams) (*ValidationReturns, error) { + params := validationParams.Params + imageRegistry := ®istries.ImageRegistry{ + Registry: params.Registry, + Auth: params.Auth, + Insecure: params.Insecure, + UseHTTP: params.Insecure, + } + registryHelper := registries.NewRegistryHelper() + var message string + ok, err := registryHelper.Auth(ctx, imageRegistry) + if err != nil { + message = err.Error() + klog.Errorf("validate image-registry %s failed, err: %v", imageRegistry, err) + } + return &ValidationReturns{ + Returns: ResponseVars{ + Result: ok, + Message: message, + }, + }, nil +} + +// HelmRepository . +func HelmRepository(ctx context.Context, validationParams *HelmRepositoryParams) (*ValidationReturns, error) { + params := validationParams.Params + helmRepository := &helm.Repository{ + URL: params.URL, + Username: params.Username, + Password: params.Password, + CAFile: params.CAFile, + } + helmHelper := helm.NewHelper() + var message string + ok, err := helmHelper.ValidateRepo(ctx, helmRepository) + if err != nil { + message = err.Error() + klog.Errorf("validate helm-repository %s failed, err: %v", helmRepository, err) + } + return &ValidationReturns{ + Returns: ResponseVars{ + Result: ok, + Message: message, + }, + }, nil +} + +// ProviderName . +const ProviderName = "config" + +//go:embed config.cue +var template string + +// Package . +var Package = runtime.Must(cuexruntime.NewInternalPackage(ProviderName, template, map[string]cuexruntime.ProviderFn{ + "image-registry": cuexruntime.GenericProviderFn[ImageRegistryParams, ValidationReturns](ImageRegistry), + "helm-repository": cuexruntime.GenericProviderFn[HelmRepositoryParams, ValidationReturns](HelmRepository), +})) diff --git a/pkg/cue/cuex/providers/config/config_test.go b/pkg/cue/cuex/providers/config/config_test.go new file mode 100644 index 000000000..4af562e81 --- /dev/null +++ b/pkg/cue/cuex/providers/config/config_test.go @@ -0,0 +1,114 @@ +/* +Copyright 2023 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestImageRegistry(t *testing.T) { + ctx := context.Background() + testCases := []struct { + name string + validationParams *ImageRegistryParams + expectResult bool + }{ + { + name: "Should authenticate with correct credential", + validationParams: &ImageRegistryParams{ + Params: ImageRegistryVars{ + Registry: "dockerhub.qingcloud.com", + Auth: struct { + Username string `json:"username"` + Password string `json:"password"` + Email string `json:"email"` + }{ + Username: "guest", Password: "guest", + }, + Insecure: false, + UseHTTP: false, + }, + }, + expectResult: true, + }, + { + name: "Shouldn't authenticate with incorrect credentials", + validationParams: &ImageRegistryParams{ + Params: ImageRegistryVars{ + Registry: "index.docker.io", + Auth: struct { + Username string `json:"username"` + Password string `json:"password"` + Email string `json:"email"` + }{ + Username: "foo", Password: "bar", + }, + Insecure: false, + UseHTTP: false, + }, + }, + expectResult: false, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + _v, err := ImageRegistry(ctx, testCase.validationParams) + require.NoError(t, err) + require.Equal(t, testCase.expectResult, _v.Returns.Result) + }) + } +} + +func TestHelmRepository(t *testing.T) { + ctx := context.Background() + testCases := []struct { + name string + validationParams *HelmRepositoryParams + expectResult bool + }{ + { + name: "Should authenticate with correct credential", + validationParams: &HelmRepositoryParams{ + Params: HelmRepositoryVars{ + URL: "https://charts.kubevela.net/core", + }, + }, + expectResult: true, + }, + { + name: "Shouldn't authenticate with incorrect helm repo URL", + validationParams: &HelmRepositoryParams{ + Params: HelmRepositoryVars{ + URL: "https://www.baidu.com", + }, + }, + expectResult: false, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + _v, err := HelmRepository(ctx, testCase.validationParams) + require.NoError(t, err) + require.Equal(t, testCase.expectResult, _v.Returns.Result) + }) + } +} diff --git a/pkg/utils/helm/helm_helper.go b/pkg/utils/helm/helm_helper.go index 7b36c8008..e3e052e44 100644 --- a/pkg/utils/helm/helm_helper.go +++ b/pkg/utils/helm/helm_helper.go @@ -21,6 +21,7 @@ import ( "context" "fmt" "io" + "net/url" "os" "path" "path/filepath" @@ -368,6 +369,32 @@ func (h *Helper) GetValuesFromChart(repoURL string, chartName string, version st return nil, fmt.Errorf("cannot load chart from chart repo") } +// ValidateRepo will validate the helm repository +func (h *Helper) ValidateRepo(ctx context.Context, repo *Repository) (bool, error) { + parsedURL, err := url.Parse(repo.URL) + if err != nil { + return false, err + } + userInfo := parsedURL.User + if len(repo.Username) > 0 && len(repo.Password) > 0 { + userInfo = url.UserPassword(repo.Username, repo.Password) + } + var cred = &RepoCredential{} + // TODO: support S3Config validation + if strings.HasPrefix(repo.URL, "https://") || strings.HasPrefix(repo.URL, "http://") { + if userInfo != nil { + cred.Username = userInfo.Username() + cred.Password, _ = userInfo.Password() + } + } + + _, err = LoadRepoIndex(ctx, repo.URL, cred) + if err != nil { + return false, err + } + return true, nil +} + func calculateCacheTimeFromIndex(length int) time.Duration { cacheTime := 3 * time.Minute if length > 20 { diff --git a/pkg/utils/helm/helm_helper_test.go b/pkg/utils/helm/helm_helper_test.go index 7a388abe0..1256b7e48 100644 --- a/pkg/utils/helm/helm_helper_test.go +++ b/pkg/utils/helm/helm_helper_test.go @@ -36,6 +36,8 @@ import ( var _ = Describe("Test helm helper", func() { + ctx := context.Background() + It("Test LoadCharts ", func() { helper := NewHelper() chart, err := helper.LoadCharts("./testdata/autoscalertrait-0.1.0.tgz", nil) @@ -118,6 +120,26 @@ var _ = Describe("Test helm helper", func() { Expect(err).Should(BeNil()) Expect(values).ShouldNot(BeNil()) }) + + It("Test validate helm repo", func() { + helper := NewHelper() + helmRepo := &Repository{ + URL: "https://charts.kubevela.net/core", + } + ok, err := helper.ValidateRepo(ctx, helmRepo) + Expect(err).Should(BeNil()) + Expect(ok).Should(BeTrue()) + }) + + It("Test validate the corrupt helm repo", func() { + helper := NewHelper() + helmRepo := &Repository{ + URL: "https://www.baidu.com", + } + ok, err := helper.ValidateRepo(ctx, helmRepo) + Expect(err).To(HaveOccurred()) + Expect(ok).Should(BeFalse()) + }) }) var _ = Describe("Test helm associated func", func() { diff --git a/pkg/utils/helm/repo_index.go b/pkg/utils/helm/repo_index.go new file mode 100644 index 000000000..21e743883 --- /dev/null +++ b/pkg/utils/helm/repo_index.go @@ -0,0 +1,99 @@ +/* +Copyright 2023 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helm + +import ( + "bytes" + "context" + "fmt" + "net/url" + "strings" + "time" + + "helm.sh/helm/v3/pkg/getter" + helmrepo "helm.sh/helm/v3/pkg/repo" + "sigs.k8s.io/yaml" +) + +// IndexYaml is the index.yaml of helm repo +const IndexYaml = "index.yaml" + +// LoadRepoIndex load helm repo index +func LoadRepoIndex(ctx context.Context, u string, cred *RepoCredential) (*helmrepo.IndexFile, error) { + + if !strings.HasSuffix(u, "/") { + u = fmt.Sprintf("%s/%s", u, IndexYaml) + } else { + u = fmt.Sprintf("%s%s", u, IndexYaml) + } + + resp, err := loadData(u, cred) + if err != nil { + return nil, err + } + + indexFile, err := loadIndex(resp.Bytes()) + if err != nil { + return nil, err + } + + return indexFile, nil +} + +func loadData(u string, cred *RepoCredential) (*bytes.Buffer, error) { + parsedURL, err := url.Parse(u) + if err != nil { + return nil, err + } + var resp *bytes.Buffer + + skipTLS := true + if cred.InsecureSkipTLSVerify != nil && !*cred.InsecureSkipTLSVerify { + skipTLS = false + } + + indexURL := parsedURL.String() + // TODO add user-agent + g, _ := getter.NewHTTPGetter() + resp, err = g.Get(indexURL, + getter.WithTimeout(5*time.Minute), + getter.WithURL(u), + getter.WithInsecureSkipVerifyTLS(skipTLS), + getter.WithTLSClientConfig(cred.CertFile, cred.KeyFile, cred.CAFile), + getter.WithBasicAuth(cred.Username, cred.Password), + ) + if err != nil { + return nil, err + } + + return resp, nil +} + +// loadIndex loads an index file and does minimal validity checking. +// +// This will fail if API Version is not set (ErrNoAPIVersion) or if the unmarshal fails. +func loadIndex(data []byte) (*helmrepo.IndexFile, error) { + i := &helmrepo.IndexFile{} + if err := yaml.Unmarshal(data, i); err != nil { + return i, err + } + i.SortEntries() + if i.APIVersion == "" { + return i, helmrepo.ErrNoAPIVersion + } + return i, nil +} diff --git a/pkg/utils/helm/repo_index_test.go b/pkg/utils/helm/repo_index_test.go new file mode 100644 index 000000000..bb8105000 --- /dev/null +++ b/pkg/utils/helm/repo_index_test.go @@ -0,0 +1,52 @@ +/* +Copyright 2023 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package helm + +import ( + "context" + "fmt" + "strings" + "testing" +) + +func TestLoadRepo(t *testing.T) { + + u := "https://charts.kubevela.net/core" + + ctx := context.Background() + index, err := LoadRepoIndex(ctx, u, &RepoCredential{}) + if err != nil { + t.Errorf("load repo failed, err: %s", err) + t.Failed() + return + } + + for _, entry := range index.Entries { + chartUrl := entry[0].URLs[0] + + if !(strings.HasPrefix(chartUrl, "https://") || strings.HasPrefix(chartUrl, "http://")) { + chartUrl = fmt.Sprintf("%s/%s", u, chartUrl) + } + chartData, err := loadData(chartUrl, &RepoCredential{}) + if err != nil { + t.Errorf("load chart data failed, err: %s", err) + t.Failed() + } + _ = chartData + break + } + +} diff --git a/pkg/utils/helm/types.go b/pkg/utils/helm/types.go new file mode 100644 index 000000000..b2689e93d --- /dev/null +++ b/pkg/utils/helm/types.go @@ -0,0 +1,41 @@ +/* +Copyright 2023 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package helm + +// RepoCredential is the helm repo credential +type RepoCredential struct { + // chart repository username + Username string `json:"username,omitempty"` + // chart repository password + Password string `json:"password,omitempty"` + // identify HTTPS client using this SSL certificate file + CertFile string `json:"certFile,omitempty"` + // identify HTTPS client using this SSL key file + KeyFile string `json:"keyFile,omitempty"` + // verify certificates of HTTPS-enabled servers using this CA bundle + CAFile string `json:"caFile,omitempty"` + // skip tls certificate checks for the repository, default is ture + InsecureSkipTLSVerify *bool `json:"insecureSkipTLSVerify,omitempty"` +} + +// Repository is the helm repository +type Repository struct { + URL string `json:"url"` + Username string `json:"username"` + Password string `json:"password"` + CAFile string `json:"caFile"` +} diff --git a/pkg/utils/registries/options.go b/pkg/utils/registries/options.go new file mode 100644 index 000000000..fa95bcfd7 --- /dev/null +++ b/pkg/utils/registries/options.go @@ -0,0 +1,86 @@ +/* +Copyright 2023 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registries + +import ( + "context" + "net/http" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +const ( + // DefaultRegistry is the registry name that will be used if no registry + // provided and the default is not overridden. + DefaultRegistry = "index.docker.io" + defaultRegistryAlias = "docker.io" + + // DefaultTag is the tag name that will be used if no tag provided and the + // default is not overridden. + DefaultTag = "latest" +) + +type options struct { + name []name.Option + remote []remote.Option + platform *v1.Platform +} + +// Option is a functional option +type Option func(*options) + +// WithTransport is a functional option for overriding the default transport +// for remote operations. +func WithTransport(t http.RoundTripper) Option { + return func(o *options) { + o.remote = append(o.remote, remote.WithTransport(t)) + } +} + +// Insecure is an Option that allows image references to be fetched without TLS. +func Insecure(o *options) { + o.name = append(o.name, name.Insecure) +} + +// WithAuth is a functional option for overriding the default authenticator +// for remote operations. +func WithAuth(auth authn.Authenticator) Option { + return func(o *options) { + // Replace the default keychain at position 0. + o.remote[0] = remote.WithAuth(auth) + } +} + +// WithContext is a functional option for setting the context. +func WithContext(ctx context.Context) Option { + return func(o *options) { + o.remote = append(o.remote, remote.WithContext(ctx)) + } +} + +// WithPlatform is an Option to specify the platform. +func WithPlatform(platform *v1.Platform) Option { + return func(o *options) { + if platform != nil { + o.remote = append(o.remote, remote.WithPlatform(*platform)) + } + o.platform = platform + } +} diff --git a/pkg/utils/registries/registry_helper.go b/pkg/utils/registries/registry_helper.go new file mode 100644 index 000000000..b62122525 --- /dev/null +++ b/pkg/utils/registries/registry_helper.go @@ -0,0 +1,41 @@ +/* +Copyright 2023 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registries + +import "context" + +// RegistryHelper provides helper functions for common Registry operations +type RegistryHelper interface { + // Auth check if secret has correct credential to authenticate with remote registry + Auth(ctx context.Context, imageRegistry *ImageRegistry) (bool, error) +} + +type registryHelper struct{} + +// NewRegistryHelper creates a registry helper +func NewRegistryHelper() RegistryHelper { + return ®istryHelper{} +} + +func (r *registryHelper) Auth(ctx context.Context, imageRegistry *ImageRegistry) (bool, error) { + secretAuth, err := NewSecretAuthenticator(imageRegistry) + if err != nil { + return false, err + } + + return secretAuth.Auth(ctx) +} diff --git a/pkg/utils/registries/secret_authenticator.go b/pkg/utils/registries/secret_authenticator.go new file mode 100644 index 000000000..3f653201a --- /dev/null +++ b/pkg/utils/registries/secret_authenticator.go @@ -0,0 +1,131 @@ +/* +Copyright 2023 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registries + +import ( + "context" + "encoding/base64" + "fmt" + "net/http" + "net/url" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" +) + +// SecretAuthenticator provides helper functions for secret authenticator operations +type SecretAuthenticator interface { + Options() []Option + + Auth(ctx context.Context) (bool, error) + + Authorization() (*authn.AuthConfig, error) +} + +type secretAuthenticator struct { + auths DockerConfig + insecure bool // force using insecure when talk to the remote registry, even registry address starts with https +} + +// NewSecretAuthenticator creates a secret authenticator +func NewSecretAuthenticator(imageRegistry *ImageRegistry) (SecretAuthenticator, error) { + + if imageRegistry == nil { + return &secretAuthenticator{}, nil + } + + sa := &secretAuthenticator{ + insecure: false, + } + + // force insecure if imageRegistry has Insecure + sa.insecure = imageRegistry.Insecure + if imageRegistry.Insecure { + sa.insecure = true + } + + auth := fmt.Sprintf("%s:%s", imageRegistry.Auth.Username, imageRegistry.Auth.Password) + + entry := DockerConfigEntry{ + Username: imageRegistry.Auth.Username, + Password: imageRegistry.Auth.Password, + Email: imageRegistry.Auth.Email, + Auth: base64.StdEncoding.EncodeToString([]byte(auth)), + } + + sa.auths = map[string]DockerConfigEntry{} + if imageRegistry.UseHTTP { + sa.auths[fmt.Sprintf("http://%s", imageRegistry.Registry)] = entry + } else { + sa.auths[fmt.Sprintf("https://%s", imageRegistry.Registry)] = entry + } + return sa, nil +} + +func (s *secretAuthenticator) Authorization() (*authn.AuthConfig, error) { + for _, v := range s.auths { + return &authn.AuthConfig{ + Username: v.Username, + Password: v.Password, + Auth: v.Auth, + }, nil + } + return &authn.AuthConfig{}, nil +} + +func (s *secretAuthenticator) Auth(ctx context.Context) (bool, error) { + for k := range s.auths { + return s.AuthRegistry(ctx, k) + } + return false, fmt.Errorf("no registry found in image-registry") +} + +func (s *secretAuthenticator) AuthRegistry(ctx context.Context, reg string) (bool, error) { + url, err := url.Parse(reg) // in case reg is unformatted like http://docker.index.io + if err != nil { + return false, err + } + + options := make([]name.Option, 0) + if url.Scheme == "http" || s.insecure { + options = append(options, name.Insecure) + } + + registry, err := name.NewRegistry(url.Host, options...) + if err != nil { + return false, err + } + + _, err = transport.NewWithContext(ctx, registry, s, http.DefaultTransport, []string{}) + if err != nil { + return false, err + } + + return true, nil +} + +func (s *secretAuthenticator) Options() []Option { + options := make([]Option, 0) + + options = append(options, WithAuth(s)) + if s.insecure { + options = append(options, Insecure) + } + + return options +} diff --git a/pkg/utils/registries/secret_authenticator_test.go b/pkg/utils/registries/secret_authenticator_test.go new file mode 100644 index 000000000..f83cfeae1 --- /dev/null +++ b/pkg/utils/registries/secret_authenticator_test.go @@ -0,0 +1,110 @@ +/* +Copyright 2023 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registries + +import ( + "context" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-containerregistry/pkg/authn" +) + +func buildImageRegistry(registry, username, password string, insecure bool, useHTTP bool) *ImageRegistry { + imageRegistry := &ImageRegistry{ + Registry: registry, + Auth: Auth{Username: username, Password: password}, + Insecure: insecure, + UseHTTP: useHTTP, + } + + return imageRegistry +} + +func TestSecretAuthenticator(t *testing.T) { + imageRegistry := buildImageRegistry("dockerhub.qingcloud.com", "guest", "guest", false, false) + + secretAuthenticator, err := NewSecretAuthenticator(imageRegistry) + if err != nil { + t.Fatal(err) + } + + auth, err := secretAuthenticator.Authorization() + if err != nil { + t.Fatal(err) + } + + expected := &authn.AuthConfig{ + Username: "guest", + Password: "guest", + Auth: "Z3Vlc3Q6Z3Vlc3Q=", + } + + if diff := cmp.Diff(auth, expected); len(diff) != 0 { + t.Errorf("%T, got+ expected-, %s", expected, diff) + } +} + +func TestAuthn(t *testing.T) { + testCases := []struct { + name string + imageRegistry *ImageRegistry + auth bool + expectErr bool + }{ + { + name: "Should authenticate with correct credential", + imageRegistry: buildImageRegistry("dockerhub.qingcloud.com", "guest", "guest", false, false), + auth: true, + expectErr: false, + }, + { + name: "Shouldn't authenticate with incorrect credentials", + imageRegistry: buildImageRegistry("index.docker.io", "foo", "bar", false, false), + auth: false, + expectErr: true, + }, + { + name: "Shouldn't authenticate with no credentials", + imageRegistry: nil, + auth: false, + expectErr: true, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + secretAuthenticator, err := NewSecretAuthenticator(testCase.imageRegistry) + if err != nil { + t.Errorf("error creating secretAuthenticator, %v", err) + } + + ok, err := secretAuthenticator.Auth(context.Background()) + if testCase.auth != ok { + t.Errorf("expected auth result: %v, but got %v", testCase.auth, ok) + } + + if testCase.expectErr && err == nil { + t.Errorf("expected error, but got nil") + } + + if !testCase.expectErr && err != nil { + t.Errorf("authentication error, %v", err) + } + }) + } +} diff --git a/pkg/utils/registries/types.go b/pkg/utils/registries/types.go new file mode 100644 index 000000000..9e570e42c --- /dev/null +++ b/pkg/utils/registries/types.go @@ -0,0 +1,45 @@ +/* +Copyright 2023 The KubeVela Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registries + +// DockerConfig represents the config file used by the docker CLI. +// This config that represents the credentials that should be used +// when pulling images from specific image repositories. +type DockerConfig map[string]DockerConfigEntry + +// DockerConfigEntry wraps a docker config as an entry +type DockerConfigEntry struct { + Username string + Password string + Email string + Auth string +} + +// ImageRegistry the request body for validating image registry +type ImageRegistry struct { + Registry string `json:"registry"` + Auth Auth `json:"auth"` + Insecure bool `json:"insecure"` + UseHTTP bool `json:"useHTTP"` +} + +// Auth the auth of image registry +type Auth struct { + Username string `json:"username"` + Password string `json:"password"` + Email string `json:"email"` +}