From e37b0276c87d4d2b97f926f6f0484f37ddcb5fa5 Mon Sep 17 00:00:00 2001 From: TIEDPAG Date: Tue, 19 Jul 2022 18:00:28 +0800 Subject: [PATCH] Feat: support addon helm repo skip tls verify (#4122) (#4146) * Feat: support addon helm repo skip tls verify (#4122) Signed-off-by: damianqin * Fix: mv http server to BeforeSuite Signed-off-by: damianqin * Fix: fix ci error Signed-off-by: damianqin * Fix: fix merge Signed-off-by: damianqin * Feat: support addon helm repo skip tls verify (#4122) Signed-off-by: damianqin * Fix: mv http server to BeforeSuite Signed-off-by: damianqin * Fix: fix ci error Signed-off-by: damianqin * Fix: fix ci Signed-off-by: damianqin Co-authored-by: damianqin Co-authored-by: Jianbo Sun --- docs/apidoc/swagger.json | 163 ++++++++++---- pkg/addon/addon.go | 5 +- pkg/addon/cache.go | 10 +- pkg/addon/cache_test.go | 37 +-- pkg/addon/helper.go | 6 +- pkg/addon/source.go | 7 +- pkg/addon/suite_test.go | 18 +- pkg/addon/testdata/tls/local-selfsign.crt | 18 ++ pkg/addon/testdata/tls/local-selfsign.key | 27 +++ pkg/addon/utils.go | 5 +- pkg/addon/versioned_registry_suite_test.go | 248 +++++++++++++++++++++ pkg/addon/versioned_registry_test.go | 104 --------- pkg/utils/common/common.go | 17 +- references/cli/addon-registry.go | 33 +-- references/cli/addon.go | 6 +- references/cli/addon_test.go | 4 + 16 files changed, 502 insertions(+), 206 deletions(-) create mode 100644 pkg/addon/testdata/tls/local-selfsign.crt create mode 100644 pkg/addon/testdata/tls/local-selfsign.key create mode 100644 pkg/addon/versioned_registry_suite_test.go diff --git a/docs/apidoc/swagger.json b/docs/apidoc/swagger.json index 21146d663..d13300d4e 100644 --- a/docs/apidoc/swagger.json +++ b/docs/apidoc/swagger.json @@ -1726,6 +1726,12 @@ "name": "policyName", "in": "path", "required": true + }, + { + "type": "boolean", + "description": "Force delete the policy and all references", + "name": "force", + "in": "query" } ], "responses": { @@ -6074,7 +6080,6 @@ }, "addon.DeployTo": { "required": [ - "runtime_cluster", "disableControlPlane", "runtimeCluster" ], @@ -6134,6 +6139,9 @@ }, "addon.HelmSource": { "properties": { + "insecureSkipTLS": { + "type": "boolean" + }, "password": { "type": "string" }, @@ -7192,8 +7200,8 @@ }, "model.ApplicationRevision": { "required": [ - "createTime", "updateTime", + "createTime", "appPrimaryKey", "version", "status", @@ -8055,10 +8063,10 @@ "v1.ApplicationDeployResponse": { "required": [ "version", - "status", + "note", "envName", "createTime", - "note", + "status", "triggerType" ], "properties": { @@ -9252,14 +9260,23 @@ "v1.CreatePolicyRequest": { "required": [ "name", + "alias", + "envName", "description", "type", - "properties" + "properties", + "workflowPolicyBind" ], "properties": { + "alias": { + "type": "string" + }, "description": { "type": "string" }, + "envName": { + "type": "string" + }, "name": { "type": "string" }, @@ -9268,6 +9285,12 @@ }, "type": { "type": "string" + }, + "workflowPolicyBind": { + "type": "array", + "items": { + "$ref": "#/definitions/v1.WorkflowPolicyBinding" + } } } }, @@ -9486,11 +9509,11 @@ }, "v1.DetailAddonResponse": { "required": [ - "icon", "name", + "description", + "icon", "version", "invisible", - "description", "schema", "uiSchema", "definitions", @@ -9570,10 +9593,10 @@ }, "v1.DetailApplicationResponse": { "required": [ - "name", - "alias", "description", "icon", + "name", + "alias", "project", "createTime", "updateTime", @@ -9633,20 +9656,20 @@ }, "v1.DetailClusterResponse": { "required": [ - "name", - "status", - "dashboardURL", "createTime", + "status", "kubeConfig", + "updateTime", + "dashboardURL", + "name", + "description", + "icon", "labels", + "reason", + "alias", "provider", "apiServerURL", - "description", - "alias", - "icon", - "reason", "kubeConfigSecret", - "updateTime", "resourceInfo" ], "properties": { @@ -9705,13 +9728,13 @@ "v1.DetailComponentResponse": { "required": [ "appPrimaryKey", - "creator", "alias", - "name", - "main", "createTime", - "updateTime", + "main", + "name", + "creator", "type", + "updateTime", "definition" ], "properties": { @@ -9799,13 +9822,13 @@ }, "v1.DetailDefinitionResponse": { "required": [ - "ownerAddon", "name", "description", - "status", - "alias", "icon", + "alias", + "status", "labels", + "ownerAddon", "schema", "uiSchema" ], @@ -9862,16 +9885,20 @@ }, "v1.DetailPolicyResponse": { "required": [ - "type", - "description", "creator", - "properties", - "createTime", "updateTime", "envName", - "name" + "name", + "alias", + "type", + "description", + "properties", + "createTime" ], "properties": { + "alias": { + "type": "string" + }, "createTime": { "type": "string", "format": "date-time" @@ -9897,22 +9924,28 @@ "updateTime": { "type": "string", "format": "date-time" + }, + "workflowPolicyBind": { + "type": "array", + "items": { + "$ref": "#/definitions/v1.WorkflowPolicyBinding" + } } } }, "v1.DetailRevisionResponse": { "required": [ - "workflowName", - "envName", - "status", - "deployUser", - "triggerType", "updateTime", - "appPrimaryKey", - "version", "createTime", + "version", + "reason", + "envName", + "appPrimaryKey", + "deployUser", "note", - "reason" + "status", + "workflowName", + "triggerType" ], "properties": { "appPrimaryKey": { @@ -10009,11 +10042,11 @@ }, "v1.DetailUserResponse": { "required": [ - "disabled", "createTime", "lastLoginTime", "name", "email", + "disabled", "projects", "roles" ], @@ -10054,12 +10087,12 @@ }, "v1.DetailWorkflowRecordResponse": { "required": [ - "applicationRevision", "status", "name", "namespace", "workflowName", "workflowAlias", + "applicationRevision", "deployTime", "deployUser", "note", @@ -10111,14 +10144,14 @@ }, "v1.DetailWorkflowResponse": { "required": [ - "alias", - "enable", "envName", - "updateTime", "name", + "alias", "default", - "createTime", - "description" + "updateTime", + "description", + "enable", + "createTime" ], "properties": { "alias": { @@ -10823,11 +10856,11 @@ }, "v1.LoginUserInfoResponse": { "required": [ - "createTime", - "lastLoginTime", "name", "email", "disabled", + "createTime", + "lastLoginTime", "projects", "platformPermissions", "projectPermissions" @@ -11031,6 +11064,7 @@ "v1.PolicyBase": { "required": [ "name", + "alias", "type", "description", "creator", @@ -11040,6 +11074,9 @@ "envName" ], "properties": { + "alias": { + "type": "string" + }, "createTime": { "type": "string", "format": "date-time" @@ -11468,19 +11505,34 @@ }, "v1.UpdatePolicyRequest": { "required": [ + "alias", + "envName", "description", "type", - "properties" + "properties", + "workflowPolicyBind" ], "properties": { + "alias": { + "type": "string" + }, "description": { "type": "string" }, + "envName": { + "type": "string" + }, "properties": { "type": "string" }, "type": { "type": "string" + }, + "workflowPolicyBind": { + "type": "array", + "items": { + "$ref": "#/definitions/v1.WorkflowPolicyBinding" + } } } }, @@ -11659,6 +11711,23 @@ } } }, + "v1.WorkflowPolicyBinding": { + "required": [ + "name", + "steps" + ], + "properties": { + "name": { + "type": "string" + }, + "steps": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "v1.WorkflowRecord": { "required": [ "name", diff --git a/pkg/addon/addon.go b/pkg/addon/addon.go index 723f18aeb..eb356dc90 100644 --- a/pkg/addon/addon.go +++ b/pkg/addon/addon.go @@ -1194,8 +1194,9 @@ func (h *Installer) loadInstallPackage(name, version string) (*InstallPackage, e } } else { versionedRegistry := BuildVersionedRegistry(h.r.Name, h.r.Helm.URL, &common.HTTPOption{ - Username: h.r.Helm.Username, - Password: h.r.Helm.Password, + Username: h.r.Helm.Username, + Password: h.r.Helm.Password, + InsecureSkipTLS: h.r.Helm.InsecureSkipTLS, }) installPackage, err = versionedRegistry.GetAddonInstallPackage(context.Background(), name, version) if err != nil { diff --git a/pkg/addon/cache.go b/pkg/addon/cache.go index de68e5fe0..ac361c362 100644 --- a/pkg/addon/cache.go +++ b/pkg/addon/cache.go @@ -107,8 +107,9 @@ func (u *Cache) GetUIData(r Registry, addonName, version string) (*UIData, error } } else { versionedRegistry := BuildVersionedRegistry(r.Name, r.Helm.URL, &common.HTTPOption{ - Username: r.Helm.Username, - Password: r.Helm.Password, + Username: r.Helm.Username, + Password: r.Helm.Password, + InsecureSkipTLS: r.Helm.InsecureSkipTLS, }) addon, err = versionedRegistry.GetAddonUIData(context.Background(), addonName, version) if err != nil { @@ -315,8 +316,9 @@ func (u *Cache) listUIDataAndCache(r Registry) ([]*UIData, error) { func (u *Cache) listVersionRegistryUIDataAndCache(r Registry) ([]*UIData, error) { versionedRegistry := BuildVersionedRegistry(r.Name, r.Helm.URL, &common.HTTPOption{ - Username: r.Helm.Username, - Password: r.Helm.Password, + Username: r.Helm.Username, + Password: r.Helm.Password, + InsecureSkipTLS: r.Helm.InsecureSkipTLS, }) uiDatas, err := versionedRegistry.ListAddon() if err != nil { diff --git a/pkg/addon/cache_test.go b/pkg/addon/cache_test.go index 87cb97b73..8b992f7ef 100644 --- a/pkg/addon/cache_test.go +++ b/pkg/addon/cache_test.go @@ -19,6 +19,8 @@ package addon import ( "testing" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" "github.com/stretchr/testify/assert" ) @@ -52,23 +54,26 @@ func TestListCachedUIData(t *testing.T) { assert.Equal(t, u.listCachedUIData(name), addons) } -func testListUIData(t *testing.T) { +var _ = Describe("Test addon cache", func() { vr := Registry{Name: "helm-repo", Helm: &HelmSource{URL: "http://127.0.0.1:18083/authReg", Username: "hello", Password: "hello"}} - uiData := UIData{Meta: Meta{ - Name: "fluxcd", - Description: "Extended workload to do continuous and progressive delivery", - Icon: "https://raw.githubusercontent.com/fluxcd/flux/master/docs/_files/weave-flux.png", - Version: "1.0.0", - Tags: []string{"extended_workload", "gitops"}, - }, - AvailableVersions: []string{"1.0.0"}, - RegistryName: "helm-repo"} - addons := []*UIData{&uiData} - u := NewCache(nil) - uiDatas, err := u.ListUIData(vr) - assert.NoError(t, err) - assert.Equal(t, uiDatas, addons) -} + + It("Test list addon helm repo UI data", func() { + uiData := UIData{Meta: Meta{ + Name: "fluxcd", + Description: "Extended workload to do continuous and progressive delivery", + Icon: "https://raw.githubusercontent.com/fluxcd/flux/master/docs/_files/weave-flux.png", + Version: "1.0.0", + Tags: []string{"extended_workload", "gitops"}, + }, + AvailableVersions: []string{"1.0.0"}, + RegistryName: "helm-repo"} + addons := []*UIData{&uiData} + u := NewCache(nil) + uiDatas, err := u.ListUIData(vr) + Expect(err).NotTo(HaveOccurred()) + Expect(uiDatas).To(Equal(addons)) + }) +}) func TestListVersionRegistryCachedUIData(t *testing.T) { name := "fluxcd" diff --git a/pkg/addon/helper.go b/pkg/addon/helper.go index 0eca75e6f..623c4a747 100644 --- a/pkg/addon/helper.go +++ b/pkg/addon/helper.go @@ -326,7 +326,11 @@ func FindWholeAddonPackagesFromRegistry(ctx context.Context, k8sClient client.Cl // Find matched addons in registries for _, r := range registries { if IsVersionRegistry(r) { - vr := BuildVersionedRegistry(r.Name, r.Helm.URL, &common.HTTPOption{Username: r.Helm.Username, Password: r.Helm.Password}) + vr := BuildVersionedRegistry(r.Name, r.Helm.URL, &common.HTTPOption{ + Username: r.Helm.Username, + Password: r.Helm.Password, + InsecureSkipTLS: r.Helm.InsecureSkipTLS, + }) for _, addonName := range addonNames { wholePackage, err := vr.GetDetailedAddon(ctx, addonName, "") if err != nil { diff --git a/pkg/addon/source.go b/pkg/addon/source.go index 78c01abe0..9f846e0b8 100644 --- a/pkg/addon/source.go +++ b/pkg/addon/source.go @@ -78,9 +78,10 @@ type GitlabAddonSource struct { // HelmSource defines the information about the helm repo addon source type HelmSource struct { - URL string `json:"url,omitempty" validate:"required"` - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` + URL string `json:"url,omitempty" validate:"required"` + InsecureSkipTLS bool `json:"insecureSkipTLS,omitempty"` + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` } // SafeCopier is an interface to copy Struct without sensitive fields, such as Token, Username, Password diff --git a/pkg/addon/suite_test.go b/pkg/addon/suite_test.go index a08843b3e..63e5a08a1 100644 --- a/pkg/addon/suite_test.go +++ b/pkg/addon/suite_test.go @@ -21,20 +21,18 @@ import ( "testing" "time" - "k8s.io/client-go/discovery" - ocmclusterv1 "open-cluster-management.io/api/cluster/v1" - ocmclusterv1alpha1 "open-cluster-management.io/api/cluster/v1alpha1" - ocmworkv1 "open-cluster-management.io/api/work/v1" - + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" v12 "k8s.io/api/core/v1" crdv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/discovery" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + ocmclusterv1 "open-cluster-management.io/api/cluster/v1" + ocmclusterv1alpha1 "open-cluster-management.io/api/cluster/v1alpha1" + ocmworkv1 "open-cluster-management.io/api/work/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/envtest/printer" @@ -42,6 +40,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" coreoam "github.com/oam-dev/kubevela/apis/core.oam.dev" + "github.com/oam-dev/kubevela/pkg/cue/packages" "github.com/oam-dev/kubevela/pkg/oam/discoverymapper" // +kubebuilder:scaffold:imports @@ -105,6 +104,9 @@ var _ = BeforeSuite(func(done Done) { Name: testns, }})) + err = stepHelmHttpServer() + Expect(err).Should(Succeed()) + close(done) }, 120) diff --git a/pkg/addon/testdata/tls/local-selfsign.crt b/pkg/addon/testdata/tls/local-selfsign.crt new file mode 100644 index 000000000..6bb95519e --- /dev/null +++ b/pkg/addon/testdata/tls/local-selfsign.crt @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC0TCCAbmgAwIBAgIRAIAefqqN1aYngJpWZ9BBwyUwDQYJKoZIhvcNAQELBQAw +ADAeFw0yMjA2MDgxMTI0NDdaFw0zMjA2MDUxMTI0NDdaMAAwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQDnBIMvpmPv3AsHTUKOQq/i4e6kgtDx4USQ7oE+ +vElPDr2SxPQEEqjlazK1jktLZG/dtCRgHNWn5ZdNmgQOlRU1KOEVVw8BZd/BMtw3 +i5O1fbnE1nmwRvAZ7li2IgzVUn7vjD5+CwBjnhbPQcbnH5tkO+5FTwT8JY1Ht7io +B9m+y2w3FgphohGEq8m9bmmWkN0VKhd2kBFaKhf42TkVAMdJ4Q6mdzZXQ6zNJ+4s +o2Eg+XPopZ2XA2ntUfjjSSpZcdPwXIyTruY51LkSR6Ak7gyrVY1mBw55qwoiMRZH +jyxwfisSwqiTrb09HYdP8eFSUyWTHHMVHdXO/nzbYidWp2spAgMBAAGjRjBEMA4G +A1UdDwEB/wQEAwIFoDAMBgNVHRMBAf8EAjAAMCQGA1UdEQEB/wQaMBiCEGhlbG0u +a3ViZXZlbGEuaW+HBH8AAAEwDQYJKoZIhvcNAQELBQADggEBAJ/IadUmFK7awBxC +diYERDtUgZghhahVFcEr8sV94SsU1ojQLTZVgFAwbPt7I1NxBkaj77ZKAmOktc/m +rIAnMQdBN1wuAfM1Rwz0TDnrfO+gTuaIf9Op2SsaGpcf221lIu3T5n+DaLS6M5a1 +BqCjhHLfrg+j1NLapMT1V6CtKqmkmDLWWh4MKruYG8odpShXfXtRsjhI/2IFf0eI +Nu4F7+lI2C4BoQ5qyv4kVZpOzqX6u82SNkp0028n7woUk28z+3juUmICGV4iDVGv +BzO1Im1LxFufZf1R2afhdz7EDfFhHCHj8LGkMGXDyfB9H9CXzPHfhatCPDFfdkB/ +D5+Z5NY= +-----END CERTIFICATE----- \ No newline at end of file diff --git a/pkg/addon/testdata/tls/local-selfsign.key b/pkg/addon/testdata/tls/local-selfsign.key new file mode 100644 index 000000000..94e5d779f --- /dev/null +++ b/pkg/addon/testdata/tls/local-selfsign.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA5wSDL6Zj79wLB01CjkKv4uHupILQ8eFEkO6BPrxJTw69ksT0 +BBKo5WsytY5LS2Rv3bQkYBzVp+WXTZoEDpUVNSjhFVcPAWXfwTLcN4uTtX25xNZ5 +sEbwGe5YtiIM1VJ+74w+fgsAY54Wz0HG5x+bZDvuRU8E/CWNR7e4qAfZvstsNxYK +YaIRhKvJvW5plpDdFSoXdpARWioX+Nk5FQDHSeEOpnc2V0OszSfuLKNhIPlz6KWd +lwNp7VH440kqWXHT8FyMk67mOdS5EkegJO4Mq1WNZgcOeasKIjEWR48scH4rEsKo +k629PR2HT/HhUlMlkxxzFR3Vzv5822InVqdrKQIDAQABAoIBAERcOHmkMhRrN7Q8 +b4JrR2LK2nRB5ZR9xW1htksbKlJDmWNR9qcx4lqpAwNojO1NVjDzuoPaZHrfXItx +FNw/z1vAsgI/lcgjHjgsVa/2FIpzHsKZeL7p7uiteADFSKjjVX6LMH7ZwI653CQO +Yc5IsxHRDiviOyZl8k2TlwfZ8twbA//DL+mbKw9psE/3+8WoIGEeOXxwGXQsZyyf +oso2191jGfzJNlC6LNvyF/mx479b+wArUEAxIysgpHYrHFrWOrTnN3pMzWNvGmhj +0MwD8RD2EOFbAIYdxlDE0OyvOfuWH9U3BfnT42zrHFZVhDDs27DS9HeeN2gvn3Wd +BISW/dUCgYEA+KG/84Ej/Sz/KZxsMB606Vfrv8b0pLJSG4mXfm7Yni+wMkeg19C4 +gDIBlK6i3bNCXX/cZvpYqHc2E/+yUONKhd7CPqUbjWgXgMJj7tsUpqw6ssel0XE6 +me1jY/ZG8gKYPCXF2P0HYp+8pAyfepXLJWZX0OlkAJ9ITg4hSB3toacCgYEA7d0h +0bHyK5gmSUX0EEl11NwV3+UFuwmVR1WtKeSPpwvr4e3t/z6AVCqxtbOSbWt5g9c0 +h2tsauNYxIYHXWOw+v7RupfAU11DbtZf5RcLHGd7CTK67NZnpBc+20JgaoxPoBLm +1E2WCN2h0X0yBsMQnoHuGyWoZPZhgWHBGeq0Bq8CgYEA6NO3h9ToGb7YV2QW39zx +dw4RQy6Ri+U9BHKNfiVZDpEDw6HV2iOnkszBNDVxaIDkZeBFSd2nxmKULpgkqKCf ++crHRZihu4K9d6MRp+xaY1YnRaG2AvKweNKXX1WRzDFn0/NfWbpg/xcstMbpVQcP +DD/Rfl4AeqCuOgH7Yqg8DkMCgYEA38cigf0/L/Q8UMMXEhuKV2+WxMTZ11T0s9/O +wZLd8vIsQKlvNqq+YHCqL595Kqfp8jcMnDpcDVNFt93AIAQiBICnHzyVG9iqECRu +o0BCm75e6hBn+1jauG2GHREdVCAc6SPfNeyoMj8Qdd2qquHpSLZEJBYRjqqcoPQu +jwtXwtECgYATq5EnWtcivAeHjmozInq31J876gRgCm4JFQtoSHVR550WKKGbrvtP +qj6b9IHZvK/vRBhmKyjyI1qpC2mHWlVSX52HgJZR5aBn1pbZj8sxtalED9NsxklN +t4BrFRWTVFv06mikMH1aQe+tQ/omvipNXRpKxEmpxgiqkxvU7aFHuQ== +-----END RSA PRIVATE KEY----- \ No newline at end of file diff --git a/pkg/addon/utils.go b/pkg/addon/utils.go index 04e7b7dba..aa51770e1 100644 --- a/pkg/addon/utils.go +++ b/pkg/addon/utils.go @@ -176,8 +176,9 @@ func findLegacyAddonDefs(ctx context.Context, k8sClient client.Client, addonName } } else { versionedRegistry := BuildVersionedRegistry(registry.Name, registry.Helm.URL, &common.HTTPOption{ - Username: registry.Helm.Username, - Password: registry.Helm.Password, + Username: registry.Helm.Username, + Password: registry.Helm.Password, + InsecureSkipTLS: registry.Helm.InsecureSkipTLS, }) uiData, err = versionedRegistry.GetAddonUIData(ctx, addonName, "") if err != nil { diff --git a/pkg/addon/versioned_registry_suite_test.go b/pkg/addon/versioned_registry_suite_test.go new file mode 100644 index 000000000..c43a16878 --- /dev/null +++ b/pkg/addon/versioned_registry_suite_test.go @@ -0,0 +1,248 @@ +/* +Copyright 2021 The KubeVela Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package addon + +import ( + "context" + "crypto/tls" + "fmt" + "net/http" + "time" + + "github.com/pkg/errors" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/oam-dev/kubevela/pkg/utils/common" +) + +var _ = Describe("Test Versioned Registry", func() { + registries := []Registry{ + { + Name: "helm-repo", + Helm: &HelmSource{URL: "http://127.0.0.1:18083"}, + }, + { + Name: "auth-helm-repo", + Helm: &HelmSource{ + URL: "http://127.0.0.1:18083", + Username: "kubevela", + Password: "versioned registry", + }, + }, + { + Name: "tls-helm-repo", + Helm: &HelmSource{URL: "https://127.0.0.1:18443", InsecureSkipTLS: true}, + }, + { + Name: "auth-tls-helm-repo", + Helm: &HelmSource{ + URL: "https://127.0.0.1:18443", + Username: "kubevela", + Password: "versioned registry", + InsecureSkipTLS: true, + }, + }, + } + + for _, registry := range registries { + registry := registry + r := BuildVersionedRegistry(registry.Name, registry.Helm.URL, &common.HTTPOption{ + InsecureSkipTLS: registry.Helm.InsecureSkipTLS, + Username: registry.Helm.Username, + Password: registry.Helm.Password, + }) + Context(fmt.Sprintf("Test %s", registry.Name), func() { + It("list addon", func() { + addon, err := r.ListAddon() + Expect(err).NotTo(HaveOccurred()) + Expect(addon).To(HaveLen(1)) + Expect(addon[0].Name).To(Equal("fluxcd")) + Expect(addon[0].AvailableVersions).To(HaveLen(1)) + }) + + It("get addon ui data", func() { + addonUIData, err := r.GetAddonUIData(context.Background(), "fluxcd", "1.0.0") + Expect(err).NotTo(HaveOccurred()) + Expect(addonUIData).NotTo(BeNil()) + Expect(addonUIData.Definitions).NotTo(BeEmpty()) + Expect(addonUIData.Icon).NotTo(BeEmpty()) + }) + + It("get addon install pkg", func() { + addonsInstallPackage, err := r.GetAddonInstallPackage(context.Background(), "fluxcd", "1.0.0") + Expect(err).NotTo(HaveOccurred()) + Expect(addonsInstallPackage).NotTo(BeNil()) + Expect(addonsInstallPackage.YAMLTemplates).NotTo(BeEmpty()) + Expect(addonsInstallPackage.DefSchemas).NotTo(BeEmpty()) + }) + + It("get addon detail", func() { + addonWholePackage, err := r.GetDetailedAddon(context.Background(), "fluxcd", "1.0.0") + Expect(err).NotTo(HaveOccurred()) + Expect(addonWholePackage).NotTo(BeNil()) + Expect(addonWholePackage.YAMLTemplates).NotTo(BeEmpty()) + Expect(addonWholePackage.DefSchemas).NotTo(BeEmpty()) + Expect(addonWholePackage.RegistryName).NotTo(BeEmpty()) + }) + }) + } + + errRegistries := []Registry{ + { + Name: "tls-err-helm-repo", + Helm: &HelmSource{URL: "https://127.0.0.1:18443", InsecureSkipTLS: false}, + }, + { + Name: "auth-tls-err-helm-repo", + Helm: &HelmSource{ + URL: "https://127.0.0.1:18443", + Username: "kubevela", + Password: "versioned registry", + InsecureSkipTLS: false, + }, + }, + } + + for _, registry := range errRegistries { + registry := registry + r := BuildVersionedRegistry(registry.Name, registry.Helm.URL, &common.HTTPOption{ + InsecureSkipTLS: registry.Helm.InsecureSkipTLS, + Username: registry.Helm.Username, + Password: registry.Helm.Password, + }) + Context(fmt.Sprintf("Test %s", registry.Name), func() { + It("list addon", func() { + addon, err := r.ListAddon() + Expect(err).To(HaveOccurred()) + Expect(addon).To(BeEmpty()) + }) + + It("get addon ui data", func() { + addonUIData, err := r.GetAddonUIData(context.Background(), "fluxcd", "1.0.0") + Expect(err).To(HaveOccurred()) + Expect(addonUIData).To(BeNil()) + }) + + It("get addon install pkg", func() { + addonsInstallPackage, err := r.GetAddonInstallPackage(context.Background(), "fluxcd", "1.0.0") + Expect(err).To(HaveOccurred()) + Expect(addonsInstallPackage).To(BeNil()) + }) + + It("get addon detail", func() { + addonWholePackage, err := r.GetDetailedAddon(context.Background(), "fluxcd", "1.0.0") + Expect(err).To(HaveOccurred()) + Expect(addonWholePackage).To(BeNil()) + }) + }) + } + + Context("Test multiversion helm repo", func() { + const repoName = "multiversion-helm-repo" + mr := BuildVersionedRegistry(repoName, "http://127.0.0.1:18083/multi", nil) + It("list addon", func() { + addons, err := mr.ListAddon() + Expect(err).To(Succeed()) + Expect(addons).To(HaveLen(2)) + }) + + It("get addon ui data", func() { + addonUIData, err := mr.GetAddonUIData(context.Background(), "fluxcd", "2.0.0") + Expect(err).To(Succeed()) + Expect(addonUIData.Definitions).NotTo(BeEmpty()) + Expect(addonUIData.Icon).NotTo(BeEmpty()) + Expect(addonUIData.Version).To(Equal("2.0.0")) + }) + + It("get addon install pkg", func() { + addonsInstallPackage, err := mr.GetAddonInstallPackage(context.Background(), "fluxcd", "1.0.0") + Expect(err).To(Succeed()) + Expect(addonsInstallPackage).NotTo(BeNil()) + Expect(addonsInstallPackage.YAMLTemplates).NotTo(BeEmpty()) + Expect(addonsInstallPackage.DefSchemas).NotTo(BeEmpty()) + Expect(addonsInstallPackage.SystemRequirements.VelaVersion).To(HaveSuffix("1.3.0")) + Expect(addonsInstallPackage.SystemRequirements.KubernetesVersion).To(HaveSuffix("1.10.0")) + }) + + It("get addon detail", func() { + addonWholePackage, err := mr.GetDetailedAddon(context.Background(), "fluxcd", "1.0.0") + Expect(err).To(Succeed()) + Expect(addonWholePackage).NotTo(BeNil()) + Expect(addonWholePackage.YAMLTemplates).NotTo(BeEmpty()) + Expect(addonWholePackage.DefSchemas).NotTo(BeEmpty()) + Expect(addonWholePackage.RegistryName).To(Equal(repoName)) + Expect(addonWholePackage.SystemRequirements.VelaVersion).To(Equal(">=1.3.0")) + Expect(addonWholePackage.SystemRequirements.KubernetesVersion).To(Equal(">=1.10.0")) + }) + + It("get addon available version", func() { + version, err := mr.GetAddonAvailableVersion("fluxcd") + Expect(err).To(Succeed()) + Expect(version).To(HaveLen(2)) + }) + }) +}) + +func stepHelmHttpServer() error { + handler := &http.ServeMux{} + handler.HandleFunc("/", versionedHandler) + handler.HandleFunc("/authReg", basicAuthVersionedHandler) + handler.HandleFunc("/multi/", multiVersionHandler) + + helmRepoHttpServer := &http.Server{ + Addr: fmt.Sprintf(":%d", 18083), + Handler: handler, + } + helmRepoHttpsServer := &http.Server{ + Addr: fmt.Sprintf(":%d", 18443), + Handler: handler, + } + + go func() { + err := helmRepoHttpsServer.ListenAndServeTLS("./testdata/tls/local-selfsign.crt", "./testdata/tls/local-selfsign.key") + Expect(err).ShouldNot(HaveOccurred()) + }() + go func() { + err := helmRepoHttpServer.ListenAndServe() + Expect(err).ShouldNot(HaveOccurred()) + }() + + err := checkHelmHttpServer("http://127.0.0.1:18083", 3, time.Second) + if err != nil { + return err + } + err = checkHelmHttpServer("http://127.0.0.1:18443", 3, time.Second) + if err != nil { + return err + } + return nil +} + +func checkHelmHttpServer(url string, maxTryNum int, interval time.Duration) error { + client := http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}} + var err error + for cur := 0; cur < maxTryNum; cur++ { + _, err = client.Get(url) + if err != nil { + time.Sleep(interval) + continue + } + } + if err != nil { + return errors.Wrap(err, "exceeded maximum number of retries.") + } + return nil +} diff --git a/pkg/addon/versioned_registry_test.go b/pkg/addon/versioned_registry_test.go index dce097631..92c0f5e98 100644 --- a/pkg/addon/versioned_registry_test.go +++ b/pkg/addon/versioned_registry_test.go @@ -17,125 +17,21 @@ limitations under the License. package addon import ( - "context" "encoding/base64" - "fmt" "io/ioutil" - "log" "net/http" "net/http/httptest" "strings" "testing" - "time" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/repo" "github.com/stretchr/testify/assert" - "github.com/oam-dev/kubevela/pkg/utils/common" "github.com/oam-dev/kubevela/pkg/utils/helm" ) -func TestVersionRegistry(t *testing.T) { - go func() { - http.HandleFunc("/", versionedHandler) - http.HandleFunc("/authReg", basicAuthVersionedHandler) - http.HandleFunc("/multi/", multiVersionHandler) - err := http.ListenAndServe(fmt.Sprintf(":%d", 18083), nil) - if err != nil { - log.Fatal("Setup server error:", err) - } - }() - - // wait server setup - time.Sleep(3 * time.Second) - r := BuildVersionedRegistry("helm-repo", "http://127.0.0.1:18083", nil) - addons, err := r.ListAddon() - assert.NoError(t, err) - assert.Equal(t, len(addons), 1) - assert.Equal(t, addons[0].Name, "fluxcd") - assert.Equal(t, len(addons[0].AvailableVersions), 1) - - addonUIData, err := r.GetAddonUIData(context.Background(), "fluxcd", "1.0.0") - assert.NoError(t, err) - assert.NotEmpty(t, addonUIData.Definitions) - assert.NotEmpty(t, addonUIData.Icon) - - addonsInstallPackage, err := r.GetAddonInstallPackage(context.Background(), "fluxcd", "1.0.0") - assert.NoError(t, err) - assert.NotEmpty(t, addonsInstallPackage) - assert.NotEmpty(t, addonsInstallPackage.YAMLTemplates) - assert.NotEmpty(t, addonsInstallPackage.DefSchemas) - - addonWholePackage, err := r.GetDetailedAddon(context.Background(), "fluxcd", "1.0.0") - assert.NoError(t, err) - assert.NotEmpty(t, addonWholePackage) - assert.NotEmpty(t, addonWholePackage.YAMLTemplates) - assert.NotEmpty(t, addonWholePackage.DefSchemas) - assert.NotEmpty(t, addonWholePackage.RegistryName) - - ar := BuildVersionedRegistry("auth-helm-repo", "http://127.0.0.1:18083/authReg", &common.HTTPOption{Username: "hello", Password: "hello"}) - addons, err = ar.ListAddon() - assert.NoError(t, err) - assert.Equal(t, len(addons), 1) - assert.Equal(t, addons[0].Name, "fluxcd") - assert.Equal(t, len(addons[0].AvailableVersions), 1) - - addonUIData, err = ar.GetAddonUIData(context.Background(), "fluxcd", "1.0.0") - assert.NoError(t, err) - assert.NotEmpty(t, addonUIData.Definitions) - assert.NotEmpty(t, addonUIData.Icon) - - addonsInstallPackage, err = ar.GetAddonInstallPackage(context.Background(), "fluxcd", "1.0.0") - assert.NoError(t, err) - assert.NotEmpty(t, addonsInstallPackage) - assert.NotEmpty(t, addonsInstallPackage.YAMLTemplates) - assert.NotEmpty(t, addonsInstallPackage.DefSchemas) - - addonWholePackage, err = ar.GetDetailedAddon(context.Background(), "fluxcd", "1.0.0") - assert.NoError(t, err) - assert.NotEmpty(t, addonWholePackage) - assert.NotEmpty(t, addonWholePackage.YAMLTemplates) - assert.NotEmpty(t, addonWholePackage.DefSchemas) - assert.NotEmpty(t, addonWholePackage.RegistryName) - - testListUIData(t) - mr := BuildVersionedRegistry("multiversion-helm-repo", "http://127.0.0.1:18083/multi", nil) - addons, err = mr.ListAddon() - assert.NoError(t, err) - assert.Equal(t, len(addons), 2) - - addonUIData, err = mr.GetAddonUIData(context.Background(), "fluxcd", "2.0.0") - assert.NoError(t, err) - assert.NotEmpty(t, addonUIData.Definitions) - assert.NotEmpty(t, addonUIData.Icon) - assert.Equal(t, addonUIData.Version, "2.0.0") - - addonsInstallPackage, err = mr.GetAddonInstallPackage(context.Background(), "fluxcd", "1.0.0") - assert.NoError(t, err) - assert.NotEmpty(t, addonsInstallPackage) - assert.NotEmpty(t, addonsInstallPackage.YAMLTemplates) - assert.NotEmpty(t, addonsInstallPackage.DefSchemas) - assert.NotEmpty(t, addonsInstallPackage.SystemRequirements.VelaVersion, "1.3.0") - assert.NotEmpty(t, addonsInstallPackage.SystemRequirements.KubernetesVersion, "1.10.0") - - addonWholePackage, err = mr.GetDetailedAddon(context.Background(), "fluxcd", "1.0.0") - assert.NoError(t, err) - assert.NotEmpty(t, addonWholePackage) - assert.NotEmpty(t, addonWholePackage.YAMLTemplates) - assert.NotEmpty(t, addonWholePackage.DefSchemas) - assert.NotEmpty(t, addonWholePackage.RegistryName) - assert.Equal(t, addonWholePackage.RegistryName, "multiversion-helm-repo") - - version, err := mr.GetAddonAvailableVersion("fluxcd") - assert.NoError(t, err) - assert.Equal(t, len(version), 2) - assert.Equal(t, addonWholePackage.SystemRequirements.VelaVersion, ">=1.3.0") - assert.Equal(t, addonWholePackage.SystemRequirements.KubernetesVersion, ">=1.10.0") - -} - func TestChooseAddonVersion(t *testing.T) { versions := []*repo.ChartVersion{ { diff --git a/pkg/utils/common/common.go b/pkg/utils/common/common.go index 887fb3894..7fbd3284d 100644 --- a/pkg/utils/common/common.go +++ b/pkg/utils/common/common.go @@ -80,6 +80,9 @@ import ( var ( // Scheme defines the default KubeVela schema Scheme = k8sruntime.NewScheme() + //nolint:gosec + // insecureHTTPClient insecure http client + insecureHTTPClient = &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}} ) const ( @@ -117,11 +120,12 @@ func init() { // HTTPOption define the https options type HTTPOption struct { - Username string - Password string - CaFile string - CertFile string - KeyFile string + Username string + Password string + CaFile string + CertFile string + KeyFile string + InsecureSkipTLS bool } // InitBaseRestConfig will return reset config for create controller runtime client @@ -167,6 +171,9 @@ func HTTPGetResponse(ctx context.Context, url string, opts *HTTPOption) (*http.R if opts != nil && len(opts.Username) != 0 && len(opts.Password) != 0 { req.SetBasicAuth(opts.Username, opts.Password) } + if opts != nil && opts.InsecureSkipTLS { + httpClient = insecureHTTPClient + } // if specify the caFile, we cannot re-use the default httpClient, so create a new one. if opts != nil && (len(opts.CaFile) != 0 || len(opts.KeyFile) != 0 || len(opts.CertFile) != 0) { // must set MinVersion of TLS, otherwise will report GoSec error G402 diff --git a/references/cli/addon-registry.go b/references/cli/addon-registry.go index 5acfc90b2..b6e1a8b1b 100644 --- a/references/cli/addon-registry.go +++ b/references/cli/addon-registry.go @@ -31,19 +31,20 @@ import ( ) const ( - addonRegistryType = "type" - addonEndpoint = "endpoint" - addonOssBucket = "bucket" - addonPath = "path" - addonGitToken = "gitToken" - addonOssType = "OSS" - addonGitType = "git" - addonGiteeType = "gitee" - addonGitlabType = "gitlab" - addonHelmType = "helm" - addonUsername = "username" - addonPassword = "password" - addonRepoName = "repoName" + addonRegistryType = "type" + addonEndpoint = "endpoint" + addonOssBucket = "bucket" + addonPath = "path" + addonGitToken = "gitToken" + addonOssType = "OSS" + addonGitType = "git" + addonGiteeType = "gitee" + addonGitlabType = "gitlab" + addonHelmType = "helm" + addonUsername = "username" + addonPassword = "password" + addonRepoName = "repoName" + addonHelmInsecureSkipTLS = "insecureSkipTLS" ) // NewAddonRegistryCommand return an addon registry command @@ -297,6 +298,8 @@ func parseArgsFromFlag(cmd *cobra.Command) { cmd.Flags().StringP(addonGitToken, "", "", "specify the github repo token") cmd.Flags().StringP(addonUsername, "", "", "specify the Helm addon registry username") cmd.Flags().StringP(addonPassword, "", "", "specify the Helm addon registry password") + cmd.Flags().BoolP(addonHelmInsecureSkipTLS, "", false, + "specify the Helm addon registry skip tls verify") } func getRegistryFromArgs(cmd *cobra.Command, args []string) (*pkgaddon.Registry, error) { @@ -389,6 +392,10 @@ func getRegistryFromArgs(cmd *cobra.Command, args []string) (*pkgaddon.Registry, if err != nil { return nil, err } + r.Helm.InsecureSkipTLS, err = cmd.Flags().GetBool(addonHelmInsecureSkipTLS) + if err != nil { + return nil, err + } default: return nil, errors.New("not support addon registry type") diff --git a/references/cli/addon.go b/references/cli/addon.go index 7848949ae..da0a2d248 100644 --- a/references/cli/addon.go +++ b/references/cli/addon.go @@ -899,7 +899,11 @@ func listAddons(ctx context.Context, clt client.Client, registry string) (*uitab continue } } else { - versionedRegistry := pkgaddon.BuildVersionedRegistry(r.Name, r.Helm.URL, &common.HTTPOption{Username: r.Helm.Username, Password: r.Helm.Password}) + versionedRegistry := pkgaddon.BuildVersionedRegistry(r.Name, r.Helm.URL, &common.HTTPOption{ + Username: r.Helm.Username, + Password: r.Helm.Password, + InsecureSkipTLS: r.Helm.InsecureSkipTLS, + }) addonList, err = versionedRegistry.ListAddon() if err != nil { continue diff --git a/references/cli/addon_test.go b/references/cli/addon_test.go index f37bee2de..777af418d 100644 --- a/references/cli/addon_test.go +++ b/references/cli/addon_test.go @@ -137,6 +137,7 @@ func testAddonRegistryAddCmd() { testcase := []struct { args []string errMsg string + result *pkgaddon.Registry }{ { args: []string{"noAuthRegistry", "--type=helm", "--endpoint=http://127.0.0.1/chartrepo/oam"}, @@ -146,6 +147,9 @@ func testAddonRegistryAddCmd() { args: []string{"basicAuthRegistry", "--type=helm", "--endpoint=http://127.0.0.1/chartrepo/oam", "--username=hello", "--password=word"}, errMsg: "fail to add basis auth addon registry", }, + { + args: []string{"skipTlsRegistry", "--type=helm", "--endpoint=https://127.0.0.1/chartrepo/oam", "--insecureSkipTLS=true"}, + }, } ioStream := util.IOStreams{}