github/flagger
1
0
Fork 0
mirror of https://github.com/fluxcd/flagger.git synced 2026-04-15 06:57:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #464 from weaveworks/cloud-watch-metrics

Browse Source 
pkg/metrics/providers: add AWS CloudWatch provider
This commit is contained in:
Stefan Prodan
2020-03-03 09:05:37 +02:00
committed by GitHub
parent 62f2851dfd ecb8207488
commit b47cfb62b2
11 changed files with 419 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
artifacts/flagger/crd.yaml
+4 -1
View File
@@ -733,7 +733,6 @@ spec:
type: object
required:
- type
- address
properties:
type:
description: Type of this provider
@@ -742,6 +741,7 @@ spec:
- prometheus
- influxdb
- datadog
- cloudwatch
address:
description: API address of this provider
type: string
@@ -754,6 +754,9 @@ spec:
name:
description: Name of the Kubernetes secret
type: string
region:
description: Region of the provider
type: string
query:
description: Query of this metric template
type: string
charts/flagger/crds/crd.yaml
+4 -1
View File
@@ -733,7 +733,6 @@ spec:
type: object
required:
- type
- address
properties:
type:
description: Type of this provider
@@ -742,6 +741,7 @@ spec:
- prometheus
- influxdb
- datadog
- cloudwatch
address:
description: API address of this provider
type: string
@@ -754,6 +754,9 @@ spec:
name:
description: Name of the Kubernetes secret
type: string
region:
description: Region of the provider
type: string
query:
description: Query of this metric template
type: string
docs/gitbook/usage/metrics.md
+83
View File
@@ -233,3 +233,86 @@ Reference the template in the canary analysis:
max: 5
interval: 1m
```
### AWS CloudWatch metrics
You can create custom metric checks using the AWS CloudWatch metrics provider.
The template example:
```yaml
apiVersion: flagger.app/v1alpha1
kind: MetricTemplate
metadata:
name: cloudwatch-error-rate
namespace: istio-system
spec:
provider:
type: cloudwatch
region: ap-northeast-1 # specify the region of your metrics
query: |
[
{
"Id": "e1",
"Expression": "m1 / m2",
"Label": "ErrorRate"
},
{
"Id": "m1",
"MetricStat": {
"Metric": {
"Namespace": "MyKubernetesCluster",
"MetricName": "ErrorCount",
"Dimensions": [
{
"Name": "appName",
"Value": "{{ name }}.{{ namespace }}"
}
]
},
"Period": 60,
"Stat": "Sum",
"Unit": "Count"
},
"ReturnData": false
},
{
"Id": "m2",
"MetricStat": {
"Metric": {
"Namespace": "MyKubernetesCluster",
"MetricName": "RequestCount",
"Dimensions": [
{
"Name": "appName",
"Value": "{{ name }}.{{ namespace }}"
}
]
},
"Period": 60,
"Stat": "Sum",
"Unit": "Count"
},
"ReturnData": false
}
]
```
where the query is in the form as in [the AWS' official document](https://aws.amazon.com/premiumsupport/knowledge-center/cloudwatch-getmetricdata-api/).
Reference the template in the canary analysis:
```yaml
analysis:
metrics:
- name: "cw custom error rate"
templateRef:
name: cloudwatch-error-rate
namespace: istio-system
thresholdRange:
max: 0.1
interval: 1m
```
Please note that the flagger need AWS IAM permission to perform `cloudwatch:GetMetricData` to use this provider.
go.mod
+2 -1
View File
@@ -4,12 +4,13 @@ go 1.13
require (
github.com/Masterminds/semver/v3 v3.0.3
github.com/aws/aws-sdk-go v1.29.12
github.com/davecgh/go-spew v1.1.1
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
github.com/google/go-cmp v0.3.0
github.com/googleapis/gnostic v0.2.0 // indirect
github.com/imdario/mergo v0.3.7 // indirect
github.com/pkg/errors v0.8.1
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.0.0
go.uber.org/atomic v1.3.2 // indirect
go.uber.org/multierr v1.1.0 // indirect
go.sum
+11 -2
View File
@@ -20,6 +20,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/aws/aws-sdk-go v1.29.12 h1:V2Z0/kQZsogK/KqkGkgcbcFh1pH7fhXWY+6aTty2G3k=
github.com/aws/aws-sdk-go v1.29.12/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
@@ -38,7 +40,9 @@ github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
@@ -51,6 +55,7 @@ github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8
github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
@@ -96,6 +101,8 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI=
github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -141,8 +148,8 @@ github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -210,6 +217,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
kustomize/base/flagger/crd.yaml
+4 -1
View File
@@ -733,7 +733,6 @@ spec:
type: object
required:
- type
- address
properties:
type:
description: Type of this provider
@@ -742,6 +741,7 @@ spec:
- prometheus
- influxdb
- datadog
- cloudwatch
address:
description: API address of this provider
type: string
@@ -754,6 +754,9 @@ spec:
name:
description: Name of the Kubernetes secret
type: string
region:
description: Region of the provider
type: string
query:
description: Query of this metric template
type: string
pkg/apis/flagger/v1beta1/metric.go
+5
View File
@@ -64,11 +64,16 @@ type MetricTemplateProvider struct {
Type string `json:"type,omitempty"`
// HTTP(S) address of this provider
// +optional
Address string `json:"address,omitempty"`
// Secret reference containing the provider credentials
// +optional
SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"`
// Region of the provider
// +optional
Region string `json:"region,omitempty"`
}
// MetricTemplateModel is the query template model
pkg/metrics/providers/cloudwatch.go
+113
View File
@@ -0,0 +1,113 @@
package providers
import (
"encoding/json"
"fmt"
"net/http"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatch"
flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
)
const (
cloudWatchMaxRetries = 3
cloudWatchStartDeltaMultiplierOnMetricInterval = 10
)
type CloudWatchProvider struct {
client cloudWatchClient
startDelta time.Duration
}
// for the testing purpose
type cloudWatchClient interface {
GetMetricData(input *cloudwatch.GetMetricDataInput) (*cloudwatch.GetMetricDataOutput, error)
}
// NewCloudWatchProvider takes a metricInterval, a provider spec and the credentials map, and
// returns a cloudWatchProvider ready to execute queries against the AWS CloudWatch metrics
func NewCloudWatchProvider(metricInterval string, provider flaggerv1.MetricTemplateProvider) (*CloudWatchProvider, error) {
if provider.Region == "" {
return nil, fmt.Errorf("region not specified")
}
sess, err := session.NewSession(aws.NewConfig().
WithRegion(provider.Region).WithMaxRetries(cloudWatchMaxRetries))
if err != nil {
return nil, fmt.Errorf("error creating aws session: %s", err.Error())
}
md, err := time.ParseDuration(metricInterval)
if err != nil {
return nil, fmt.Errorf("error parsing metric interval: %s", err.Error())
}
return &CloudWatchProvider{
client: cloudwatch.New(sess),
startDelta: cloudWatchStartDeltaMultiplierOnMetricInterval * md,
}, err
}
// RunQuery executes the aws cloud watch metrics query against GetMetricData endpoint
// and returns the the first result as float64
func (p *CloudWatchProvider) RunQuery(query string) (float64, error) {
var cq []*cloudwatch.MetricDataQuery
if err := json.Unmarshal([]byte(query), &cq); err != nil {
return 0, fmt.Errorf("error unmarshaling query: %s", err.Error())
}
end := time.Now()
start := end.Add(-p.startDelta)
res, err := p.client.GetMetricData(&cloudwatch.GetMetricDataInput{
EndTime: aws.Time(end),
MaxDatapoints: aws.Int64(20),
StartTime: aws.Time(start),
MetricDataQueries: cq,
})
if err != nil {
return 0, fmt.Errorf("error requesting cloudwatch: %s", err.Error())
}
mr := res.MetricDataResults
if len(mr) < 1 {
return 0, fmt.Errorf("no values found in response: %s", res.String())
}
vs := res.MetricDataResults[0].Values
if len(vs) < 1 {
return 0, fmt.Errorf("no values found in response: %s", res.String())
}
return aws.Float64Value(vs[0]), nil
}
// IsOnline calls GetMetricData endpoint with the empty query
// and returns an error if the returned status code is NOT http.StatusBadRequests.
// For example, if the flagger does not have permission to perform `cloudwatch:GetMetricData`,
// the returned status code would be http.StatusForbidden
func (p *CloudWatchProvider) IsOnline() (bool, error) {
_, err := p.client.GetMetricData(&cloudwatch.GetMetricDataInput{
EndTime: aws.Time(time.Time{}),
MetricDataQueries: []*cloudwatch.MetricDataQuery{},
StartTime: aws.Time(time.Time{}),
})
if err == nil {
return true, nil
}
ae, ok := err.(awserr.RequestFailure)
if !ok {
return false, fmt.Errorf("unexpected error: %v", err)
} else if ae.StatusCode() != http.StatusBadRequest {
return false, fmt.Errorf("unexpected status code: %v", ae)
}
return true, nil
}
pkg/metrics/providers/cloudwatch_test.go
+187
View File
@@ -0,0 +1,187 @@
package providers
import (
"net/http"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/aws/aws-sdk-go/service/costandusagereportservice"
flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
)
type cloudWatchClientMock struct {
o *cloudwatch.GetMetricDataOutput
err error
}
func (c cloudWatchClientMock) GetMetricData(_ *cloudwatch.GetMetricDataInput) (*cloudwatch.GetMetricDataOutput, error) {
return c.o, c.err
}
func TestNewCloudWatchProvider(t *testing.T) {
t.Run("ok", func(t *testing.T) {
p, err := NewCloudWatchProvider(
"5m",
flaggerv1.MetricTemplateProvider{
Region: costandusagereportservice.AWSRegionApEast1,
})
if err != nil {
t.Fatal(err)
}
if exp := 5 * 60 * time.Second * cloudWatchStartDeltaMultiplierOnMetricInterval; p.startDelta != exp {
t.Fatalf("expected %d but got %d", exp, p.startDelta)
}
})
t.Run("ng", func(t *testing.T) {
_, err := NewCloudWatchProvider("5m", flaggerv1.MetricTemplateProvider{})
if err == nil {
t.Fatal("error expected since region was not specified")
}
})
}
func TestCloudWatchProvider_IsOnline(t *testing.T) {
t.Run("forbidden", func(t *testing.T) {
p := CloudWatchProvider{client: cloudWatchClientMock{
o: nil,
err: awserr.NewRequestFailure(nil, http.StatusForbidden, "request-id"),
}}
actual, err := p.IsOnline()
if err == nil {
t.Error("error expected")
}
if actual {
t.Error("false expected")
}
})
t.Run("ok", func(t *testing.T) {
// no error
p := CloudWatchProvider{client: cloudWatchClientMock{}}
actual, err := p.IsOnline()
if err != nil {
t.Errorf("no error expected but got %v", err)
}
if !actual {
t.Error("true expected")
}
// with error but bad request
p = CloudWatchProvider{client: cloudWatchClientMock{
err: awserr.NewRequestFailure(nil, http.StatusBadRequest, "request-id"),
}}
if err != nil {
t.Errorf("no error expected but got %v", err)
}
if !actual {
t.Error("true expected")
}
})
}
func TestCloudWatchProvider_RunQuery(t *testing.T) {
// ref: https://aws.amazon.com/premiumsupport/knowledge-center/cloudwatch-getmetricdata-api/
query := `
[
{
"Id": "e1",
"Expression": "m1 / m2",
"Label": "ErrorRate"
},
{
"Id": "m1",
"MetricStat": {
"Metric": {
"Namespace": "MyApplication",
"MetricName": "Errors",
"Dimensions": [
{
"Name": "FunctionName",
"Value": "MyFunc"
}
]
},
"Period": 300,
"Stat": "Sum",
"Unit": "Count"
},
"ReturnData": false
},
{
"Id": "m2",
"MetricStat": {
"Metric": {
"Namespace": "MyApplication",
"MetricName": "Invocations",
"Dimensions": [
{
"Name": "FunctionName",
"Value": "MyFunc"
}
]
},
"Period": 300,
"Stat": "Sum",
"Unit": "Count"
},
"ReturnData": false
}
]`
t.Run("ok", func(t *testing.T) {
var exp float64 = 100
p := CloudWatchProvider{client: cloudWatchClientMock{
o: &cloudwatch.GetMetricDataOutput{
MetricDataResults: []*cloudwatch.MetricDataResult{
{Values: []*float64{aws.Float64(exp)}},
},
},
}}
actual, err := p.RunQuery(query)
if err != nil {
t.Fatalf("no error expected but got %v", err)
}
if actual != exp {
t.Errorf("expected %f but got %f", exp, actual)
}
})
t.Run("no values", func(t *testing.T) {
assert := func(err error) {
if err == nil {
t.Fatal("error expected")
}
if !strings.Contains(err.Error(), "no values") {
t.Fatalf("no values expected in %v", err)
}
}
p := CloudWatchProvider{client: cloudWatchClientMock{
o: &cloudwatch.GetMetricDataOutput{
MetricDataResults: []*cloudwatch.MetricDataResult{
{Values: []*float64{}},
},
},
}}
_, err := p.RunQuery(query)
assert(err)
p = CloudWatchProvider{client: cloudWatchClientMock{
o: &cloudwatch.GetMetricDataOutput{}}}
_, err = p.RunQuery(query)
assert(err)
})
}
pkg/metrics/providers/factory.go
+5 -4
View File
@@ -11,12 +11,13 @@ func (factory Factory) Provider(
provider flaggerv1.MetricTemplateProvider,
credentials map[string][]byte,
) (Interface, error) {
switch {
case provider.Type == "prometheus":
switch provider.Type {
case "prometheus":
return NewPrometheusProvider(provider, credentials)
case provider.Type == "datadog":
case "datadog":
return NewDatadogProvider(metricInterval, provider, credentials)
case "cloudwatch":
return NewCloudWatchProvider(metricInterval, provider)
default:
return NewPrometheusProvider(provider, credentials)
}
pkg/metrics/providers/prometheus.go
+1 -1
View File
@@ -39,7 +39,7 @@ type prometheusResponse struct {
// returns a Prometheus client ready to execute queries against the API
func NewPrometheusProvider(provider flaggerv1.MetricTemplateProvider, credentials map[string][]byte) (*PrometheusProvider, error) {
promURL, err := url.Parse(provider.Address)
if err != nil {
if provider.Address == "" || err != nil {
return nil, fmt.Errorf("%s address %s is not a valid URL", provider.Type, provider.Address)
}