mirror of
https://github.com/fluxcd/flagger.git
synced 2026-09-05 02:47:17 +00:00
Merge pull request #1954 from chriscur/chriscur/azure-monitor-workspace-provider
Add Azure Monitor Workspace metrics provider
This commit is contained in:
Sanskar Jaiswal
@@ -1388,6 +1388,7 @@ spec:
|
||||
- dynatrace
|
||||
- keptn
|
||||
- splunk
|
||||
- azuremonitor
|
||||
address:
|
||||
description: API address of this provider
|
||||
type: string
|
||||
|
||||
@@ -1388,6 +1388,7 @@ spec:
|
||||
- dynatrace
|
||||
- keptn
|
||||
- splunk
|
||||
- azuremonitor
|
||||
address:
|
||||
description: API address of this provider
|
||||
type: string
|
||||
|
||||
@@ -798,6 +798,111 @@ Reference the template in the canary analysis:
|
||||
interval: 1m
|
||||
```
|
||||
|
||||
## Azure Monitor Workspace
|
||||
|
||||
You can create custom metric checks using the Azure Monitor provider, which queries the
|
||||
[Prometheus compatible API](https://learn.microsoft.com/azure/azure-monitor/metrics/prometheus-api-promql)
|
||||
of an [Azure Monitor Workspace](https://learn.microsoft.com/azure/azure-monitor/metrics/azure-monitor-workspace-overview).
|
||||
|
||||
Queries are authenticated with a Microsoft Entra ID token, so no authentication proxy is
|
||||
required in front of the workspace. Set `address` to the workspace **Query endpoint** and grant
|
||||
the identity that Flagger uses the `Monitoring Data Reader` role on the workspace. Azure
|
||||
Government and Azure China workspaces are detected from the endpoint.
|
||||
|
||||
With no `secretRef`, Flagger authenticates with the
|
||||
[workload identity](https://learn.microsoft.com/azure/aks/workload-identity-overview) of its own
|
||||
pod. This requires a federated credential for the `system:serviceaccount:flagger-system:flagger`
|
||||
subject, and a Flagger install that carries the annotation and label the webhook looks for:
|
||||
|
||||
```yaml
|
||||
serviceAccount:
|
||||
annotations:
|
||||
azure.workload.identity/client-id: your-managed-identity-client-id
|
||||
podLabels:
|
||||
azure.workload.identity/use: "true"
|
||||
```
|
||||
|
||||
The metric template then needs no credentials:
|
||||
|
||||
```yaml
|
||||
apiVersion: flagger.app/v1beta1
|
||||
kind: MetricTemplate
|
||||
metadata:
|
||||
name: request-success-rate
|
||||
namespace: istio-system
|
||||
spec:
|
||||
provider:
|
||||
type: azuremonitor
|
||||
address: https://flagger-abc1.eastus.prometheus.monitor.azure.com
|
||||
query: |
|
||||
100 - sum(
|
||||
rate(
|
||||
istio_requests_total{
|
||||
reporter="destination",
|
||||
destination_workload_namespace="{{ namespace }}",
|
||||
destination_workload="{{ target }}",
|
||||
response_code=~"5.."
|
||||
}[{{ interval }}]
|
||||
)
|
||||
)
|
||||
/
|
||||
sum(
|
||||
rate(
|
||||
istio_requests_total{
|
||||
reporter="destination",
|
||||
destination_workload_namespace="{{ namespace }}",
|
||||
destination_workload="{{ target }}"
|
||||
}[{{ interval }}]
|
||||
)
|
||||
) * 100
|
||||
```
|
||||
|
||||
Reference the template in the canary analysis:
|
||||
|
||||
```yaml
|
||||
analysis:
|
||||
metrics:
|
||||
- name: "request success rate"
|
||||
templateRef:
|
||||
name: request-success-rate
|
||||
namespace: istio-system
|
||||
thresholdRange:
|
||||
min: 99
|
||||
interval: 1m
|
||||
```
|
||||
|
||||
To use a different identity, reference a secret with `secretRef`. The keys it contains select
|
||||
the identity:
|
||||
|
||||
| Secret keys | Identity used |
|
||||
| --- | --- |
|
||||
| `clientId`, `tenantId`, `clientSecret` | Microsoft Entra ID application (service principal) |
|
||||
| `clientId`, `tenantId` | Workload identity, for an identity other than the pod default |
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: azure-monitor
|
||||
namespace: istio-system
|
||||
stringData:
|
||||
clientId: your-application-id
|
||||
tenantId: your-tenant-id
|
||||
clientSecret: your-client-secret
|
||||
```
|
||||
|
||||
Because Flagger can authenticate with its own identity, `address` is restricted to an `https`
|
||||
workspace query endpoint and `insecureSkipVerify` is not supported.
|
||||
|
||||
Troubleshooting:
|
||||
|
||||
* `no token file specified` or `no client ID specified` means the workload identity webhook did
|
||||
not project a token into the Flagger pod. Check the service account annotation and pod label.
|
||||
* A `403` response means the identity is missing the `Monitoring Data Reader` role assignment
|
||||
on the Azure Monitor Workspace.
|
||||
* A `429` response means the workspace query limits have been reached. Flagger treats this as a
|
||||
failed metric check, which can cause a canary to be rolled back.
|
||||
|
||||
## Kubernetes External Metrics
|
||||
|
||||
You can query an external metrics provider that implements the
|
||||
|
||||
@@ -4,6 +4,8 @@ go 1.26.0
|
||||
|
||||
require (
|
||||
cloud.google.com/go/monitoring v1.29.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0
|
||||
github.com/Masterminds/semver/v3 v3.5.0
|
||||
github.com/aws/aws-sdk-go v1.55.8
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
|
||||
@@ -39,6 +41,8 @@ require (
|
||||
cloud.google.com/go/auth v0.20.0 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect
|
||||
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/blendle/zapdriver v1.3.1 // indirect
|
||||
@@ -51,6 +55,7 @@ require (
|
||||
github.com/go-openapi/jsonpointer v0.21.0 // indirect
|
||||
github.com/go-openapi/jsonreference v0.21.0 // indirect
|
||||
github.com/go-openapi/swag v0.23.0 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
|
||||
github.com/google/gnostic-models v0.7.0 // indirect
|
||||
github.com/google/go-containerregistry v0.20.3 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
@@ -69,6 +74,7 @@ require (
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/oapi-codegen/runtime v1.0.0 // indirect
|
||||
github.com/opencontainers/go-digest v1.0.0 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.67.5 // indirect
|
||||
|
||||
@@ -9,6 +9,15 @@ cloud.google.com/go/monitoring v1.27.0 h1:BhYwMqao+e5Nn7JtWMM9m6zRtKtVUK6kJWMizX
|
||||
cloud.google.com/go/monitoring v1.27.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM=
|
||||
cloud.google.com/go/monitoring v1.29.0 h1:AHhDsFaSax1/4k+qlIDX/SDGe6hggnfXJ9dkgD9qBPY=
|
||||
cloud.google.com/go/monitoring v1.29.0/go.mod h1:72NOVjJXHY/HBfoLT0+qlCZBT059+9VXLeAnL2PeeVM=
|
||||
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
|
||||
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
|
||||
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
|
||||
github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE=
|
||||
@@ -64,6 +73,8 @@ github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF
|
||||
github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4=
|
||||
github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
|
||||
github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
|
||||
@@ -134,6 +145,8 @@ github.com/oapi-codegen/runtime v1.0.0 h1:P4rqFX5fMFWqRzY9M/3YF9+aPSPPB06IzP2P7o
|
||||
github.com/oapi-codegen/runtime v1.0.0/go.mod h1:LmCUMQuPB4M/nLXilQXhHw+BLZdDb18B34OO356yJ/A=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
@@ -231,6 +244,7 @@ golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
|
||||
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
|
||||
@@ -1388,6 +1388,7 @@ spec:
|
||||
- dynatrace
|
||||
- keptn
|
||||
- splunk
|
||||
- azuremonitor
|
||||
address:
|
||||
description: API address of this provider
|
||||
type: string
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
/*
|
||||
Copyright 2025 The Flux authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package providers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
|
||||
flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
|
||||
)
|
||||
|
||||
const (
|
||||
azureClientIDSecretKey = "clientId"
|
||||
azureTenantIDSecretKey = "tenantId"
|
||||
azureClientSecretSecretKey = "clientSecret"
|
||||
|
||||
azureTokenTimeout = 30 * time.Second
|
||||
|
||||
azureCredentialCacheSize = 100
|
||||
)
|
||||
|
||||
// credentials are cached because a provider is built for every metric on every
|
||||
// analysis interval and each credential holds its own token cache
|
||||
var (
|
||||
azureCredentialCacheMu sync.Mutex
|
||||
azureCredentialCache = map[string]azcore.TokenCredential{}
|
||||
)
|
||||
|
||||
var azureClouds = []struct {
|
||||
suffix string
|
||||
config cloud.Configuration
|
||||
}{
|
||||
{suffix: ".prometheus.monitor.azure.us", config: cloud.AzureGovernment},
|
||||
{suffix: ".prometheus.monitor.azure.cn", config: cloud.AzureChina},
|
||||
{suffix: ".prometheus.monitor.azure.com", config: cloud.AzurePublic},
|
||||
}
|
||||
|
||||
// AzureMonitorProvider executes promQL queries against an Azure Monitor Workspace
|
||||
type AzureMonitorProvider struct {
|
||||
*PrometheusProvider
|
||||
}
|
||||
|
||||
// NewAzureMonitorProvider takes a provider spec and the credentials map, validates the
|
||||
// address and returns a Prometheus client holding a Microsoft Entra ID token
|
||||
// for the workspace
|
||||
func NewAzureMonitorProvider(provider flaggerv1.MetricTemplateProvider, credentials map[string][]byte) (*AzureMonitorProvider, error) {
|
||||
if provider.InsecureSkipVerify {
|
||||
return nil, fmt.Errorf("%s provider does not support insecureSkipVerify", provider.Type)
|
||||
}
|
||||
|
||||
address, err := url.Parse(provider.Address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s address %s is not a valid URL: %w", provider.Type, provider.Address, err)
|
||||
}
|
||||
|
||||
if !strings.EqualFold(address.Scheme, "https") {
|
||||
return nil, fmt.Errorf("%s address %s must use https", provider.Type, provider.Address)
|
||||
}
|
||||
|
||||
cloudConfig, audience, err := azureCloudForHost(address.Hostname())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s address %s %w", provider.Type, provider.Address, err)
|
||||
}
|
||||
|
||||
cred, err := azureCredential(provider.Type, credentials, cloudConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), azureTokenTimeout)
|
||||
defer cancel()
|
||||
|
||||
token, err := cred.GetToken(ctx, policy.TokenRequestOptions{Scopes: []string{azureScope(audience)}})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s token request failed: %w", provider.Type, err)
|
||||
}
|
||||
|
||||
// RunQuery adds the bearer token to the header map it was given, and that map
|
||||
// is shared by every canary using this template, so it is copied first
|
||||
promProvider := provider
|
||||
promProvider.Headers = provider.Headers.Clone()
|
||||
|
||||
// the Prometheus provider only reads credentials when a secret is referenced
|
||||
if promProvider.SecretRef == nil {
|
||||
promProvider.SecretRef = &corev1.LocalObjectReference{}
|
||||
}
|
||||
|
||||
prom, err := NewPrometheusProvider(promProvider, map[string][]byte{"token": []byte(token.Token)})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &AzureMonitorProvider{PrometheusProvider: prom}, nil
|
||||
}
|
||||
|
||||
func azureCredential(providerType string, credentials map[string][]byte,
|
||||
cloudConfig cloud.Configuration,
|
||||
) (azcore.TokenCredential, error) {
|
||||
key := azureCredentialKey(credentials, cloudConfig)
|
||||
|
||||
azureCredentialCacheMu.Lock()
|
||||
defer azureCredentialCacheMu.Unlock()
|
||||
|
||||
if cred, ok := azureCredentialCache[key]; ok {
|
||||
return cred, nil
|
||||
}
|
||||
|
||||
cred, err := newAzureCredential(providerType, credentials, cloudConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// the cache is emptied instead of grown without limit, rebuilding a
|
||||
// credential provider is not expensive
|
||||
if len(azureCredentialCache) >= azureCredentialCacheSize {
|
||||
clear(azureCredentialCache)
|
||||
}
|
||||
|
||||
azureCredentialCache[key] = cred
|
||||
return cred, nil
|
||||
}
|
||||
|
||||
func azureCredentialKey(credentials map[string][]byte, cloudConfig cloud.Configuration) string {
|
||||
return fmt.Sprintf("%s|%s|%s|%x",
|
||||
cloudConfig.ActiveDirectoryAuthorityHost,
|
||||
credentials[azureClientIDSecretKey],
|
||||
credentials[azureTenantIDSecretKey],
|
||||
sha256.Sum256(credentials[azureClientSecretSecretKey]))
|
||||
}
|
||||
|
||||
// newAzureCredential selects an identity from the keys found in the provider
|
||||
// secret, defaulting to the workload identity of the Flagger pod
|
||||
func newAzureCredential(providerType string, credentials map[string][]byte,
|
||||
cloudConfig cloud.Configuration,
|
||||
) (azcore.TokenCredential, error) {
|
||||
clientOptions := azcore.ClientOptions{Cloud: cloudConfig}
|
||||
|
||||
clientID := string(credentials[azureClientIDSecretKey])
|
||||
tenantID := string(credentials[azureTenantIDSecretKey])
|
||||
clientSecret := string(credentials[azureClientSecretSecretKey])
|
||||
|
||||
switch {
|
||||
case clientSecret != "":
|
||||
if clientID == "" {
|
||||
return nil, fmt.Errorf("%s credentials does not contain a clientId", providerType)
|
||||
}
|
||||
if tenantID == "" {
|
||||
return nil, fmt.Errorf("%s credentials does not contain a tenantId", providerType)
|
||||
}
|
||||
return azidentity.NewClientSecretCredential(tenantID, clientID, clientSecret,
|
||||
&azidentity.ClientSecretCredentialOptions{ClientOptions: clientOptions})
|
||||
case clientID != "" && tenantID != "":
|
||||
return azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{
|
||||
ClientOptions: clientOptions,
|
||||
ClientID: clientID,
|
||||
TenantID: tenantID,
|
||||
})
|
||||
case clientID != "":
|
||||
return nil, fmt.Errorf("%s credentials does not contain a tenantId", providerType)
|
||||
case tenantID != "":
|
||||
return nil, fmt.Errorf("%s credentials does not contain a clientId", providerType)
|
||||
default:
|
||||
return azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{
|
||||
ClientOptions: clientOptions,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// azureCloudForHost returns the Azure cloud and the token audience matching
|
||||
// the query endpoint
|
||||
func azureCloudForHost(host string) (cloud.Configuration, string, error) {
|
||||
host = strings.ToLower(host)
|
||||
for _, c := range azureClouds {
|
||||
if strings.HasSuffix(host, c.suffix) {
|
||||
return c.config, "https://" + strings.TrimPrefix(c.suffix, "."), nil
|
||||
}
|
||||
}
|
||||
suffixes := make([]string, len(azureClouds))
|
||||
for i, c := range azureClouds {
|
||||
suffixes[i] = c.suffix
|
||||
}
|
||||
return cloud.Configuration{}, "", fmt.Errorf(
|
||||
"is not an Azure Monitor Workspace query endpoint, expected a host ending in %s", strings.Join(suffixes, ", "))
|
||||
}
|
||||
|
||||
func azureScope(audience string) string {
|
||||
if strings.HasSuffix(audience, "/.default") {
|
||||
return audience
|
||||
}
|
||||
return strings.TrimSuffix(audience, "/") + "/.default"
|
||||
}
|
||||
@@ -0,0 +1,442 @@
|
||||
/*
|
||||
Copyright 2025 The Flux authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package providers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
|
||||
flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
|
||||
)
|
||||
|
||||
const azureMonitorTestAddress = "https://flagger-abc1.eastus.prometheus.monitor.azure.com"
|
||||
|
||||
type fakeAzureCredential struct {
|
||||
token string
|
||||
scopes []string
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *fakeAzureCredential) GetToken(_ context.Context, opts policy.TokenRequestOptions) (azcore.AccessToken, error) {
|
||||
c.scopes = opts.Scopes
|
||||
if c.err != nil {
|
||||
return azcore.AccessToken{}, c.err
|
||||
}
|
||||
return azcore.AccessToken{Token: c.token, ExpiresOn: time.Now().Add(time.Hour)}, nil
|
||||
}
|
||||
|
||||
// azureWorkloadIdentityEnv sets the variables injected by the workload identity webhook
|
||||
func azureWorkloadIdentityEnv(t *testing.T) {
|
||||
t.Helper()
|
||||
t.Setenv("AZURE_FEDERATED_TOKEN_FILE", "/var/run/secrets/azure/tokens/azure-identity-token")
|
||||
t.Setenv("AZURE_CLIENT_ID", "pod-client-id")
|
||||
t.Setenv("AZURE_TENANT_ID", "pod-tenant-id")
|
||||
}
|
||||
|
||||
func azureMonitorTestProvider(address string) flaggerv1.MetricTemplateProvider {
|
||||
return flaggerv1.MetricTemplateProvider{
|
||||
Type: "azuremonitor",
|
||||
Address: address,
|
||||
SecretRef: &corev1.LocalObjectReference{Name: "azuremonitor"},
|
||||
}
|
||||
}
|
||||
|
||||
// azureUseCredential seeds the credential cache so the constructor mints its token from cred
|
||||
func azureUseCredential(t *testing.T, cred azcore.TokenCredential) {
|
||||
t.Helper()
|
||||
azureWorkloadIdentityEnv(t)
|
||||
|
||||
key := azureCredentialKey(nil, cloud.AzurePublic)
|
||||
|
||||
azureCredentialCacheMu.Lock()
|
||||
azureCredentialCache[key] = cred
|
||||
azureCredentialCacheMu.Unlock()
|
||||
|
||||
t.Cleanup(func() {
|
||||
azureCredentialCacheMu.Lock()
|
||||
delete(azureCredentialCache, key)
|
||||
azureCredentialCacheMu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
// azureMonitorTestClient points a provider at the test server, since only workspace addresses are accepted
|
||||
func azureMonitorTestClient(t *testing.T, serverURL string, provider flaggerv1.MetricTemplateProvider,
|
||||
cred azcore.TokenCredential,
|
||||
) *AzureMonitorProvider {
|
||||
t.Helper()
|
||||
azureUseCredential(t, cred)
|
||||
|
||||
az, err := NewAzureMonitorProvider(provider, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
u, err := url.Parse(serverURL)
|
||||
require.NoError(t, err)
|
||||
az.url = *u
|
||||
|
||||
return az
|
||||
}
|
||||
|
||||
func TestNewAzureMonitorProvider(t *testing.T) {
|
||||
azureWorkloadIdentityEnv(t)
|
||||
|
||||
t.Run("ok", func(t *testing.T) {
|
||||
cred := &fakeAzureCredential{token: "token"}
|
||||
azureUseCredential(t, cred)
|
||||
|
||||
az, err := NewAzureMonitorProvider(azureMonitorTestProvider(azureMonitorTestAddress), nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, azureMonitorTestAddress, az.url.String())
|
||||
assert.Equal(t, "token", az.token)
|
||||
assert.Equal(t, []string{"https://prometheus.monitor.azure.com/.default"}, cred.scopes)
|
||||
})
|
||||
|
||||
t.Run("no secret", func(t *testing.T) {
|
||||
azureUseCredential(t, &fakeAzureCredential{token: "token"})
|
||||
|
||||
provider := azureMonitorTestProvider(azureMonitorTestAddress)
|
||||
provider.SecretRef = nil
|
||||
|
||||
az, err := NewAzureMonitorProvider(provider, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "token", az.token)
|
||||
})
|
||||
|
||||
t.Run("token error", func(t *testing.T) {
|
||||
azureUseCredential(t, &fakeAzureCredential{err: assert.AnError})
|
||||
|
||||
_, err := NewAzureMonitorProvider(azureMonitorTestProvider(azureMonitorTestAddress), nil)
|
||||
require.ErrorContains(t, err, "azuremonitor token request failed")
|
||||
})
|
||||
|
||||
t.Run("invalid address", func(t *testing.T) {
|
||||
_, err := NewAzureMonitorProvider(azureMonitorTestProvider(""), nil)
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("address outside of azure monitor", func(t *testing.T) {
|
||||
_, err := NewAzureMonitorProvider(azureMonitorTestProvider("https://prometheus.example.com"), nil)
|
||||
require.EqualError(t, err,
|
||||
"azuremonitor address https://prometheus.example.com is not an Azure Monitor Workspace query endpoint, "+
|
||||
"expected a host ending in .prometheus.monitor.azure.us, .prometheus.monitor.azure.cn, .prometheus.monitor.azure.com")
|
||||
})
|
||||
|
||||
t.Run("address without tls", func(t *testing.T) {
|
||||
address := "http://flagger-abc1.eastus.prometheus.monitor.azure.com"
|
||||
|
||||
_, err := NewAzureMonitorProvider(azureMonitorTestProvider(address), nil)
|
||||
require.EqualError(t, err, "azuremonitor address "+address+" must use https")
|
||||
})
|
||||
|
||||
t.Run("insecure skip verify", func(t *testing.T) {
|
||||
provider := azureMonitorTestProvider(azureMonitorTestAddress)
|
||||
provider.InsecureSkipVerify = true
|
||||
|
||||
_, err := NewAzureMonitorProvider(provider, nil)
|
||||
require.EqualError(t, err, "azuremonitor provider does not support insecureSkipVerify")
|
||||
})
|
||||
}
|
||||
|
||||
func TestAzureMonitorProvider_RunQuery(t *testing.T) {
|
||||
t.Run("ok", func(t *testing.T) {
|
||||
expected := `sum(envoy_cluster_upstream_rq)`
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "/api/v1/query", r.URL.Path)
|
||||
assert.Equal(t, expected, r.URL.Query()["query"][0])
|
||||
assert.Equal(t, "Bearer token", r.Header.Get("Authorization"))
|
||||
|
||||
json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1545905245.458,"100"]}]}}`
|
||||
w.Write([]byte(json))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
cred := &fakeAzureCredential{token: "token"}
|
||||
az := azureMonitorTestClient(t, ts.URL, azureMonitorTestProvider(azureMonitorTestAddress), cred)
|
||||
|
||||
val, err := az.RunQuery(expected)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, float64(100), val)
|
||||
assert.Equal(t, []string{"https://prometheus.monitor.azure.com/.default"}, cred.scopes)
|
||||
})
|
||||
}
|
||||
|
||||
func TestAzureMonitorProvider_IsOnline(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "Bearer token", r.Header.Get("Authorization"))
|
||||
|
||||
json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1545905245.458,"1"]}]}}`
|
||||
w.Write([]byte(json))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
az := azureMonitorTestClient(t, ts.URL, azureMonitorTestProvider(azureMonitorTestAddress), &fakeAzureCredential{token: "token"})
|
||||
|
||||
ok, err := az.IsOnline()
|
||||
require.NoError(t, err)
|
||||
assert.True(t, ok)
|
||||
}
|
||||
|
||||
func TestAzureMonitorProvider_Headers(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "Bearer token", r.Header.Get("Authorization"))
|
||||
assert.Equal(t, "bar", r.Header.Get("Foo"))
|
||||
|
||||
json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1545905245.458,"1"]}]}}`
|
||||
w.Write([]byte(json))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
provider := azureMonitorTestProvider(azureMonitorTestAddress)
|
||||
provider.Headers = http.Header{"Foo": []string{"bar"}}
|
||||
|
||||
az := azureMonitorTestClient(t, ts.URL, provider, &fakeAzureCredential{token: "token"})
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
_, err := az.RunQuery("vector(1)")
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
assert.Empty(t, provider.Headers.Get("Authorization"),
|
||||
"the header map comes from the informer cache and must not be modified")
|
||||
}
|
||||
|
||||
// TestAzureMonitorProvider_Redirect relies on the http client dropping the Authorization
|
||||
// header when a redirect leaves the workspace host. The hosts are faked because the
|
||||
// client compares host names and every test server listens on the loopback address.
|
||||
func TestAzureMonitorProvider_Redirect(t *testing.T) {
|
||||
var received string
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
received = r.Header.Get("Authorization")
|
||||
|
||||
json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1545905245.458,"1"]}]}}`
|
||||
w.Write([]byte(json))
|
||||
}))
|
||||
defer target.Close()
|
||||
|
||||
workspace := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, "Bearer token", r.Header.Get("Authorization"))
|
||||
http.Redirect(w, r, "http://elsewhere.invalid/api/v1/query?query=vector(1)", http.StatusFound)
|
||||
}))
|
||||
defer workspace.Close()
|
||||
|
||||
az := azureMonitorTestClient(t, "http://workspace.invalid", azureMonitorTestProvider(azureMonitorTestAddress),
|
||||
&fakeAzureCredential{token: "token"})
|
||||
|
||||
hosts := map[string]string{
|
||||
"workspace.invalid:80": workspace.Listener.Addr().String(),
|
||||
"elsewhere.invalid:80": target.Listener.Addr().String(),
|
||||
}
|
||||
az.client = &http.Client{Transport: &http.Transport{
|
||||
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
if mapped, ok := hosts[addr]; ok {
|
||||
addr = mapped
|
||||
}
|
||||
return (&net.Dialer{}).DialContext(ctx, network, addr)
|
||||
},
|
||||
}}
|
||||
|
||||
val, err := az.RunQuery("vector(1)")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, float64(1), val, "the redirect must have been followed")
|
||||
assert.Empty(t, received, "the token must not be sent to another host")
|
||||
}
|
||||
|
||||
func TestNewAzureCredential(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
credentials map[string][]byte
|
||||
expectedType any
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
name: "service principal",
|
||||
credentials: map[string][]byte{
|
||||
azureClientIDSecretKey: []byte("client-id"),
|
||||
azureTenantIDSecretKey: []byte("tenant-id"),
|
||||
azureClientSecretSecretKey: []byte("client-secret"),
|
||||
},
|
||||
expectedType: &azidentity.ClientSecretCredential{},
|
||||
},
|
||||
{
|
||||
name: "workload identity",
|
||||
credentials: map[string][]byte{
|
||||
azureClientIDSecretKey: []byte("client-id"),
|
||||
azureTenantIDSecretKey: []byte("tenant-id"),
|
||||
},
|
||||
expectedType: &azidentity.WorkloadIdentityCredential{},
|
||||
},
|
||||
{
|
||||
name: "pod workload identity",
|
||||
credentials: nil,
|
||||
expectedType: &azidentity.WorkloadIdentityCredential{},
|
||||
},
|
||||
{
|
||||
name: "client secret without client id",
|
||||
credentials: map[string][]byte{
|
||||
azureTenantIDSecretKey: []byte("tenant-id"),
|
||||
azureClientSecretSecretKey: []byte("client-secret"),
|
||||
},
|
||||
expectedErr: "azuremonitor credentials does not contain a clientId",
|
||||
},
|
||||
{
|
||||
name: "client secret without tenant id",
|
||||
credentials: map[string][]byte{
|
||||
azureClientIDSecretKey: []byte("client-id"),
|
||||
azureClientSecretSecretKey: []byte("client-secret"),
|
||||
},
|
||||
expectedErr: "azuremonitor credentials does not contain a tenantId",
|
||||
},
|
||||
{
|
||||
name: "client id without tenant id",
|
||||
credentials: map[string][]byte{
|
||||
azureClientIDSecretKey: []byte("client-id"),
|
||||
},
|
||||
expectedErr: "azuremonitor credentials does not contain a tenantId",
|
||||
},
|
||||
{
|
||||
name: "tenant id without client id",
|
||||
credentials: map[string][]byte{
|
||||
azureTenantIDSecretKey: []byte("tenant-id"),
|
||||
},
|
||||
expectedErr: "azuremonitor credentials does not contain a clientId",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
azureWorkloadIdentityEnv(t)
|
||||
|
||||
cred, err := newAzureCredential("azuremonitor", tt.credentials, cloud.AzurePublic)
|
||||
if tt.expectedErr != "" {
|
||||
require.EqualError(t, err, tt.expectedErr)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.IsType(t, tt.expectedType, cred)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAzureCredentialCache(t *testing.T) {
|
||||
azureWorkloadIdentityEnv(t)
|
||||
|
||||
credentials := map[string][]byte{
|
||||
azureClientIDSecretKey: []byte("cache-test-client-id"),
|
||||
azureTenantIDSecretKey: []byte("tenant-id"),
|
||||
}
|
||||
|
||||
first, err := azureCredential("azuremonitor", credentials, cloud.AzurePublic)
|
||||
require.NoError(t, err)
|
||||
|
||||
second, err := azureCredential("azuremonitor", credentials, cloud.AzurePublic)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Same(t, first, second)
|
||||
|
||||
other, err := azureCredential("azuremonitor", map[string][]byte{
|
||||
azureClientIDSecretKey: []byte("another-client-id"),
|
||||
azureTenantIDSecretKey: []byte("tenant-id"),
|
||||
}, cloud.AzurePublic)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.NotSame(t, first, other)
|
||||
}
|
||||
|
||||
func TestAzureCloudForHost(t *testing.T) {
|
||||
tests := []struct {
|
||||
host string
|
||||
expectedCloud cloud.Configuration
|
||||
expectedAudience string
|
||||
expectedErr bool
|
||||
}{
|
||||
{
|
||||
host: "flagger-abcd.eastus.prometheus.monitor.azure.com",
|
||||
expectedCloud: cloud.AzurePublic,
|
||||
expectedAudience: "https://prometheus.monitor.azure.com",
|
||||
},
|
||||
{
|
||||
host: "flagger-abcd.usgovvirginia.prometheus.monitor.azure.us",
|
||||
expectedCloud: cloud.AzureGovernment,
|
||||
expectedAudience: "https://prometheus.monitor.azure.us",
|
||||
},
|
||||
{
|
||||
host: "flagger-abcd.chinaeast.prometheus.monitor.azure.cn",
|
||||
expectedCloud: cloud.AzureChina,
|
||||
expectedAudience: "https://prometheus.monitor.azure.cn",
|
||||
},
|
||||
{
|
||||
host: "FLAGGER-ABCD.EASTUS.PROMETHEUS.MONITOR.AZURE.US",
|
||||
expectedCloud: cloud.AzureGovernment,
|
||||
expectedAudience: "https://prometheus.monitor.azure.us",
|
||||
},
|
||||
{
|
||||
host: "prometheus.example.com",
|
||||
expectedErr: true,
|
||||
},
|
||||
{
|
||||
host: "prometheus.monitor.azure.com.example.com",
|
||||
expectedErr: true,
|
||||
},
|
||||
{
|
||||
host: "notprometheus.monitor.azure.com",
|
||||
expectedErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.host, func(t *testing.T) {
|
||||
config, audience, err := azureCloudForHost(tt.host)
|
||||
if tt.expectedErr {
|
||||
require.Error(t, err)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.expectedCloud, config)
|
||||
assert.Equal(t, tt.expectedAudience, audience)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAzureScope(t *testing.T) {
|
||||
tests := []struct {
|
||||
audience string
|
||||
expected string
|
||||
}{
|
||||
{audience: "https://prometheus.monitor.azure.com", expected: "https://prometheus.monitor.azure.com/.default"},
|
||||
{audience: "https://prometheus.monitor.azure.com/", expected: "https://prometheus.monitor.azure.com/.default"},
|
||||
{audience: "https://prometheus.monitor.azure.com/.default", expected: "https://prometheus.monitor.azure.com/.default"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.audience, func(t *testing.T) {
|
||||
assert.Equal(t, tt.expected, azureScope(tt.audience))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,8 @@ func (factory Factory) Provider(metricInterval string, provider flaggerv1.Metric
|
||||
return NewKeptnProvider(config)
|
||||
case "splunk":
|
||||
return NewSplunkProvider(metricInterval, provider, credentials)
|
||||
case "azuremonitor":
|
||||
return NewAzureMonitorProvider(provider, credentials)
|
||||
default:
|
||||
factory.logger.Warnf("unknown metrics provider '%s', using prometheus", provider.Type)
|
||||
return NewPrometheusProvider(provider, credentials)
|
||||
|
||||
Reference in New Issue
Block a user