github/flagger
1
0
Fork 0
mirror of https://github.com/fluxcd/flagger.git synced 2026-04-15 06:57:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add metrics verification to controller tests

Browse Source 
Enhance existing scheduler tests for deployments, daemonsets, and
services by adding prometheus metrics verification using testutil.
This ensures that status metrics are correctly recorded during
canary promotion workflows and provides better test coverage for
the metrics recording functionality.

Signed-off-by: cappyzawa <cappyzawa@gmail.com>
This commit is contained in:
cappyzawa
2025-10-08 13:47:11 +09:00
parent f9f10e842e
commit 16f54923b2
5 changed files with 459 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/scheduler.go
+40
View File
@@ -27,6 +27,7 @@ import (
flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
"github.com/fluxcd/flagger/pkg/canary"
"github.com/fluxcd/flagger/pkg/metrics"
"github.com/fluxcd/flagger/pkg/router"
)
@@ -37,6 +38,27 @@ func (c *Controller) min(a int, b int) int {
return b
}
// getDeploymentStrategy determines the deployment strategy based on canary analysis configuration
func (c *Controller) getDeploymentStrategy(canary *flaggerv1.Canary) string {
analysis := canary.GetAnalysis()
if analysis == nil {
return metrics.CanaryStrategy
}
// A/B Testing: has match conditions and iterations
if len(analysis.Match) > 0 && analysis.Iterations > 0 {
return metrics.ABTestingStrategy
}
// Blue/Green: has iterations but no match conditions
if analysis.Iterations > 0 {
return metrics.BlueGreenStrategy
}
// Canary Release: default (has maxWeight, stepWeight, or stepWeights)
return metrics.CanaryStrategy
}
func (c *Controller) maxWeight(canary *flaggerv1.Canary) int {
var stepWeightsLen = len(canary.GetAnalysis().StepWeights)
if stepWeightsLen > 0 {
@@ -400,6 +422,12 @@ func (c *Controller) advanceCanary(name string, namespace string) {
return
}
c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseSucceeded)
c.recorder.IncSuccesses(metrics.CanaryMetricLabels{
Name: cd.Spec.TargetRef.Name,
Namespace: cd.Namespace,
DeploymentStrategy: c.getDeploymentStrategy(cd),
AnalysisStatus: metrics.AnalysisStatusCompleted,
})
c.runPostRolloutHooks(cd, flaggerv1.CanaryPhaseSucceeded)
c.recordEventInfof(cd, "Promotion completed! Scaling down %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)
c.alert(cd, "Canary analysis completed successfully, promotion finished.",
@@ -814,6 +842,12 @@ func (c *Controller) shouldSkipAnalysis(canary *flaggerv1.Canary, canaryControll
// notify
c.recorder.SetStatus(canary, flaggerv1.CanaryPhaseSucceeded)
c.recorder.IncSuccesses(metrics.CanaryMetricLabels{
Name: canary.Spec.TargetRef.Name,
Namespace: canary.Namespace,
DeploymentStrategy: c.getDeploymentStrategy(canary),
AnalysisStatus: metrics.AnalysisStatusSkipped,
})
c.recordEventInfof(canary, "Promotion completed! Canary analysis was skipped for %s.%s",
canary.Spec.TargetRef.Name, canary.Namespace)
c.alert(canary, "Canary analysis was skipped, promotion finished.",
@@ -961,6 +995,12 @@ func (c *Controller) rollback(canary *flaggerv1.Canary, canaryController canary.
}
c.recorder.SetStatus(canary, flaggerv1.CanaryPhaseFailed)
c.recorder.IncFailures(metrics.CanaryMetricLabels{
Name: canary.Spec.TargetRef.Name,
Namespace: canary.Namespace,
DeploymentStrategy: c.getDeploymentStrategy(canary),
AnalysisStatus: metrics.AnalysisStatusCompleted,
})
c.runPostRolloutHooks(canary, flaggerv1.CanaryPhaseFailed)
}
pkg/controller/scheduler_metrics_test.go
+202 -1
View File
@@ -17,16 +17,21 @@ limitations under the License.
package controller
import (
"context"
"testing"
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/record"
flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
istiov1alpha1 "github.com/fluxcd/flagger/pkg/apis/istio/common/v1alpha1"
istiov1beta1 "github.com/fluxcd/flagger/pkg/apis/istio/v1beta1"
"github.com/fluxcd/flagger/pkg/metrics"
"github.com/fluxcd/flagger/pkg/metrics/observers"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestController_checkMetricProviderAvailability(t *testing.T) {
@@ -183,3 +188,199 @@ func TestController_runMetricChecks(t *testing.T) {
assert.Equal(t, true, ctrl.runMetricChecks(canary))
})
}
func TestController_MetricsStateTransition(t *testing.T) {
t.Run("initialization and progression metrics", func(t *testing.T) {
mocks := newDeploymentFixture(nil)
mocks.ctrl.advanceCanary("podinfo", "default")
mocks.makePrimaryReady(t)
mocks.ctrl.advanceCanary("podinfo", "default")
actualStatus := testutil.ToFloat64(mocks.ctrl.recorder.GetStatusMetric().WithLabelValues("podinfo", "default"))
assert.Equal(t, float64(1), actualStatus)
actualTotal := testutil.ToFloat64(mocks.ctrl.recorder.GetTotalMetric().WithLabelValues("default"))
assert.GreaterOrEqual(t, actualTotal, float64(0))
dep2 := newDeploymentTestDeploymentV2()
_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
require.NoError(t, err)
mocks.ctrl.advanceCanary("podinfo", "default")
mocks.makeCanaryReady(t)
mocks.ctrl.advanceCanary("podinfo", "default")
actualStatus = testutil.ToFloat64(mocks.ctrl.recorder.GetStatusMetric().WithLabelValues("podinfo", "default"))
assert.Equal(t, float64(0), actualStatus)
actualPrimaryWeight := testutil.ToFloat64(mocks.ctrl.recorder.GetWeightMetric().WithLabelValues("podinfo-primary", "default"))
actualCanaryWeight := testutil.ToFloat64(mocks.ctrl.recorder.GetWeightMetric().WithLabelValues("podinfo", "default"))
t.Logf("Progression weights - Primary: %f, Canary: %f", actualPrimaryWeight, actualCanaryWeight)
assert.GreaterOrEqual(t, actualPrimaryWeight, float64(50))
assert.GreaterOrEqual(t, actualCanaryWeight, float64(10))
assert.LessOrEqual(t, actualPrimaryWeight, float64(100))
assert.LessOrEqual(t, actualCanaryWeight, float64(50))
totalWeight := actualPrimaryWeight + actualCanaryWeight
assert.InDelta(t, 100.0, totalWeight, 1.0)
})
t.Run("failed canary rollback", func(t *testing.T) {
mocks := newDeploymentFixture(nil)
mocks.ctrl.advanceCanary("podinfo", "default")
mocks.makePrimaryReady(t)
mocks.ctrl.advanceCanary("podinfo", "default")
err := mocks.deployer.SyncStatus(mocks.canary, flaggerv1.CanaryStatus{
Phase: flaggerv1.CanaryPhaseProgressing,
FailedChecks: 10,
})
require.NoError(t, err)
c, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
require.NoError(t, err)
cd := c.DeepCopy()
cd.Spec.Analysis.Metrics = append(c.Spec.Analysis.Metrics, flaggerv1.CanaryMetric{
Name: "fail",
Interval: "1m",
ThresholdRange: &flaggerv1.CanaryThresholdRange{
Min: toFloatPtr(0),
Max: toFloatPtr(50),
},
Query: "fail",
})
_, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{})
require.NoError(t, err)
mocks.ctrl.advanceCanary("podinfo", "default")
mocks.ctrl.advanceCanary("podinfo", "default")
actualStatus := testutil.ToFloat64(mocks.ctrl.recorder.GetStatusMetric().WithLabelValues("podinfo", "default"))
assert.Equal(t, float64(2), actualStatus)
actualPrimaryWeight := testutil.ToFloat64(mocks.ctrl.recorder.GetWeightMetric().WithLabelValues("podinfo-primary", "default"))
actualCanaryWeight := testutil.ToFloat64(mocks.ctrl.recorder.GetWeightMetric().WithLabelValues("podinfo", "default"))
assert.Equal(t, float64(100), actualPrimaryWeight)
assert.Equal(t, float64(0), actualCanaryWeight)
})
}
func TestController_AnalysisMetricsRecording(t *testing.T) {
t.Run("builtin metrics analysis recording", func(t *testing.T) {
mocks := newDeploymentFixture(nil)
analysis := &flaggerv1.CanaryAnalysis{
Metrics: []flaggerv1.CanaryMetric{
{
Name: "request-success-rate",
ThresholdRange: &flaggerv1.CanaryThresholdRange{
Min: toFloatPtr(99),
Max: toFloatPtr(100),
},
},
{
Name: "request-duration",
ThresholdRange: &flaggerv1.CanaryThresholdRange{
Min: toFloatPtr(0),
Max: toFloatPtr(500),
},
},
},
}
canary := &flaggerv1.Canary{
ObjectMeta: metav1.ObjectMeta{
Name: "podinfo",
Namespace: "default",
},
Spec: flaggerv1.CanarySpec{
TargetRef: flaggerv1.LocalObjectReference{
Name: "podinfo",
},
Analysis: analysis,
},
}
result := mocks.ctrl.runMetricChecks(canary)
assert.True(t, result)
successRateMetric := mocks.ctrl.recorder.GetAnalysisMetric().WithLabelValues("podinfo", "default", "request-success-rate")
assert.NotNil(t, successRateMetric)
durationMetric := mocks.ctrl.recorder.GetAnalysisMetric().WithLabelValues("podinfo", "default", "request-duration")
assert.NotNil(t, durationMetric)
})
}
func TestController_getDeploymentStrategy(t *testing.T) {
ctrl := newDeploymentFixture(nil).ctrl
tests := []struct {
name string
analysis *flaggerv1.CanaryAnalysis
expected string
}{
{
name: "canary strategy with maxWeight",
analysis: &flaggerv1.CanaryAnalysis{
MaxWeight: 30,
StepWeight: 10,
},
expected: metrics.CanaryStrategy,
},
{
name: "canary strategy with stepWeights",
analysis: &flaggerv1.CanaryAnalysis{
StepWeights: []int{10, 20, 30},
},
expected: metrics.CanaryStrategy,
},
{
name: "blue_green strategy with iterations",
analysis: &flaggerv1.CanaryAnalysis{
Iterations: 5,
},
expected: metrics.BlueGreenStrategy,
},
{
name: "ab_testing strategy with iterations and match",
analysis: &flaggerv1.CanaryAnalysis{
Iterations: 10,
Match: []istiov1beta1.HTTPMatchRequest{
{
Headers: map[string]istiov1alpha1.StringMatch{
"x-canary": {
Exact: "insider",
},
},
},
},
},
expected: metrics.ABTestingStrategy,
},
{
name: "default to canary when analysis is nil",
analysis: nil,
expected: metrics.CanaryStrategy,
},
{
name: "default to canary when analysis is empty",
analysis: &flaggerv1.CanaryAnalysis{},
expected: metrics.CanaryStrategy,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
canary := &flaggerv1.Canary{
Spec: flaggerv1.CanarySpec{
Analysis: tt.analysis,
},
}
result := ctrl.getDeploymentStrategy(canary)
assert.Equal(t, tt.expected, result)
})
}
}