github/flagger
1
0
Fork 0
mirror of https://github.com/fluxcd/flagger.git synced 2026-04-15 06:57:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add count metrics for canary successes and failures

Browse Source 
Implement flagger_canary_successes_total and flagger_canary_failures_total
counter metrics with deployment strategy detection and analysis status
tracking for better observability of canary deployment outcomes.

Signed-off-by: cappyzawa <cappyzawa@gmail.com>
This commit is contained in:
cappyzawa
2025-10-08 13:53:03 +09:00
parent 16f54923b2
commit ffbc25efda
8 changed files with 227 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/events.go
+21 -16
View File
@@ -209,30 +209,35 @@ func alertMetadata(canary *flaggerv1.Canary) []notifier.Field {
},
)
if canary.GetAnalysis().StepWeight > 0 {
fields = append(fields, notifier.Field{
Name: "Traffic routing",
Value: fmt.Sprintf("Weight step: %v max: %v",
canary.GetAnalysis().StepWeight,
canary.GetAnalysis().MaxWeight),
})
} else if len(canary.GetAnalysis().StepWeights) > 0 {
fields = append(fields, notifier.Field{
Name: "Traffic routing",
Value: fmt.Sprintf("Weight steps: %s max: %v",
strings.Trim(strings.Join(strings.Fields(fmt.Sprint(canary.GetAnalysis().StepWeights)), ","), "[]"),
canary.GetAnalysis().MaxWeight),
})
} else if len(canary.GetAnalysis().Match) > 0 {
strategy := canary.DeploymentStrategy()
switch strategy {
case flaggerv1.DeploymentStrategyABTesting:
fields = append(fields, notifier.Field{
Name: "Traffic routing",
Value: "A/B Testing",
})
} else if canary.GetAnalysis().Iterations > 0 {
case flaggerv1.DeploymentStrategyBlueGreen:
fields = append(fields, notifier.Field{
Name: "Traffic routing",
Value: "Blue/Green",
})
default:
// Canary strategy
if canary.GetAnalysis().StepWeight > 0 {
fields = append(fields, notifier.Field{
Name: "Traffic routing",
Value: fmt.Sprintf("Weight step: %v max: %v",
canary.GetAnalysis().StepWeight,
canary.GetAnalysis().MaxWeight),
})
} else if len(canary.GetAnalysis().StepWeights) > 0 {
fields = append(fields, notifier.Field{
Name: "Traffic routing",
Value: fmt.Sprintf("Weight steps: %s max: %v",
strings.Trim(strings.Join(strings.Fields(fmt.Sprint(canary.GetAnalysis().StepWeights)), ","), "[]"),
canary.GetAnalysis().MaxWeight),
})
}
}
return fields
}
pkg/controller/scheduler.go
+7 -30
View File
@@ -38,27 +38,6 @@ func (c *Controller) min(a int, b int) int {
return b
}
// getDeploymentStrategy determines the deployment strategy based on canary analysis configuration
func (c *Controller) getDeploymentStrategy(canary *flaggerv1.Canary) string {
analysis := canary.GetAnalysis()
if analysis == nil {
return metrics.CanaryStrategy
}
// A/B Testing: has match conditions and iterations
if len(analysis.Match) > 0 && analysis.Iterations > 0 {
return metrics.ABTestingStrategy
}
// Blue/Green: has iterations but no match conditions
if analysis.Iterations > 0 {
return metrics.BlueGreenStrategy
}
// Canary Release: default (has maxWeight, stepWeight, or stepWeights)
return metrics.CanaryStrategy
}
func (c *Controller) maxWeight(canary *flaggerv1.Canary) int {
var stepWeightsLen = len(canary.GetAnalysis().StepWeights)
if stepWeightsLen > 0 {
@@ -425,7 +404,7 @@ func (c *Controller) advanceCanary(name string, namespace string) {
c.recorder.IncSuccesses(metrics.CanaryMetricLabels{
Name: cd.Spec.TargetRef.Name,
Namespace: cd.Namespace,
DeploymentStrategy: c.getDeploymentStrategy(cd),
DeploymentStrategy: cd.DeploymentStrategy(),
AnalysisStatus: metrics.AnalysisStatusCompleted,
})
c.runPostRolloutHooks(cd, flaggerv1.CanaryPhaseSucceeded)
@@ -488,14 +467,12 @@ func (c *Controller) advanceCanary(name string, namespace string) {
}
}
// strategy: A/B testing
if len(cd.GetAnalysis().Match) > 0 && cd.GetAnalysis().Iterations > 0 {
strategy := cd.DeploymentStrategy()
switch strategy {
case flaggerv1.DeploymentStrategyABTesting:
c.runAB(cd, canaryController, meshRouter)
return
}
// strategy: Blue/Green
if cd.GetAnalysis().Iterations > 0 {
case flaggerv1.DeploymentStrategyBlueGreen:
c.runBlueGreen(cd, canaryController, meshRouter, provider, mirrored)
return
}
@@ -845,7 +822,7 @@ func (c *Controller) shouldSkipAnalysis(canary *flaggerv1.Canary, canaryControll
c.recorder.IncSuccesses(metrics.CanaryMetricLabels{
Name: canary.Spec.TargetRef.Name,
Namespace: canary.Namespace,
DeploymentStrategy: c.getDeploymentStrategy(canary),
DeploymentStrategy: canary.DeploymentStrategy(),
AnalysisStatus: metrics.AnalysisStatusSkipped,
})
c.recordEventInfof(canary, "Promotion completed! Canary analysis was skipped for %s.%s",
@@ -998,7 +975,7 @@ func (c *Controller) rollback(canary *flaggerv1.Canary, canaryController canary.
c.recorder.IncFailures(metrics.CanaryMetricLabels{
Name: canary.Spec.TargetRef.Name,
Namespace: canary.Namespace,
DeploymentStrategy: c.getDeploymentStrategy(canary),
DeploymentStrategy: canary.DeploymentStrategy(),
AnalysisStatus: metrics.AnalysisStatusCompleted,
})
c.runPostRolloutHooks(canary, flaggerv1.CanaryPhaseFailed)
pkg/controller/scheduler_metrics_test.go
+46 -76
View File
@@ -28,9 +28,6 @@ import (
"k8s.io/client-go/tools/record"
flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
istiov1alpha1 "github.com/fluxcd/flagger/pkg/apis/istio/common/v1alpha1"
istiov1beta1 "github.com/fluxcd/flagger/pkg/apis/istio/v1beta1"
"github.com/fluxcd/flagger/pkg/metrics"
"github.com/fluxcd/flagger/pkg/metrics/observers"
)
@@ -190,7 +187,7 @@ func TestController_runMetricChecks(t *testing.T) {
}
func TestController_MetricsStateTransition(t *testing.T) {
t.Run("initialization and progression metrics", func(t *testing.T) {
t.Run("successful canary promotion with count metrics", func(t *testing.T) {
mocks := newDeploymentFixture(nil)
mocks.ctrl.advanceCanary("podinfo", "default")
@@ -224,9 +221,22 @@ func TestController_MetricsStateTransition(t *testing.T) {
totalWeight := actualPrimaryWeight + actualCanaryWeight
assert.InDelta(t, 100.0, totalWeight, 1.0)
const maxAdvanceAttempts = 10 // sufficient attempts to reach canary completion
for range maxAdvanceAttempts {
mocks.ctrl.advanceCanary("podinfo", "default")
c, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
require.NoError(t, err)
if c.Status.Phase == flaggerv1.CanaryPhaseSucceeded {
break
}
}
successCount := testutil.ToFloat64(mocks.ctrl.recorder.GetSuccessesMetric().WithLabelValues("podinfo", "default", "canary", "completed"))
assert.Equal(t, float64(1), successCount)
})
t.Run("failed canary rollback", func(t *testing.T) {
t.Run("failed canary rollback with count metrics", func(t *testing.T) {
mocks := newDeploymentFixture(nil)
mocks.ctrl.advanceCanary("podinfo", "default")
@@ -264,6 +274,37 @@ func TestController_MetricsStateTransition(t *testing.T) {
actualCanaryWeight := testutil.ToFloat64(mocks.ctrl.recorder.GetWeightMetric().WithLabelValues("podinfo", "default"))
assert.Equal(t, float64(100), actualPrimaryWeight)
assert.Equal(t, float64(0), actualCanaryWeight)
failureCount := testutil.ToFloat64(mocks.ctrl.recorder.GetFailuresMetric().WithLabelValues("podinfo", "default", "canary", "completed"))
assert.Equal(t, float64(1), failureCount)
})
t.Run("skipped analysis with count metrics", func(t *testing.T) {
mocks := newDeploymentFixture(nil)
mocks.ctrl.advanceCanary("podinfo", "default")
mocks.makePrimaryReady(t)
mocks.ctrl.advanceCanary("podinfo", "default")
// Enable skip analysis
cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
require.NoError(t, err)
cd.Spec.SkipAnalysis = true
_, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{})
require.NoError(t, err)
// Trigger deployment update
dep2 := newDeploymentTestDeploymentV2()
_, err = mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
require.NoError(t, err)
// Skip analysis should succeed immediately
mocks.ctrl.advanceCanary("podinfo", "default")
mocks.makeCanaryReady(t)
mocks.ctrl.advanceCanary("podinfo", "default")
successCount := testutil.ToFloat64(mocks.ctrl.recorder.GetSuccessesMetric().WithLabelValues("podinfo", "default", "canary", "skipped"))
assert.Equal(t, float64(1), successCount)
})
}
@@ -313,74 +354,3 @@ func TestController_AnalysisMetricsRecording(t *testing.T) {
assert.NotNil(t, durationMetric)
})
}
func TestController_getDeploymentStrategy(t *testing.T) {
ctrl := newDeploymentFixture(nil).ctrl
tests := []struct {
name string
analysis *flaggerv1.CanaryAnalysis
expected string
}{
{
name: "canary strategy with maxWeight",
analysis: &flaggerv1.CanaryAnalysis{
MaxWeight: 30,
StepWeight: 10,
},
expected: metrics.CanaryStrategy,
},
{
name: "canary strategy with stepWeights",
analysis: &flaggerv1.CanaryAnalysis{
StepWeights: []int{10, 20, 30},
},
expected: metrics.CanaryStrategy,
},
{
name: "blue_green strategy with iterations",
analysis: &flaggerv1.CanaryAnalysis{
Iterations: 5,
},
expected: metrics.BlueGreenStrategy,
},
{
name: "ab_testing strategy with iterations and match",
analysis: &flaggerv1.CanaryAnalysis{
Iterations: 10,
Match: []istiov1beta1.HTTPMatchRequest{
{
Headers: map[string]istiov1alpha1.StringMatch{
"x-canary": {
Exact: "insider",
},
},
},
},
},
expected: metrics.ABTestingStrategy,
},
{
name: "default to canary when analysis is nil",
analysis: nil,
expected: metrics.CanaryStrategy,
},
{
name: "default to canary when analysis is empty",
analysis: &flaggerv1.CanaryAnalysis{},
expected: metrics.CanaryStrategy,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
canary := &flaggerv1.Canary{
Spec: flaggerv1.CanarySpec{
Analysis: tt.analysis,
},
}
result := ctrl.getDeploymentStrategy(canary)
assert.Equal(t, tt.expected, result)
})
}
}