Merge pull request #1092 from northwesternmutual/main

Update metadata during subsequent promote

pkg/canary/daemonset_controller.go

@@ -27,6 +27,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/util/retry"
 
 	flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
 	clientset "github.com/fluxcd/flagger/pkg/client/clientset/versioned"
@@ -119,59 +120,76 @@ func (c *DaemonSetController) Promote(cd *flaggerv1.Canary) error {
 	targetName := cd.Spec.TargetRef.Name
 	primaryName := fmt.Sprintf("%s-primary", targetName)
 
-	canary, err := c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{})
-	if err != nil {
-		return fmt.Errorf("damonset %s.%s get query error: %v", targetName, cd.Namespace, err)
-	}
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		canary, err := c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("damonset %s.%s get query error: %v", targetName, cd.Namespace, err)
+		}
 
-	label, labelValue, err := c.getSelectorLabel(canary)
-	primaryLabelValue := fmt.Sprintf("%s-primary", labelValue)
-	if err != nil {
-		return fmt.Errorf("getSelectorLabel failed: %w", err)
-	}
+		label, labelValue, err := c.getSelectorLabel(canary)
+		primaryLabelValue := fmt.Sprintf("%s-primary", labelValue)
+		if err != nil {
+			return fmt.Errorf("getSelectorLabel failed: %w", err)
+		}
 
-	primary, err := c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{})
-	if err != nil {
-		return fmt.Errorf("daemonset %s.%s get query error: %w", primaryName, cd.Namespace, err)
-	}
+		primary, err := c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("daemonset %s.%s get query error: %w", primaryName, cd.Namespace, err)
+		}
 
-	// promote secrets and config maps
-	configRefs, err := c.configTracker.GetTargetConfigs(cd)
-	if err != nil {
-		return fmt.Errorf("GetTargetConfigs failed: %w", err)
-	}
-	if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs, c.includeLabelPrefix); err != nil {
-		return fmt.Errorf("CreatePrimaryConfigs failed: %w", err)
-	}
+		// promote secrets and config maps
+		configRefs, err := c.configTracker.GetTargetConfigs(cd)
+		if err != nil {
+			return fmt.Errorf("GetTargetConfigs failed: %w", err)
+		}
+		if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs, c.includeLabelPrefix); err != nil {
+			return fmt.Errorf("CreatePrimaryConfigs failed: %w", err)
+		}
 
-	primaryCopy := primary.DeepCopy()
-	primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds
-	primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit
-	primaryCopy.Spec.UpdateStrategy = canary.Spec.UpdateStrategy
+		primaryCopy := primary.DeepCopy()
+		primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds
+		primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit
+		primaryCopy.Spec.UpdateStrategy = canary.Spec.UpdateStrategy
 
-	// update spec with primary secrets and config maps
-	primaryCopy.Spec.Template.Spec = c.configTracker.ApplyPrimaryConfigs(canary.Spec.Template.Spec, configRefs)
+		// update spec with primary secrets and config maps
+		primaryCopy.Spec.Template.Spec = c.configTracker.ApplyPrimaryConfigs(canary.Spec.Template.Spec, configRefs)
 
-	// ignore `daemonSetScaleDownNodeSelector` node selector
-	for key := range daemonSetScaleDownNodeSelector {
-		delete(primaryCopy.Spec.Template.Spec.NodeSelector, key)
-	}
+		// ignore `daemonSetScaleDownNodeSelector` node selector
+		for key := range daemonSetScaleDownNodeSelector {
+			delete(primaryCopy.Spec.Template.Spec.NodeSelector, key)
+		}
 
-	// update pod annotations to ensure a rolling update
-	annotations, err := makeAnnotations(canary.Spec.Template.Annotations)
-	if err != nil {
-		return fmt.Errorf("makeAnnotations failed: %w", err)
-	}
+		// update pod annotations to ensure a rolling update
+		annotations, err := makeAnnotations(canary.Spec.Template.Annotations)
+		if err != nil {
+			return fmt.Errorf("makeAnnotations failed: %w", err)
+		}
 
-	primaryCopy.Spec.Template.Annotations = annotations
-	primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryLabelValue, label)
+		primaryCopy.Spec.Template.Annotations = annotations
+		primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryLabelValue, label)
 
-	// apply update
-	_, err = c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{})
+		// update ds annotations
+		primaryCopy.ObjectMeta.Annotations = make(map[string]string)
+		filteredAnnotations := includeLabelsByPrefix(canary.ObjectMeta.Annotations, c.includeLabelPrefix)
+		for k, v := range filteredAnnotations {
+			primaryCopy.ObjectMeta.Annotations[k] = v
+		}
+		// update ds labels
+		primaryCopy.ObjectMeta.Labels = make(map[string]string)
+		filteredLabels := includeLabelsByPrefix(canary.ObjectMeta.Labels, c.includeLabelPrefix)
+		for k, v := range filteredLabels {
+			primaryCopy.ObjectMeta.Labels[k] = v
+		}
+
+		// apply update
+		_, err = c.kubeClient.AppsV1().DaemonSets(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{})
+		return err
+	})
 	if err != nil {
 		return fmt.Errorf("updating daemonset %s.%s template spec failed: %w",
-			primaryCopy.GetName(), primaryCopy.Namespace, err)
+			primaryName, cd.Namespace, err)
 	}
+
 	return nil
 }
 

pkg/canary/daemonset_controller_test.go

@@ -96,6 +96,14 @@ func TestDaemonSetController_Promote(t *testing.T) {
 	sourceImage := dae2.Spec.Template.Spec.Containers[0].Image
 	assert.Equal(t, primaryImage, sourceImage)
 
+	daePrimaryLabels := daePrimary.ObjectMeta.Labels
+	daeSourceLabels := dae2.ObjectMeta.Labels
+	assert.Equal(t, daeSourceLabels["app.kubernetes.io/test-label-1"], daePrimaryLabels["app.kubernetes.io/test-label-1"])
+
+	daePrimaryAnnotations := daePrimary.ObjectMeta.Annotations
+	daeSourceAnnotations := dae2.ObjectMeta.Annotations
+	assert.Equal(t, daeSourceAnnotations["app.kubernetes.io/test-annotation-1"], daePrimaryAnnotations["app.kubernetes.io/test-annotation-1"])
+
 	configPrimary, err := mocks.kubeClient.CoreV1().ConfigMaps("default").Get(context.TODO(), "podinfo-config-env-primary", metav1.GetOptions{})
 	if assert.NoError(t, err) {
 		assert.Equal(t, configPrimary.Data["color"], config2.Data["color"])

pkg/canary/daemonset_fixture_test.go

@@ -369,6 +369,12 @@ func newDaemonSetControllerTestPodInfo(dc daemonsetConfigs) *appsv1.DaemonSet {
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: "default",
 			Name:      dc.name,
+			Annotations: map[string]string{
+				"test-annotation-1": "test-annotation-value-1",
+			},
+			Labels: map[string]string{
+				"test-label-1": "test-label-value-1",
+			},
 		},
 		Spec: appsv1.DaemonSetSpec{
 			Selector: &metav1.LabelSelector{
@@ -379,7 +385,11 @@ func newDaemonSetControllerTestPodInfo(dc daemonsetConfigs) *appsv1.DaemonSet {
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: map[string]string{
-						dc.label: dc.labelValue,
+						dc.label:       dc.labelValue,
+						"test-label-1": "test-label-value-1",
+					},
+					Annotations: map[string]string{
+						"test-annotation-1": "test-annotation-value-1",
 					},
 				},
 				Spec: corev1.PodSpec{

pkg/canary/deployment_controller.go

@@ -29,6 +29,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/util/retry"
 
 	flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
 	clientset "github.com/fluxcd/flagger/pkg/client/clientset/versioned"
@@ -84,54 +85,70 @@ func (c *DeploymentController) Promote(cd *flaggerv1.Canary) error {
 	targetName := cd.Spec.TargetRef.Name
 	primaryName := fmt.Sprintf("%s-primary", targetName)
 
-	canary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{})
-	if err != nil {
-		return fmt.Errorf("deployment %s.%s get query error: %w", targetName, cd.Namespace, err)
-	}
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		canary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("deployment %s.%s get query error: %w", targetName, cd.Namespace, err)
+		}
 
-	label, labelValue, err := c.getSelectorLabel(canary)
-	primaryLabelValue := fmt.Sprintf("%s-primary", labelValue)
-	if err != nil {
-		return fmt.Errorf("getSelectorLabel failed: %w", err)
-	}
+		label, labelValue, err := c.getSelectorLabel(canary)
+		primaryLabelValue := fmt.Sprintf("%s-primary", labelValue)
+		if err != nil {
+			return fmt.Errorf("getSelectorLabel failed: %w", err)
+		}
 
-	primary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{})
-	if err != nil {
-		return fmt.Errorf("deployment %s.%s get query error: %w", primaryName, cd.Namespace, err)
-	}
+		primary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("deployment %s.%s get query error: %w", primaryName, cd.Namespace, err)
+		}
 
-	// promote secrets and config maps
-	configRefs, err := c.configTracker.GetTargetConfigs(cd)
-	if err != nil {
-		return fmt.Errorf("GetTargetConfigs failed: %w", err)
-	}
-	if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs, c.includeLabelPrefix); err != nil {
-		return fmt.Errorf("CreatePrimaryConfigs failed: %w", err)
-	}
+		// promote secrets and config maps
+		configRefs, err := c.configTracker.GetTargetConfigs(cd)
+		if err != nil {
+			return fmt.Errorf("GetTargetConfigs failed: %w", err)
+		}
+		if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs, c.includeLabelPrefix); err != nil {
+			return fmt.Errorf("CreatePrimaryConfigs failed: %w", err)
+		}
 
-	primaryCopy := primary.DeepCopy()
-	primaryCopy.Spec.ProgressDeadlineSeconds = canary.Spec.ProgressDeadlineSeconds
-	primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds
-	primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit
-	primaryCopy.Spec.Strategy = canary.Spec.Strategy
+		primaryCopy := primary.DeepCopy()
+		primaryCopy.Spec.ProgressDeadlineSeconds = canary.Spec.ProgressDeadlineSeconds
+		primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds
+		primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit
+		primaryCopy.Spec.Strategy = canary.Spec.Strategy
 
-	// update spec with primary secrets and config maps
-	primaryCopy.Spec.Template.Spec = c.getPrimaryDeploymentTemplateSpec(canary, configRefs)
+		// update spec with primary secrets and config maps
+		primaryCopy.Spec.Template.Spec = c.getPrimaryDeploymentTemplateSpec(canary, configRefs)
 
-	// update pod annotations to ensure a rolling update
-	annotations, err := makeAnnotations(canary.Spec.Template.Annotations)
-	if err != nil {
-		return fmt.Errorf("makeAnnotations failed: %w", err)
-	}
+		// update pod annotations to ensure a rolling update
+		podAnnotations, err := makeAnnotations(canary.Spec.Template.Annotations)
+		if err != nil {
+			return fmt.Errorf("makeAnnotations for podAnnotations failed: %w", err)
+		}
 
-	primaryCopy.Spec.Template.Annotations = annotations
-	primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryLabelValue, label)
+		primaryCopy.Spec.Template.Annotations = podAnnotations
+		primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryLabelValue, label)
 
-	// apply update
-	_, err = c.kubeClient.AppsV1().Deployments(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{})
+		// update deploy annotations
+		primaryCopy.ObjectMeta.Annotations = make(map[string]string)
+		filteredAnnotations := includeLabelsByPrefix(canary.ObjectMeta.Annotations, c.includeLabelPrefix)
+		for k, v := range filteredAnnotations {
+			primaryCopy.ObjectMeta.Annotations[k] = v
+		}
+		// update deploy labels
+		primaryCopy.ObjectMeta.Labels = make(map[string]string)
+		filteredLabels := includeLabelsByPrefix(canary.ObjectMeta.Labels, c.includeLabelPrefix)
+		for k, v := range filteredLabels {
+			primaryCopy.ObjectMeta.Labels[k] = v
+		}
+
+		// apply update
+		_, err = c.kubeClient.AppsV1().Deployments(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{})
+		return err
+	})
 	if err != nil {
 		return fmt.Errorf("updating deployment %s.%s template spec failed: %w",
-			primaryCopy.GetName(), primaryCopy.Namespace, err)
+			primaryName, cd.Namespace, err)
 	}
 
 	// update HPA
@@ -364,18 +381,41 @@ func (c *DeploymentController) reconcilePrimaryHpa(cd *flaggerv1.Canary, init bo
 	if !init && primaryHpa != nil {
 		diffMetrics := cmp.Diff(hpaSpec.Metrics, primaryHpa.Spec.Metrics)
 		diffBehavior := cmp.Diff(hpaSpec.Behavior, primaryHpa.Spec.Behavior)
-		if diffMetrics != "" || diffBehavior != "" || int32Default(hpaSpec.MinReplicas) != int32Default(primaryHpa.Spec.MinReplicas) || hpaSpec.MaxReplicas != primaryHpa.Spec.MaxReplicas {
+		diffLabels := cmp.Diff(hpa.ObjectMeta.Labels, primaryHpa.ObjectMeta.Labels)
+		diffAnnotations := cmp.Diff(hpa.ObjectMeta.Annotations, primaryHpa.ObjectMeta.Annotations)
+		if diffMetrics != "" || diffBehavior != "" || diffLabels != "" || diffAnnotations != "" || int32Default(hpaSpec.MinReplicas) != int32Default(primaryHpa.Spec.MinReplicas) || hpaSpec.MaxReplicas != primaryHpa.Spec.MaxReplicas {
 			fmt.Println(diffMetrics, diffBehavior, hpaSpec.MinReplicas, primaryHpa.Spec.MinReplicas, hpaSpec.MaxReplicas, primaryHpa.Spec.MaxReplicas)
-			hpaClone := primaryHpa.DeepCopy()
-			hpaClone.Spec.MaxReplicas = hpaSpec.MaxReplicas
-			hpaClone.Spec.MinReplicas = hpaSpec.MinReplicas
-			hpaClone.Spec.Metrics = hpaSpec.Metrics
-			hpaClone.Spec.Behavior = hpaSpec.Behavior
 
-			_, err := c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Update(context.TODO(), hpaClone, metav1.UpdateOptions{})
+			err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
+				primaryHpa, err := c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Get(context.TODO(), primaryHpaName, metav1.GetOptions{})
+				if err != nil {
+					return err
+				}
+				hpaClone := primaryHpa.DeepCopy()
+				hpaClone.Spec.MaxReplicas = hpaSpec.MaxReplicas
+				hpaClone.Spec.MinReplicas = hpaSpec.MinReplicas
+				hpaClone.Spec.Metrics = hpaSpec.Metrics
+				hpaClone.Spec.Behavior = hpaSpec.Behavior
+
+				// update hpa annotations
+				hpaClone.ObjectMeta.Annotations = make(map[string]string)
+				filteredAnnotations := includeLabelsByPrefix(hpa.ObjectMeta.Annotations, c.includeLabelPrefix)
+				for k, v := range filteredAnnotations {
+					hpaClone.ObjectMeta.Annotations[k] = v
+				}
+				// update hpa labels
+				hpaClone.ObjectMeta.Labels = make(map[string]string)
+				filteredLabels := includeLabelsByPrefix(hpa.ObjectMeta.Labels, c.includeLabelPrefix)
+				for k, v := range filteredLabels {
+					hpaClone.ObjectMeta.Labels[k] = v
+				}
+
+				_, err = c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Update(context.TODO(), hpaClone, metav1.UpdateOptions{})
+				return err
+			})
 			if err != nil {
 				return fmt.Errorf("updating HorizontalPodAutoscaler %s.%s failed: %w",
-					hpaClone.Name, hpaClone.Namespace, err)
+					primaryHpa.Name, primaryHpa.Namespace, err)
 			}
 			c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).
 				Infof("HorizontalPodAutoscaler %s.%s updated", primaryHpa.GetName(), cd.Namespace)

pkg/canary/deployment_controller_test.go

@@ -109,6 +109,14 @@ func TestDeploymentController_Promote(t *testing.T) {
 	sourceImage := dep2.Spec.Template.Spec.Containers[0].Image
 	assert.Equal(t, sourceImage, primaryImage)
 
+	depPrimaryLabels := depPrimary.ObjectMeta.Labels
+	depSourceLabels := dep2.ObjectMeta.Labels
+	assert.Equal(t, depSourceLabels["app.kubernetes.io/test-label-1"], depPrimaryLabels["app.kubernetes.io/test-label-1"])
+
+	depPrimaryAnnotations := depPrimary.ObjectMeta.Annotations
+	depSourceAnnotations := dep2.ObjectMeta.Annotations
+	assert.Equal(t, depSourceAnnotations["app.kubernetes.io/test-annotation-1"], depPrimaryAnnotations["app.kubernetes.io/test-annotation-1"])
+
 	configPrimary, err := mocks.kubeClient.CoreV1().ConfigMaps("default").Get(context.TODO(), "podinfo-config-env-primary", metav1.GetOptions{})
 	require.NoError(t, err)
 	assert.Equal(t, config2.Data["color"], configPrimary.Data["color"])
@@ -117,6 +125,14 @@ func TestDeploymentController_Promote(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, int32(2), hpaPrimary.Spec.MaxReplicas)
 
+	hpaPrimaryLabels := hpaPrimary.ObjectMeta.Labels
+	hpaSourceLabels := hpa.ObjectMeta.Labels
+	assert.Equal(t, hpaSourceLabels["app.kubernetes.io/test-label-1"], hpaPrimaryLabels["app.kubernetes.io/test-label-1"])
+
+	hpaPrimaryAnnotations := hpaPrimary.ObjectMeta.Annotations
+	hpaSourceAnnotations := hpa.ObjectMeta.Annotations
+	assert.Equal(t, hpaSourceAnnotations["app.kubernetes.io/test-annotation-1"], hpaPrimaryAnnotations["app.kubernetes.io/test-annotation-1"])
+
 	value := depPrimary.Spec.Template.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm.LabelSelector.MatchExpressions[0].Values[0]
 	assert.Equal(t, "podinfo-primary", value)
 

pkg/canary/deployment_fixture_test.go

@@ -122,6 +122,7 @@ func newCustomizableFixture(dc deploymentConfigs) (deploymentControllerFixture,
 			KubeClient:    kubeClient,
 			FlaggerClient: flaggerClient,
 		},
+		includeLabelPrefix: []string{"app.kubernetes.io"},
 	}
 
 	return deploymentControllerFixture{
@@ -421,6 +422,10 @@ func newDeploymentControllerTest(dc deploymentConfigs) *appsv1.Deployment {
 			Name:      dc.name,
 			Annotations: map[string]string{
 				"kustomize.toolkit.fluxcd.io/checksum": "0a40893bfdc545d62125bd3e74eeb2ebaa7097c2",
+				"test-annotation-1":                    "test-annotation-value-1",
+			},
+			Labels: map[string]string{
+				"test-label-1": "test-label-value-1",
 			},
 		},
 		Spec: appsv1.DeploymentSpec{
@@ -431,8 +436,12 @@ func newDeploymentControllerTest(dc deploymentConfigs) *appsv1.Deployment {
 			},
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						"test-annotation-1": "test-annotation-value-1",
+					},
 					Labels: map[string]string{
-						dc.label: dc.labelValue,
+						dc.label:       dc.labelValue,
+						"test-label-1": "test-label-value-1",
 					},
 				},
 				Spec: corev1.PodSpec{
@@ -757,6 +766,12 @@ func newDeploymentControllerTestV2() *appsv1.Deployment {
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: "default",
 			Name:      "podinfo",
+			Annotations: map[string]string{
+				"test-annotation-1": "test-annotation-value-1",
+			},
+			Labels: map[string]string{
+				"test-label-1": "test-label-value-1",
+			},
 		},
 		Spec: appsv1.DeploymentSpec{
 			Selector: &metav1.LabelSelector{
@@ -767,7 +782,11 @@ func newDeploymentControllerTestV2() *appsv1.Deployment {
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels: map[string]string{
-						"name": "podinfo",
+						"name":         "podinfo",
+						"test-label-1": "test-label-value-1",
+					},
+					Annotations: map[string]string{
+						"test-annotation-1": "test-annotation-value-1",
 					},
 				},
 				Spec: corev1.PodSpec{

pkg/canary/factory.go

@@ -58,16 +58,18 @@ func (factory *Factory) Controller(kind string) Controller {
 		includeLabelPrefix: factory.includeLabelPrefix,
 	}
 	daemonSetCtrl := &DaemonSetController{
-		logger:        factory.logger,
-		kubeClient:    factory.kubeClient,
-		flaggerClient: factory.flaggerClient,
-		labels:        factory.labels,
-		configTracker: factory.configTracker,
+		logger:             factory.logger,
+		kubeClient:         factory.kubeClient,
+		flaggerClient:      factory.flaggerClient,
+		labels:             factory.labels,
+		configTracker:      factory.configTracker,
+		includeLabelPrefix: factory.includeLabelPrefix,
 	}
 	serviceCtrl := &ServiceController{
-		logger:        factory.logger,
-		kubeClient:    factory.kubeClient,
-		flaggerClient: factory.flaggerClient,
+		logger:             factory.logger,
+		kubeClient:         factory.kubeClient,
+		flaggerClient:      factory.flaggerClient,
+		includeLabelPrefix: factory.includeLabelPrefix,
 	}
 
 	switch kind {

pkg/canary/service_controller.go

@@ -26,6 +26,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/util/retry"
 
 	flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
 	clientset "github.com/fluxcd/flagger/pkg/client/clientset/versioned"
@@ -33,9 +34,10 @@ import (
 
 // ServiceController is managing the operations for Kubernetes service kind
 type ServiceController struct {
-	kubeClient    kubernetes.Interface
-	flaggerClient clientset.Interface
-	logger        *zap.SugaredLogger
+	kubeClient         kubernetes.Interface
+	flaggerClient      clientset.Interface
+	logger             *zap.SugaredLogger
+	includeLabelPrefix []string
 }
 
 // SetStatusFailedChecks updates the canary failed checks counter
@@ -171,29 +173,45 @@ func (c *ServiceController) Promote(cd *flaggerv1.Canary) error {
 	targetName := cd.Spec.TargetRef.Name
 	primaryName := fmt.Sprintf("%s-primary", targetName)
 
-	canary, err := c.kubeClient.CoreV1().Services(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{})
-	if err != nil {
-		return fmt.Errorf("service %s.%s get query error: %w", targetName, cd.Namespace, err)
-	}
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		canary, err := c.kubeClient.CoreV1().Services(cd.Namespace).Get(context.TODO(), targetName, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("service %s.%s get query error: %w", targetName, cd.Namespace, err)
+		}
 
-	primary, err := c.kubeClient.CoreV1().Services(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{})
-	if err != nil {
-		return fmt.Errorf("service %s.%s get query error: %w", primaryName, cd.Namespace, err)
-	}
+		primary, err := c.kubeClient.CoreV1().Services(cd.Namespace).Get(context.TODO(), primaryName, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("service %s.%s get query error: %w", primaryName, cd.Namespace, err)
+		}
 
-	primaryCopy := canary.DeepCopy()
-	primaryCopy.ObjectMeta.Name = primary.ObjectMeta.Name
-	if primaryCopy.Spec.Type == "ClusterIP" {
-		primaryCopy.Spec.ClusterIP = primary.Spec.ClusterIP
-	}
-	primaryCopy.ObjectMeta.ResourceVersion = primary.ObjectMeta.ResourceVersion
-	primaryCopy.ObjectMeta.UID = primary.ObjectMeta.UID
+		primaryCopy := canary.DeepCopy()
+		primaryCopy.ObjectMeta.Name = primary.ObjectMeta.Name
+		if primaryCopy.Spec.Type == "ClusterIP" {
+			primaryCopy.Spec.ClusterIP = primary.Spec.ClusterIP
+		}
+		primaryCopy.ObjectMeta.ResourceVersion = primary.ObjectMeta.ResourceVersion
+		primaryCopy.ObjectMeta.UID = primary.ObjectMeta.UID
 
-	// apply update
-	_, err = c.kubeClient.CoreV1().Services(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{})
+		// update service annotations
+		primaryCopy.ObjectMeta.Annotations = make(map[string]string)
+		filteredAnnotations := includeLabelsByPrefix(canary.ObjectMeta.Annotations, c.includeLabelPrefix)
+		for k, v := range filteredAnnotations {
+			primaryCopy.ObjectMeta.Annotations[k] = v
+		}
+		// update service labels
+		primaryCopy.ObjectMeta.Labels = make(map[string]string)
+		filteredLabels := includeLabelsByPrefix(canary.ObjectMeta.Labels, c.includeLabelPrefix)
+		for k, v := range filteredLabels {
+			primaryCopy.ObjectMeta.Labels[k] = v
+		}
+
+		// apply update
+		_, err = c.kubeClient.CoreV1().Services(cd.Namespace).Update(context.TODO(), primaryCopy, metav1.UpdateOptions{})
+		return err
+	})
 	if err != nil {
 		return fmt.Errorf("updating service %s.%s spec failed: %w",
-			primaryCopy.GetName(), primaryCopy.Namespace, err)
+			primaryName, cd.Namespace, err)
 	}
 
 	return nil