diff --git a/pkg/canary/config_tracker.go b/pkg/canary/config_tracker.go
index fafc5af9..23e80feb 100644
--- a/pkg/canary/config_tracker.go
+++ b/pkg/canary/config_tracker.go
@@ -438,29 +438,5 @@ func (ct *ConfigTracker) ApplyPrimaryConfigs(spec corev1.PodSpec, refs map[strin
 		}
 	}
 
-	// update affinity
-	if affinity := spec.Affinity; affinity != nil {
-		if podAntiAffinity := affinity.PodAntiAffinity; podAntiAffinity != nil {
-			for _, preferredAntiAffinity := range podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution {
-				term := preferredAntiAffinity.PodAffinityTerm
-				appendPrimarySuffixToValuesIfNeeded(term.TopologyKey, term.LabelSelector)
-			}
-
-			for _, requiredAntiAffinity := range podAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution {
-				appendPrimarySuffixToValuesIfNeeded(requiredAntiAffinity.TopologyKey, requiredAntiAffinity.LabelSelector)
-			}
-		}
-	}
-
 	return spec
 }
-
-func appendPrimarySuffixToValuesIfNeeded(topologyKey string, labelSelector *metav1.LabelSelector) {
-	if labelSelector != nil && topologyKey == "failure-domain.beta.kubernetes.io/zone" {
-		for _, matchExpression := range labelSelector.MatchExpressions {
-			for i := range matchExpression.Values {
-				matchExpression.Values[i] += "-primary"
-			}
-		}
-	}
-}
diff --git a/pkg/canary/config_tracker_test.go b/pkg/canary/config_tracker_test.go
index d30ea1b0..3399db95 100644
--- a/pkg/canary/config_tracker_test.go
+++ b/pkg/canary/config_tracker_test.go
@@ -39,23 +39,6 @@ func TestConfigIsDisabled(t *testing.T) {
 	}
 }
 
-func TestConfigTracker_AntiAffinity(t *testing.T) {
-	t.Run("deployment", func(t *testing.T) {
-		dc := deploymentConfigs{name: "podinfo", label: "name", labelValue: "podinfo"}
-		mocks := newDeploymentFixture(dc)
-		mocks.initializeCanary(t)
-
-		depPrimary, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
-		require.NoError(t, err)
-
-		value := depPrimary.Spec.Template.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm.LabelSelector.MatchExpressions[0].Values[0]
-		assert.Equal(t, "podinfo-primary", value)
-
-		value = depPrimary.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector.MatchExpressions[0].Values[0]
-		assert.Equal(t, "podinfo-primary", value)
-	})
-}
-
 func TestConfigTracker_ConfigMaps(t *testing.T) {
 	t.Run("deployment", func(t *testing.T) {
 		dc := deploymentConfigs{name: "podinfo", label: "name", labelValue: "podinfo"}
diff --git a/pkg/canary/deployment_controller.go b/pkg/canary/deployment_controller.go
index bec31c7e..463b0d8d 100644
--- a/pkg/canary/deployment_controller.go
+++ b/pkg/canary/deployment_controller.go
@@ -116,7 +116,7 @@ func (c *DeploymentController) Promote(cd *flaggerv1.Canary) error {
 	primaryCopy.Spec.Strategy = canary.Spec.Strategy
 
 	// update spec with primary secrets and config maps
-	primaryCopy.Spec.Template.Spec = c.configTracker.ApplyPrimaryConfigs(canary.Spec.Template.Spec, configRefs)
+	primaryCopy.Spec.Template.Spec = c.getPrimaryDeploymentTemplateSpec(canary, configRefs)
 
 	// update pod annotations to ensure a rolling update
 	annotations, err := makeAnnotations(canary.Spec.Template.Annotations)
@@ -289,7 +289,7 @@ func (c *DeploymentController) createPrimaryDeployment(cd *flaggerv1.Canary, inc
 						Annotations: annotations,
 					},
 					// update spec with the primary secrets and config maps
-					Spec: c.configTracker.ApplyPrimaryConfigs(canaryDep.Spec.Template.Spec, configRefs),
+					Spec: c.getPrimaryDeploymentTemplateSpec(canaryDep, configRefs),
 				},
 			},
 		}
@@ -452,3 +452,43 @@ func (c *DeploymentController) scale(cd *flaggerv1.Canary, replicas int32) error
 	}
 	return nil
 }
+
+func (c *DeploymentController) getPrimaryDeploymentTemplateSpec(canaryDep *appsv1.Deployment, refs map[string]ConfigRef) corev1.PodSpec {
+	spec := c.configTracker.ApplyPrimaryConfigs(canaryDep.Spec.Template.Spec, refs)
+
+	// update affinity
+	if affinity := spec.Affinity; affinity != nil {
+		if podAntiAffinity := affinity.PodAntiAffinity; podAntiAffinity != nil {
+			for _, preferredAntiAffinity := range podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution {
+				c.appendPrimarySuffixToValuesIfNeeded(preferredAntiAffinity.PodAffinityTerm.LabelSelector)
+			}
+
+			for _, requiredAntiAffinity := range podAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution {
+				c.appendPrimarySuffixToValuesIfNeeded(requiredAntiAffinity.LabelSelector)
+			}
+		}
+	}
+
+	return spec
+}
+
+func (c *DeploymentController) appendPrimarySuffixToValuesIfNeeded(labelSelector *metav1.LabelSelector) {
+	if labelSelector != nil {
+		for _, matchExpression := range labelSelector.MatchExpressions {
+			if contains(c.labels, matchExpression.Key) {
+				for i := range matchExpression.Values {
+					matchExpression.Values[i] += "-primary"
+				}
+			}
+		}
+	}
+}
+
+func contains(slice []string, val string) bool {
+	for _, item := range slice {
+		if item == val {
+			return true
+		}
+	}
+	return false
+}
diff --git a/pkg/canary/deployment_controller_test.go b/pkg/canary/deployment_controller_test.go
index c4e22380..decf006a 100644
--- a/pkg/canary/deployment_controller_test.go
+++ b/pkg/canary/deployment_controller_test.go
@@ -112,6 +112,12 @@ func TestDeploymentController_Promote(t *testing.T) {
 	hpaPrimary, err := mocks.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
 	require.NoError(t, err)
 	assert.Equal(t, int32(2), hpaPrimary.Spec.MaxReplicas)
+
+	value := depPrimary.Spec.Template.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm.LabelSelector.MatchExpressions[0].Values[0]
+	assert.Equal(t, "podinfo-primary", value)
+
+	value = depPrimary.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector.MatchExpressions[0].Values[0]
+	assert.Equal(t, "podinfo-primary", value)
 }
 
 func TestDeploymentController_ScaleToZero(t *testing.T) {
@@ -256,3 +262,20 @@ func TestDeploymentController_Finalize(t *testing.T) {
 		require.Equal(t, int32(1), *c.Spec.Replicas)
 	}
 }
+
+func TestDeploymentController_AntiAffinity(t *testing.T) {
+	t.Run("deployment", func(t *testing.T) {
+		dc := deploymentConfigs{name: "podinfo", label: "name", labelValue: "podinfo"}
+		mocks := newDeploymentFixture(dc)
+		mocks.initializeCanary(t)
+
+		depPrimary, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
+		require.NoError(t, err)
+
+		value := depPrimary.Spec.Template.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm.LabelSelector.MatchExpressions[0].Values[0]
+		assert.Equal(t, "podinfo-primary", value)
+
+		value = depPrimary.Spec.Template.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector.MatchExpressions[0].Values[0]
+		assert.Equal(t, "podinfo-primary", value)
+	})
+}
diff --git a/pkg/canary/deployment_fixture_test.go b/pkg/canary/deployment_fixture_test.go
index 997e5c1f..05bc1959 100644
--- a/pkg/canary/deployment_fixture_test.go
+++ b/pkg/canary/deployment_fixture_test.go
@@ -561,10 +561,10 @@ func newDeploymentControllerTest(dc deploymentConfigs) *appsv1.Deployment {
 							PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{
 								{
 									PodAffinityTerm: corev1.PodAffinityTerm{
-										TopologyKey: "failure-domain.beta.kubernetes.io/zone",
 										LabelSelector: &metav1.LabelSelector{
 											MatchExpressions: []metav1.LabelSelectorRequirement{
 												{
+													Key:    "app",
 													Values: []string{"podinfo"},
 												},
 											},
@@ -574,10 +574,10 @@ func newDeploymentControllerTest(dc deploymentConfigs) *appsv1.Deployment {
 							},
 							RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
 								{
-									TopologyKey: "failure-domain.beta.kubernetes.io/zone",
 									LabelSelector: &metav1.LabelSelector{
 										MatchExpressions: []metav1.LabelSelectorRequirement{
 											{
+												Key:    "app",
 												Values: []string{"podinfo"},
 											},
 										},
@@ -787,6 +787,36 @@ func newDeploymentControllerTestV2() *appsv1.Deployment {
 							},
 						},
 					},
+					Affinity: &corev1.Affinity{
+						PodAntiAffinity: &corev1.PodAntiAffinity{
+							PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{
+								{
+									PodAffinityTerm: corev1.PodAffinityTerm{
+										LabelSelector: &metav1.LabelSelector{
+											MatchExpressions: []metav1.LabelSelectorRequirement{
+												{
+													Key:    "app",
+													Values: []string{"podinfo"},
+												},
+											},
+										},
+									},
+								},
+							},
+							RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
+								{
+									LabelSelector: &metav1.LabelSelector{
+										MatchExpressions: []metav1.LabelSelectorRequirement{
+											{
+												Key:    "app",
+												Values: []string{"podinfo"},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
 				},
 			},
 		},