Merge pull request #1783 from fluxcd/session-affinity-primary-cookie

feat: add support for primary backend cookies in session affinity (Gateway API)

artifacts/flagger/crd.yaml

@@ -1155,6 +1155,9 @@ spec:
                         cookieName:
                           description: CookieName is the key that will be used for the session affinity cookie.
                           type: string
+                        primaryCookieName:
+                          description: CookieName is the key that will be used for the session affinity cookie.
+                          type: string
                         maxAge:
                           description: MaxAge indicates the number of seconds until the session affinity cookie will expire.
                           default: 86400

charts/flagger/crds/crd.yaml

@@ -1155,6 +1155,9 @@ spec:
                         cookieName:
                           description: CookieName is the key that will be used for the session affinity cookie.
                           type: string
+                        primaryCookieName:
+                          description: CookieName is the key that will be used for the session affinity cookie.
+                          type: string
                         maxAge:
                           description: MaxAge indicates the number of seconds until the session affinity cookie will expire.
                           default: 86400

docs/gitbook/tutorials/gatewayapi-progressive-delivery.md

@@ -509,6 +509,9 @@ You can load `www.example.com` in your browser and refresh it until you see the
 All subsequent requests after that will be served by `podinfo:6.0.1` and not `podinfo:6.0.0` because of the session affinity
 configured by Flagger in the HTTPRoute object.
 
+To configure stickiness for the Primary deployment to ensure fair weighted traffic routing, please
+checkout the [deployment strategies docs](../usage/deployment-strategies.md#canary-release-with-session-affinity).
+
 # A/B Testing
 
 Besides weighted routing, Flagger can be configured to route traffic to the canary based on HTTP match conditions. In an A/B testing scenario, you'll be using HTTP headers or cookies to target a certain segment of your users. This is particularly useful for frontend applications that require session affinity.

docs/gitbook/usage/deployment-strategies.md

@@ -407,7 +407,7 @@ cookie based routing with regular weight based routing. This means once a user i
 version of our application (based on the traffic weights), they're always routed to that version, i.e.
 they're never routed back to the old version of our application.
 
-You can enable this, by specifying `.spec.analsyis.sessionAffinity` in the Canary:
+You can enable this, by specifying `.spec.analysis.sessionAffinity` in the Canary:
 
 ```yaml
   analysis:
@@ -450,3 +450,47 @@ the Canary deployment:
 ```
 Set-Cookie: flagger-cookie=McxKdLQoIN; Max-Age=21600
 ```
+
+### Configuring stickiness for Primary deployment
+
+The above strategy is helpful because it makes sure that any user that's routed to the Canary deployment
+once is always routed to that deployment. But, this can results in an imbalance in the traffic shifting,
+as over time, most of the traffic flows to the Canary deployment. To ensure fair traffic distribution, we
+can also configure stickiness for the Primary deployment. You can configure this by specifying a
+`primaryCookieName` field:
+
+```yaml
+  analysis:
+    # schedule interval (default 60s)
+    interval: 1m
+    sessionAffinity:
+      # name of the cookie used
+      cookieName: flagger-cookie
+      # max age of the cookie (in seconds)
+      # optional; defaults to 86400
+      maxAge: 21600
+      # name of the cookie to use for the primary backend
+      # optional; unset means no primary stickiness
+      primaryCookieName: primary-flagger-cookie
+```
+
+> Note: This is only supported for the Gateway API provider for now.
+
+Let's understand what the above configuration does. All the session affinity stuff in the above section
+still occurs, but now the response header for requests routed to the primary deployment also include a
+`Set-Cookie` header:
+```
+Set-Cookie: primary-flagger-cookie=ApvLdqCoMF; Max-Age=60
+```
+
+Note that the age of the cookie is the same as the Canary analysis's interval. This means that the cookie
+expires when a new steps of the analysis begins and a new cookie is generated like so:
+```
+Set-Cookie: primary-flagger-cookie=BRtlVaQoPC; Max-Age=60
+```
+
+This ensures that, if the first request of a user during a particular step is routed to the primary deployment,
+then all subsequent requests will be routed to the same until the next step starts. During a new step, a new cookie 
+value is generated which is then included in the headers of responses from the primary workload. This allows for
+weighted traffic routing to happen while ensuring that users don't ever switch back to the primary deployment from
+the canary deployment during a Canary analysis.

kustomize/base/flagger/crd.yaml

@@ -1155,6 +1155,9 @@ spec:
                         cookieName:
                           description: CookieName is the key that will be used for the session affinity cookie.
                           type: string
+                        primaryCookieName:
+                          description: CookieName is the key that will be used for the session affinity cookie.
+                          type: string
                         maxAge:
                           description: MaxAge indicates the number of seconds until the session affinity cookie will expire.
                           default: 86400

pkg/apis/flagger/v1beta1/canary.go

@@ -295,6 +295,9 @@ type SessionAffinity struct {
 	// The default value is 86,400 seconds, i.e. a day.
 	// +optional
 	MaxAge int `json:"maxAge,omitempty"`
+	// PrimaryCookieName is the key that will be used for the primary session affinity cookie.
+	// +optional
+	PrimaryCookieName string `json:"primaryCookieName,omitempty"`
 }
 
 // CanaryMetric holds the reference to metrics used for canary analysis

pkg/controller/controller.go

@@ -337,6 +337,9 @@ func (c *Controller) verifyCanary(canary *flaggerv1.Canary) error {
 	if err := verifyKnativeCanary(canary); err != nil {
 		return err
 	}
+	if err := verifySessionAffinity(canary); err != nil {
+		return err
+	}
 
 	return nil
 }
@@ -378,6 +381,16 @@ func verifyKnativeCanary(canary *flaggerv1.Canary) error {
 	return nil
 }
 
+func verifySessionAffinity(canary *flaggerv1.Canary) error {
+	if canary.Spec.Analysis.SessionAffinity != nil {
+		if canary.Spec.Analysis.SessionAffinity.CookieName == canary.Spec.Analysis.SessionAffinity.PrimaryCookieName {
+			return fmt.Errorf("can't use the same cookie name for both primary and cookie name; please update them to be different")
+		}
+	}
+
+	return nil
+}
+
 func checkCustomResourceType(obj interface{}, logger *zap.SugaredLogger) (flaggerv1.Canary, bool) {
 	var roll *flaggerv1.Canary
 	var ok bool

pkg/controller/controller_test.go

@@ -26,6 +26,7 @@ func TestController_verifyCanary(t *testing.T) {
 						Name:      "upstream",
 						Namespace: "test",
 					},
+					Analysis: &flaggerv1.CanaryAnalysis{},
 				},
 			},
 			wantErr: true,
@@ -42,6 +43,7 @@ func TestController_verifyCanary(t *testing.T) {
 						Name:      "upstream",
 						Namespace: "default",
 					},
+					Analysis: &flaggerv1.CanaryAnalysis{},
 				},
 			},
 			wantErr: false,
@@ -103,6 +105,7 @@ func TestController_verifyCanary(t *testing.T) {
 						Kind: "Deployment",
 						Name: "podinfo",
 					},
+					Analysis: &flaggerv1.CanaryAnalysis{},
 				},
 			},
 			wantErr: true,
@@ -121,6 +124,7 @@ func TestController_verifyCanary(t *testing.T) {
 						APIVersion: "serving.knative.dev/v1",
 						Name:       "podinfo",
 					},
+					Analysis: &flaggerv1.CanaryAnalysis{},
 				},
 			},
 			wantErr: true,
@@ -140,6 +144,7 @@ func TestController_verifyCanary(t *testing.T) {
 						APIVersion: "serving.knative.dev/v1",
 						Name:       "podinfo",
 					},
+					Analysis: &flaggerv1.CanaryAnalysis{},
 				},
 			},
 			wantErr: true,
@@ -158,10 +163,29 @@ func TestController_verifyCanary(t *testing.T) {
 						APIVersion: "serving.knative.dev/v1",
 						Name:       "podinfo",
 					},
+					Analysis: &flaggerv1.CanaryAnalysis{},
 				},
 			},
 			wantErr: false,
 		},
+		{
+			name: "session affinity with same cookie names should return an error",
+			canary: flaggerv1.Canary{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "cd-1",
+					Namespace: "default",
+				},
+				Spec: flaggerv1.CanarySpec{
+					Analysis: &flaggerv1.CanaryAnalysis{
+						SessionAffinity: &flaggerv1.SessionAffinity{
+							CookieName:        "smth",
+							PrimaryCookieName: "smth",
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
 	}
 
 	ctrl := &Controller{

pkg/router/gateway_api.go

@@ -22,6 +22,7 @@ import (
 	"reflect"
 	"slices"
 	"strings"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
@@ -190,20 +191,26 @@ func (gwr *GatewayAPIRouter) Reconcile(canary *flaggerv1.Canary) error {
 		cmpopts.IgnoreFields(v1.BackendRef{}, "Weight"),
 		cmpopts.EquateEmpty(),
 	}
+
 	if canary.Spec.Analysis.SessionAffinity != nil {
-		ignoreRoute := cmpopts.IgnoreSliceElements(func(r v1.HTTPRouteRule) bool {
-			// Ignore the rule that does sticky routing, i.e. matches against the `Cookie` header.
-			for _, match := range r.Matches {
-				for _, headerMatch := range match.Headers {
-					if *headerMatch.Type == headerMatchRegex && headerMatch.Name == cookieHeader &&
-						strings.Contains(headerMatch.Value, canary.Spec.Analysis.SessionAffinity.CookieName) {
-						return true
+		ignoreCookieRouteFunc := func(name string) func(r v1.HTTPRouteRule) bool {
+			return func(r v1.HTTPRouteRule) bool {
+				// Ignore the rule that does sticky routing, i.e. matches against the `Cookie` header.
+				for _, match := range r.Matches {
+					for _, headerMatch := range match.Headers {
+						if *headerMatch.Type == headerMatchRegex && headerMatch.Name == cookieHeader &&
+							strings.Contains(headerMatch.Value, name) {
+							return true
+						}
 					}
 				}
+				return false
 			}
-			return false
-		})
-		ignoreCmpOptions = append(ignoreCmpOptions, ignoreRoute)
+		}
+		ignoreCanaryRoute := cmpopts.IgnoreSliceElements(ignoreCookieRouteFunc(canary.Spec.Analysis.SessionAffinity.CookieName))
+		ignorePrimaryRoute := cmpopts.IgnoreSliceElements(ignoreCookieRouteFunc(canary.Spec.Analysis.SessionAffinity.PrimaryCookieName))
+
+		ignoreCmpOptions = append(ignoreCmpOptions, ignoreCanaryRoute, ignorePrimaryRoute)
 		// Ignore backend specific filters, since we use that to insert the `Set-Cookie` header in responses.
 		ignoreCmpOptions = append(ignoreCmpOptions, cmpopts.IgnoreFields(v1.HTTPBackendRef{}, "Filters"))
 	}
@@ -439,42 +446,76 @@ func (gwr *GatewayAPIRouter) Finalize(_ *flaggerv1.Canary) error {
 	return nil
 }
 
+func getBackendByServiceName(rule *v1.HTTPRouteRule, svcName string) *v1.HTTPBackendRef {
+	for i, backendRef := range rule.BackendRefs {
+		if string(backendRef.BackendObjectReference.Name) == svcName {
+
+			return &rule.BackendRefs[i]
+		}
+	}
+	return nil
+}
+
 // getSessionAffinityRouteRules returns the HTTPRouteRule objects required to perform
 // session affinity based Canary releases.
 func (gwr *GatewayAPIRouter) getSessionAffinityRouteRules(canary *flaggerv1.Canary, canaryWeight int,
 	weightedRouteRule *v1.HTTPRouteRule) ([]v1.HTTPRouteRule, error) {
 	_, primarySvcName, canarySvcName := canary.GetServiceNames()
-	stickyRouteRule := *weightedRouteRule
+	stickyCanaryRouteRule := *weightedRouteRule
+	stickyPrimaryRouteRule := *weightedRouteRule
 
 	// If a canary run is active, we want all responses corresponding to requests hitting the canary deployment
 	// (due to weighted routing) to include a `Set-Cookie` header. All requests that have the `Cookie` header
 	// and match the value of the `Set-Cookie` header will be routed to the canary deployment.
 	if canaryWeight != 0 {
+		// if the status doesn't have the canary cookie, then generate a new canary cookie.
 		if canary.Status.SessionAffinityCookie == "" {
 			canary.Status.SessionAffinityCookie = fmt.Sprintf("%s=%s", canary.Spec.Analysis.SessionAffinity.CookieName, randSeq())
 		}
+		primaryCookie := fmt.Sprintf("%s=%s", canary.Spec.Analysis.SessionAffinity.PrimaryCookieName, randSeq())
 
-		// Add `Set-Cookie` header modifier to the primary backend in the weighted routing rule.
-		for i, backendRef := range weightedRouteRule.BackendRefs {
-			if string(backendRef.BackendObjectReference.Name) == canarySvcName {
-				backendRef.Filters = append(backendRef.Filters, v1.HTTPRouteFilter{
-					Type: v1.HTTPRouteFilterResponseHeaderModifier,
-					ResponseHeaderModifier: &v1.HTTPHeaderFilter{
-						Add: []v1.HTTPHeader{
-							{
-								Name: setCookieHeader,
-								Value: fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr,
-									canary.Spec.Analysis.SessionAffinity.GetMaxAge(),
-								),
-							},
+		// add response modifier to the canary backend ref in the rule that does weighted routing
+		// to include the canary cookie.
+		canaryBackendRef := getBackendByServiceName(weightedRouteRule, canarySvcName)
+		canaryBackendRef.Filters = append(canaryBackendRef.Filters, v1.HTTPRouteFilter{
+			Type: v1.HTTPRouteFilterResponseHeaderModifier,
+			ResponseHeaderModifier: &v1.HTTPHeaderFilter{
+				Add: []v1.HTTPHeader{
+					{
+						Name: setCookieHeader,
+						Value: fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr,
+							canary.Spec.Analysis.SessionAffinity.GetMaxAge(),
+						),
+					},
+				},
+			},
+		})
+
+		// add response modifier to the primary backend ref in the rule that does weighted routing
+		// to include the primary cookie, only if a primary cookie name has been specified.
+		if canary.Spec.Analysis.SessionAffinity.PrimaryCookieName != "" {
+			primaryBackendRef := getBackendByServiceName(weightedRouteRule, primarySvcName)
+			interval, err := time.ParseDuration(canary.Spec.Analysis.Interval)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse canary interval: %w", err)
+			}
+			primaryBackendRef.Filters = append(primaryBackendRef.Filters, v1.HTTPRouteFilter{
+				Type: v1.HTTPRouteFilterResponseHeaderModifier,
+				ResponseHeaderModifier: &v1.HTTPHeaderFilter{
+					Add: []v1.HTTPHeader{
+						{
+							Name: setCookieHeader,
+							Value: fmt.Sprintf("%s; %s=%d", primaryCookie, maxAgeAttr,
+								int(interval.Seconds()),
+							),
 						},
 					},
-				})
-			}
-			weightedRouteRule.BackendRefs[i] = backendRef
+				},
+			})
 		}
 
-		// Add `Cookie` header matcher to the sticky routing rule.
+		// configure the sticky canary rule to match against requests that match against the
+		// canary cookie and send them to the canary backend.
 		cookieKeyAndVal := strings.Split(canary.Status.SessionAffinityCookie, "=")
 		regexMatchType := v1.HeaderMatchRegularExpression
 		cookieMatch := v1.HTTPRouteMatch{
@@ -493,8 +534,8 @@ func (gwr *GatewayAPIRouter) getSessionAffinityRouteRules(canary *flaggerv1.Cana
 		}
 
 		mergedMatches := gwr.mergeMatchConditions([]v1.HTTPRouteMatch{cookieMatch}, svcMatches)
-		stickyRouteRule.Matches = mergedMatches
-		stickyRouteRule.BackendRefs = []v1.HTTPBackendRef{
+		stickyCanaryRouteRule.Matches = mergedMatches
+		stickyCanaryRouteRule.BackendRefs = []v1.HTTPBackendRef{
 			{
 				BackendRef: gwr.makeBackendRef(primarySvcName, 0, canary.Spec.Service.Port),
 			},
@@ -502,6 +543,42 @@ func (gwr *GatewayAPIRouter) getSessionAffinityRouteRules(canary *flaggerv1.Cana
 				BackendRef: gwr.makeBackendRef(canarySvcName, 100, canary.Spec.Service.Port),
 			},
 		}
+
+		// add a sticky primary rule to match against requests that match against the
+		// primary cookie and send them to the primary backend, only if a primary cookie name has
+		// been specified.
+		if canary.Spec.Analysis.SessionAffinity.PrimaryCookieName != "" {
+			cookieKeyAndVal = strings.Split(primaryCookie, "=")
+			regexMatchType = v1.HeaderMatchRegularExpression
+			primaryCookieMatch := v1.HTTPRouteMatch{
+				Headers: []v1.HTTPHeaderMatch{
+					{
+						Type:  &regexMatchType,
+						Name:  cookieHeader,
+						Value: fmt.Sprintf(".*%s.*%s.*", cookieKeyAndVal[0], cookieKeyAndVal[1]),
+					},
+				},
+			}
+
+			svcMatches, err = gwr.mapRouteMatches(canary.Spec.Service.Match)
+			if err != nil {
+				return nil, err
+			}
+
+			mergedMatches = gwr.mergeMatchConditions([]v1.HTTPRouteMatch{primaryCookieMatch}, svcMatches)
+			stickyPrimaryRouteRule.Matches = mergedMatches
+			stickyPrimaryRouteRule.BackendRefs = []v1.HTTPBackendRef{
+				{
+					BackendRef: gwr.makeBackendRef(primarySvcName, 100, canary.Spec.Service.Port),
+				},
+				{
+					BackendRef: gwr.makeBackendRef(canarySvcName, 0, canary.Spec.Service.Port),
+				},
+			}
+			return []v1.HTTPRouteRule{stickyCanaryRouteRule, stickyPrimaryRouteRule, *weightedRouteRule}, nil
+		}
+
+		return []v1.HTTPRouteRule{stickyCanaryRouteRule, *weightedRouteRule}, nil
 	} else {
 		// If canary weight is 0 and SessionAffinityCookie is non-blank, then it belongs to a previous canary run.
 		if canary.Status.SessionAffinityCookie != "" {
@@ -524,9 +601,9 @@ func (gwr *GatewayAPIRouter) getSessionAffinityRouteRules(canary *flaggerv1.Cana
 			}
 			svcMatches, _ := gwr.mapRouteMatches(canary.Spec.Service.Match)
 			mergedMatches := gwr.mergeMatchConditions([]v1.HTTPRouteMatch{cookieMatch}, svcMatches)
-			stickyRouteRule.Matches = mergedMatches
+			stickyCanaryRouteRule.Matches = mergedMatches
 
-			stickyRouteRule.Filters = append(stickyRouteRule.Filters, v1.HTTPRouteFilter{
+			stickyCanaryRouteRule.Filters = append(stickyCanaryRouteRule.Filters, v1.HTTPRouteFilter{
 				Type: v1.HTTPRouteFilterResponseHeaderModifier,
 				ResponseHeaderModifier: &v1.HTTPHeaderFilter{
 					Add: []v1.HTTPHeader{
@@ -540,9 +617,8 @@ func (gwr *GatewayAPIRouter) getSessionAffinityRouteRules(canary *flaggerv1.Cana
 		}
 
 		canary.Status.SessionAffinityCookie = ""
+		return []v1.HTTPRouteRule{stickyCanaryRouteRule, *weightedRouteRule}, nil
 	}
-
-	return []v1.HTTPRouteRule{stickyRouteRule, *weightedRouteRule}, nil
 }
 
 func (gwr *GatewayAPIRouter) mapRouteMatches(requestMatches []istiov1beta1.HTTPMatchRequest) ([]v1.HTTPRouteMatch, error) {

pkg/router/gateway_api_test.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"strings"
 	"testing"
+	"time"
 
 	flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
 	v1 "github.com/fluxcd/flagger/pkg/apis/gatewayapi/v1"
@@ -104,6 +105,7 @@ func TestGatewayAPIRouter_Routes(t *testing.T) {
 		_, pSvcName, cSvcName := canary.GetServiceNames()
 
 		err := router.SetRoutes(canary, 90, 10, false)
+		require.NoError(t, err)
 
 		hr, err := mocks.meshClient.GatewayapiV1().HTTPRoutes("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
 		require.NoError(t, err)
@@ -290,80 +292,192 @@ func TestGatewayAPIRouter_Routes(t *testing.T) {
 }
 
 func TestGatewayAPIRouter_getSessionAffinityRouteRules(t *testing.T) {
-	canary := newTestGatewayAPICanary()
-	mocks := newFixture(canary)
-	cookieKey := "flagger-cookie"
-	canary.Spec.Analysis.SessionAffinity = &flaggerv1.SessionAffinity{
-		CookieName: cookieKey,
-		MaxAge:     300,
-	}
+	t.Run("without primary cookie", func(t *testing.T) {
+		canary := newTestGatewayAPICanary()
+		mocks := newFixture(canary)
+		cookieKey := "flagger-cookie"
+		canary.Spec.Analysis.SessionAffinity = &flaggerv1.SessionAffinity{
+			CookieName: cookieKey,
+			MaxAge:     300,
+		}
 
-	router := &GatewayAPIRouter{
-		gatewayAPIClient: mocks.meshClient,
-		kubeClient:       mocks.kubeClient,
-		logger:           mocks.logger,
-	}
-	_, pSvcName, cSvcName := canary.GetServiceNames()
-	weightedRouteRule := &v1.HTTPRouteRule{
-		BackendRefs: []v1.HTTPBackendRef{
-			{
-				BackendRef: router.makeBackendRef(pSvcName, initialPrimaryWeight, canary.Spec.Service.Port),
+		router := &GatewayAPIRouter{
+			gatewayAPIClient: mocks.meshClient,
+			kubeClient:       mocks.kubeClient,
+			logger:           mocks.logger,
+		}
+		_, pSvcName, cSvcName := canary.GetServiceNames()
+		weightedRouteRule := &v1.HTTPRouteRule{
+			BackendRefs: []v1.HTTPBackendRef{
+				{
+					BackendRef: router.makeBackendRef(pSvcName, initialPrimaryWeight, canary.Spec.Service.Port),
+				},
+				{
+					BackendRef: router.makeBackendRef(cSvcName, initialCanaryWeight, canary.Spec.Service.Port),
+				},
 			},
-			{
-				BackendRef: router.makeBackendRef(cSvcName, initialCanaryWeight, canary.Spec.Service.Port),
+		}
+		rules, err := router.getSessionAffinityRouteRules(canary, 10, weightedRouteRule)
+		require.NoError(t, err)
+		assert.Equal(t, len(rules), 2)
+		assert.True(t, strings.HasPrefix(canary.Status.SessionAffinityCookie, cookieKey))
+
+		stickyRule := rules[0]
+		cookieMatch := stickyRule.Matches[0].Headers[0]
+		assert.Equal(t, *cookieMatch.Type, v1.HeaderMatchRegularExpression)
+		assert.Equal(t, string(cookieMatch.Name), cookieHeader)
+		assert.Contains(t, cookieMatch.Value, cookieKey)
+
+		assert.Equal(t, len(stickyRule.BackendRefs), 2)
+		for _, backendRef := range stickyRule.BackendRefs {
+			if string(backendRef.BackendRef.Name) == pSvcName {
+				assert.Equal(t, *backendRef.BackendRef.Weight, int32(0))
+			}
+			if string(backendRef.BackendRef.Name) == cSvcName {
+				assert.Equal(t, *backendRef.BackendRef.Weight, int32(100))
+			}
+		}
+
+		weightedRule := rules[1]
+		var found bool
+		for _, backendRef := range weightedRule.BackendRefs {
+			if string(backendRef.Name) == cSvcName {
+				found = true
+				filter := backendRef.Filters[0]
+				assert.Equal(t, filter.Type, v1.HTTPRouteFilterResponseHeaderModifier)
+				assert.NotNil(t, filter.ResponseHeaderModifier)
+				assert.Equal(t, string(filter.ResponseHeaderModifier.Add[0].Name), setCookieHeader)
+				assert.Equal(t, filter.ResponseHeaderModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr, 300))
+			}
+		}
+		assert.True(t, found)
+
+		rules, err = router.getSessionAffinityRouteRules(canary, 0, weightedRouteRule)
+		require.NoError(t, err)
+		assert.Empty(t, canary.Status.SessionAffinityCookie)
+		assert.Contains(t, canary.Status.PreviousSessionAffinityCookie, cookieKey)
+
+		stickyRule = rules[0]
+		cookieMatch = stickyRule.Matches[0].Headers[0]
+		assert.Equal(t, *cookieMatch.Type, v1.HeaderMatchRegularExpression)
+		assert.Equal(t, string(cookieMatch.Name), cookieHeader)
+		assert.Contains(t, cookieMatch.Value, cookieKey)
+
+		assert.Equal(t, stickyRule.Filters[0].Type, v1.HTTPRouteFilterResponseHeaderModifier)
+		headerModifier := stickyRule.Filters[0].ResponseHeaderModifier
+		assert.NotNil(t, headerModifier)
+		assert.Equal(t, string(headerModifier.Add[0].Name), setCookieHeader)
+		assert.Equal(t, headerModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.PreviousSessionAffinityCookie, maxAgeAttr, -1))
+	})
+
+	t.Run("with primary cookie", func(t *testing.T) {
+		canary := newTestGatewayAPICanary()
+		mocks := newFixture(canary)
+		canaryCookieKey := "canary-flagger-cookie"
+		primaryCookieKey := "primary-flagger-cookie"
+		canary.Spec.Analysis.Interval = "15s"
+		canary.Spec.Analysis.SessionAffinity = &flaggerv1.SessionAffinity{
+			CookieName:        canaryCookieKey,
+			PrimaryCookieName: primaryCookieKey,
+			MaxAge:            300,
+		}
+
+		router := &GatewayAPIRouter{
+			gatewayAPIClient: mocks.meshClient,
+			kubeClient:       mocks.kubeClient,
+			logger:           mocks.logger,
+		}
+		_, pSvcName, cSvcName := canary.GetServiceNames()
+		weightedRouteRule := &v1.HTTPRouteRule{
+			BackendRefs: []v1.HTTPBackendRef{
+				{
+					BackendRef: router.makeBackendRef(pSvcName, initialPrimaryWeight, canary.Spec.Service.Port),
+				},
+				{
+					BackendRef: router.makeBackendRef(cSvcName, initialCanaryWeight, canary.Spec.Service.Port),
+				},
 			},
-		},
-	}
-	rules, err := router.getSessionAffinityRouteRules(canary, 10, weightedRouteRule)
-	require.NoError(t, err)
-	assert.Equal(t, len(rules), 2)
-	assert.True(t, strings.HasPrefix(canary.Status.SessionAffinityCookie, cookieKey))
-
-	stickyRule := rules[0]
-	cookieMatch := stickyRule.Matches[0].Headers[0]
-	assert.Equal(t, *cookieMatch.Type, v1.HeaderMatchRegularExpression)
-	assert.Equal(t, string(cookieMatch.Name), cookieHeader)
-	assert.Contains(t, cookieMatch.Value, cookieKey)
-
-	assert.Equal(t, len(stickyRule.BackendRefs), 2)
-	for _, backendRef := range stickyRule.BackendRefs {
-		if string(backendRef.BackendRef.Name) == pSvcName {
-			assert.Equal(t, *backendRef.BackendRef.Weight, int32(0))
 		}
-		if string(backendRef.BackendRef.Name) == cSvcName {
-			assert.Equal(t, *backendRef.BackendRef.Weight, int32(100))
+		rules, err := router.getSessionAffinityRouteRules(canary, 10, weightedRouteRule)
+		require.NoError(t, err)
+		assert.Equal(t, len(rules), 3)
+		assert.True(t, strings.HasPrefix(canary.Status.SessionAffinityCookie, canaryCookieKey))
+
+		canaryStickyRule := rules[0]
+		cookieMatch := canaryStickyRule.Matches[0].Headers[0]
+		assert.Equal(t, *cookieMatch.Type, v1.HeaderMatchRegularExpression)
+		assert.Equal(t, string(cookieMatch.Name), cookieHeader)
+		assert.Contains(t, cookieMatch.Value, canaryCookieKey)
+
+		assert.Equal(t, len(canaryStickyRule.BackendRefs), 2)
+		for _, backendRef := range canaryStickyRule.BackendRefs {
+			if string(backendRef.BackendRef.Name) == pSvcName {
+				assert.Equal(t, *backendRef.BackendRef.Weight, int32(0))
+			}
+			if string(backendRef.BackendRef.Name) == cSvcName {
+				assert.Equal(t, *backendRef.BackendRef.Weight, int32(100))
+			}
 		}
-	}
 
-	weightedRule := rules[1]
-	var found bool
-	for _, backendRef := range weightedRule.BackendRefs {
-		if string(backendRef.Name) == cSvcName {
-			found = true
-			filter := backendRef.Filters[0]
-			assert.Equal(t, filter.Type, v1.HTTPRouteFilterResponseHeaderModifier)
-			assert.NotNil(t, filter.ResponseHeaderModifier)
-			assert.Equal(t, string(filter.ResponseHeaderModifier.Add[0].Name), setCookieHeader)
-			assert.Equal(t, filter.ResponseHeaderModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr, 300))
+		primaryStickyRule := rules[1]
+		cookieMatch = primaryStickyRule.Matches[0].Headers[0]
+		assert.Equal(t, *cookieMatch.Type, v1.HeaderMatchRegularExpression)
+		assert.Equal(t, string(cookieMatch.Name), cookieHeader)
+		assert.Contains(t, cookieMatch.Value, primaryCookieKey)
+
+		assert.Equal(t, len(primaryStickyRule.BackendRefs), 2)
+		for _, backendRef := range primaryStickyRule.BackendRefs {
+			if string(backendRef.BackendRef.Name) == pSvcName {
+				assert.Equal(t, *backendRef.BackendRef.Weight, int32(100))
+			}
+			if string(backendRef.BackendRef.Name) == cSvcName {
+				assert.Equal(t, *backendRef.BackendRef.Weight, int32(0))
+			}
 		}
-	}
-	assert.True(t, found)
 
-	rules, err = router.getSessionAffinityRouteRules(canary, 0, weightedRouteRule)
-	assert.Empty(t, canary.Status.SessionAffinityCookie)
-	assert.Contains(t, canary.Status.PreviousSessionAffinityCookie, cookieKey)
+		weightedRule := rules[2]
+		var c int
+		for _, backendRef := range weightedRule.BackendRefs {
+			if string(backendRef.Name) == cSvcName {
+				c += 1
+				filter := backendRef.Filters[0]
+				assert.Equal(t, filter.Type, v1.HTTPRouteFilterResponseHeaderModifier)
+				assert.NotNil(t, filter.ResponseHeaderModifier)
+				assert.Equal(t, string(filter.ResponseHeaderModifier.Add[0].Name), setCookieHeader)
+				assert.Equal(t, filter.ResponseHeaderModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr, 300))
+			}
 
-	stickyRule = rules[0]
-	cookieMatch = stickyRule.Matches[0].Headers[0]
-	assert.Equal(t, *cookieMatch.Type, v1.HeaderMatchRegularExpression)
-	assert.Equal(t, string(cookieMatch.Name), cookieHeader)
-	assert.Contains(t, cookieMatch.Value, cookieKey)
+			if string(backendRef.Name) == pSvcName {
+				c += 1
+				filter := backendRef.Filters[0]
+				assert.Equal(t, filter.Type, v1.HTTPRouteFilterResponseHeaderModifier)
+				assert.NotNil(t, filter.ResponseHeaderModifier)
+				assert.Equal(t, string(filter.ResponseHeaderModifier.Add[0].Name), setCookieHeader)
+				assert.Contains(t, filter.ResponseHeaderModifier.Add[0].Value, canary.Spec.Analysis.SessionAffinity.PrimaryCookieName)
+				interval, err := time.ParseDuration(canary.Spec.Analysis.Interval)
+				require.NoError(t, err)
+				assert.Contains(t, filter.ResponseHeaderModifier.Add[0].Value, fmt.Sprintf("%s=%d", maxAgeAttr, int(interval.Seconds())))
+			}
+		}
+		assert.Equal(t, 2, c)
 
-	assert.Equal(t, stickyRule.Filters[0].Type, v1.HTTPRouteFilterResponseHeaderModifier)
-	headerModifier := stickyRule.Filters[0].ResponseHeaderModifier
-	assert.NotNil(t, headerModifier)
-	assert.Equal(t, string(headerModifier.Add[0].Name), setCookieHeader)
-	assert.Equal(t, headerModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.PreviousSessionAffinityCookie, maxAgeAttr, -1))
+		rules, err = router.getSessionAffinityRouteRules(canary, 0, weightedRouteRule)
+		require.NoError(t, err)
+		assert.Empty(t, canary.Status.SessionAffinityCookie)
+		assert.Contains(t, canary.Status.PreviousSessionAffinityCookie, canaryCookieKey)
+
+		canaryStickyRule = rules[0]
+		cookieMatch = canaryStickyRule.Matches[0].Headers[0]
+		assert.Equal(t, *cookieMatch.Type, v1.HeaderMatchRegularExpression)
+		assert.Equal(t, string(cookieMatch.Name), cookieHeader)
+		assert.Contains(t, cookieMatch.Value, canaryCookieKey)
+
+		assert.Equal(t, canaryStickyRule.Filters[0].Type, v1.HTTPRouteFilterResponseHeaderModifier)
+		headerModifier := canaryStickyRule.Filters[0].ResponseHeaderModifier
+		assert.NotNil(t, headerModifier)
+		assert.Equal(t, string(headerModifier.Add[0].Name), setCookieHeader)
+		assert.Equal(t, headerModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.PreviousSessionAffinityCookie, maxAgeAttr, -1))
+	})
 }
 
 func TestGatewayAPIRouter_makeFilters(t *testing.T) {

test/gatewayapi/test-session-affinity.sh

@@ -44,7 +44,8 @@ spec:
     maxWeight: 50
     stepWeight: 10
     sessionAffinity:
-      cookieName: flagger-cookie
+      cookieName: canary-flagger-cookie
+      primaryCookieName: primary-flagger-cookie
     metrics:
       - name: error-rate
         templateRef:
@@ -63,7 +64,7 @@ spec:
     webhooks:
       - name: load-test
         type: rollout
-        url: http://flagger-loadtester.test/
+        url: http://localhost:8080
         timeout: 5s
         metadata:
           cmd: "hey -z 2m -q 10 -c 2 -host www.example.com http://gateway-istio.istio-ingress"
@@ -104,7 +105,8 @@ until ${ok}; do
 done
 
 echo '>>> Verifying session affinity'
-if ! URL=http://localhost:8888 HOST=www.example.com VERSION=6.1.0 COOKIE_NAME=flagger-cookie \
+if ! URL=http://localhost:8888 HOST=www.example.com CANARY_VERSION=6.1.0 \
+  CANARY_COOKIE_NAME=canary-flagger-cookie PRIMARY_VERSION=6.0.4 PRIMARY_COOKIE_NAME=primary-flagger-cookie \
     go run ${REPO_ROOT}/test/gatewayapi/verify_session_affinity.go; then
     echo "failed to verify session affinity"
     exit $?

test/gatewayapi/verify_session_affinity.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"log"
 	"net/http"
 	"os"
@@ -11,30 +11,39 @@ import (
 	"time"
 )
 
-var c = make(chan string, 1)
-var mu sync.Mutex
-var try = true
+// channel for canary cookie
+var cc = make(chan string, 1)
+
+// channel for primary cookie
+var pc = make(chan string, 1)
 var timeout = time.Second * 10
 
 func main() {
 	url := os.Getenv("URL")
 	host := os.Getenv("HOST")
-	version := os.Getenv("VERSION")
-	cookieName := os.Getenv("COOKIE_NAME")
+	canaryVersion := os.Getenv("CANARY_VERSION")
+	canaryCookieName := os.Getenv("CANARY_COOKIE_NAME")
+	primaryVersion := os.Getenv("PRIMARY_VERSION")
+	primaryCookieName := os.Getenv("PRIMARY_COOKIE_NAME")
 
-	// Generate traffic
-	for i := 0; i < 10; i++ {
-		go tryUntilCanaryIsHit(url, host, version, cookieName)
-	}
+	go tryUntilWorkloadIsHit(url, host, canaryVersion, canaryCookieName, cc)
+	go tryUntilWorkloadIsHit(url, host, primaryVersion, primaryCookieName, pc)
 
+	wg := &sync.WaitGroup{}
+	wg.Add(2)
+
+	go verifySessionAffinity(cc, wg, url, host, primaryVersion)
+	go verifySessionAffinity(pc, wg, url, host, canaryVersion)
+
+	wg.Wait()
+	log.Println("✔ successfully verified session affinity")
+}
+
+func verifySessionAffinity(cc chan string, wg *sync.WaitGroup, url, host, wrongVersion string) {
 	select {
 	// If we receive a cookie, then try to verify that we are always routed to the
 	// Canary deployment based on the cookie.
-	case cookie := <-c:
-		mu.Lock()
-		try = false
-		mu.Unlock()
-
+	case cookie := <-cc:
 		for i := 0; i < 5; i++ {
 			headers := map[string]string{
 				"Cookie": cookie,
@@ -43,15 +52,15 @@ func main() {
 			if err != nil {
 				log.Fatalf("failed to send request to verify cookie based routing: %v", err)
 			}
-			if !strings.Contains(body, version) {
-				log.Fatalf("received response from primary deployment instead of canary deployment")
+			if strings.Contains(body, wrongVersion) {
+				log.Fatalf("received response from the wrong deployment")
 			}
 		}
-
-		log.Println("✔ successfully verified session affinity")
+		wg.Done()
 	case <-time.After(timeout):
-		log.Fatal("timed out waiting for canary hit")
+		log.Fatal("timed out waiting for workload hit")
 	}
+
 }
 
 // sendRequest sends a request to the URL with the provided host and headers.
@@ -74,7 +83,7 @@ func sendRequest(url, host string, headers map[string]string) (string, []*http.C
 	}
 	defer resp.Body.Close()
 
-	body, err := ioutil.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return "", nil, err
 	}
@@ -82,17 +91,10 @@ func sendRequest(url, host string, headers map[string]string) (string, []*http.C
 	return string(body), resp.Cookies(), nil
 }
 
-// tryUntilCanaryIsHit is a recursive function that tries to send request and
+// tryUntilWorkloadIsHit is a recursive function that tries to send request and
 // either sends the cookie back to the main thread (if received) or re-sends
 // the request.
-func tryUntilCanaryIsHit(url, host, version, cookieName string) {
-	mu.Lock()
-	if !try {
-		mu.Unlock()
-		return
-	}
-	mu.Unlock()
-
+func tryUntilWorkloadIsHit(url, host, version, cookieName string, c chan string) {
 	body, cookies, err := sendRequest(url, host, nil)
 	if err != nil {
 		log.Printf("warning: failed to send request: %s", err)
@@ -105,6 +107,5 @@ func tryUntilCanaryIsHit(url, host, version, cookieName string) {
 		}
 	}
 
-	tryUntilCanaryIsHit(url, host, version, cookieName)
-	return
+	tryUntilWorkloadIsHit(url, host, version, cookieName, c)
 }