From 1dc270c2e678eccde4d9e901bbd6cf0a3129d71f Mon Sep 17 00:00:00 2001 From: Sanskar Jaiswal Date: Sun, 23 Mar 2025 01:27:45 +0530 Subject: [PATCH] feat: add support for primary backend cookies in session affinity Signed-off-by: Sanskar Jaiswal --- artifacts/flagger/crd.yaml | 3 + charts/flagger/crds/crd.yaml | 3 + kustomize/base/flagger/crd.yaml | 3 + pkg/apis/flagger/v1beta1/canary.go | 3 + pkg/controller/controller.go | 13 ++ pkg/controller/controller_test.go | 24 ++ pkg/router/gateway_api.go | 144 +++++++++--- pkg/router/gateway_api_test.go | 242 +++++++++++++++------ test/gatewayapi/test-session-affinity.sh | 8 +- test/gatewayapi/verify_session_affinity.go | 65 +++--- 10 files changed, 375 insertions(+), 133 deletions(-) diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index 90482172..1dd9e35e 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -1155,6 +1155,9 @@ spec: cookieName: description: CookieName is the key that will be used for the session affinity cookie. type: string + primaryCookieName: + description: CookieName is the key that will be used for the session affinity cookie. + type: string maxAge: description: MaxAge indicates the number of seconds until the session affinity cookie will expire. default: 86400 diff --git a/charts/flagger/crds/crd.yaml b/charts/flagger/crds/crd.yaml index 90482172..1dd9e35e 100644 --- a/charts/flagger/crds/crd.yaml +++ b/charts/flagger/crds/crd.yaml @@ -1155,6 +1155,9 @@ spec: cookieName: description: CookieName is the key that will be used for the session affinity cookie. type: string + primaryCookieName: + description: CookieName is the key that will be used for the session affinity cookie. + type: string maxAge: description: MaxAge indicates the number of seconds until the session affinity cookie will expire. default: 86400 diff --git a/kustomize/base/flagger/crd.yaml b/kustomize/base/flagger/crd.yaml index 90482172..1dd9e35e 100644 --- a/kustomize/base/flagger/crd.yaml +++ b/kustomize/base/flagger/crd.yaml @@ -1155,6 +1155,9 @@ spec: cookieName: description: CookieName is the key that will be used for the session affinity cookie. type: string + primaryCookieName: + description: CookieName is the key that will be used for the session affinity cookie. + type: string maxAge: description: MaxAge indicates the number of seconds until the session affinity cookie will expire. default: 86400 diff --git a/pkg/apis/flagger/v1beta1/canary.go b/pkg/apis/flagger/v1beta1/canary.go index 13efecd5..6998546e 100644 --- a/pkg/apis/flagger/v1beta1/canary.go +++ b/pkg/apis/flagger/v1beta1/canary.go @@ -295,6 +295,9 @@ type SessionAffinity struct { // The default value is 86,400 seconds, i.e. a day. // +optional MaxAge int `json:"maxAge,omitempty"` + // PrimaryCookieName is the key that will be used for the primary session affinity cookie. + // +optional + PrimaryCookieName string `json:"primaryCookieName,omitempty"` } // CanaryMetric holds the reference to metrics used for canary analysis diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index bb7e7249..6a0ead6d 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -337,6 +337,9 @@ func (c *Controller) verifyCanary(canary *flaggerv1.Canary) error { if err := verifyKnativeCanary(canary); err != nil { return err } + if err := verifySessionAffinity(canary); err != nil { + return err + } return nil } @@ -378,6 +381,16 @@ func verifyKnativeCanary(canary *flaggerv1.Canary) error { return nil } +func verifySessionAffinity(canary *flaggerv1.Canary) error { + if canary.Spec.Analysis.SessionAffinity != nil { + if canary.Spec.Analysis.SessionAffinity.CookieName == canary.Spec.Analysis.SessionAffinity.PrimaryCookieName { + return fmt.Errorf("can't use the same cookie name for both primary and cookie name; please update them to be different") + } + } + + return nil +} + func checkCustomResourceType(obj interface{}, logger *zap.SugaredLogger) (flaggerv1.Canary, bool) { var roll *flaggerv1.Canary var ok bool diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index 9bfc37b3..db5ec0ee 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -26,6 +26,7 @@ func TestController_verifyCanary(t *testing.T) { Name: "upstream", Namespace: "test", }, + Analysis: &flaggerv1.CanaryAnalysis{}, }, }, wantErr: true, @@ -42,6 +43,7 @@ func TestController_verifyCanary(t *testing.T) { Name: "upstream", Namespace: "default", }, + Analysis: &flaggerv1.CanaryAnalysis{}, }, }, wantErr: false, @@ -103,6 +105,7 @@ func TestController_verifyCanary(t *testing.T) { Kind: "Deployment", Name: "podinfo", }, + Analysis: &flaggerv1.CanaryAnalysis{}, }, }, wantErr: true, @@ -121,6 +124,7 @@ func TestController_verifyCanary(t *testing.T) { APIVersion: "serving.knative.dev/v1", Name: "podinfo", }, + Analysis: &flaggerv1.CanaryAnalysis{}, }, }, wantErr: true, @@ -140,6 +144,7 @@ func TestController_verifyCanary(t *testing.T) { APIVersion: "serving.knative.dev/v1", Name: "podinfo", }, + Analysis: &flaggerv1.CanaryAnalysis{}, }, }, wantErr: true, @@ -158,10 +163,29 @@ func TestController_verifyCanary(t *testing.T) { APIVersion: "serving.knative.dev/v1", Name: "podinfo", }, + Analysis: &flaggerv1.CanaryAnalysis{}, }, }, wantErr: false, }, + { + name: "session affinity with same cookie names should return an error", + canary: flaggerv1.Canary{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cd-1", + Namespace: "default", + }, + Spec: flaggerv1.CanarySpec{ + Analysis: &flaggerv1.CanaryAnalysis{ + SessionAffinity: &flaggerv1.SessionAffinity{ + CookieName: "smth", + PrimaryCookieName: "smth", + }, + }, + }, + }, + wantErr: true, + }, } ctrl := &Controller{ diff --git a/pkg/router/gateway_api.go b/pkg/router/gateway_api.go index 65a32289..be0cfef8 100644 --- a/pkg/router/gateway_api.go +++ b/pkg/router/gateway_api.go @@ -22,6 +22,7 @@ import ( "reflect" "slices" "strings" + "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -190,20 +191,26 @@ func (gwr *GatewayAPIRouter) Reconcile(canary *flaggerv1.Canary) error { cmpopts.IgnoreFields(v1.BackendRef{}, "Weight"), cmpopts.EquateEmpty(), } + if canary.Spec.Analysis.SessionAffinity != nil { - ignoreRoute := cmpopts.IgnoreSliceElements(func(r v1.HTTPRouteRule) bool { - // Ignore the rule that does sticky routing, i.e. matches against the `Cookie` header. - for _, match := range r.Matches { - for _, headerMatch := range match.Headers { - if *headerMatch.Type == headerMatchRegex && headerMatch.Name == cookieHeader && - strings.Contains(headerMatch.Value, canary.Spec.Analysis.SessionAffinity.CookieName) { - return true + ignoreCookieRouteFunc := func(name string) func(r v1.HTTPRouteRule) bool { + return func(r v1.HTTPRouteRule) bool { + // Ignore the rule that does sticky routing, i.e. matches against the `Cookie` header. + for _, match := range r.Matches { + for _, headerMatch := range match.Headers { + if *headerMatch.Type == headerMatchRegex && headerMatch.Name == cookieHeader && + strings.Contains(headerMatch.Value, name) { + return true + } } } + return false } - return false - }) - ignoreCmpOptions = append(ignoreCmpOptions, ignoreRoute) + } + ignoreCanaryRoute := cmpopts.IgnoreSliceElements(ignoreCookieRouteFunc(canary.Spec.Analysis.SessionAffinity.CookieName)) + ignorePrimaryRoute := cmpopts.IgnoreSliceElements(ignoreCookieRouteFunc(canary.Spec.Analysis.SessionAffinity.PrimaryCookieName)) + + ignoreCmpOptions = append(ignoreCmpOptions, ignoreCanaryRoute, ignorePrimaryRoute) // Ignore backend specific filters, since we use that to insert the `Set-Cookie` header in responses. ignoreCmpOptions = append(ignoreCmpOptions, cmpopts.IgnoreFields(v1.HTTPBackendRef{}, "Filters")) } @@ -439,42 +446,76 @@ func (gwr *GatewayAPIRouter) Finalize(_ *flaggerv1.Canary) error { return nil } +func getBackendByServiceName(rule *v1.HTTPRouteRule, svcName string) *v1.HTTPBackendRef { + for i, backendRef := range rule.BackendRefs { + if string(backendRef.BackendObjectReference.Name) == svcName { + + return &rule.BackendRefs[i] + } + } + return nil +} + // getSessionAffinityRouteRules returns the HTTPRouteRule objects required to perform // session affinity based Canary releases. func (gwr *GatewayAPIRouter) getSessionAffinityRouteRules(canary *flaggerv1.Canary, canaryWeight int, weightedRouteRule *v1.HTTPRouteRule) ([]v1.HTTPRouteRule, error) { _, primarySvcName, canarySvcName := canary.GetServiceNames() - stickyRouteRule := *weightedRouteRule + stickyCanaryRouteRule := *weightedRouteRule + stickyPrimaryRouteRule := *weightedRouteRule // If a canary run is active, we want all responses corresponding to requests hitting the canary deployment // (due to weighted routing) to include a `Set-Cookie` header. All requests that have the `Cookie` header // and match the value of the `Set-Cookie` header will be routed to the canary deployment. if canaryWeight != 0 { + // if the status doesn't have the canary cookie, then generate a new canary cookie. if canary.Status.SessionAffinityCookie == "" { canary.Status.SessionAffinityCookie = fmt.Sprintf("%s=%s", canary.Spec.Analysis.SessionAffinity.CookieName, randSeq()) } + primaryCookie := fmt.Sprintf("%s=%s", canary.Spec.Analysis.SessionAffinity.PrimaryCookieName, randSeq()) - // Add `Set-Cookie` header modifier to the primary backend in the weighted routing rule. - for i, backendRef := range weightedRouteRule.BackendRefs { - if string(backendRef.BackendObjectReference.Name) == canarySvcName { - backendRef.Filters = append(backendRef.Filters, v1.HTTPRouteFilter{ - Type: v1.HTTPRouteFilterResponseHeaderModifier, - ResponseHeaderModifier: &v1.HTTPHeaderFilter{ - Add: []v1.HTTPHeader{ - { - Name: setCookieHeader, - Value: fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr, - canary.Spec.Analysis.SessionAffinity.GetMaxAge(), - ), - }, + // add response modifier to the canary backend ref in the rule that does weighted routing + // to include the canary cookie. + canaryBackendRef := getBackendByServiceName(weightedRouteRule, canarySvcName) + canaryBackendRef.Filters = append(canaryBackendRef.Filters, v1.HTTPRouteFilter{ + Type: v1.HTTPRouteFilterResponseHeaderModifier, + ResponseHeaderModifier: &v1.HTTPHeaderFilter{ + Add: []v1.HTTPHeader{ + { + Name: setCookieHeader, + Value: fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr, + canary.Spec.Analysis.SessionAffinity.GetMaxAge(), + ), + }, + }, + }, + }) + + // add response modifier to the primary backend ref in the rule that does weighted routing + // to include the primary cookie, only if a primary cookie name has been specified. + if canary.Spec.Analysis.SessionAffinity.PrimaryCookieName != "" { + primaryBackendRef := getBackendByServiceName(weightedRouteRule, primarySvcName) + interval, err := time.ParseDuration(canary.Spec.Analysis.Interval) + if err != nil { + return nil, fmt.Errorf("failed to parse canary interval: %w", err) + } + primaryBackendRef.Filters = append(primaryBackendRef.Filters, v1.HTTPRouteFilter{ + Type: v1.HTTPRouteFilterResponseHeaderModifier, + ResponseHeaderModifier: &v1.HTTPHeaderFilter{ + Add: []v1.HTTPHeader{ + { + Name: setCookieHeader, + Value: fmt.Sprintf("%s; %s=%d", primaryCookie, maxAgeAttr, + int(interval.Seconds()), + ), }, }, - }) - } - weightedRouteRule.BackendRefs[i] = backendRef + }, + }) } - // Add `Cookie` header matcher to the sticky routing rule. + // configure the sticky canary rule to match against requests that match against the + // canary cookie and send them to the canary backend. cookieKeyAndVal := strings.Split(canary.Status.SessionAffinityCookie, "=") regexMatchType := v1.HeaderMatchRegularExpression cookieMatch := v1.HTTPRouteMatch{ @@ -493,8 +534,8 @@ func (gwr *GatewayAPIRouter) getSessionAffinityRouteRules(canary *flaggerv1.Cana } mergedMatches := gwr.mergeMatchConditions([]v1.HTTPRouteMatch{cookieMatch}, svcMatches) - stickyRouteRule.Matches = mergedMatches - stickyRouteRule.BackendRefs = []v1.HTTPBackendRef{ + stickyCanaryRouteRule.Matches = mergedMatches + stickyCanaryRouteRule.BackendRefs = []v1.HTTPBackendRef{ { BackendRef: gwr.makeBackendRef(primarySvcName, 0, canary.Spec.Service.Port), }, @@ -502,6 +543,42 @@ func (gwr *GatewayAPIRouter) getSessionAffinityRouteRules(canary *flaggerv1.Cana BackendRef: gwr.makeBackendRef(canarySvcName, 100, canary.Spec.Service.Port), }, } + + // add a sticky primary rule to match against requests that match against the + // primary cookie and send them to the primary backend, only if a primary cookie name has + // been specified. + if canary.Spec.Analysis.SessionAffinity.PrimaryCookieName != "" { + cookieKeyAndVal = strings.Split(primaryCookie, "=") + regexMatchType = v1.HeaderMatchRegularExpression + primaryCookieMatch := v1.HTTPRouteMatch{ + Headers: []v1.HTTPHeaderMatch{ + { + Type: ®exMatchType, + Name: cookieHeader, + Value: fmt.Sprintf(".*%s.*%s.*", cookieKeyAndVal[0], cookieKeyAndVal[1]), + }, + }, + } + + svcMatches, err = gwr.mapRouteMatches(canary.Spec.Service.Match) + if err != nil { + return nil, err + } + + mergedMatches = gwr.mergeMatchConditions([]v1.HTTPRouteMatch{primaryCookieMatch}, svcMatches) + stickyPrimaryRouteRule.Matches = mergedMatches + stickyPrimaryRouteRule.BackendRefs = []v1.HTTPBackendRef{ + { + BackendRef: gwr.makeBackendRef(primarySvcName, 100, canary.Spec.Service.Port), + }, + { + BackendRef: gwr.makeBackendRef(canarySvcName, 0, canary.Spec.Service.Port), + }, + } + return []v1.HTTPRouteRule{stickyCanaryRouteRule, stickyPrimaryRouteRule, *weightedRouteRule}, nil + } + + return []v1.HTTPRouteRule{stickyCanaryRouteRule, *weightedRouteRule}, nil } else { // If canary weight is 0 and SessionAffinityCookie is non-blank, then it belongs to a previous canary run. if canary.Status.SessionAffinityCookie != "" { @@ -524,9 +601,9 @@ func (gwr *GatewayAPIRouter) getSessionAffinityRouteRules(canary *flaggerv1.Cana } svcMatches, _ := gwr.mapRouteMatches(canary.Spec.Service.Match) mergedMatches := gwr.mergeMatchConditions([]v1.HTTPRouteMatch{cookieMatch}, svcMatches) - stickyRouteRule.Matches = mergedMatches + stickyCanaryRouteRule.Matches = mergedMatches - stickyRouteRule.Filters = append(stickyRouteRule.Filters, v1.HTTPRouteFilter{ + stickyCanaryRouteRule.Filters = append(stickyCanaryRouteRule.Filters, v1.HTTPRouteFilter{ Type: v1.HTTPRouteFilterResponseHeaderModifier, ResponseHeaderModifier: &v1.HTTPHeaderFilter{ Add: []v1.HTTPHeader{ @@ -540,9 +617,8 @@ func (gwr *GatewayAPIRouter) getSessionAffinityRouteRules(canary *flaggerv1.Cana } canary.Status.SessionAffinityCookie = "" + return []v1.HTTPRouteRule{stickyCanaryRouteRule, *weightedRouteRule}, nil } - - return []v1.HTTPRouteRule{stickyRouteRule, *weightedRouteRule}, nil } func (gwr *GatewayAPIRouter) mapRouteMatches(requestMatches []istiov1beta1.HTTPMatchRequest) ([]v1.HTTPRouteMatch, error) { diff --git a/pkg/router/gateway_api_test.go b/pkg/router/gateway_api_test.go index f00ea554..425034a9 100644 --- a/pkg/router/gateway_api_test.go +++ b/pkg/router/gateway_api_test.go @@ -21,6 +21,7 @@ import ( "fmt" "strings" "testing" + "time" flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" v1 "github.com/fluxcd/flagger/pkg/apis/gatewayapi/v1" @@ -104,6 +105,7 @@ func TestGatewayAPIRouter_Routes(t *testing.T) { _, pSvcName, cSvcName := canary.GetServiceNames() err := router.SetRoutes(canary, 90, 10, false) + require.NoError(t, err) hr, err := mocks.meshClient.GatewayapiV1().HTTPRoutes("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) @@ -290,80 +292,192 @@ func TestGatewayAPIRouter_Routes(t *testing.T) { } func TestGatewayAPIRouter_getSessionAffinityRouteRules(t *testing.T) { - canary := newTestGatewayAPICanary() - mocks := newFixture(canary) - cookieKey := "flagger-cookie" - canary.Spec.Analysis.SessionAffinity = &flaggerv1.SessionAffinity{ - CookieName: cookieKey, - MaxAge: 300, - } + t.Run("without primary cookie", func(t *testing.T) { + canary := newTestGatewayAPICanary() + mocks := newFixture(canary) + cookieKey := "flagger-cookie" + canary.Spec.Analysis.SessionAffinity = &flaggerv1.SessionAffinity{ + CookieName: cookieKey, + MaxAge: 300, + } - router := &GatewayAPIRouter{ - gatewayAPIClient: mocks.meshClient, - kubeClient: mocks.kubeClient, - logger: mocks.logger, - } - _, pSvcName, cSvcName := canary.GetServiceNames() - weightedRouteRule := &v1.HTTPRouteRule{ - BackendRefs: []v1.HTTPBackendRef{ - { - BackendRef: router.makeBackendRef(pSvcName, initialPrimaryWeight, canary.Spec.Service.Port), + router := &GatewayAPIRouter{ + gatewayAPIClient: mocks.meshClient, + kubeClient: mocks.kubeClient, + logger: mocks.logger, + } + _, pSvcName, cSvcName := canary.GetServiceNames() + weightedRouteRule := &v1.HTTPRouteRule{ + BackendRefs: []v1.HTTPBackendRef{ + { + BackendRef: router.makeBackendRef(pSvcName, initialPrimaryWeight, canary.Spec.Service.Port), + }, + { + BackendRef: router.makeBackendRef(cSvcName, initialCanaryWeight, canary.Spec.Service.Port), + }, }, - { - BackendRef: router.makeBackendRef(cSvcName, initialCanaryWeight, canary.Spec.Service.Port), + } + rules, err := router.getSessionAffinityRouteRules(canary, 10, weightedRouteRule) + require.NoError(t, err) + assert.Equal(t, len(rules), 2) + assert.True(t, strings.HasPrefix(canary.Status.SessionAffinityCookie, cookieKey)) + + stickyRule := rules[0] + cookieMatch := stickyRule.Matches[0].Headers[0] + assert.Equal(t, *cookieMatch.Type, v1.HeaderMatchRegularExpression) + assert.Equal(t, string(cookieMatch.Name), cookieHeader) + assert.Contains(t, cookieMatch.Value, cookieKey) + + assert.Equal(t, len(stickyRule.BackendRefs), 2) + for _, backendRef := range stickyRule.BackendRefs { + if string(backendRef.BackendRef.Name) == pSvcName { + assert.Equal(t, *backendRef.BackendRef.Weight, int32(0)) + } + if string(backendRef.BackendRef.Name) == cSvcName { + assert.Equal(t, *backendRef.BackendRef.Weight, int32(100)) + } + } + + weightedRule := rules[1] + var found bool + for _, backendRef := range weightedRule.BackendRefs { + if string(backendRef.Name) == cSvcName { + found = true + filter := backendRef.Filters[0] + assert.Equal(t, filter.Type, v1.HTTPRouteFilterResponseHeaderModifier) + assert.NotNil(t, filter.ResponseHeaderModifier) + assert.Equal(t, string(filter.ResponseHeaderModifier.Add[0].Name), setCookieHeader) + assert.Equal(t, filter.ResponseHeaderModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr, 300)) + } + } + assert.True(t, found) + + rules, err = router.getSessionAffinityRouteRules(canary, 0, weightedRouteRule) + require.NoError(t, err) + assert.Empty(t, canary.Status.SessionAffinityCookie) + assert.Contains(t, canary.Status.PreviousSessionAffinityCookie, cookieKey) + + stickyRule = rules[0] + cookieMatch = stickyRule.Matches[0].Headers[0] + assert.Equal(t, *cookieMatch.Type, v1.HeaderMatchRegularExpression) + assert.Equal(t, string(cookieMatch.Name), cookieHeader) + assert.Contains(t, cookieMatch.Value, cookieKey) + + assert.Equal(t, stickyRule.Filters[0].Type, v1.HTTPRouteFilterResponseHeaderModifier) + headerModifier := stickyRule.Filters[0].ResponseHeaderModifier + assert.NotNil(t, headerModifier) + assert.Equal(t, string(headerModifier.Add[0].Name), setCookieHeader) + assert.Equal(t, headerModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.PreviousSessionAffinityCookie, maxAgeAttr, -1)) + }) + + t.Run("with primary cookie", func(t *testing.T) { + canary := newTestGatewayAPICanary() + mocks := newFixture(canary) + canaryCookieKey := "canary-flagger-cookie" + primaryCookieKey := "primary-flagger-cookie" + canary.Spec.Analysis.Interval = "15s" + canary.Spec.Analysis.SessionAffinity = &flaggerv1.SessionAffinity{ + CookieName: canaryCookieKey, + PrimaryCookieName: primaryCookieKey, + MaxAge: 300, + } + + router := &GatewayAPIRouter{ + gatewayAPIClient: mocks.meshClient, + kubeClient: mocks.kubeClient, + logger: mocks.logger, + } + _, pSvcName, cSvcName := canary.GetServiceNames() + weightedRouteRule := &v1.HTTPRouteRule{ + BackendRefs: []v1.HTTPBackendRef{ + { + BackendRef: router.makeBackendRef(pSvcName, initialPrimaryWeight, canary.Spec.Service.Port), + }, + { + BackendRef: router.makeBackendRef(cSvcName, initialCanaryWeight, canary.Spec.Service.Port), + }, }, - }, - } - rules, err := router.getSessionAffinityRouteRules(canary, 10, weightedRouteRule) - require.NoError(t, err) - assert.Equal(t, len(rules), 2) - assert.True(t, strings.HasPrefix(canary.Status.SessionAffinityCookie, cookieKey)) - - stickyRule := rules[0] - cookieMatch := stickyRule.Matches[0].Headers[0] - assert.Equal(t, *cookieMatch.Type, v1.HeaderMatchRegularExpression) - assert.Equal(t, string(cookieMatch.Name), cookieHeader) - assert.Contains(t, cookieMatch.Value, cookieKey) - - assert.Equal(t, len(stickyRule.BackendRefs), 2) - for _, backendRef := range stickyRule.BackendRefs { - if string(backendRef.BackendRef.Name) == pSvcName { - assert.Equal(t, *backendRef.BackendRef.Weight, int32(0)) } - if string(backendRef.BackendRef.Name) == cSvcName { - assert.Equal(t, *backendRef.BackendRef.Weight, int32(100)) + rules, err := router.getSessionAffinityRouteRules(canary, 10, weightedRouteRule) + require.NoError(t, err) + assert.Equal(t, len(rules), 3) + assert.True(t, strings.HasPrefix(canary.Status.SessionAffinityCookie, canaryCookieKey)) + + canaryStickyRule := rules[0] + cookieMatch := canaryStickyRule.Matches[0].Headers[0] + assert.Equal(t, *cookieMatch.Type, v1.HeaderMatchRegularExpression) + assert.Equal(t, string(cookieMatch.Name), cookieHeader) + assert.Contains(t, cookieMatch.Value, canaryCookieKey) + + assert.Equal(t, len(canaryStickyRule.BackendRefs), 2) + for _, backendRef := range canaryStickyRule.BackendRefs { + if string(backendRef.BackendRef.Name) == pSvcName { + assert.Equal(t, *backendRef.BackendRef.Weight, int32(0)) + } + if string(backendRef.BackendRef.Name) == cSvcName { + assert.Equal(t, *backendRef.BackendRef.Weight, int32(100)) + } } - } - weightedRule := rules[1] - var found bool - for _, backendRef := range weightedRule.BackendRefs { - if string(backendRef.Name) == cSvcName { - found = true - filter := backendRef.Filters[0] - assert.Equal(t, filter.Type, v1.HTTPRouteFilterResponseHeaderModifier) - assert.NotNil(t, filter.ResponseHeaderModifier) - assert.Equal(t, string(filter.ResponseHeaderModifier.Add[0].Name), setCookieHeader) - assert.Equal(t, filter.ResponseHeaderModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr, 300)) + primaryStickyRule := rules[1] + cookieMatch = primaryStickyRule.Matches[0].Headers[0] + assert.Equal(t, *cookieMatch.Type, v1.HeaderMatchRegularExpression) + assert.Equal(t, string(cookieMatch.Name), cookieHeader) + assert.Contains(t, cookieMatch.Value, primaryCookieKey) + + assert.Equal(t, len(primaryStickyRule.BackendRefs), 2) + for _, backendRef := range primaryStickyRule.BackendRefs { + if string(backendRef.BackendRef.Name) == pSvcName { + assert.Equal(t, *backendRef.BackendRef.Weight, int32(100)) + } + if string(backendRef.BackendRef.Name) == cSvcName { + assert.Equal(t, *backendRef.BackendRef.Weight, int32(0)) + } } - } - assert.True(t, found) - rules, err = router.getSessionAffinityRouteRules(canary, 0, weightedRouteRule) - assert.Empty(t, canary.Status.SessionAffinityCookie) - assert.Contains(t, canary.Status.PreviousSessionAffinityCookie, cookieKey) + weightedRule := rules[2] + var c int + for _, backendRef := range weightedRule.BackendRefs { + if string(backendRef.Name) == cSvcName { + c += 1 + filter := backendRef.Filters[0] + assert.Equal(t, filter.Type, v1.HTTPRouteFilterResponseHeaderModifier) + assert.NotNil(t, filter.ResponseHeaderModifier) + assert.Equal(t, string(filter.ResponseHeaderModifier.Add[0].Name), setCookieHeader) + assert.Equal(t, filter.ResponseHeaderModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr, 300)) + } - stickyRule = rules[0] - cookieMatch = stickyRule.Matches[0].Headers[0] - assert.Equal(t, *cookieMatch.Type, v1.HeaderMatchRegularExpression) - assert.Equal(t, string(cookieMatch.Name), cookieHeader) - assert.Contains(t, cookieMatch.Value, cookieKey) + if string(backendRef.Name) == pSvcName { + c += 1 + filter := backendRef.Filters[0] + assert.Equal(t, filter.Type, v1.HTTPRouteFilterResponseHeaderModifier) + assert.NotNil(t, filter.ResponseHeaderModifier) + assert.Equal(t, string(filter.ResponseHeaderModifier.Add[0].Name), setCookieHeader) + assert.Contains(t, filter.ResponseHeaderModifier.Add[0].Value, canary.Spec.Analysis.SessionAffinity.PrimaryCookieName) + interval, err := time.ParseDuration(canary.Spec.Analysis.Interval) + require.NoError(t, err) + assert.Contains(t, filter.ResponseHeaderModifier.Add[0].Value, fmt.Sprintf("%s=%d", maxAgeAttr, int(interval.Seconds()))) + } + } + assert.Equal(t, 2, c) - assert.Equal(t, stickyRule.Filters[0].Type, v1.HTTPRouteFilterResponseHeaderModifier) - headerModifier := stickyRule.Filters[0].ResponseHeaderModifier - assert.NotNil(t, headerModifier) - assert.Equal(t, string(headerModifier.Add[0].Name), setCookieHeader) - assert.Equal(t, headerModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.PreviousSessionAffinityCookie, maxAgeAttr, -1)) + rules, err = router.getSessionAffinityRouteRules(canary, 0, weightedRouteRule) + require.NoError(t, err) + assert.Empty(t, canary.Status.SessionAffinityCookie) + assert.Contains(t, canary.Status.PreviousSessionAffinityCookie, canaryCookieKey) + + canaryStickyRule = rules[0] + cookieMatch = canaryStickyRule.Matches[0].Headers[0] + assert.Equal(t, *cookieMatch.Type, v1.HeaderMatchRegularExpression) + assert.Equal(t, string(cookieMatch.Name), cookieHeader) + assert.Contains(t, cookieMatch.Value, canaryCookieKey) + + assert.Equal(t, canaryStickyRule.Filters[0].Type, v1.HTTPRouteFilterResponseHeaderModifier) + headerModifier := canaryStickyRule.Filters[0].ResponseHeaderModifier + assert.NotNil(t, headerModifier) + assert.Equal(t, string(headerModifier.Add[0].Name), setCookieHeader) + assert.Equal(t, headerModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.PreviousSessionAffinityCookie, maxAgeAttr, -1)) + }) } func TestGatewayAPIRouter_makeFilters(t *testing.T) { diff --git a/test/gatewayapi/test-session-affinity.sh b/test/gatewayapi/test-session-affinity.sh index bc7d7668..e74b7c9d 100755 --- a/test/gatewayapi/test-session-affinity.sh +++ b/test/gatewayapi/test-session-affinity.sh @@ -44,7 +44,8 @@ spec: maxWeight: 50 stepWeight: 10 sessionAffinity: - cookieName: flagger-cookie + cookieName: canary-flagger-cookie + primaryCookieName: primary-flagger-cookie metrics: - name: error-rate templateRef: @@ -63,7 +64,7 @@ spec: webhooks: - name: load-test type: rollout - url: http://flagger-loadtester.test/ + url: http://localhost:8080 timeout: 5s metadata: cmd: "hey -z 2m -q 10 -c 2 -host www.example.com http://gateway-istio.istio-ingress" @@ -104,7 +105,8 @@ until ${ok}; do done echo '>>> Verifying session affinity' -if ! URL=http://localhost:8888 HOST=www.example.com VERSION=6.1.0 COOKIE_NAME=flagger-cookie \ +if ! URL=http://localhost:8888 HOST=www.example.com CANARY_VERSION=6.1.0 \ + CANARY_COOKIE_NAME=canary-flagger-cookie PRIMARY_VERSION=6.0.4 PRIMARY_COOKIE_NAME=primary-flagger-cookie \ go run ${REPO_ROOT}/test/gatewayapi/verify_session_affinity.go; then echo "failed to verify session affinity" exit $? diff --git a/test/gatewayapi/verify_session_affinity.go b/test/gatewayapi/verify_session_affinity.go index 7677483b..dcf05b96 100644 --- a/test/gatewayapi/verify_session_affinity.go +++ b/test/gatewayapi/verify_session_affinity.go @@ -2,7 +2,7 @@ package main import ( "fmt" - "io/ioutil" + "io" "log" "net/http" "os" @@ -11,30 +11,39 @@ import ( "time" ) -var c = make(chan string, 1) -var mu sync.Mutex -var try = true +// channel for canary cookie +var cc = make(chan string, 1) + +// channel for primary cookie +var pc = make(chan string, 1) var timeout = time.Second * 10 func main() { url := os.Getenv("URL") host := os.Getenv("HOST") - version := os.Getenv("VERSION") - cookieName := os.Getenv("COOKIE_NAME") + canaryVersion := os.Getenv("CANARY_VERSION") + canaryCookieName := os.Getenv("CANARY_COOKIE_NAME") + primaryVersion := os.Getenv("PRIMARY_VERSION") + primaryCookieName := os.Getenv("PRIMARY_COOKIE_NAME") - // Generate traffic - for i := 0; i < 10; i++ { - go tryUntilCanaryIsHit(url, host, version, cookieName) - } + go tryUntilWorkloadIsHit(url, host, canaryVersion, canaryCookieName, cc) + go tryUntilWorkloadIsHit(url, host, primaryVersion, primaryCookieName, pc) + wg := &sync.WaitGroup{} + wg.Add(2) + + go verifySessionAffinity(cc, wg, url, host, primaryVersion) + go verifySessionAffinity(pc, wg, url, host, canaryVersion) + + wg.Wait() + log.Println("✔ successfully verified session affinity") +} + +func verifySessionAffinity(cc chan string, wg *sync.WaitGroup, url, host, wrongVersion string) { select { // If we receive a cookie, then try to verify that we are always routed to the // Canary deployment based on the cookie. - case cookie := <-c: - mu.Lock() - try = false - mu.Unlock() - + case cookie := <-cc: for i := 0; i < 5; i++ { headers := map[string]string{ "Cookie": cookie, @@ -43,15 +52,15 @@ func main() { if err != nil { log.Fatalf("failed to send request to verify cookie based routing: %v", err) } - if !strings.Contains(body, version) { - log.Fatalf("received response from primary deployment instead of canary deployment") + if strings.Contains(body, wrongVersion) { + log.Fatalf("received response from the wrong deployment") } } - - log.Println("✔ successfully verified session affinity") + wg.Done() case <-time.After(timeout): - log.Fatal("timed out waiting for canary hit") + log.Fatal("timed out waiting for workload hit") } + } // sendRequest sends a request to the URL with the provided host and headers. @@ -74,7 +83,7 @@ func sendRequest(url, host string, headers map[string]string) (string, []*http.C } defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return "", nil, err } @@ -82,17 +91,10 @@ func sendRequest(url, host string, headers map[string]string) (string, []*http.C return string(body), resp.Cookies(), nil } -// tryUntilCanaryIsHit is a recursive function that tries to send request and +// tryUntilWorkloadIsHit is a recursive function that tries to send request and // either sends the cookie back to the main thread (if received) or re-sends // the request. -func tryUntilCanaryIsHit(url, host, version, cookieName string) { - mu.Lock() - if !try { - mu.Unlock() - return - } - mu.Unlock() - +func tryUntilWorkloadIsHit(url, host, version, cookieName string, c chan string) { body, cookies, err := sendRequest(url, host, nil) if err != nil { log.Printf("warning: failed to send request: %s", err) @@ -105,6 +107,5 @@ func tryUntilCanaryIsHit(url, host, version, cookieName string) { } } - tryUntilCanaryIsHit(url, host, version, cookieName) - return + tryUntilWorkloadIsHit(url, host, version, cookieName, c) }