diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index ed2ce257..6cfba256 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -1020,6 +1020,18 @@ spec: type: object additionalProperties: type: string + sessionAffinity: + description: SessionAffinity represents the session affinity settings for a canary run. + type: object + required: [ "cookieName" ] + properties: + cookieName: + description: CookieName is the key that will be used for the session affinity cookie. + type: string + maxAge: + description: MaxAge indicates the number of seconds until the session affinity cookie will expire. + default: 86400 + type: number status: description: CanaryStatus defines the observed state of a canary. type: object @@ -1064,6 +1076,12 @@ spec: description: LastTransitionTime of this canary format: date-time type: string + sessionAffinityCookie: + description: Session affinity cookie of the current canary run + type: string + previousSessionAffinityCookie: + description: Session affinity cookie of the previous canary run + type: string conditions: description: Status conditions of this canary type: array diff --git a/charts/flagger/crds/crd.yaml b/charts/flagger/crds/crd.yaml index ed2ce257..6cfba256 100644 --- a/charts/flagger/crds/crd.yaml +++ b/charts/flagger/crds/crd.yaml @@ -1020,6 +1020,18 @@ spec: type: object additionalProperties: type: string + sessionAffinity: + description: SessionAffinity represents the session affinity settings for a canary run. + type: object + required: [ "cookieName" ] + properties: + cookieName: + description: CookieName is the key that will be used for the session affinity cookie. + type: string + maxAge: + description: MaxAge indicates the number of seconds until the session affinity cookie will expire. + default: 86400 + type: number status: description: CanaryStatus defines the observed state of a canary. type: object @@ -1064,6 +1076,12 @@ spec: description: LastTransitionTime of this canary format: date-time type: string + sessionAffinityCookie: + description: Session affinity cookie of the current canary run + type: string + previousSessionAffinityCookie: + description: Session affinity cookie of the previous canary run + type: string conditions: description: Status conditions of this canary type: array diff --git a/docs/gitbook/tutorials/istio-progressive-delivery.md b/docs/gitbook/tutorials/istio-progressive-delivery.md index b130e646..12f86246 100644 --- a/docs/gitbook/tutorials/istio-progressive-delivery.md +++ b/docs/gitbook/tutorials/istio-progressive-delivery.md @@ -292,6 +292,118 @@ Events: Warning Synced 1m flagger Canary failed! Scaling down podinfo.test ``` +## Session Affinity + +While Flagger can perform weighted routing and A/B testing individually, with Istio it can combine the two leading to a Canary +release with session affinity. For more information you can read the [deployment strategies docs](../usage/deployment-strategies.md#canary-release-with-session-affinity). + +Create a canary custom resource \(replace app.example.com with your own domain\): + +```yaml +apiVersion: flagger.app/v1beta1 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + # deployment reference + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + # the maximum time in seconds for the canary deployment + # to make progress before it is rollback (default 600s) + progressDeadlineSeconds: 60 + # HPA reference (optional) + autoscalerRef: + apiVersion: autoscaling/v2beta2 + kind: HorizontalPodAutoscaler + name: podinfo + service: + # service port number + port: 9898 + # container port number or name (optional) + targetPort: 9898 + # Istio gateways (optional) + gateways: + - public-gateway.istio-system.svc.cluster.local + # Istio virtual service host names (optional) + hosts: + - app.example.com + # Istio traffic policy (optional) + trafficPolicy: + tls: + # use ISTIO_MUTUAL when mTLS is enabled + mode: DISABLE + # Istio retry policy (optional) + retries: + attempts: 3 + perTryTimeout: 1s + retryOn: "gateway-error,connect-failure,refused-stream" + analysis: + # schedule interval (default 60s) + interval: 1m + # max number of failed metric checks before rollback + threshold: 5 + # max traffic percentage routed to canary + # percentage (0-100) + maxWeight: 50 + # canary increment step + # percentage (0-100) + stepWeight: 10 + # session affinity config + sessionAffinity: + # name of the cookie used + cookieName: flagger-cookie + # max age of the cookie (in seconds) + # optional; defaults to 86400 + maxAge: 21600 + metrics: + - name: request-success-rate + # minimum req success rate (non 5xx responses) + # percentage (0-100) + thresholdRange: + min: 99 + interval: 1m + - name: request-duration + # maximum req duration P99 + # milliseconds + thresholdRange: + max: 500 + interval: 30s + # testing (optional) + webhooks: + - name: acceptance-test + type: pre-rollout + url: http://flagger-loadtester.test/ + timeout: 30s + metadata: + type: bash + cmd: "curl -sd 'test' http://podinfo-canary:9898/token | grep token" + - name: load-test + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898/" +``` + +Save the above resource as podinfo-canary-session-affinity.yaml and then apply it: + +```bash +kubectl apply -f ./podinfo-canary-session-affinity.yaml +``` + +Trigger a canary deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=ghcr.io/stefanprodan/podinfo:6.0.1 +``` + +You can load `app.example.com` in your browser and refresh it until you see the requests being served by `podinfo:6.0.1`. +All subsequent requests after that will be served by `podinfo:6.0.1` and not `podinfo:6.0.0` because of the session affinity +configured by Flagger with Istio. + ## Traffic mirroring ![Flagger Canary Traffic Shadowing](https://raw.githubusercontent.com/fluxcd/flagger/main/docs/diagrams/flagger-canary-traffic-mirroring.png) diff --git a/docs/gitbook/usage/deployment-strategies.md b/docs/gitbook/usage/deployment-strategies.md index 605dccb0..d814f5ca 100644 --- a/docs/gitbook/usage/deployment-strategies.md +++ b/docs/gitbook/usage/deployment-strategies.md @@ -10,6 +10,8 @@ Flagger can run automated application analysis, promotion and rollback for the f * Kubernetes CNI, Istio, Linkerd, App Mesh, NGINX, Contour, Gloo Edge, Open Service Mesh, Gateway API * **Blue/Green Mirroring** \(traffic shadowing\) * Istio +* **Canary Release with Session Affinity** \(progressive traffic shifting combined with cookie based routing\) + * Istio For Canary releases and A/B testing you'll need a Layer 7 traffic management solution like a service mesh or an ingress controller. For Blue/Green deployments no service mesh or ingress controller is required. @@ -393,3 +395,59 @@ After the analysis finishes, the traffic is routed to the canary (green) before triggering the primary (blue) rolling update, this ensures a smooth transition to the new version avoiding dropping in-flight requests during the Kubernetes deployment rollout. +## Canary Release with Session Affinity + +This deployment strategy mixes a Canary Release with A/B testing. A Canary Release is helpful when +we're trying to expose new features to users progressively, but because of the very nature of its +routing (weight based), users can land on the application's old version even after they have been +routed to the new version previously. This can be annoying, or worse break how other services interact +with our application. To address this issue, we borrow some things from A/B testing. + +Since A/B testing is particularly helpful for applications that require session affinity, we integrate +cookie based routing with regular weight based routing. This means once a user is exposed to the new +version of our application (based on the traffic weights), they're always routed to that version, i.e. +they're never routed back to the old version of our application. + +You can enable this, by specifying `.spec.analsyis.sessionAffinity` in the Canary (only Istio is supported): + +```yaml + analysis: + # schedule interval (default 60s) + interval: 1m + # max number of failed metric checks before rollback + threshold: 10 + # max traffic percentage routed to canary + # percentage (0-100) + maxWeight: 50 + # canary increment step + # percentage (0-100) + stepWeight: 2 + # session affinity config + sessionAffinity: + # name of the cookie used + cookieName: flagger-cookie + # max age of the cookie (in seconds) + # optional; defaults to 86400 + maxAge: 21600 +``` + +`.spec.analysis.sessionAffinity.cookieName` is the name of the Cookie that is stored. The value of the +cookie is a randomly generated string of characters that act as a unique identifier. For the above +config, the response header of a request routed to the canary deployment during a Canary run will look like: +``` +Set-Cookie: flagger-cookie=LpsIaLdoNZ; Max-Age=21600 +``` + +After a Canary run is over and all traffic is shifted back to the primary deployment, all responses will +have the following header: +``` +Set-Cookie: flagger-cookie=LpsIaLdoNZ; Max-Age=-1 +``` +This tells the client to delete the cookie, making sure there are no junk cookies lying around in the user's +system. + +If a new Canary run is triggered, the response header will set a new cookie for all requests routed to +the Canary deployment: +``` +Set-Cookie: flagger-cookie=McxKdLQoIN; Max-Age=21600 +``` diff --git a/kustomize/base/flagger/crd.yaml b/kustomize/base/flagger/crd.yaml index ed2ce257..6cfba256 100644 --- a/kustomize/base/flagger/crd.yaml +++ b/kustomize/base/flagger/crd.yaml @@ -1020,6 +1020,18 @@ spec: type: object additionalProperties: type: string + sessionAffinity: + description: SessionAffinity represents the session affinity settings for a canary run. + type: object + required: [ "cookieName" ] + properties: + cookieName: + description: CookieName is the key that will be used for the session affinity cookie. + type: string + maxAge: + description: MaxAge indicates the number of seconds until the session affinity cookie will expire. + default: 86400 + type: number status: description: CanaryStatus defines the observed state of a canary. type: object @@ -1064,6 +1076,12 @@ spec: description: LastTransitionTime of this canary format: date-time type: string + sessionAffinityCookie: + description: Session affinity cookie of the current canary run + type: string + previousSessionAffinityCookie: + description: Session affinity cookie of the previous canary run + type: string conditions: description: Status conditions of this canary type: array diff --git a/pkg/apis/flagger/v1beta1/canary.go b/pkg/apis/flagger/v1beta1/canary.go index 693beb74..cbd3351e 100644 --- a/pkg/apis/flagger/v1beta1/canary.go +++ b/pkg/apis/flagger/v1beta1/canary.go @@ -262,6 +262,20 @@ type CanaryAnalysis struct { // A/B testing HTTP header match conditions // +optional Match []istiov1alpha3.HTTPMatchRequest `json:"match,omitempty"` + + // SessionAffinity represents the session affinity settings for a canary run. + // +optional + SessionAffinity *SessionAffinity `json:"sessionAffinity,omitempty"` +} + +type SessionAffinity struct { + // CookieName is the key that will be used for the session affinity cookie. + CookieName string `json:"cookieName,omitempty"` + // MaxAge indicates the number of seconds until the session affinity cookie will expire. + // ref: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie#attributes + // The default value is 86,400 seconds, i.e. a day. + // +optional + MaxAge int `json:"maxAge,omitempty"` } // CanaryMetric holds the reference to metrics used for canary analysis @@ -437,6 +451,15 @@ type CustomMetadata struct { Annotations map[string]string `json:"annotations,omitempty"` } +// GetMaxAge returns the max age of a cookie in seconds. +func (s *SessionAffinity) GetMaxAge() int { + if s.MaxAge == 0 { + // 24 hours * 60 mins * 60 seconds + return 86400 + } + return s.MaxAge +} + // GetServiceNames returns the apex, primary and canary Kubernetes service names func (c *Canary) GetServiceNames() (apexName, primaryName, canaryName string) { apexName = c.Spec.TargetRef.Name diff --git a/pkg/apis/flagger/v1beta1/status.go b/pkg/apis/flagger/v1beta1/status.go index 2a487fb5..fd92f08d 100644 --- a/pkg/apis/flagger/v1beta1/status.go +++ b/pkg/apis/flagger/v1beta1/status.go @@ -74,6 +74,10 @@ type CanaryStatus struct { CanaryWeight int `json:"canaryWeight"` Iterations int `json:"iterations"` // +optional + PreviousSessionAffinityCookie string `json:"previousSessionAffinityCookie,omitempty"` + // +optional + SessionAffinityCookie string `json:"sessionAffinityCookie,omitempty"` + // +optional TrackedConfigs *map[string]string `json:"trackedConfigs,omitempty"` // +optional LastAppliedSpec string `json:"lastAppliedSpec,omitempty"` diff --git a/pkg/apis/flagger/v1beta1/zz_generated.deepcopy.go b/pkg/apis/flagger/v1beta1/zz_generated.deepcopy.go index cd960dfe..c6eaa2c1 100644 --- a/pkg/apis/flagger/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/flagger/v1beta1/zz_generated.deepcopy.go @@ -263,6 +263,11 @@ func (in *CanaryAnalysis) DeepCopyInto(out *CanaryAnalysis) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.SessionAffinity != nil { + in, out := &in.SessionAffinity, &out.SessionAffinity + *out = new(SessionAffinity) + **out = **in + } return } @@ -815,3 +820,19 @@ func (in *MetricTemplateStatus) DeepCopy() *MetricTemplateStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SessionAffinity) DeepCopyInto(out *SessionAffinity) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SessionAffinity. +func (in *SessionAffinity) DeepCopy() *SessionAffinity { + if in == nil { + return nil + } + out := new(SessionAffinity) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/apis/istio/v1alpha3/virtual_service.go b/pkg/apis/istio/v1alpha3/virtual_service.go index a41ca2c9..5060bd1b 100644 --- a/pkg/apis/istio/v1alpha3/virtual_service.go +++ b/pkg/apis/istio/v1alpha3/virtual_service.go @@ -311,6 +311,10 @@ type Destination struct { // Describes match conditions and actions for routing HTTP/1.1, HTTP2, and // gRPC traffic. See VirtualService for usage examples. type HTTPRoute struct { + // The name assigned to the route for debugging purposes. The route’s name will + // be concatenated with the match’s name and will be logged in the access logs + // for requests matching this route/match. + Name string `json:"name,omitempty"` // Match conditions to be satisfied for the rule to be // activated. All conditions inside a single match block have AND // semantics, while the list of match blocks have OR semantics. The rule @@ -321,7 +325,7 @@ type HTTPRoute struct { // forwarding target can be one of several versions of a service (see // glossary in beginning of document). Weights associated with the // service version determine the proportion of traffic it receives. - Route []DestinationWeight `json:"route,omitempty"` + Route []HTTPRouteDestination `json:"route,omitempty"` // A http rule can either redirect or forward (default) traffic. If // traffic passthrough option is specified in the rule, @@ -528,7 +532,7 @@ type HTTPMatchRequest struct { SourceNamespace string `json:"sourceNamespace,omitempty"` } -type DestinationWeight struct { +type HTTPRouteDestination struct { // REQUIRED. Destination uniquely identifies the instances of a service // to which the request/connection should be forwarded to. Destination Destination `json:"destination"` @@ -538,6 +542,9 @@ type DestinationWeight struct { // If there is only destination in a rule, the weight value is assumed to // be 100. Weight int `json:"weight"` + + // Header manipulation rules + Headers *Headers `json:"headers,omitempty"` } // PortSelector specifies the number of a port to be used for @@ -590,7 +597,7 @@ type TCPRoute struct { // Currently, only one destination is allowed for TCP services. When TCP // weighted routing support is introduced in Envoy, multiple destinations // with weights can be specified. - Route DestinationWeight `json:"route"` + Route HTTPRouteDestination `json:"route"` } // L4 connection match attributes. Note that L4 connection matching support diff --git a/pkg/apis/istio/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/istio/v1alpha3/zz_generated.deepcopy.go index 6a6fcfc5..b69a7ab9 100644 --- a/pkg/apis/istio/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/istio/v1alpha3/zz_generated.deepcopy.go @@ -229,23 +229,6 @@ func (in *DestinationRuleSpec) DeepCopy() *DestinationRuleSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DestinationWeight) DeepCopyInto(out *DestinationWeight) { - *out = *in - in.Destination.DeepCopyInto(&out.Destination) - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DestinationWeight. -func (in *DestinationWeight) DeepCopy() *DestinationWeight { - if in == nil { - return nil - } - out := new(DestinationWeight) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Distribute) DeepCopyInto(out *Distribute) { *out = *in @@ -456,7 +439,7 @@ func (in *HTTPRoute) DeepCopyInto(out *HTTPRoute) { } if in.Route != nil { in, out := &in.Route, &out.Route - *out = make([]DestinationWeight, len(*in)) + *out = make([]HTTPRouteDestination, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -514,6 +497,28 @@ func (in *HTTPRoute) DeepCopy() *HTTPRoute { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HTTPRouteDestination) DeepCopyInto(out *HTTPRouteDestination) { + *out = *in + in.Destination.DeepCopyInto(&out.Destination) + if in.Headers != nil { + in, out := &in.Headers, &out.Headers + *out = new(Headers) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPRouteDestination. +func (in *HTTPRouteDestination) DeepCopy() *HTTPRouteDestination { + if in == nil { + return nil + } + out := new(HTTPRouteDestination) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HTTPSettings) DeepCopyInto(out *HTTPSettings) { *out = *in diff --git a/pkg/router/istio.go b/pkg/router/istio.go index 6c5b89e3..7c78c6d9 100644 --- a/pkg/router/istio.go +++ b/pkg/router/istio.go @@ -20,6 +20,8 @@ import ( "context" "encoding/json" "fmt" + "math/rand" + "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -30,6 +32,7 @@ import ( "k8s.io/client-go/kubernetes" flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" + istiov1alpha1 "github.com/fluxcd/flagger/pkg/apis/istio/common/v1alpha1" istiov1alpha3 "github.com/fluxcd/flagger/pkg/apis/istio/v1alpha3" clientset "github.com/fluxcd/flagger/pkg/client/clientset/versioned" ) @@ -43,6 +46,13 @@ type IstioRouter struct { setOwnerRefs bool } +const cookieHeader = "Cookie" +const setCookieHeader = "Set-Cookie" +const stickyRouteName = "sticky-route" +const maxAgeAttr = "Max-Age" + +var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + // Reconcile creates or updates the Istio virtual service and destination rules func (ir *IstioRouter) Reconcile(canary *flaggerv1.Canary) error { _, primaryName, canaryName := canary.GetServiceNames() @@ -153,7 +163,7 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error { } // create destinations with primary weight 100% and canary weight 0% - canaryRoute := []istiov1alpha3.DestinationWeight{ + canaryRoute := []istiov1alpha3.HTTPRouteDestination{ makeDestination(canary, primaryName, 100), makeDestination(canary, canaryName, 0), } @@ -199,7 +209,7 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error { Retries: canary.Spec.Service.Retries, CorsPolicy: canary.Spec.Service.CorsPolicy, Headers: canary.Spec.Service.Headers, - Route: []istiov1alpha3.DestinationWeight{ + Route: []istiov1alpha3.HTTPRouteDestination{ makeDestination(canary, primaryName, 100), }, }, @@ -255,13 +265,28 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error { virtualService.Spec.Hosts = []string{} } + ignoreCmpOptions := []cmp.Option{ + cmpopts.IgnoreFields(istiov1alpha3.HTTPRouteDestination{}, "Weight"), + cmpopts.IgnoreFields(istiov1alpha3.HTTPRoute{}, "Mirror", "MirrorPercentage"), + } + if canary.Spec.Analysis.SessionAffinity != nil { + // We ignore this route as this does not do weighted routing and is handled exclusively + // by SetRoutes(). + ignoreSlice := cmpopts.IgnoreSliceElements(func(t istiov1alpha3.HTTPRoute) bool { + if t.Name == stickyRouteName { + return true + } + return false + }) + ignoreCmpOptions = append(ignoreCmpOptions, ignoreSlice) + ignoreCmpOptions = append(ignoreCmpOptions, cmpopts.IgnoreFields(istiov1alpha3.HTTPRouteDestination{}, "Headers")) + } // update service but keep the original destination weights and mirror if virtualService != nil { if diff := cmp.Diff( newSpec, virtualService.Spec, - cmpopts.IgnoreFields(istiov1alpha3.DestinationWeight{}, "Weight"), - cmpopts.IgnoreFields(istiov1alpha3.HTTPRoute{}, "Mirror", "MirrorPercentage"), + ignoreCmpOptions..., ); diff != "" { vtClone := virtualService.DeepCopy() vtClone.Spec = newSpec @@ -333,6 +358,23 @@ func (ir *IstioRouter) GetRoutes(canary *flaggerv1.Canary) ( mirrored = true } + if canary.Spec.Analysis.SessionAffinity != nil { + for _, http := range vs.Spec.Http { + for _, routeDest := range http.Route { + // we are interested in the route that sets the cookie as that's the route + // that does weighted routing. + if routeDest.Headers != nil { + if routeDest.Destination.Host == primaryName { + primaryWeight = routeDest.Weight + } + if routeDest.Destination.Host == canaryName { + canaryWeight = routeDest.Weight + } + } + } + } + } + if primaryWeight == 0 && canaryWeight == 0 { err = fmt.Errorf("VirtualService %s.%s does not contain routes for %s-primary and %s-canary", apexName, canary.Namespace, apexName, apexName) @@ -358,20 +400,103 @@ func (ir *IstioRouter) SetRoutes( vsCopy := vs.DeepCopy() // weighted routing (progressive canary) - vsCopy.Spec.Http = []istiov1alpha3.HTTPRoute{ - { - Match: canary.Spec.Service.Match, - Rewrite: canary.Spec.Service.Rewrite, - Timeout: canary.Spec.Service.Timeout, - Retries: canary.Spec.Service.Retries, - CorsPolicy: canary.Spec.Service.CorsPolicy, - Headers: canary.Spec.Service.Headers, - Route: []istiov1alpha3.DestinationWeight{ - makeDestination(canary, primaryName, primaryWeight), - makeDestination(canary, canaryName, canaryWeight), - }, + weightedRoute := istiov1alpha3.HTTPRoute{ + Match: canary.Spec.Service.Match, + Rewrite: canary.Spec.Service.Rewrite, + Timeout: canary.Spec.Service.Timeout, + Retries: canary.Spec.Service.Retries, + CorsPolicy: canary.Spec.Service.CorsPolicy, + Headers: canary.Spec.Service.Headers, + Route: []istiov1alpha3.HTTPRouteDestination{ + makeDestination(canary, primaryName, primaryWeight), + makeDestination(canary, canaryName, canaryWeight), }, } + vsCopy.Spec.Http = []istiov1alpha3.HTTPRoute{ + weightedRoute, + } + + if canary.Spec.Analysis.SessionAffinity != nil { + // If a canary run is active, we want all responses corresponding to requests hitting the canary deployment + // (due to weighted routing) to include a `Set-Cookie` header. All requests that have the `Cookie` header + // and match the value of the `Set-Cookie` header will be routed to the canary deployment. + stickyRoute := weightedRoute + stickyRoute.Name = stickyRouteName + if canaryWeight != 0 { + if canary.Status.SessionAffinityCookie == "" { + canary.Status.SessionAffinityCookie = fmt.Sprintf("%s=%s", canary.Spec.Analysis.SessionAffinity.CookieName, randSeq()) + } + + for i, routeDest := range weightedRoute.Route { + if routeDest.Destination.Host == canaryName { + if routeDest.Headers == nil { + routeDest.Headers = &istiov1alpha3.Headers{ + Response: &istiov1alpha3.HeaderOperations{}, + } + } + routeDest.Headers.Response.Add = map[string]string{ + setCookieHeader: fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr, + canary.Spec.Analysis.SessionAffinity.GetMaxAge(), + ), + } + } + weightedRoute.Route[i] = routeDest + } + + cookieMatch := istiov1alpha3.HTTPMatchRequest{ + Headers: map[string]istiov1alpha1.StringMatch{ + cookieHeader: { + Exact: canary.Status.SessionAffinityCookie, + }, + }, + } + canaryMatch := mergeMatchConditions([]istiov1alpha3.HTTPMatchRequest{cookieMatch}, canary.Spec.Service.Match) + stickyRoute.Match = canaryMatch + stickyRoute.Route = []istiov1alpha3.HTTPRouteDestination{ + makeDestination(canary, primaryName, 0), + makeDestination(canary, canaryName, 100), + } + } else { + // If canary weight is 0 and SessionAffinityCookie is non-blank, then it belongs to a previous canary run. + if canary.Status.SessionAffinityCookie != "" { + canary.Status.PreviousSessionAffinityCookie = canary.Status.SessionAffinityCookie + } + previousCookie := canary.Status.PreviousSessionAffinityCookie + + // Match against the previous session cookie and delete that cookie + if previousCookie != "" { + cookieMatch := istiov1alpha3.HTTPMatchRequest{ + Headers: map[string]istiov1alpha1.StringMatch{ + cookieHeader: { + Exact: previousCookie, + }, + }, + } + canaryMatch := mergeMatchConditions([]istiov1alpha3.HTTPMatchRequest{cookieMatch}, canary.Spec.Service.Match) + stickyRoute.Match = canaryMatch + + if stickyRoute.Headers == nil { + stickyRoute.Headers = &istiov1alpha3.Headers{ + Response: &istiov1alpha3.HeaderOperations{ + Add: map[string]string{}, + }, + } + } else if stickyRoute.Headers.Response == nil { + stickyRoute.Headers.Response = &istiov1alpha3.HeaderOperations{ + Add: map[string]string{}, + } + } else if stickyRoute.Headers.Response.Add == nil { + stickyRoute.Headers.Response.Add = map[string]string{} + } + stickyRoute.Headers.Response.Add[setCookieHeader] = fmt.Sprintf("%s; %s=%d", previousCookie, maxAgeAttr, -1) + } + + canary.Status.SessionAffinityCookie = "" + } + vsCopy.Spec.Http = []istiov1alpha3.HTTPRoute{ + stickyRoute, weightedRoute, + } + } if mirrored { vsCopy.Spec.Http[0].Mirror = &istiov1alpha3.Destination{ @@ -395,7 +520,7 @@ func (ir *IstioRouter) SetRoutes( Retries: canary.Spec.Service.Retries, CorsPolicy: canary.Spec.Service.CorsPolicy, Headers: canary.Spec.Service.Headers, - Route: []istiov1alpha3.DestinationWeight{ + Route: []istiov1alpha3.HTTPRouteDestination{ makeDestination(canary, primaryName, primaryWeight), makeDestination(canary, canaryName, canaryWeight), }, @@ -407,7 +532,7 @@ func (ir *IstioRouter) SetRoutes( Retries: canary.Spec.Service.Retries, CorsPolicy: canary.Spec.Service.CorsPolicy, Headers: canary.Spec.Service.Headers, - Route: []istiov1alpha3.DestinationWeight{ + Route: []istiov1alpha3.HTTPRouteDestination{ makeDestination(canary, primaryName, primaryWeight), }, }, @@ -483,8 +608,8 @@ func mergeMatchConditions(canary, defaults []istiov1alpha3.HTTPMatchRequest) []i } // makeDestination returns a an destination weight for the specified host -func makeDestination(canary *flaggerv1.Canary, host string, weight int) istiov1alpha3.DestinationWeight { - dest := istiov1alpha3.DestinationWeight{ +func makeDestination(canary *flaggerv1.Canary, host string, weight int) istiov1alpha3.HTTPRouteDestination { + dest := istiov1alpha3.HTTPRouteDestination{ Destination: istiov1alpha3.Destination{ Host: host, }, @@ -495,7 +620,7 @@ func makeDestination(canary *flaggerv1.Canary, host string, weight int) istiov1a if canary.Spec.Service.PortDiscovery && (len(canary.Spec.Service.Gateways) > 0 && canary.Spec.Service.Gateways[0] != "mesh" || canary.Spec.Service.Delegation) { - dest = istiov1alpha3.DestinationWeight{ + dest = istiov1alpha3.HTTPRouteDestination{ Destination: istiov1alpha3.Destination{ Host: host, Port: &istiov1alpha3.PortSelector{ @@ -508,3 +633,13 @@ func makeDestination(canary *flaggerv1.Canary, host string, weight int) istiov1a return dest } + +func randSeq() string { + rand.Seed(time.Now().UnixNano()) + + b := make([]rune, 10) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) +} diff --git a/pkg/router/istio_test.go b/pkg/router/istio_test.go index 9b549846..f2bb63f0 100644 --- a/pkg/router/istio_test.go +++ b/pkg/router/istio_test.go @@ -20,8 +20,10 @@ import ( "context" "encoding/json" "fmt" + "strings" "testing" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -122,7 +124,7 @@ func TestIstioRouter_SetRoutes(t *testing.T) { vs, err := mocks.meshClient.NetworkingV1alpha3().VirtualServices("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) - var pRoute, cRoute istiov1alpha3.DestinationWeight + var pRoute, cRoute istiov1alpha3.HTTPRouteDestination var mirror *istiov1alpha3.Destination for _, http := range vs.Spec.Http { for _, route := range http.Route { @@ -142,6 +144,164 @@ func TestIstioRouter_SetRoutes(t *testing.T) { }) + t.Run("session affinity", func(t *testing.T) { + canary := mocks.canary.DeepCopy() + cookieKey := "flagger-cookie" + // enable session affinity and start canary run + canary.Spec.Analysis.SessionAffinity = &v1beta1.SessionAffinity{ + CookieName: cookieKey, + MaxAge: 300, + } + err := router.SetRoutes(canary, 0, 10, false) + + vs, err := mocks.meshClient.NetworkingV1alpha3().VirtualServices("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) + + assert.Len(t, vs.Spec.Http, 2) + stickyRoute := vs.Spec.Http[0] + weightedRoute := vs.Spec.Http[1] + + // stickyRoute should match against a cookie and direct all traffic to the canary when a canary run is active. + var found bool + for _, match := range stickyRoute.Match { + if val, ok := match.Headers[cookieHeader]; ok { + found = true + assert.True(t, strings.HasPrefix(val.Exact, cookieKey)) + for _, routeDest := range stickyRoute.Route { + if routeDest.Destination.Host == pHost { + assert.Equal(t, 0, routeDest.Weight) + } + if routeDest.Destination.Host == cHost { + assert.Equal(t, 100, routeDest.Weight) + } + } + } + } + assert.True(t, found) + + // weightedRoute should do regular weight based routing and inject the Set-Cookie header + // for all responses returned from the canary deployment. + for _, routeDest := range weightedRoute.Route { + if routeDest.Destination.Host == pHost { + assert.Equal(t, 0, routeDest.Weight) + } + if routeDest.Destination.Host == cHost { + assert.Equal(t, 10, routeDest.Weight) + val, ok := routeDest.Headers.Response.Add[setCookieHeader] + assert.True(t, ok) + assert.True(t, strings.HasPrefix(val, cookieKey)) + assert.True(t, strings.Contains(val, "Max-Age=300")) + } + } + assert.True(t, strings.HasPrefix(canary.Status.SessionAffinityCookie, cookieKey)) + + // reconcile canary, destination rules, virtual services + err = router.Reconcile(canary) + require.NoError(t, err) + + reconciledVS, err := mocks.meshClient.NetworkingV1alpha3().VirtualServices("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) + + // routes should not be changed. + assert.Len(t, vs.Spec.Http, 2) + assert.NotNil(t, reconciledVS) + assert.Equal(t, cmp.Diff(reconciledVS.Spec.Http[0], stickyRoute), "") + assert.Equal(t, cmp.Diff(reconciledVS.Spec.Http[1], weightedRoute), "") + + // further continue the canary run + err = router.SetRoutes(canary, 50, 50, false) + require.NoError(t, err) + + vs, err = mocks.meshClient.NetworkingV1alpha3().VirtualServices("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) + + assert.Len(t, vs.Spec.Http, 2) + stickyRoute = vs.Spec.Http[0] + weightedRoute = vs.Spec.Http[1] + + found = false + for _, match := range stickyRoute.Match { + if val, ok := match.Headers[cookieHeader]; ok { + found = true + assert.True(t, strings.HasPrefix(val.Exact, cookieKey)) + for _, routeDest := range stickyRoute.Route { + if routeDest.Destination.Host == pHost { + assert.Equal(t, 0, routeDest.Weight) + } + if routeDest.Destination.Host == cHost { + assert.Equal(t, 100, routeDest.Weight) + } + } + } + } + assert.True(t, found) + + for _, routeDest := range weightedRoute.Route { + if routeDest.Destination.Host == pHost { + assert.Equal(t, 50, routeDest.Weight) + } + if routeDest.Destination.Host == cHost { + assert.Equal(t, 50, routeDest.Weight) + val, ok := routeDest.Headers.Response.Add[setCookieHeader] + assert.True(t, ok) + assert.True(t, strings.HasPrefix(val, cookieKey)) + assert.True(t, strings.Contains(val, "Max-Age=300")) + } + } + assert.True(t, strings.HasPrefix(canary.Status.SessionAffinityCookie, cookieKey)) + sessionAffinityCookie := canary.Status.SessionAffinityCookie + + // promotion + err = router.SetRoutes(canary, 100, 0, false) + require.NoError(t, err) + + vs, err = mocks.meshClient.NetworkingV1alpha3().VirtualServices("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) + + assert.Len(t, vs.Spec.Http, 2) + stickyRoute = vs.Spec.Http[0] + weightedRoute = vs.Spec.Http[1] + + found = false + for _, match := range stickyRoute.Match { + if val, ok := match.Headers[cookieHeader]; ok { + found = true + assert.True(t, strings.HasPrefix(val.Exact, cookieKey)) + for _, routeDest := range stickyRoute.Route { + if routeDest.Destination.Host == pHost { + assert.Equal(t, 100, routeDest.Weight) + } + if routeDest.Destination.Host == cHost { + assert.Equal(t, 0, routeDest.Weight) + } + } + } + } + assert.True(t, found) + + assert.Equal(t, canary.Status.SessionAffinityCookie, "") + assert.Equal(t, canary.Status.PreviousSessionAffinityCookie, sessionAffinityCookie) + + val, ok := stickyRoute.Headers.Response.Add[setCookieHeader] + assert.True(t, ok) + assert.True(t, strings.HasPrefix(val, sessionAffinityCookie)) + assert.True(t, strings.Contains(val, "Max-Age=-1")) + + // delete the Set-Cookie header from responses returned by the weighted route + for _, routeDest := range weightedRoute.Route { + if routeDest.Destination.Host == pHost { + assert.Equal(t, 100, routeDest.Weight) + } + if routeDest.Destination.Host == cHost { + assert.Equal(t, 0, routeDest.Weight) + if routeDest.Headers != nil && routeDest.Headers.Response != nil { + _, ok := routeDest.Headers.Response.Add[setCookieHeader] + assert.False(t, ok) + } + } + } + }) + t.Run("mirror", func(t *testing.T) { for _, w := range []int{0, 10, 50} { p, c := 100, 0 @@ -154,7 +314,7 @@ func TestIstioRouter_SetRoutes(t *testing.T) { vs, err := mocks.meshClient.NetworkingV1alpha3().VirtualServices("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) - var pRoute, cRoute istiov1alpha3.DestinationWeight + var pRoute, cRoute istiov1alpha3.HTTPRouteDestination var mirror *istiov1alpha3.Destination var mirrorWeight *istiov1alpha3.Percent for _, http := range vs.Spec.Http { @@ -310,8 +470,8 @@ func TestIstioRouter_ABTest(t *testing.T) { pHost := fmt.Sprintf("%s-primary", mocks.abtest.Spec.TargetRef.Name) cHost := fmt.Sprintf("%s-canary", mocks.abtest.Spec.TargetRef.Name) - pRoute := istiov1alpha3.DestinationWeight{} - cRoute := istiov1alpha3.DestinationWeight{} + pRoute := istiov1alpha3.HTTPRouteDestination{} + cRoute := istiov1alpha3.HTTPRouteDestination{} var mirror *istiov1alpha3.Destination for _, http := range vs.Spec.Http { @@ -427,7 +587,7 @@ func TestIstioRouter_Finalize(t *testing.T) { Http: []istiov1alpha3.HTTPRoute{ { Match: nil, - Route: []istiov1alpha3.DestinationWeight{ + Route: []istiov1alpha3.HTTPRouteDestination{ { Destination: istiov1alpha3.Destination{Host: "podinfo"}, },