add session affinity support for weighted routing with istio

Add `.spec.analysis.sessionAffinity` to configure session affinity for
weighted routing. Add support for session affinity in the Istio router,
using the `Set-Cookie` and `Cookie` headers.

Signed-off-by: Sanskar Jaiswal <jaiswalsanskar078@gmail.com>
This commit is contained in:
Sanskar Jaiswal
2022-11-10 13:17:16 +05:30
parent 882286dce7
commit a496b99d6e
10 changed files with 296 additions and 47 deletions
+18
View File
@@ -1020,6 +1020,18 @@ spec:
type: object
additionalProperties:
type: string
sessionAffinity:
description: SessionAffinity represents the session affinity settings for a canary run.
type: object
required: [ "cookieName" ]
properties:
cookieName:
description: CookieName is the key that will be used for the session affinity cookie.
type: string
maxAge:
description: MaxAge indicates the number of seconds until the session affinity cookie will expire.
default: 86400
type: number
status:
description: CanaryStatus defines the observed state of a canary.
type: object
@@ -1064,6 +1076,12 @@ spec:
description: LastTransitionTime of this canary
format: date-time
type: string
sessionAffinityCookie:
description: Session affinity cookie of the current canary run
type: string
previousSessionAffinityCookie:
description: Session affinity cookie of the previous canary run
type: string
conditions:
description: Status conditions of this canary
type: array
+18
View File
@@ -1020,6 +1020,18 @@ spec:
type: object
additionalProperties:
type: string
sessionAffinity:
description: SessionAffinity represents the session affinity settings for a canary run.
type: object
required: [ "cookieName" ]
properties:
cookieName:
description: CookieName is the key that will be used for the session affinity cookie.
type: string
maxAge:
description: MaxAge indicates the number of seconds until the session affinity cookie will expire.
default: 86400
type: number
status:
description: CanaryStatus defines the observed state of a canary.
type: object
@@ -1064,6 +1076,12 @@ spec:
description: LastTransitionTime of this canary
format: date-time
type: string
sessionAffinityCookie:
description: Session affinity cookie of the current canary run
type: string
previousSessionAffinityCookie:
description: Session affinity cookie of the previous canary run
type: string
conditions:
description: Status conditions of this canary
type: array
+18
View File
@@ -1020,6 +1020,18 @@ spec:
type: object
additionalProperties:
type: string
sessionAffinity:
description: SessionAffinity represents the session affinity settings for a canary run.
type: object
required: [ "cookieName" ]
properties:
cookieName:
description: CookieName is the key that will be used for the session affinity cookie.
type: string
maxAge:
description: MaxAge indicates the number of seconds until the session affinity cookie will expire.
default: 86400
type: number
status:
description: CanaryStatus defines the observed state of a canary.
type: object
@@ -1064,6 +1076,12 @@ spec:
description: LastTransitionTime of this canary
format: date-time
type: string
sessionAffinityCookie:
description: Session affinity cookie of the current canary run
type: string
previousSessionAffinityCookie:
description: Session affinity cookie of the previous canary run
type: string
conditions:
description: Status conditions of this canary
type: array
+23
View File
@@ -262,6 +262,20 @@ type CanaryAnalysis struct {
// A/B testing HTTP header match conditions
// +optional
Match []istiov1alpha3.HTTPMatchRequest `json:"match,omitempty"`
// SessionAffinity represents the session affinity settings for a canary run.
// +optional
SessionAffinity *SessionAffinity `json:"sessionAffinity,omitempty"`
}
type SessionAffinity struct {
// CookieName is the key that will be used for the session affinity cookie.
CookieName string `json:"cookieName,omitempty"`
// MaxAge indicates the number of seconds until the session affinity cookie will expire.
// ref: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie#attributes
// The default value is 86,400 seconds, i.e. a day.
// +optional
MaxAge int `json:"maxAge,omitempty"`
}
// CanaryMetric holds the reference to metrics used for canary analysis
@@ -437,6 +451,15 @@ type CustomMetadata struct {
Annotations map[string]string `json:"annotations,omitempty"`
}
// GetMaxAge returns the max age of a cookie in seconds.
func (s *SessionAffinity) GetMaxAge() int {
if s.MaxAge == 0 {
// 24 hours * 60 mins * 60 seconds
return 86400
}
return s.MaxAge
}
// GetServiceNames returns the apex, primary and canary Kubernetes service names
func (c *Canary) GetServiceNames() (apexName, primaryName, canaryName string) {
apexName = c.Spec.TargetRef.Name
+4
View File
@@ -74,6 +74,10 @@ type CanaryStatus struct {
CanaryWeight int `json:"canaryWeight"`
Iterations int `json:"iterations"`
// +optional
PreviousSessionAffinityCookie string `json:"previousSessionAffinityCookie,omitempty"`
// +optional
SessionAffinityCookie string `json:"sessionAffinityCookie,omitempty"`
// +optional
TrackedConfigs *map[string]string `json:"trackedConfigs,omitempty"`
// +optional
LastAppliedSpec string `json:"lastAppliedSpec,omitempty"`
@@ -263,6 +263,11 @@ func (in *CanaryAnalysis) DeepCopyInto(out *CanaryAnalysis) {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.SessionAffinity != nil {
in, out := &in.SessionAffinity, &out.SessionAffinity
*out = new(SessionAffinity)
**out = **in
}
return
}
@@ -815,3 +820,19 @@ func (in *MetricTemplateStatus) DeepCopy() *MetricTemplateStatus {
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SessionAffinity) DeepCopyInto(out *SessionAffinity) {
*out = *in
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SessionAffinity.
func (in *SessionAffinity) DeepCopy() *SessionAffinity {
if in == nil {
return nil
}
out := new(SessionAffinity)
in.DeepCopyInto(out)
return out
}
+10 -3
View File
@@ -311,6 +311,10 @@ type Destination struct {
// Describes match conditions and actions for routing HTTP/1.1, HTTP2, and
// gRPC traffic. See VirtualService for usage examples.
type HTTPRoute struct {
// The name assigned to the route for debugging purposes. The routes name will
// be concatenated with the matchs name and will be logged in the access logs
// for requests matching this route/match.
Name string `json:"name,omitempty"`
// Match conditions to be satisfied for the rule to be
// activated. All conditions inside a single match block have AND
// semantics, while the list of match blocks have OR semantics. The rule
@@ -321,7 +325,7 @@ type HTTPRoute struct {
// forwarding target can be one of several versions of a service (see
// glossary in beginning of document). Weights associated with the
// service version determine the proportion of traffic it receives.
Route []DestinationWeight `json:"route,omitempty"`
Route []HTTPRouteDestination `json:"route,omitempty"`
// A http rule can either redirect or forward (default) traffic. If
// traffic passthrough option is specified in the rule,
@@ -528,7 +532,7 @@ type HTTPMatchRequest struct {
SourceNamespace string `json:"sourceNamespace,omitempty"`
}
type DestinationWeight struct {
type HTTPRouteDestination struct {
// REQUIRED. Destination uniquely identifies the instances of a service
// to which the request/connection should be forwarded to.
Destination Destination `json:"destination"`
@@ -538,6 +542,9 @@ type DestinationWeight struct {
// If there is only destination in a rule, the weight value is assumed to
// be 100.
Weight int `json:"weight"`
// Header manipulation rules
Headers *Headers `json:"headers,omitempty"`
}
// PortSelector specifies the number of a port to be used for
@@ -590,7 +597,7 @@ type TCPRoute struct {
// Currently, only one destination is allowed for TCP services. When TCP
// weighted routing support is introduced in Envoy, multiple destinations
// with weights can be specified.
Route DestinationWeight `json:"route"`
Route HTTPRouteDestination `json:"route"`
}
// L4 connection match attributes. Note that L4 connection matching support
@@ -229,23 +229,6 @@ func (in *DestinationRuleSpec) DeepCopy() *DestinationRuleSpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DestinationWeight) DeepCopyInto(out *DestinationWeight) {
*out = *in
in.Destination.DeepCopyInto(&out.Destination)
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DestinationWeight.
func (in *DestinationWeight) DeepCopy() *DestinationWeight {
if in == nil {
return nil
}
out := new(DestinationWeight)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Distribute) DeepCopyInto(out *Distribute) {
*out = *in
@@ -456,7 +439,7 @@ func (in *HTTPRoute) DeepCopyInto(out *HTTPRoute) {
}
if in.Route != nil {
in, out := &in.Route, &out.Route
*out = make([]DestinationWeight, len(*in))
*out = make([]HTTPRouteDestination, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
@@ -514,6 +497,28 @@ func (in *HTTPRoute) DeepCopy() *HTTPRoute {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *HTTPRouteDestination) DeepCopyInto(out *HTTPRouteDestination) {
*out = *in
in.Destination.DeepCopyInto(&out.Destination)
if in.Headers != nil {
in, out := &in.Headers, &out.Headers
*out = new(Headers)
(*in).DeepCopyInto(*out)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPRouteDestination.
func (in *HTTPRouteDestination) DeepCopy() *HTTPRouteDestination {
if in == nil {
return nil
}
out := new(HTTPRouteDestination)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *HTTPSettings) DeepCopyInto(out *HTTPSettings) {
*out = *in
+156 -21
View File
@@ -20,6 +20,8 @@ import (
"context"
"encoding/json"
"fmt"
"math/rand"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
@@ -30,6 +32,7 @@ import (
"k8s.io/client-go/kubernetes"
flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
istiov1alpha1 "github.com/fluxcd/flagger/pkg/apis/istio/common/v1alpha1"
istiov1alpha3 "github.com/fluxcd/flagger/pkg/apis/istio/v1alpha3"
clientset "github.com/fluxcd/flagger/pkg/client/clientset/versioned"
)
@@ -43,6 +46,13 @@ type IstioRouter struct {
setOwnerRefs bool
}
const cookieHeader = "Cookie"
const setCookieHeader = "Set-Cookie"
const stickyRouteName = "sticky-route"
const maxAgeAttr = "Max-Age"
var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
// Reconcile creates or updates the Istio virtual service and destination rules
func (ir *IstioRouter) Reconcile(canary *flaggerv1.Canary) error {
_, primaryName, canaryName := canary.GetServiceNames()
@@ -153,7 +163,7 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
}
// create destinations with primary weight 100% and canary weight 0%
canaryRoute := []istiov1alpha3.DestinationWeight{
canaryRoute := []istiov1alpha3.HTTPRouteDestination{
makeDestination(canary, primaryName, 100),
makeDestination(canary, canaryName, 0),
}
@@ -199,7 +209,7 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
Retries: canary.Spec.Service.Retries,
CorsPolicy: canary.Spec.Service.CorsPolicy,
Headers: canary.Spec.Service.Headers,
Route: []istiov1alpha3.DestinationWeight{
Route: []istiov1alpha3.HTTPRouteDestination{
makeDestination(canary, primaryName, 100),
},
},
@@ -255,13 +265,28 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
virtualService.Spec.Hosts = []string{}
}
ignoreCmpOptions := []cmp.Option{
cmpopts.IgnoreFields(istiov1alpha3.HTTPRouteDestination{}, "Weight"),
cmpopts.IgnoreFields(istiov1alpha3.HTTPRoute{}, "Mirror", "MirrorPercentage"),
}
if canary.Spec.Analysis.SessionAffinity != nil {
// We ignore this route as this does not do weighted routing and is handled exclusively
// by SetRoutes().
ignoreSlice := cmpopts.IgnoreSliceElements(func(t istiov1alpha3.HTTPRoute) bool {
if t.Name == stickyRouteName {
return true
}
return false
})
ignoreCmpOptions = append(ignoreCmpOptions, ignoreSlice)
ignoreCmpOptions = append(ignoreCmpOptions, cmpopts.IgnoreFields(istiov1alpha3.HTTPRouteDestination{}, "Headers"))
}
// update service but keep the original destination weights and mirror
if virtualService != nil {
if diff := cmp.Diff(
newSpec,
virtualService.Spec,
cmpopts.IgnoreFields(istiov1alpha3.DestinationWeight{}, "Weight"),
cmpopts.IgnoreFields(istiov1alpha3.HTTPRoute{}, "Mirror", "MirrorPercentage"),
ignoreCmpOptions...,
); diff != "" {
vtClone := virtualService.DeepCopy()
vtClone.Spec = newSpec
@@ -333,6 +358,23 @@ func (ir *IstioRouter) GetRoutes(canary *flaggerv1.Canary) (
mirrored = true
}
if canary.Spec.Analysis.SessionAffinity != nil {
for _, http := range vs.Spec.Http {
for _, routeDest := range http.Route {
// we are interested in the route that sets the cookie as that's the route
// that does weighted routing.
if routeDest.Headers != nil {
if routeDest.Destination.Host == primaryName {
primaryWeight = routeDest.Weight
}
if routeDest.Destination.Host == canaryName {
canaryWeight = routeDest.Weight
}
}
}
}
}
if primaryWeight == 0 && canaryWeight == 0 {
err = fmt.Errorf("VirtualService %s.%s does not contain routes for %s-primary and %s-canary",
apexName, canary.Namespace, apexName, apexName)
@@ -358,20 +400,103 @@ func (ir *IstioRouter) SetRoutes(
vsCopy := vs.DeepCopy()
// weighted routing (progressive canary)
vsCopy.Spec.Http = []istiov1alpha3.HTTPRoute{
{
Match: canary.Spec.Service.Match,
Rewrite: canary.Spec.Service.Rewrite,
Timeout: canary.Spec.Service.Timeout,
Retries: canary.Spec.Service.Retries,
CorsPolicy: canary.Spec.Service.CorsPolicy,
Headers: canary.Spec.Service.Headers,
Route: []istiov1alpha3.DestinationWeight{
makeDestination(canary, primaryName, primaryWeight),
makeDestination(canary, canaryName, canaryWeight),
},
weightedRoute := istiov1alpha3.HTTPRoute{
Match: canary.Spec.Service.Match,
Rewrite: canary.Spec.Service.Rewrite,
Timeout: canary.Spec.Service.Timeout,
Retries: canary.Spec.Service.Retries,
CorsPolicy: canary.Spec.Service.CorsPolicy,
Headers: canary.Spec.Service.Headers,
Route: []istiov1alpha3.HTTPRouteDestination{
makeDestination(canary, primaryName, primaryWeight),
makeDestination(canary, canaryName, canaryWeight),
},
}
vsCopy.Spec.Http = []istiov1alpha3.HTTPRoute{
weightedRoute,
}
if canary.Spec.Analysis.SessionAffinity != nil {
// If a canary run is active, we want all responses corresponding to requests hitting the canary deployment
// (due to weighted routing) to include a `Set-Cookie` header. All requests that have the `Cookie` header
// and match the value of the `Set-Cookie` header will be routed to the canary deployment.
stickyRoute := weightedRoute
stickyRoute.Name = stickyRouteName
if canaryWeight != 0 {
if canary.Status.SessionAffinityCookie == "" {
canary.Status.SessionAffinityCookie = fmt.Sprintf("%s=%s", canary.Spec.Analysis.SessionAffinity.CookieName, randSeq())
}
for i, routeDest := range weightedRoute.Route {
if routeDest.Destination.Host == canaryName {
if routeDest.Headers == nil {
routeDest.Headers = &istiov1alpha3.Headers{
Response: &istiov1alpha3.HeaderOperations{},
}
}
routeDest.Headers.Response.Add = map[string]string{
setCookieHeader: fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr,
canary.Spec.Analysis.SessionAffinity.GetMaxAge(),
),
}
}
weightedRoute.Route[i] = routeDest
}
cookieMatch := istiov1alpha3.HTTPMatchRequest{
Headers: map[string]istiov1alpha1.StringMatch{
cookieHeader: {
Exact: canary.Status.SessionAffinityCookie,
},
},
}
canaryMatch := mergeMatchConditions([]istiov1alpha3.HTTPMatchRequest{cookieMatch}, canary.Spec.Service.Match)
stickyRoute.Match = canaryMatch
stickyRoute.Route = []istiov1alpha3.HTTPRouteDestination{
makeDestination(canary, primaryName, 0),
makeDestination(canary, canaryName, 100),
}
} else {
// If canary weight is 0 and SessionAffinityCookie is non-blank, then it belongs to a previous canary run.
if canary.Status.SessionAffinityCookie != "" {
canary.Status.PreviousSessionAffinityCookie = canary.Status.SessionAffinityCookie
}
previousCookie := canary.Status.PreviousSessionAffinityCookie
// Match against the previous session cookie and delete that cookie
if previousCookie != "" {
cookieMatch := istiov1alpha3.HTTPMatchRequest{
Headers: map[string]istiov1alpha1.StringMatch{
cookieHeader: {
Exact: previousCookie,
},
},
}
canaryMatch := mergeMatchConditions([]istiov1alpha3.HTTPMatchRequest{cookieMatch}, canary.Spec.Service.Match)
stickyRoute.Match = canaryMatch
if stickyRoute.Headers == nil {
stickyRoute.Headers = &istiov1alpha3.Headers{
Response: &istiov1alpha3.HeaderOperations{
Add: map[string]string{},
},
}
} else if stickyRoute.Headers.Response == nil {
stickyRoute.Headers.Response = &istiov1alpha3.HeaderOperations{
Add: map[string]string{},
}
} else if stickyRoute.Headers.Response.Add == nil {
stickyRoute.Headers.Response.Add = map[string]string{}
}
stickyRoute.Headers.Response.Add[setCookieHeader] = fmt.Sprintf("%s; %s=%d", previousCookie, maxAgeAttr, -1)
}
canary.Status.SessionAffinityCookie = ""
}
vsCopy.Spec.Http = []istiov1alpha3.HTTPRoute{
stickyRoute, weightedRoute,
}
}
if mirrored {
vsCopy.Spec.Http[0].Mirror = &istiov1alpha3.Destination{
@@ -395,7 +520,7 @@ func (ir *IstioRouter) SetRoutes(
Retries: canary.Spec.Service.Retries,
CorsPolicy: canary.Spec.Service.CorsPolicy,
Headers: canary.Spec.Service.Headers,
Route: []istiov1alpha3.DestinationWeight{
Route: []istiov1alpha3.HTTPRouteDestination{
makeDestination(canary, primaryName, primaryWeight),
makeDestination(canary, canaryName, canaryWeight),
},
@@ -407,7 +532,7 @@ func (ir *IstioRouter) SetRoutes(
Retries: canary.Spec.Service.Retries,
CorsPolicy: canary.Spec.Service.CorsPolicy,
Headers: canary.Spec.Service.Headers,
Route: []istiov1alpha3.DestinationWeight{
Route: []istiov1alpha3.HTTPRouteDestination{
makeDestination(canary, primaryName, primaryWeight),
},
},
@@ -483,8 +608,8 @@ func mergeMatchConditions(canary, defaults []istiov1alpha3.HTTPMatchRequest) []i
}
// makeDestination returns a an destination weight for the specified host
func makeDestination(canary *flaggerv1.Canary, host string, weight int) istiov1alpha3.DestinationWeight {
dest := istiov1alpha3.DestinationWeight{
func makeDestination(canary *flaggerv1.Canary, host string, weight int) istiov1alpha3.HTTPRouteDestination {
dest := istiov1alpha3.HTTPRouteDestination{
Destination: istiov1alpha3.Destination{
Host: host,
},
@@ -495,7 +620,7 @@ func makeDestination(canary *flaggerv1.Canary, host string, weight int) istiov1a
if canary.Spec.Service.PortDiscovery &&
(len(canary.Spec.Service.Gateways) > 0 &&
canary.Spec.Service.Gateways[0] != "mesh" || canary.Spec.Service.Delegation) {
dest = istiov1alpha3.DestinationWeight{
dest = istiov1alpha3.HTTPRouteDestination{
Destination: istiov1alpha3.Destination{
Host: host,
Port: &istiov1alpha3.PortSelector{
@@ -508,3 +633,13 @@ func makeDestination(canary *flaggerv1.Canary, host string, weight int) istiov1a
return dest
}
func randSeq() string {
rand.Seed(time.Now().UnixNano())
b := make([]rune, 10)
for i := range b {
b[i] = letters[rand.Intn(len(letters))]
}
return string(b)
}
+5 -5
View File
@@ -122,7 +122,7 @@ func TestIstioRouter_SetRoutes(t *testing.T) {
vs, err := mocks.meshClient.NetworkingV1alpha3().VirtualServices("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
require.NoError(t, err)
var pRoute, cRoute istiov1alpha3.DestinationWeight
var pRoute, cRoute istiov1alpha3.HTTPRouteDestination
var mirror *istiov1alpha3.Destination
for _, http := range vs.Spec.Http {
for _, route := range http.Route {
@@ -154,7 +154,7 @@ func TestIstioRouter_SetRoutes(t *testing.T) {
vs, err := mocks.meshClient.NetworkingV1alpha3().VirtualServices("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
require.NoError(t, err)
var pRoute, cRoute istiov1alpha3.DestinationWeight
var pRoute, cRoute istiov1alpha3.HTTPRouteDestination
var mirror *istiov1alpha3.Destination
var mirrorWeight *istiov1alpha3.Percent
for _, http := range vs.Spec.Http {
@@ -310,8 +310,8 @@ func TestIstioRouter_ABTest(t *testing.T) {
pHost := fmt.Sprintf("%s-primary", mocks.abtest.Spec.TargetRef.Name)
cHost := fmt.Sprintf("%s-canary", mocks.abtest.Spec.TargetRef.Name)
pRoute := istiov1alpha3.DestinationWeight{}
cRoute := istiov1alpha3.DestinationWeight{}
pRoute := istiov1alpha3.HTTPRouteDestination{}
cRoute := istiov1alpha3.HTTPRouteDestination{}
var mirror *istiov1alpha3.Destination
for _, http := range vs.Spec.Http {
@@ -427,7 +427,7 @@ func TestIstioRouter_Finalize(t *testing.T) {
Http: []istiov1alpha3.HTTPRoute{
{
Match: nil,
Route: []istiov1alpha3.DestinationWeight{
Route: []istiov1alpha3.HTTPRouteDestination{
{
Destination: istiov1alpha3.Destination{Host: "podinfo"},
},