diff --git a/README.md b/README.md index 7e7c9539..1c41e026 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ spec: progressDeadlineSeconds: 60 # HPA reference (optional) autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: diff --git a/artifacts/examples/appmesh-abtest.yaml b/artifacts/examples/appmesh-abtest.yaml index 6fddb07e..726fcab6 100644 --- a/artifacts/examples/appmesh-abtest.yaml +++ b/artifacts/examples/appmesh-abtest.yaml @@ -11,7 +11,7 @@ spec: kind: Deployment name: podinfo autoscalerRef: - apiVersion: autoscaling/v2beta1 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: diff --git a/artifacts/examples/appmesh-canary.yaml b/artifacts/examples/appmesh-canary.yaml index dcf40c43..16741963 100644 --- a/artifacts/examples/appmesh-canary.yaml +++ b/artifacts/examples/appmesh-canary.yaml @@ -11,7 +11,7 @@ spec: kind: Deployment name: podinfo autoscalerRef: - apiVersion: autoscaling/v2beta1 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: diff --git a/artifacts/examples/istio-abtest.yaml b/artifacts/examples/istio-abtest.yaml index 1586eac2..abf7c14d 100644 --- a/artifacts/examples/istio-abtest.yaml +++ b/artifacts/examples/istio-abtest.yaml @@ -10,7 +10,7 @@ spec: kind: Deployment name: podinfo autoscalerRef: - apiVersion: autoscaling/v2beta1 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: diff --git a/artifacts/examples/istio-canary.yaml b/artifacts/examples/istio-canary.yaml index f8d41704..da5870a8 100644 --- a/artifacts/examples/istio-canary.yaml +++ b/artifacts/examples/istio-canary.yaml @@ -11,7 +11,7 @@ spec: kind: Deployment name: podinfo autoscalerRef: - apiVersion: autoscaling/v2beta1 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: diff --git a/artifacts/examples/linkerd-canary-steps.yaml b/artifacts/examples/linkerd-canary-steps.yaml index 1e1a09af..9110271c 100644 --- a/artifacts/examples/linkerd-canary-steps.yaml +++ b/artifacts/examples/linkerd-canary-steps.yaml @@ -11,7 +11,7 @@ spec: kind: Deployment name: podinfo autoscalerRef: - apiVersion: autoscaling/v2beta1 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: diff --git a/artifacts/examples/linkerd-canary.yaml b/artifacts/examples/linkerd-canary.yaml index 63884b9f..7c85e7bc 100644 --- a/artifacts/examples/linkerd-canary.yaml +++ b/artifacts/examples/linkerd-canary.yaml @@ -11,7 +11,7 @@ spec: kind: Deployment name: podinfo autoscalerRef: - apiVersion: autoscaling/v2beta1 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: diff --git a/charts/podinfo/templates/canary.yaml b/charts/podinfo/templates/canary.yaml index bb644142..bb006154 100644 --- a/charts/podinfo/templates/canary.yaml +++ b/charts/podinfo/templates/canary.yaml @@ -14,7 +14,7 @@ spec: kind: Deployment name: {{ template "podinfo.fullname" . }} autoscalerRef: - apiVersion: autoscaling/v2beta1 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: {{ template "podinfo.fullname" . }} service: @@ -57,4 +57,4 @@ spec: metadata: cmd: "hey -z 1m -q 5 -c 2 http://{{ template "podinfo.fullname" . }}.{{ .Release.Namespace }}:{{ .Values.service.port }}" {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/podinfo/templates/hpa.yaml b/charts/podinfo/templates/hpa.yaml index 8526f162..224fb344 100644 --- a/charts/podinfo/templates/hpa.yaml +++ b/charts/podinfo/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.hpa.enabled -}} -apiVersion: autoscaling/v2beta1 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ template "podinfo.fullname" . }} diff --git a/docs/gitbook/tutorials/appmesh-progressive-delivery.md b/docs/gitbook/tutorials/appmesh-progressive-delivery.md index 1188a8b2..a7013d89 100644 --- a/docs/gitbook/tutorials/appmesh-progressive-delivery.md +++ b/docs/gitbook/tutorials/appmesh-progressive-delivery.md @@ -80,7 +80,7 @@ spec: progressDeadlineSeconds: 60 # HPA reference (optional) autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: diff --git a/docs/gitbook/tutorials/contour-progressive-delivery.md b/docs/gitbook/tutorials/contour-progressive-delivery.md index 29abec2d..de7afad6 100644 --- a/docs/gitbook/tutorials/contour-progressive-delivery.md +++ b/docs/gitbook/tutorials/contour-progressive-delivery.md @@ -76,7 +76,7 @@ spec: name: podinfo # HPA reference autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: diff --git a/docs/gitbook/tutorials/gatewayapi-progressive-delivery.md b/docs/gitbook/tutorials/gatewayapi-progressive-delivery.md index a4925877..fc6f9683 100644 --- a/docs/gitbook/tutorials/gatewayapi-progressive-delivery.md +++ b/docs/gitbook/tutorials/gatewayapi-progressive-delivery.md @@ -137,7 +137,7 @@ Save the above resource as metric-templates.yaml and then apply it: kubectl apply -f metric-templates.yaml ``` -Create a canary custom resource \(replace "loaclproject.contour.io" with your own domain\): +Create a canary custom resource \(replace "localproject.contour.io" with your own domain\): ```yaml apiVersion: flagger.app/v1beta1 @@ -156,7 +156,7 @@ spec: progressDeadlineSeconds: 60 # HPA reference (optional) autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: @@ -382,13 +382,15 @@ Events: Warning Synced 1m flagger Canary failed! Scaling down podinfo.test ``` -# A/B Testing +## Session Affinity -Besides weighted routing, Flagger can be configured to route traffic to the canary based on HTTP match conditions. In an A/B testing scenario, you'll be using HTTP headers or cookies to target a certain segment of your users. This is particularly useful for frontend applications that require session affinity. +While Flagger can perform weighted routing and A/B testing individually, with Gateway API it can combine the two leading to a Canary +release with session affinity. +For more information you can read the [deployment strategies docs](../usage/deployment-strategies.md#canary-release-with-session-affinity). -![Flagger A/B Testing Stages](https://raw.githubusercontent.com/fluxcd/flagger/main/docs/diagrams/flagger-abtest-steps.png) +> **Note:** The implementation must have support for the [`ResponseHeaderModifier`](https://github.com/kubernetes-sigs/gateway-api/blob/3d22aa5a08413222cb79e6b2e245870360434614/apis/v1beta1/httproute_types.go#L651) API. -Create a canary custom resource \(replace "loaclproject.contour.io" with your own domain\): +Create a canary custom resource \(replace localproject.contour.io with your own domain\): ```yaml apiVersion: flagger.app/v1beta1 @@ -407,7 +409,116 @@ spec: progressDeadlineSeconds: 60 # HPA reference (optional) autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 + kind: HorizontalPodAutoscaler + name: podinfo + service: + # service port number + port: 9898 + # container port number or name (optional) + targetPort: 9898 + # Gateway API HTTPRoute host names + hosts: + - localproject.contour.io + # Reference to the Gateway that the generated HTTPRoute would attach to. + gatewayRefs: + - name: contour + namespace: projectcontour + analysis: + # schedule interval (default 60s) + interval: 1m + # max number of failed metric checks before rollback + threshold: 5 + # max traffic percentage routed to canary + # percentage (0-100) + maxWeight: 50 + # canary increment step + # percentage (0-100) + stepWeight: 10 + # session affinity config + sessionAffinity: + # name of the cookie used + cookieName: flagger-cookie + # max age of the cookie (in seconds) + # optional; defaults to 86400 + maxAge: 21600 + metrics: + - name: error-rate + # max error rate (5xx responses) + # percentage (0-100) + templateRef: + name: error-rate + namespace: flagger-system + thresholdRange: + max: 1 + interval: 1m + - name: latency + templateRef: + name: latency + namespace: flagger-system + # seconds + thresholdRange: + max: 0.5 + interval: 30s + # testing (optional) + webhooks: + - name: smoke-test + type: pre-rollout + url: http://flagger-loadtester.test/ + timeout: 15s + metadata: + type: bash + cmd: "curl -sd 'anon' http://podinfo-canary.test:9898/token | grep token" + - name: load-test + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + cmd: "hey -z 2m -q 10 -c 2 -host localproject.contour.io http://envoy.projectcontour/" +``` + +Save the above resource as podinfo-canary-session-affinity.yaml and then apply it: + +```bash +kubectl apply -f ./podinfo-canary-session-affinity.yaml +``` + +Trigger a canary deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=ghcr.io/stefanprodan/podinfo:6.0.1 +``` + +You can load `localproject.contour.io` in your browser and refresh it until you see the requests being served by `podinfo:6.0.1`. +All subsequent requests after that will be served by `podinfo:6.0.1` and not `podinfo:6.0.0` because of the session affinity +configured by Flagger in the HTTPRoute object. + +# A/B Testing + +Besides weighted routing, Flagger can be configured to route traffic to the canary based on HTTP match conditions. In an A/B testing scenario, you'll be using HTTP headers or cookies to target a certain segment of your users. This is particularly useful for frontend applications that require session affinity. + +![Flagger A/B Testing Stages](https://raw.githubusercontent.com/fluxcd/flagger/main/docs/diagrams/flagger-abtest-steps.png) + +Create a canary custom resource \(replace "localproject.contour.io" with your own domain\): + +```yaml +apiVersion: flagger.app/v1beta1 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + # deployment reference + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + # the maximum time in seconds for the canary deployment + # to make progress before it is rollback (default 600s) + progressDeadlineSeconds: 60 + # HPA reference (optional) + autoscalerRef: + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: diff --git a/docs/gitbook/tutorials/gloo-progressive-delivery.md b/docs/gitbook/tutorials/gloo-progressive-delivery.md index ccccb331..fcc1f062 100644 --- a/docs/gitbook/tutorials/gloo-progressive-delivery.md +++ b/docs/gitbook/tutorials/gloo-progressive-delivery.md @@ -110,7 +110,7 @@ spec: name: podinfo # HPA reference (optional) autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: diff --git a/docs/gitbook/tutorials/istio-ab-testing.md b/docs/gitbook/tutorials/istio-ab-testing.md index 2e1241b8..d4b276fc 100644 --- a/docs/gitbook/tutorials/istio-ab-testing.md +++ b/docs/gitbook/tutorials/istio-ab-testing.md @@ -84,7 +84,7 @@ spec: progressDeadlineSeconds: 60 # HPA reference (optional) autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: diff --git a/docs/gitbook/tutorials/istio-progressive-delivery.md b/docs/gitbook/tutorials/istio-progressive-delivery.md index 49aa55bc..cefbf3fb 100644 --- a/docs/gitbook/tutorials/istio-progressive-delivery.md +++ b/docs/gitbook/tutorials/istio-progressive-delivery.md @@ -85,7 +85,7 @@ spec: progressDeadlineSeconds: 60 # HPA reference (optional) autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: @@ -316,7 +316,7 @@ spec: progressDeadlineSeconds: 60 # HPA reference (optional) autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: diff --git a/docs/gitbook/tutorials/kubernetes-blue-green.md b/docs/gitbook/tutorials/kubernetes-blue-green.md index 9a3cf3d7..ad8db524 100644 --- a/docs/gitbook/tutorials/kubernetes-blue-green.md +++ b/docs/gitbook/tutorials/kubernetes-blue-green.md @@ -84,7 +84,7 @@ spec: progressDeadlineSeconds: 60 # HPA reference (optional) autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: diff --git a/docs/gitbook/tutorials/linkerd-progressive-delivery.md b/docs/gitbook/tutorials/linkerd-progressive-delivery.md index 69bfae57..a3b11748 100644 --- a/docs/gitbook/tutorials/linkerd-progressive-delivery.md +++ b/docs/gitbook/tutorials/linkerd-progressive-delivery.md @@ -98,7 +98,7 @@ spec: name: podinfo # HPA reference (optional) autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo # the maximum time in seconds for the canary deployment @@ -426,7 +426,7 @@ spec: kind: Deployment name: podinfo autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo service: diff --git a/docs/gitbook/tutorials/nginx-progressive-delivery.md b/docs/gitbook/tutorials/nginx-progressive-delivery.md index 384b1551..b61c5c8c 100644 --- a/docs/gitbook/tutorials/nginx-progressive-delivery.md +++ b/docs/gitbook/tutorials/nginx-progressive-delivery.md @@ -110,7 +110,7 @@ spec: name: podinfo # HPA reference (optional) autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo # the maximum time in seconds for the canary deployment diff --git a/docs/gitbook/tutorials/osm-progressive-delivery.md b/docs/gitbook/tutorials/osm-progressive-delivery.md index 36292f20..9f1edbe1 100644 --- a/docs/gitbook/tutorials/osm-progressive-delivery.md +++ b/docs/gitbook/tutorials/osm-progressive-delivery.md @@ -86,7 +86,7 @@ spec: name: podinfo # HPA reference (optional) autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo # the maximum time in seconds for the canary deployment diff --git a/docs/gitbook/tutorials/skipper-progressive-delivery.md b/docs/gitbook/tutorials/skipper-progressive-delivery.md index dc1a9c99..38380f09 100644 --- a/docs/gitbook/tutorials/skipper-progressive-delivery.md +++ b/docs/gitbook/tutorials/skipper-progressive-delivery.md @@ -113,7 +113,7 @@ spec: name: podinfo # HPA reference (optional) autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo # the maximum time in seconds for the canary deployment diff --git a/docs/gitbook/tutorials/traefik-progressive-delivery.md b/docs/gitbook/tutorials/traefik-progressive-delivery.md index 7a4be213..410333a3 100644 --- a/docs/gitbook/tutorials/traefik-progressive-delivery.md +++ b/docs/gitbook/tutorials/traefik-progressive-delivery.md @@ -103,7 +103,7 @@ spec: name: podinfo # HPA reference (optional) autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo # the maximum time in seconds for the canary deployment diff --git a/docs/gitbook/tutorials/zero-downtime-deployments.md b/docs/gitbook/tutorials/zero-downtime-deployments.md index 432ab38c..f99b205a 100644 --- a/docs/gitbook/tutorials/zero-downtime-deployments.md +++ b/docs/gitbook/tutorials/zero-downtime-deployments.md @@ -139,7 +139,7 @@ Note that without resource requests the horizontal pod autoscaler can't determin A production environment should be able to handle traffic bursts without impacting the quality of service. This can be achieved with Kubernetes autoscaling capabilities. Autoscaling in Kubernetes has two dimensions: the Cluster Autoscaler that deals with node scaling operations and the Horizontal Pod Autoscaler that automatically scales the number of pods in a deployment. ```yaml -apiVersion: autoscaling/v2beta2 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler spec: scaleTargetRef: diff --git a/docs/gitbook/usage/deployment-strategies.md b/docs/gitbook/usage/deployment-strategies.md index 5899b43f..d49b09d8 100644 --- a/docs/gitbook/usage/deployment-strategies.md +++ b/docs/gitbook/usage/deployment-strategies.md @@ -11,7 +11,7 @@ Flagger can run automated application analysis, promotion and rollback for the f * **Blue/Green Mirroring** \(traffic shadowing\) * Istio * **Canary Release with Session Affinity** \(progressive traffic shifting combined with cookie based routing\) - * Istio + * Istio, Gateway API For Canary releases and A/B testing you'll need a Layer 7 traffic management solution like a service mesh or an ingress controller. For Blue/Green deployments no service mesh or ingress controller is required. @@ -408,7 +408,7 @@ cookie based routing with regular weight based routing. This means once a user i version of our application (based on the traffic weights), they're always routed to that version, i.e. they're never routed back to the old version of our application. -You can enable this, by specifying `.spec.analsyis.sessionAffinity` in the Canary (only Istio is supported): +You can enable this, by specifying `.spec.analsyis.sessionAffinity` in the Canary: ```yaml analysis: diff --git a/docs/gitbook/usage/how-it-works.md b/docs/gitbook/usage/how-it-works.md index 2a586504..e418cf88 100644 --- a/docs/gitbook/usage/how-it-works.md +++ b/docs/gitbook/usage/how-it-works.md @@ -65,7 +65,7 @@ spec: kind: Deployment name: podinfo autoscalerRef: - apiVersion: autoscaling/v2beta2 + apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: podinfo primaryScalerReplicas: diff --git a/pkg/canary/deployment_controller.go b/pkg/canary/deployment_controller.go index ad95426a..3d605bad 100644 --- a/pkg/canary/deployment_controller.go +++ b/pkg/canary/deployment_controller.go @@ -20,10 +20,8 @@ import ( "context" "fmt" - "github.com/google/go-cmp/cmp" "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" - hpav2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -330,105 +328,6 @@ func (c *DeploymentController) createPrimaryDeployment(cd *flaggerv1.Canary, inc return nil } -func (c *DeploymentController) reconcilePrimaryHpa(cd *flaggerv1.Canary, init bool) error { - primaryName := fmt.Sprintf("%s-primary", cd.Spec.TargetRef.Name) - hpa, err := c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Get(context.TODO(), cd.Spec.AutoscalerRef.Name, metav1.GetOptions{}) - if err != nil { - return fmt.Errorf("HorizontalPodAutoscaler %s.%s get query error: %w", - cd.Spec.AutoscalerRef.Name, cd.Namespace, err) - } - - hpaSpec := hpav2.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: hpav2.CrossVersionObjectReference{ - Name: primaryName, - Kind: hpa.Spec.ScaleTargetRef.Kind, - APIVersion: hpa.Spec.ScaleTargetRef.APIVersion, - }, - MinReplicas: hpa.Spec.MinReplicas, - MaxReplicas: hpa.Spec.MaxReplicas, - Metrics: hpa.Spec.Metrics, - Behavior: hpa.Spec.Behavior, - } - - primaryHpaName := fmt.Sprintf("%s-primary", cd.Spec.AutoscalerRef.Name) - primaryHpa, err := c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Get(context.TODO(), primaryHpaName, metav1.GetOptions{}) - - // create HPA - if errors.IsNotFound(err) { - primaryHpa = &hpav2.HorizontalPodAutoscaler{ - ObjectMeta: metav1.ObjectMeta{ - Name: primaryHpaName, - Namespace: cd.Namespace, - Labels: filterMetadata(hpa.Labels), - OwnerReferences: []metav1.OwnerReference{ - *metav1.NewControllerRef(cd, schema.GroupVersionKind{ - Group: flaggerv1.SchemeGroupVersion.Group, - Version: flaggerv1.SchemeGroupVersion.Version, - Kind: flaggerv1.CanaryKind, - }), - }, - }, - Spec: hpaSpec, - } - - _, err = c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Create(context.TODO(), primaryHpa, metav1.CreateOptions{}) - if err != nil { - return fmt.Errorf("creating HorizontalPodAutoscaler %s.%s failed: %w", - primaryHpa.Name, primaryHpa.Namespace, err) - } - c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof( - "HorizontalPodAutoscaler %s.%s created", primaryHpa.GetName(), cd.Namespace) - return nil - } else if err != nil { - return fmt.Errorf("HorizontalPodAutoscaler %s.%s get query failed: %w", - primaryHpa.Name, primaryHpa.Namespace, err) - } - - // update HPA - if !init && primaryHpa != nil { - diffMetrics := cmp.Diff(hpaSpec.Metrics, primaryHpa.Spec.Metrics) - diffBehavior := cmp.Diff(hpaSpec.Behavior, primaryHpa.Spec.Behavior) - diffLabels := cmp.Diff(hpa.ObjectMeta.Labels, primaryHpa.ObjectMeta.Labels) - diffAnnotations := cmp.Diff(hpa.ObjectMeta.Annotations, primaryHpa.ObjectMeta.Annotations) - if diffMetrics != "" || diffBehavior != "" || diffLabels != "" || diffAnnotations != "" || int32Default(hpaSpec.MinReplicas) != int32Default(primaryHpa.Spec.MinReplicas) || hpaSpec.MaxReplicas != primaryHpa.Spec.MaxReplicas { - err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - primaryHpa, err := c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Get(context.TODO(), primaryHpaName, metav1.GetOptions{}) - if err != nil { - return err - } - hpaClone := primaryHpa.DeepCopy() - hpaClone.Spec.MaxReplicas = hpaSpec.MaxReplicas - hpaClone.Spec.MinReplicas = hpaSpec.MinReplicas - hpaClone.Spec.Metrics = hpaSpec.Metrics - hpaClone.Spec.Behavior = hpaSpec.Behavior - - // update hpa annotations - hpaClone.ObjectMeta.Annotations = make(map[string]string) - filteredAnnotations := includeLabelsByPrefix(hpa.ObjectMeta.Annotations, c.includeLabelPrefix) - for k, v := range filteredAnnotations { - hpaClone.ObjectMeta.Annotations[k] = v - } - // update hpa labels - hpaClone.ObjectMeta.Labels = make(map[string]string) - filteredLabels := includeLabelsByPrefix(hpa.ObjectMeta.Labels, c.includeLabelPrefix) - for k, v := range filteredLabels { - hpaClone.ObjectMeta.Labels[k] = v - } - - _, err = c.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Update(context.TODO(), hpaClone, metav1.UpdateOptions{}) - return err - }) - if err != nil { - return fmt.Errorf("updating HorizontalPodAutoscaler %s.%s failed: %w", - primaryHpa.Name, primaryHpa.Namespace, err) - } - c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)). - Infof("HorizontalPodAutoscaler %s.%s updated", primaryHpa.GetName(), cd.Namespace) - } - } - return nil -} - // getSelectorLabel returns the selector match label func (c *DeploymentController) getSelectorLabel(deployment *appsv1.Deployment) (string, string, error) { for _, l := range c.labels { diff --git a/pkg/router/gateway_api_v1beta1.go b/pkg/router/gateway_api_v1beta1.go index 19ed8c70..b966dc1f 100644 --- a/pkg/router/gateway_api_v1beta1.go +++ b/pkg/router/gateway_api_v1beta1.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "reflect" + "strings" flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" "github.com/fluxcd/flagger/pkg/apis/gatewayapi/v1beta1" @@ -162,10 +163,32 @@ func (gwr *GatewayAPIV1Beta1Router) Reconcile(canary *flaggerv1.Canary) error { return fmt.Errorf("HTTPRoute %s.%s get error: %w", apexSvcName, hrNamespace, err) } + ignoreCmpOptions := []cmp.Option{ + cmpopts.IgnoreFields(v1beta1.BackendRef{}, "Weight"), + cmpopts.EquateEmpty(), + } + if canary.Spec.Analysis.SessionAffinity != nil { + ignoreRoute := cmpopts.IgnoreSliceElements(func(r v1beta1.HTTPRouteRule) bool { + // Ignore the rule that does sticky routing, i.e. matches against the `Cookie` header. + for _, match := range r.Matches { + for _, headerMatch := range match.Headers { + if *headerMatch.Type == v1beta1HeaderMatchRegex && headerMatch.Name == cookieHeader && + strings.Contains(headerMatch.Value, canary.Spec.Analysis.SessionAffinity.CookieName) { + return true + } + } + } + return false + }) + ignoreCmpOptions = append(ignoreCmpOptions, ignoreRoute) + // Ignore backend specific filters, since we use that to insert the `Set-Cookie` header in responses. + ignoreCmpOptions = append(ignoreCmpOptions, cmpopts.IgnoreFields(v1beta1.HTTPBackendRef{}, "Filters")) + } + if httpRoute != nil { specDiff := cmp.Diff( httpRoute.Spec, httpRouteSpec, - cmpopts.IgnoreFields(v1beta1.BackendRef{}, "Weight"), + ignoreCmpOptions..., ) labelsDiff := cmp.Diff(newMetadata.Labels, httpRoute.Labels, cmpopts.EquateEmpty()) annotationsDiff := cmp.Diff(newMetadata.Annotations, httpRoute.Annotations, cmpopts.EquateEmpty()) @@ -200,7 +223,19 @@ func (gwr *GatewayAPIV1Beta1Router) GetRoutes(canary *flaggerv1.Canary) ( err = fmt.Errorf("HTTPRoute %s.%s get error: %w", apexSvcName, hrNamespace, err) return } + var weightedRule *v1beta1.HTTPRouteRule for _, rule := range httpRoute.Spec.Rules { + // If session affinity is enabled, then we are only interested in the rule + // that has backend-specific filters, as that's the rule that does weighted + // routing. + if canary.Spec.Analysis.SessionAffinity != nil { + for _, backendRef := range rule.BackendRefs { + if len(backendRef.Filters) > 0 { + weightedRule = &rule + } + } + } + // A/B testing: Avoid reading the rule with only for backendRef. if len(rule.BackendRefs) == 2 { for _, backendRef := range rule.BackendRefs { @@ -212,7 +247,17 @@ func (gwr *GatewayAPIV1Beta1Router) GetRoutes(canary *flaggerv1.Canary) ( } } } + } + if weightedRule != nil { + for _, backendRef := range weightedRule.BackendRefs { + if backendRef.Name == v1beta1.ObjectName(primarySvcName) { + primaryWeight = int(*backendRef.Weight) + } + if backendRef.Name == v1beta1.ObjectName(canarySvcName) { + canaryWeight = int(*backendRef.Weight) + } + } } return } @@ -248,25 +293,35 @@ func (gwr *GatewayAPIV1Beta1Router) SetRoutes( }, }) } + weightedRouteRule := &v1beta1.HTTPRouteRule{ + Matches: matches, + BackendRefs: []v1beta1.HTTPBackendRef{ + { + BackendRef: gwr.makeBackendRef(primarySvcName, pWeight, canary.Spec.Service.Port), + }, + { + BackendRef: gwr.makeBackendRef(canarySvcName, cWeight, canary.Spec.Service.Port), + }, + }, + } httpRouteSpec := v1beta1.HTTPRouteSpec{ CommonRouteSpec: v1beta1.CommonRouteSpec{ ParentRefs: canary.Spec.Service.GatewayRefs, }, Hostnames: hostNames, Rules: []v1beta1.HTTPRouteRule{ - { - Matches: matches, - BackendRefs: []v1beta1.HTTPBackendRef{ - { - BackendRef: gwr.makeBackendRef(primarySvcName, pWeight, canary.Spec.Service.Port), - }, - { - BackendRef: gwr.makeBackendRef(canarySvcName, cWeight, canary.Spec.Service.Port), - }, - }, - }, + *weightedRouteRule, }, } + + if canary.Spec.Analysis.SessionAffinity != nil { + rules, err := gwr.getSessionAffinityRouteRules(canary, canaryWeight, weightedRouteRule) + if err != nil { + return err + } + httpRouteSpec.Rules = rules + } + hrClone.Spec = httpRouteSpec // A/B testing @@ -295,6 +350,112 @@ func (gwr *GatewayAPIV1Beta1Router) Finalize(_ *flaggerv1.Canary) error { return nil } +// getSessionAffinityRouteRules returns the HTTPRouteRule objects required to perform +// session affinity based Canary releases. +func (gwr *GatewayAPIV1Beta1Router) getSessionAffinityRouteRules(canary *flaggerv1.Canary, canaryWeight int, + weightedRouteRule *v1beta1.HTTPRouteRule) ([]v1beta1.HTTPRouteRule, error) { + _, primarySvcName, canarySvcName := canary.GetServiceNames() + stickyRouteRule := *weightedRouteRule + + // If a canary run is active, we want all responses corresponding to requests hitting the canary deployment + // (due to weighted routing) to include a `Set-Cookie` header. All requests that have the `Cookie` header + // and match the value of the `Set-Cookie` header will be routed to the canary deployment. + if canaryWeight != 0 { + if canary.Status.SessionAffinityCookie == "" { + canary.Status.SessionAffinityCookie = fmt.Sprintf("%s=%s", canary.Spec.Analysis.SessionAffinity.CookieName, randSeq()) + } + + // Add `Set-Cookie` header modifier to the primary backend in the weighted routing rule. + for i, backendRef := range weightedRouteRule.BackendRefs { + if string(backendRef.BackendObjectReference.Name) == canarySvcName { + backendRef.Filters = append(backendRef.Filters, v1beta1.HTTPRouteFilter{ + Type: v1beta1.HTTPRouteFilterResponseHeaderModifier, + ResponseHeaderModifier: &v1beta1.HTTPHeaderFilter{ + Add: []v1beta1.HTTPHeader{ + { + Name: setCookieHeader, + Value: fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr, + canary.Spec.Analysis.SessionAffinity.GetMaxAge(), + ), + }, + }, + }, + }) + } + weightedRouteRule.BackendRefs[i] = backendRef + } + + // Add `Cookie` header matcher to the sticky routing rule. + cookieKeyAndVal := strings.Split(canary.Status.SessionAffinityCookie, "=") + regexMatchType := v1beta1.HeaderMatchRegularExpression + cookieMatch := v1beta1.HTTPRouteMatch{ + Headers: []v1beta1.HTTPHeaderMatch{ + { + Type: ®exMatchType, + Name: cookieHeader, + Value: fmt.Sprintf(".*%s.*%s.*", cookieKeyAndVal[0], cookieKeyAndVal[1]), + }, + }, + } + + svcMatches, err := gwr.mapRouteMatches(canary.Spec.Service.Match) + if err != nil { + return nil, err + } + + mergedMatches := gwr.mergeMatchConditions([]v1beta1.HTTPRouteMatch{cookieMatch}, svcMatches) + stickyRouteRule.Matches = mergedMatches + stickyRouteRule.BackendRefs = []v1beta1.HTTPBackendRef{ + { + BackendRef: gwr.makeBackendRef(primarySvcName, 0, canary.Spec.Service.Port), + }, + { + BackendRef: gwr.makeBackendRef(canarySvcName, 100, canary.Spec.Service.Port), + }, + } + } else { + // If canary weight is 0 and SessionAffinityCookie is non-blank, then it belongs to a previous canary run. + if canary.Status.SessionAffinityCookie != "" { + canary.Status.PreviousSessionAffinityCookie = canary.Status.SessionAffinityCookie + } + previousCookie := canary.Status.PreviousSessionAffinityCookie + + // Match against the previous session cookie and delete that cookie + if previousCookie != "" { + cookieKeyAndVal := strings.Split(previousCookie, "=") + regexMatchType := v1beta1.HeaderMatchRegularExpression + cookieMatch := v1beta1.HTTPRouteMatch{ + Headers: []v1beta1.HTTPHeaderMatch{ + { + Type: ®exMatchType, + Name: cookieHeader, + Value: fmt.Sprintf(".*%s.*%s.*", cookieKeyAndVal[0], cookieKeyAndVal[1]), + }, + }, + } + svcMatches, _ := gwr.mapRouteMatches(canary.Spec.Service.Match) + mergedMatches := gwr.mergeMatchConditions([]v1beta1.HTTPRouteMatch{cookieMatch}, svcMatches) + stickyRouteRule.Matches = mergedMatches + + stickyRouteRule.Filters = append(stickyRouteRule.Filters, v1beta1.HTTPRouteFilter{ + Type: v1beta1.HTTPRouteFilterResponseHeaderModifier, + ResponseHeaderModifier: &v1beta1.HTTPHeaderFilter{ + Add: []v1beta1.HTTPHeader{ + { + Name: setCookieHeader, + Value: fmt.Sprintf("%s; %s=%d", previousCookie, maxAgeAttr, -1), + }, + }, + }, + }) + } + + canary.Status.SessionAffinityCookie = "" + } + + return []v1beta1.HTTPRouteRule{stickyRouteRule, *weightedRouteRule}, nil +} + func (gwr *GatewayAPIV1Beta1Router) mapRouteMatches(requestMatches []v1alpha3.HTTPMatchRequest) ([]v1beta1.HTTPRouteMatch, error) { matches := []v1beta1.HTTPRouteMatch{} @@ -389,6 +550,9 @@ func (gwr *GatewayAPIV1Beta1Router) mergeMatchConditions(analysis, service []v1b if len(analysis) == 0 { return service } + if len(service) == 0 { + return analysis + } merged := make([]v1beta1.HTTPRouteMatch, len(service)*len(analysis)) num := 0 diff --git a/pkg/router/gateway_api_v1beta1_test.go b/pkg/router/gateway_api_v1beta1_test.go index f317fb44..37a4627d 100644 --- a/pkg/router/gateway_api_v1beta1_test.go +++ b/pkg/router/gateway_api_v1beta1_test.go @@ -18,8 +18,13 @@ package router import ( "context" + "fmt" + "strings" "testing" + flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" + "github.com/fluxcd/flagger/pkg/apis/gatewayapi/v1beta1" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -61,12 +66,248 @@ func TestGatewayAPIV1Beta1Router_Routes(t *testing.T) { err := router.Reconcile(canary) require.NoError(t, err) - err = router.SetRoutes(canary, 50, 50, false) - require.NoError(t, err) + t.Run("normal", func(t *testing.T) { + err = router.SetRoutes(canary, 50, 50, false) + require.NoError(t, err) - httpRoute, err := router.gatewayAPIClient.GatewayapiV1beta1().HTTPRoutes("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) - require.NoError(t, err) + httpRoute, err := router.gatewayAPIClient.GatewayapiV1beta1().HTTPRoutes("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) - primary := httpRoute.Spec.Rules[0].BackendRefs[0] - assert.Equal(t, int32(50), *primary.Weight) + primary := httpRoute.Spec.Rules[0].BackendRefs[0] + assert.Equal(t, int32(50), *primary.Weight) + }) + + t.Run("session affinity", func(t *testing.T) { + canary := mocks.canary.DeepCopy() + cookieKey := "flagger-cookie" + // enable session affinity and start canary run + canary.Spec.Analysis.SessionAffinity = &flaggerv1.SessionAffinity{ + CookieName: cookieKey, + MaxAge: 300, + } + _, pSvcName, cSvcName := canary.GetServiceNames() + + err := router.SetRoutes(canary, 90, 10, false) + + hr, err := mocks.meshClient.GatewayapiV1beta1().HTTPRoutes("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) + assert.Len(t, hr.Spec.Rules, 2) + + stickyRule := hr.Spec.Rules[0] + weightedRule := hr.Spec.Rules[1] + + // stickyRoute should match against a cookie and direct all traffic to the canary when a canary run is active. + cookieMatch := stickyRule.Matches[0].Headers[0] + assert.Equal(t, *cookieMatch.Type, v1beta1.HeaderMatchRegularExpression) + assert.Equal(t, string(cookieMatch.Name), cookieHeader) + assert.Contains(t, cookieMatch.Value, cookieKey) + + assert.Equal(t, len(stickyRule.BackendRefs), 2) + for _, backendRef := range stickyRule.BackendRefs { + if string(backendRef.BackendRef.Name) == pSvcName { + assert.Equal(t, *backendRef.BackendRef.Weight, int32(0)) + } + if string(backendRef.BackendRef.Name) == cSvcName { + assert.Equal(t, *backendRef.BackendRef.Weight, int32(100)) + } + } + + // weightedRoute should do regular weight based routing and inject the Set-Cookie header + // for all responses returned from the canary deployment. + var found bool + for _, backendRef := range weightedRule.BackendRefs { + if string(backendRef.Name) == cSvcName { + found = true + filter := backendRef.Filters[0] + assert.Equal(t, filter.Type, v1beta1.HTTPRouteFilterResponseHeaderModifier) + assert.NotNil(t, filter.ResponseHeaderModifier) + assert.Equal(t, string(filter.ResponseHeaderModifier.Add[0].Name), setCookieHeader) + assert.Equal(t, filter.ResponseHeaderModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr, 300)) + assert.Equal(t, *backendRef.Weight, int32(10)) + } + if string(backendRef.Name) == pSvcName { + assert.Equal(t, *backendRef.Weight, int32(90)) + } + } + assert.True(t, found) + assert.True(t, strings.HasPrefix(canary.Status.SessionAffinityCookie, cookieKey)) + + // reconcile Canary and HTTPRoute + err = router.Reconcile(canary) + require.NoError(t, err) + + // HTTPRoute should be unchanged + hr, err = mocks.meshClient.GatewayapiV1beta1().HTTPRoutes("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) + assert.Len(t, hr.Spec.Rules, 2) + assert.Empty(t, cmp.Diff(hr.Spec.Rules[0], stickyRule)) + assert.Empty(t, cmp.Diff(hr.Spec.Rules[1], weightedRule)) + + // further continue the canary run + err = router.SetRoutes(canary, 50, 50, false) + require.NoError(t, err) + hr, err = mocks.meshClient.GatewayapiV1beta1().HTTPRoutes("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) + + stickyRule = hr.Spec.Rules[0] + weightedRule = hr.Spec.Rules[1] + + // stickyRoute should match against a cookie and direct all traffic to the canary when a canary run is active. + cookieMatch = stickyRule.Matches[0].Headers[0] + assert.Equal(t, *cookieMatch.Type, v1beta1.HeaderMatchRegularExpression) + assert.Equal(t, string(cookieMatch.Name), cookieHeader) + assert.Contains(t, cookieMatch.Value, cookieKey) + + assert.Equal(t, len(stickyRule.BackendRefs), 2) + for _, backendRef := range stickyRule.BackendRefs { + if string(backendRef.BackendRef.Name) == pSvcName { + assert.Equal(t, *backendRef.BackendRef.Weight, int32(0)) + } + if string(backendRef.BackendRef.Name) == cSvcName { + assert.Equal(t, *backendRef.BackendRef.Weight, int32(100)) + } + } + + // weightedRoute should do regular weight based routing and inject the Set-Cookie header + // for all responses returned from the canary deployment. + found = false + for _, backendRef := range weightedRule.BackendRefs { + if string(backendRef.Name) == cSvcName { + found = true + filter := backendRef.Filters[0] + assert.Equal(t, filter.Type, v1beta1.HTTPRouteFilterResponseHeaderModifier) + assert.NotNil(t, filter.ResponseHeaderModifier) + assert.Equal(t, string(filter.ResponseHeaderModifier.Add[0].Name), setCookieHeader) + assert.Equal(t, filter.ResponseHeaderModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr, 300)) + + assert.Equal(t, *backendRef.Weight, int32(50)) + } + if string(backendRef.Name) == pSvcName { + assert.Equal(t, *backendRef.Weight, int32(50)) + } + } + assert.True(t, found) + + // promotion + err = router.SetRoutes(canary, 100, 0, false) + require.NoError(t, err) + hr, err = mocks.meshClient.GatewayapiV1beta1().HTTPRoutes("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) + + assert.Empty(t, canary.Status.SessionAffinityCookie) + assert.Contains(t, canary.Status.PreviousSessionAffinityCookie, cookieKey) + + stickyRule = hr.Spec.Rules[0] + weightedRule = hr.Spec.Rules[1] + + // Assert that the stucky rule matches against the previous cookie and tells clients to delete it. + cookieMatch = stickyRule.Matches[0].Headers[0] + assert.Equal(t, *cookieMatch.Type, v1beta1.HeaderMatchRegularExpression) + assert.Equal(t, string(cookieMatch.Name), cookieHeader) + assert.Contains(t, cookieMatch.Value, cookieKey) + + assert.Equal(t, stickyRule.Filters[0].Type, v1beta1.HTTPRouteFilterResponseHeaderModifier) + headerModifier := stickyRule.Filters[0].ResponseHeaderModifier + assert.NotNil(t, headerModifier) + assert.Equal(t, string(headerModifier.Add[0].Name), setCookieHeader) + assert.Equal(t, headerModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.PreviousSessionAffinityCookie, maxAgeAttr, -1)) + + for _, backendRef := range stickyRule.BackendRefs { + if string(backendRef.BackendRef.Name) == pSvcName { + assert.Equal(t, *backendRef.BackendRef.Weight, int32(100)) + } + if string(backendRef.BackendRef.Name) == cSvcName { + assert.Equal(t, *backendRef.BackendRef.Weight, int32(0)) + } + } + + for _, backendRef := range weightedRule.BackendRefs { + if string(backendRef.Name) == cSvcName { + // Assert the weighted rule does not send Set-Cookie headers anymore + assert.Len(t, backendRef.Filters, 0) + assert.Equal(t, *backendRef.Weight, int32(0)) + } + if string(backendRef.Name) == pSvcName { + assert.Equal(t, *backendRef.Weight, int32(100)) + } + } + assert.True(t, found) + }) +} + +func TestGatewayAPIV1Beta1Router_getSessionAffinityRouteRules(t *testing.T) { + canary := newTestGatewayAPICanary() + mocks := newFixture(canary) + cookieKey := "flagger-cookie" + canary.Spec.Analysis.SessionAffinity = &flaggerv1.SessionAffinity{ + CookieName: cookieKey, + MaxAge: 300, + } + + router := &GatewayAPIV1Beta1Router{ + gatewayAPIClient: mocks.meshClient, + kubeClient: mocks.kubeClient, + logger: mocks.logger, + } + _, pSvcName, cSvcName := canary.GetServiceNames() + weightedRouteRule := &v1beta1.HTTPRouteRule{ + BackendRefs: []v1beta1.HTTPBackendRef{ + { + BackendRef: router.makeBackendRef(pSvcName, initialPrimaryWeight, canary.Spec.Service.Port), + }, + { + BackendRef: router.makeBackendRef(cSvcName, initialCanaryWeight, canary.Spec.Service.Port), + }, + }, + } + rules, err := router.getSessionAffinityRouteRules(canary, 10, weightedRouteRule) + require.NoError(t, err) + assert.Equal(t, len(rules), 2) + assert.True(t, strings.HasPrefix(canary.Status.SessionAffinityCookie, cookieKey)) + + stickyRule := rules[0] + cookieMatch := stickyRule.Matches[0].Headers[0] + assert.Equal(t, *cookieMatch.Type, v1beta1.HeaderMatchRegularExpression) + assert.Equal(t, string(cookieMatch.Name), cookieHeader) + assert.Contains(t, cookieMatch.Value, cookieKey) + + assert.Equal(t, len(stickyRule.BackendRefs), 2) + for _, backendRef := range stickyRule.BackendRefs { + if string(backendRef.BackendRef.Name) == pSvcName { + assert.Equal(t, *backendRef.BackendRef.Weight, int32(0)) + } + if string(backendRef.BackendRef.Name) == cSvcName { + assert.Equal(t, *backendRef.BackendRef.Weight, int32(100)) + } + } + + weightedRule := rules[1] + var found bool + for _, backendRef := range weightedRule.BackendRefs { + if string(backendRef.Name) == cSvcName { + found = true + filter := backendRef.Filters[0] + assert.Equal(t, filter.Type, v1beta1.HTTPRouteFilterResponseHeaderModifier) + assert.NotNil(t, filter.ResponseHeaderModifier) + assert.Equal(t, string(filter.ResponseHeaderModifier.Add[0].Name), setCookieHeader) + assert.Equal(t, filter.ResponseHeaderModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr, 300)) + } + } + assert.True(t, found) + + rules, err = router.getSessionAffinityRouteRules(canary, 0, weightedRouteRule) + assert.Empty(t, canary.Status.SessionAffinityCookie) + assert.Contains(t, canary.Status.PreviousSessionAffinityCookie, cookieKey) + + stickyRule = rules[0] + cookieMatch = stickyRule.Matches[0].Headers[0] + assert.Equal(t, *cookieMatch.Type, v1beta1.HeaderMatchRegularExpression) + assert.Equal(t, string(cookieMatch.Name), cookieHeader) + assert.Contains(t, cookieMatch.Value, cookieKey) + + assert.Equal(t, stickyRule.Filters[0].Type, v1beta1.HTTPRouteFilterResponseHeaderModifier) + headerModifier := stickyRule.Filters[0].ResponseHeaderModifier + assert.NotNil(t, headerModifier) + assert.Equal(t, string(headerModifier.Add[0].Name), setCookieHeader) + assert.Equal(t, headerModifier.Add[0].Value, fmt.Sprintf("%s; %s=%d", canary.Status.PreviousSessionAffinityCookie, maxAgeAttr, -1)) } diff --git a/test/gatewayapi/install.sh b/test/gatewayapi/install.sh index 00d82ea6..c7ab9337 100755 --- a/test/gatewayapi/install.sh +++ b/test/gatewayapi/install.sh @@ -2,7 +2,7 @@ set -o errexit -CONTOUR_VER="v1.23.0" +CONTOUR_VER="v1.26.0" GATEWAY_API_VER="v1beta1" REPO_ROOT=$(git rev-parse --show-toplevel) KUSTOMIZE_VERSION=4.5.2 diff --git a/test/gatewayapi/run.sh b/test/gatewayapi/run.sh index 8c8571f3..fe8d95b7 100755 --- a/test/gatewayapi/run.sh +++ b/test/gatewayapi/run.sh @@ -11,3 +11,4 @@ DIR="$(cd "$(dirname "$0")" && pwd)" "$DIR"/test-canary.sh "$DIR"/test-bg.sh "$DIR"/test-ab.sh +"$DIR"/test-session-affinity.sh diff --git a/test/gatewayapi/test-session-affinity.sh b/test/gatewayapi/test-session-affinity.sh new file mode 100755 index 00000000..8b2ed621 --- /dev/null +++ b/test/gatewayapi/test-session-affinity.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for progressive traffic shifting with session affinity, Canary analysis and promotion +# Prerequisites: Kubernetes Kind and Contour with GatewayAPI + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +source ${REPO_ROOT}/test/gatewayapi/test-utils.sh + +create_latency_metric_template +create_error_rate_metric_template + +echo '>>> Deploy podinfo in sa-test namespace' +kubectl create ns sa-test +kubectl apply -f ${REPO_ROOT}/test/workloads/secret.yaml -n sa-test +kubectl apply -f ${REPO_ROOT}/test/workloads/deployment.yaml -n sa-test + +echo '>>> Installing Canary' +cat <>> Port forwarding load balancer' +kubectl port-forward -n projectcontour svc/envoy-contour 8888:80 2>&1 > /dev/null & +pf_pid=$! + +cleanup() { + echo ">> Killing port forward process ${pf_pid}" + kill -9 $pf_pid +} +trap "cleanup" EXIT SIGINT + +echo '>>> Triggering canary deployment' +kubectl -n sa-test set image deployment/podinfo podinfod=stefanprodan/podinfo:6.1.0 + +echo '>>> Waiting for initial traffic shift' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n sa-test get canary podinfo -o=jsonpath='{.status.canaryWeight}' | grep '10' && ok=true || ok=false + sleep 5 + kubectl -n flagger-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Verifying session affinity' +if ! URL=http://localhost:8888 HOST=localproject.contour.io VERSION=6.1.0 COOKIE_NAME=flagger-cookie \ + go run ${REPO_ROOT}/test/gatewayapi/verify_session_affinity.go; then + echo "failed to verify session affinity" + exit $? +fi + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n sa-test describe deployment/podinfo-primary | grep '6.1.0' && ok=true || ok=false + sleep 10 + kubectl -n flagger-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +display_httproute "sa-test" + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n sa-test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Verifying cookie cleanup' +canary_cookie=$(kubectl -n sa-test get canary podinfo -o=jsonpath='{.status.previousSessionAffinityCookie}' | xargs) +response=$(curl -H "Host: localproject.contour.io" -H "Cookie: $canary_cookie" -D - http://localhost:8888) + +if [[ $response == *"$canary_cookie"* ]]; then + echo "✔ Found previous cookie in response" +else + echo "⨯ Previous cookie ${canary_cookie} not found in response" + exit 1 +fi + +if [[ $response == *"Max-Age=-1"* ]]; then + echo "✔ Found Max-Age attribute in cookie" +else + echo "⨯ Max-Age attribute not present in cookie" + exit 1 +fi + +echo '✔ Canary release with session affinity promotion test passed' + +kubectl delete -n sa-test canary podinfo diff --git a/test/gatewayapi/verify_session_affinity.go b/test/gatewayapi/verify_session_affinity.go new file mode 100644 index 00000000..7677483b --- /dev/null +++ b/test/gatewayapi/verify_session_affinity.go @@ -0,0 +1,110 @@ +package main + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "strings" + "sync" + "time" +) + +var c = make(chan string, 1) +var mu sync.Mutex +var try = true +var timeout = time.Second * 10 + +func main() { + url := os.Getenv("URL") + host := os.Getenv("HOST") + version := os.Getenv("VERSION") + cookieName := os.Getenv("COOKIE_NAME") + + // Generate traffic + for i := 0; i < 10; i++ { + go tryUntilCanaryIsHit(url, host, version, cookieName) + } + + select { + // If we receive a cookie, then try to verify that we are always routed to the + // Canary deployment based on the cookie. + case cookie := <-c: + mu.Lock() + try = false + mu.Unlock() + + for i := 0; i < 5; i++ { + headers := map[string]string{ + "Cookie": cookie, + } + body, _, err := sendRequest(url, host, headers) + if err != nil { + log.Fatalf("failed to send request to verify cookie based routing: %v", err) + } + if !strings.Contains(body, version) { + log.Fatalf("received response from primary deployment instead of canary deployment") + } + } + + log.Println("✔ successfully verified session affinity") + case <-time.After(timeout): + log.Fatal("timed out waiting for canary hit") + } +} + +// sendRequest sends a request to the URL with the provided host and headers. +// It returns the response body and cookies or an error. +func sendRequest(url, host string, headers map[string]string) (string, []*http.Cookie, error) { + client := http.DefaultClient + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", nil, err + } + + for key, value := range headers { + req.Header.Add(key, value) + } + req.Host = host + + resp, err := client.Do(req) + if err != nil { + return "", nil, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", nil, err + } + + return string(body), resp.Cookies(), nil +} + +// tryUntilCanaryIsHit is a recursive function that tries to send request and +// either sends the cookie back to the main thread (if received) or re-sends +// the request. +func tryUntilCanaryIsHit(url, host, version, cookieName string) { + mu.Lock() + if !try { + mu.Unlock() + return + } + mu.Unlock() + + body, cookies, err := sendRequest(url, host, nil) + if err != nil { + log.Printf("warning: failed to send request: %s", err) + return + } + if strings.Contains(body, version) { + if cookies[0].Name == cookieName { + c <- fmt.Sprintf("%s=%s", cookies[0].Name, cookies[0].Value) + return + } + } + + tryUntilCanaryIsHit(url, host, version, cookieName) + return +}