✨ Skipper Router Implementation

Router implementation for zalan.do/Skipper Ingress - An HTTP router and reverse proxy for service composition, including use cases like Kubernetes Ingress https://github.com/zalando/skipper/ * The concept is to define routes with specific weights via the skipper specific annotation predicate of "zalando.org/backend-weights". * A new "canary ingress" is created that has higher "weight" thus receiving all traffic, which distributes progressively * After the canary process is finished, this ingress is disabled via the "False()" annotation predicate to route traffic again back to the apex Ingress. There are certain Skipper principles which are taken into account: ``` Skipper Principles: * if only one backend has a weight, only one backend will get 100% traffic * if two of three or more backends have a weight, only those two should get traffic. * if two backends don't have any weight, it's undefined and right now they get equal amount of traffic. * weights can be int or float, but always treated as a ratio. Implementation: * apex Ingress is immutable * new canary Ingress contains two paths for primary and canary service * canary Ingress manages weights on primary & canary service, hence no traffic to apex service ```
2026-04-15 06:57:34 +00:00 · 2020-08-17 08:23:38 +02:00
parent be16bd8768
commit ca14a08f9c
6 changed files with 358 additions and 3 deletions
@@ -10,6 +10,8 @@ require (
 	github.com/prometheus/client_golang v1.5.1
 	github.com/stretchr/testify v1.5.1
 	go.uber.org/zap v1.14.1
+	golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect
+	golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d // indirect
 	gopkg.in/h2non/gock.v1 v1.0.15
 	k8s.io/api v0.18.8
 	k8s.io/apimachinery v0.18.8
@@ -188,6 +188,7 @@ github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLk
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -220,6 +221,7 @@ golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo=
 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -229,6 +231,8 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee h1:WG0RUwxtNT4qqaXX3DPA8zHFNm/D9xaBpxzHt1WcA/E=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -245,6 +249,8 @@ golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ym
 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
+golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -293,7 +299,10 @@ golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs=
 golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d h1:7M9AXzLrJWWGdDYtBblPHBTnHtaN6KKQ98OYb35mLlY=
+golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
@@ -122,6 +122,11 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
 			kubeClient:        factory.kubeClient,
 			annotationsPrefix: factory.ingressAnnotationsPrefix,
 		}
+	case provider == flaggerv1.SkipperProvider:
+		return &SkipperRouter{
+			logger:     factory.logger,
+			kubeClient: factory.kubeClient,
+		}
 	case provider == flaggerv1.KubernetesProvider:
 		return &NopRouter{}
 	default:
@@ -125,9 +125,6 @@ func (c *KubernetesDefaultRouter) reconcileService(canary *flaggerv1.Canary, nam
 		metadata.Annotations = make(map[string]string)
 	}

-	c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
-		Debugw(fmt.Sprintf("Creating Service %s", name), "metadata", metadata, "service_configuration", canary.Spec.Service)
-
 	// create service if it doesn't exists
 	svc, err := c.kubeClient.CoreV1().Services(canary.Namespace).Get(context.TODO(), name, metav1.GetOptions{})
 	if errors.IsNotFound(err) {
@@ -0,0 +1,235 @@
+package router
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/google/go-cmp/cmp"
+	"go.uber.org/zap"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/kubernetes"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
+)
+
+/*
+Skipper Principles:
+* if only one backend has a weight, only one backend will get 100% traffic
+* if two of three or more backends have a weight, only those two should get traffic.
+* if two backends don't have any weight, it's undefined and right now they get equal amount of traffic.
+* weights can be int or float, but always treated as a ratio.
+
+Implementation:
+* apex Ingress is immutable
+* new canary Ingress contains two paths for primary and canary service
+* canary Ingress manages weights on primary & canary service, hence no traffic to apex service
+
+*/
+
+const (
+	skipperpredicateAnnotationKey      = "zalando.org/skipper-predicate"
+	skipperBackendWeightsAnnotationKey = "zalando.org/backend-weights"
+	canaryPatternf                     = "%s-canary"
+	canaryRouteWeight                  = "Weight(100)"
+	canaryRouteDisable                 = "False()"
+)
+
+type SkipperRouter struct {
+	kubeClient kubernetes.Interface
+	logger     *zap.SugaredLogger
+}
+
+// Reconcile creates or updates the ingresses
+func (skp *SkipperRouter) Reconcile(canary *flaggerv1.Canary) error {
+	if canary.Spec.IngressRef == nil || canary.Spec.IngressRef.Name == "" {
+		return fmt.Errorf("ingress selector is empty")
+	}
+
+	apexSvcName, primarySvcName, canarySvcName := canary.GetServiceNames()
+	apexIngressName, canaryIngressName := skp.getIngressNames(canary.Spec.IngressRef.Name)
+
+	// retrieving apex ingress
+	apexIngress, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Get(
+		context.TODO(), apexIngressName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("apexIngress %s.%s get query error: %w", apexIngressName, canary.Namespace, err)
+	}
+
+	// building the canary ingress from apex
+	iClone := apexIngress.DeepCopy()
+	for x := range iClone.Spec.Rules {
+		rule := &iClone.Spec.Rules[x] // ref not value
+		for y := range rule.HTTP.Paths {
+			path := &rule.HTTP.Paths[y] // ref not value
+			if path.Backend.ServiceName == apexSvcName {
+				// flipping to primary service
+				path.Backend.ServiceName = primarySvcName
+				// adding second canary service
+				canaryBackend := path.DeepCopy()
+				canaryBackend.Backend.ServiceName = canarySvcName
+				rule.HTTP.Paths = append(rule.HTTP.Paths, *canaryBackend)
+			}
+		}
+	}
+	if apexIngress.DeepCopy() == iClone {
+		return fmt.Errorf("backend %s not found in ingress %s", apexSvcName, apexIngressName)
+	}
+
+	iClone.Annotations = skp.makeAnnotations(iClone.Annotations, map[string]int{primarySvcName: 100, canarySvcName: 0})
+	iClone.Name = canaryIngressName
+	iClone.Namespace = canary.Namespace
+	iClone.OwnerReferences = []metav1.OwnerReference{
+		*metav1.NewControllerRef(canary, schema.GroupVersionKind{
+			Group:   flaggerv1.SchemeGroupVersion.Group,
+			Version: flaggerv1.SchemeGroupVersion.Version,
+			Kind:    flaggerv1.CanaryKind,
+		}),
+	}
+
+	// search for existence
+	canaryIngress, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Get(
+		context.TODO(), canaryIngressName, metav1.GetOptions{})
+
+	// new ingress
+	if errors.IsNotFound(err) {
+		// Let K8s set this. Otherwise K8s API complains with "resourceVersion should not be set on objects to be created"
+		iClone.ObjectMeta.ResourceVersion = ""
+		_, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Create(context.TODO(), iClone, metav1.CreateOptions{})
+		if err != nil {
+			return fmt.Errorf("ingress %s.%s create error: %w", iClone.Name, iClone.Namespace, err)
+		}
+		skp.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
+			Infof("Ingress %s.%s created", iClone.GetName(), canary.Namespace)
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("ingress %s.%s query error: %w", canaryIngressName, canary.Namespace, err)
+	}
+
+	// existant, updating
+	if cmp.Diff(iClone.Spec, canaryIngress.Spec) != "" {
+		ingressClone := canaryIngress.DeepCopy()
+		ingressClone.Spec = iClone.Spec
+		ingressClone.Annotations = iClone.Annotations
+
+		_, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Update(context.TODO(), ingressClone, metav1.UpdateOptions{})
+		if err != nil {
+			return fmt.Errorf("ingress %s.%s update error: %w", canaryIngressName, ingressClone.Namespace, err)
+		}
+		skp.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
+			Infof("Ingress %s updated", canaryIngressName)
+	}
+	return nil
+}
+
+func (skp *SkipperRouter) GetRoutes(canary *flaggerv1.Canary) (primaryWeight, canaryWeight int, mirrored bool, err error) {
+	_, primarySvcName, canarySvcName := canary.GetServiceNames()
+
+	_, canaryIngressName := skp.getIngressNames(canary.Spec.IngressRef.Name)
+	canaryIngress, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Get(context.TODO(), canaryIngressName, metav1.GetOptions{})
+	if err != nil {
+		err = fmt.Errorf("ingress %s.%s get query error: %w", canaryIngressName, canary.Namespace, err)
+		return
+	}
+
+	weights, err := skp.backendWeights(canaryIngress.Annotations)
+	if err != nil {
+		err = fmt.Errorf("ingress %s.%s get backendWeights error: %w", canaryIngressName, canary.Namespace, err)
+		return
+	}
+	var ok bool
+	primaryWeight, ok = weights[primarySvcName]
+	if !ok {
+		err = fmt.Errorf("ingress %s.%s could not get weights[primarySvcName]", canaryIngressName, canary.Namespace)
+		return
+	}
+	canaryWeight, ok = weights[canarySvcName]
+	if !ok {
+		err = fmt.Errorf("ingress %s.%s could not get weights[canarySvcName]", canaryIngressName, canary.Namespace)
+		return
+	}
+	mirrored = false
+	skp.logger.With("GetRoutes", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
+		Debugf("GetRoutes primaryWeight: %d, canaryWeight: %d", primaryWeight, canaryWeight)
+	return
+}
+
+func (skp *SkipperRouter) SetRoutes(canary *flaggerv1.Canary, primaryWeight, canaryWeight int, _ bool) (err error) {
+	_, primarySvcName, canarySvcName := canary.GetServiceNames()
+	_, canaryIngressName := skp.getIngressNames(canary.Spec.IngressRef.Name)
+	canaryIngress, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Get(context.TODO(), canaryIngressName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("ingress %s.%s get query error: %w", canaryIngressName, canary.Namespace, err)
+	}
+
+	iClone := canaryIngress.DeepCopy()
+
+	// TODO: A/B testing
+
+	// Canary
+	iClone.Annotations = skp.makeAnnotations(iClone.Annotations, map[string]int{
+		primarySvcName: primaryWeight,
+		canarySvcName:  canaryWeight,
+	})
+
+	// Disable the canary-ingress route after the canary process
+	if canaryWeight == 0 {
+		iClone.Annotations[skipperpredicateAnnotationKey] = canaryRouteDisable
+	}
+
+	_, err = skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Update(
+		context.TODO(), iClone, metav1.UpdateOptions{})
+	if err != nil {
+		return fmt.Errorf("ingress %s.%s update error %w", iClone.Name, iClone.Namespace, err)
+	}
+	skp.logger.With("SetRoutes", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
+		Debugf("primaryWeight: %d, canaryWeight: %d", primaryWeight, canaryWeight)
+
+	return err
+}
+
+func (skp *SkipperRouter) Finalize(canary *flaggerv1.Canary) error {
+	gracePeriodSeconds := int64(2)
+	_, canaryIngressName := skp.getIngressNames(canary.Spec.IngressRef.Name)
+	skp.logger.With("deleteCanaryIngress", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).
+		Debugf("Deleting Canary Ingress: %s", canaryIngressName)
+
+	err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Delete(
+		context.TODO(), canaryIngressName, metav1.DeleteOptions{GracePeriodSeconds: &gracePeriodSeconds})
+	if err != nil {
+		return fmt.Errorf("ingress %s.%s unable to remove canary ingress: %w", canaryIngressName, canary.Namespace, err)
+	}
+	return nil
+}
+
+func (skp *SkipperRouter) makeAnnotations(annotations map[string]string, backendWeights map[string]int) map[string]string {
+	b, err := json.Marshal(backendWeights)
+	if err != nil {
+		skp.logger.Errorf("Skipper:makeAnnotations: unable to marshal backendWeights %w", err)
+		return annotations
+	}
+	annotations[skipperBackendWeightsAnnotationKey] = string(b)
+	// adding more weight to canary route solves traffic bypassing through apexIngress
+	annotations[skipperpredicateAnnotationKey] = canaryRouteWeight
+
+	return annotations
+}
+
+// parse backend-weights annotation if it exists
+func (skp *SkipperRouter) backendWeights(annotation map[string]string) (backendWeights map[string]int, err error) {
+	backends, ok := annotation[skipperBackendWeightsAnnotationKey]
+	if ok {
+		err = json.Unmarshal([]byte(backends), &backendWeights)
+	} else {
+		err = errors.NewNotFound(schema.GroupResource{Group: "Skipper Canary Ingress", Resource: "Annotation"},
+			skipperBackendWeightsAnnotationKey)
+	}
+	return
+}
+
+// getIngressNames returns the primary and canary Kubernetes Ingress names
+func (skp *SkipperRouter) getIngressNames(name string) (apexName, canaryName string) {
+	return name, fmt.Sprintf(canaryPatternf, name)
+}
@@ -0,0 +1,107 @@
+package router
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestSkipperRouter_Reconcile(t *testing.T) {
+	assert := assert.New(t)
+	mocks := newFixture(nil)
+
+	for _, tt := range []struct {
+		name    string
+		mocks   func() fixture
+		wantErr bool
+	}{
+		{
+			"creating new canary ingress w/ default settings",
+			func() fixture { return mocks },
+			false,
+		}, {
+			"updating existing canary ingress",
+			func() fixture {
+				ti := newTestIngress()
+				ti.Annotations["something"] = "changed"
+				_, err := mocks.kubeClient.NetworkingV1beta1().Ingresses("default").Update(
+					context.TODO(), ti, metav1.UpdateOptions{})
+				assert.NoError(err)
+				return mocks
+			},
+			false,
+		},
+	} {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			mocks := tt.mocks()
+			router := &SkipperRouter{
+				kubeClient: mocks.kubeClient,
+				logger:     mocks.logger,
+			}
+			assert.NoError(router.Reconcile(mocks.ingressCanary))
+			canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name)
+			inCanary, err := router.kubeClient.NetworkingV1beta1().Ingresses("default").Get(
+				context.TODO(), canaryName, metav1.GetOptions{})
+			assert.NoError(err)
+			// test initialisation
+			assert.JSONEq(`{ "podinfo-primary":  100, "podinfo-canary":  0 }`, inCanary.Annotations["zalando.org/backend-weights"])
+			assert.Equal("podinfo-primary", inCanary.Spec.Rules[0].HTTP.Paths[0].Backend.ServiceName, "backend flipped over")
+			assert.Equal("podinfo-canary", inCanary.Spec.Rules[0].HTTP.Paths[1].Backend.ServiceName, "backend flipped over")
+			assert.Len(inCanary.Spec.Rules[0].HTTP.Paths, 2)
+			inApex, err := router.kubeClient.NetworkingV1beta1().Ingresses("default").Get(
+				context.TODO(), mocks.ingressCanary.Spec.IngressRef.Name, metav1.GetOptions{})
+			assert.NoError(err)
+			assert.Equal(inCanary.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort,
+				inApex.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort, "canary backend not cloned")
+			assert.Equal(inCanary.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort,
+				inCanary.Spec.Rules[0].HTTP.Paths[1].Backend.ServicePort, "canary backend not cloned")
+		})
+	}
+}
+
+func TestSkipperRouter_GetSetRoutes(t *testing.T) {
+	assert := assert.New(t)
+	mocks := newFixture(nil)
+
+	router := &SkipperRouter{logger: mocks.logger, kubeClient: mocks.kubeClient}
+	assert.NoError(router.Reconcile(mocks.ingressCanary))
+
+	p, c, m, err := router.GetRoutes(mocks.ingressCanary)
+	assert.NoError(err)
+	assert.Equal(100, p)
+	assert.Equal(0, c)
+	assert.Equal(false, m)
+
+	tests := []struct {
+		name            string
+		primary, canary int
+	}{
+		{name: "0%", primary: 100, canary: 0},
+		{name: "10%", primary: 90, canary: 10},
+		{name: "20%", primary: 80, canary: 20},
+		{name: "30%", primary: 70, canary: 30},
+		{name: "85%", primary: 15, canary: 85},
+		{name: "100%", primary: 0, canary: 100},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			assert.NoError(router.SetRoutes(mocks.ingressCanary, tt.primary, tt.canary, false))
+			inCanary, err := router.kubeClient.NetworkingV1beta1().Ingresses("default").Get(
+				context.TODO(), fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name), metav1.GetOptions{})
+			assert.NoError(err)
+			assert.JSONEq(fmt.Sprintf(`{"podinfo-primary": %d,"podinfo-canary": %d}`, tt.primary, tt.canary),
+				inCanary.Annotations["zalando.org/backend-weights"])
+			p, c, m, err = router.GetRoutes(mocks.ingressCanary)
+			assert.NoError(err)
+			assert.Equal(tt.primary, p)
+			assert.Equal(tt.canary, c)
+			assert.Equal(false, m)
+		})
+	}
+
+}