diff --git a/go.mod b/go.mod index e41fd666..6255481b 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,8 @@ require ( github.com/prometheus/client_golang v1.5.1 github.com/stretchr/testify v1.5.1 go.uber.org/zap v1.14.1 + golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect + golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d // indirect gopkg.in/h2non/gock.v1 v1.0.15 k8s.io/api v0.18.8 k8s.io/apimachinery v0.18.8 diff --git a/go.sum b/go.sum index c5d33ed8..01e48ac3 100644 --- a/go.sum +++ b/go.sum @@ -188,6 +188,7 @@ github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLk github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -220,6 +221,7 @@ golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -229,6 +231,8 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee h1:WG0RUwxtNT4qqaXX3DPA8zHFNm/D9xaBpxzHt1WcA/E= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -245,6 +249,8 @@ golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ym golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -293,7 +299,10 @@ golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d h1:7M9AXzLrJWWGdDYtBblPHBTnHtaN6KKQ98OYb35mLlY= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= diff --git a/pkg/router/factory.go b/pkg/router/factory.go index 15df37fe..2a970c90 100644 --- a/pkg/router/factory.go +++ b/pkg/router/factory.go @@ -122,6 +122,11 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf kubeClient: factory.kubeClient, annotationsPrefix: factory.ingressAnnotationsPrefix, } + case provider == flaggerv1.SkipperProvider: + return &SkipperRouter{ + logger: factory.logger, + kubeClient: factory.kubeClient, + } case provider == flaggerv1.KubernetesProvider: return &NopRouter{} default: diff --git a/pkg/router/kubernetes_default.go b/pkg/router/kubernetes_default.go index df4a8bc4..739dc4c7 100644 --- a/pkg/router/kubernetes_default.go +++ b/pkg/router/kubernetes_default.go @@ -125,9 +125,6 @@ func (c *KubernetesDefaultRouter) reconcileService(canary *flaggerv1.Canary, nam metadata.Annotations = make(map[string]string) } - c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). - Debugw(fmt.Sprintf("Creating Service %s", name), "metadata", metadata, "service_configuration", canary.Spec.Service) - // create service if it doesn't exists svc, err := c.kubeClient.CoreV1().Services(canary.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) if errors.IsNotFound(err) { diff --git a/pkg/router/skipper.go b/pkg/router/skipper.go new file mode 100644 index 00000000..6b5a8c7c --- /dev/null +++ b/pkg/router/skipper.go @@ -0,0 +1,235 @@ +package router + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/google/go-cmp/cmp" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" + + flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1" +) + +/* +Skipper Principles: +* if only one backend has a weight, only one backend will get 100% traffic +* if two of three or more backends have a weight, only those two should get traffic. +* if two backends don't have any weight, it's undefined and right now they get equal amount of traffic. +* weights can be int or float, but always treated as a ratio. + +Implementation: +* apex Ingress is immutable +* new canary Ingress contains two paths for primary and canary service +* canary Ingress manages weights on primary & canary service, hence no traffic to apex service + +*/ + +const ( + skipperpredicateAnnotationKey = "zalando.org/skipper-predicate" + skipperBackendWeightsAnnotationKey = "zalando.org/backend-weights" + canaryPatternf = "%s-canary" + canaryRouteWeight = "Weight(100)" + canaryRouteDisable = "False()" +) + +type SkipperRouter struct { + kubeClient kubernetes.Interface + logger *zap.SugaredLogger +} + +// Reconcile creates or updates the ingresses +func (skp *SkipperRouter) Reconcile(canary *flaggerv1.Canary) error { + if canary.Spec.IngressRef == nil || canary.Spec.IngressRef.Name == "" { + return fmt.Errorf("ingress selector is empty") + } + + apexSvcName, primarySvcName, canarySvcName := canary.GetServiceNames() + apexIngressName, canaryIngressName := skp.getIngressNames(canary.Spec.IngressRef.Name) + + // retrieving apex ingress + apexIngress, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Get( + context.TODO(), apexIngressName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("apexIngress %s.%s get query error: %w", apexIngressName, canary.Namespace, err) + } + + // building the canary ingress from apex + iClone := apexIngress.DeepCopy() + for x := range iClone.Spec.Rules { + rule := &iClone.Spec.Rules[x] // ref not value + for y := range rule.HTTP.Paths { + path := &rule.HTTP.Paths[y] // ref not value + if path.Backend.ServiceName == apexSvcName { + // flipping to primary service + path.Backend.ServiceName = primarySvcName + // adding second canary service + canaryBackend := path.DeepCopy() + canaryBackend.Backend.ServiceName = canarySvcName + rule.HTTP.Paths = append(rule.HTTP.Paths, *canaryBackend) + } + } + } + if apexIngress.DeepCopy() == iClone { + return fmt.Errorf("backend %s not found in ingress %s", apexSvcName, apexIngressName) + } + + iClone.Annotations = skp.makeAnnotations(iClone.Annotations, map[string]int{primarySvcName: 100, canarySvcName: 0}) + iClone.Name = canaryIngressName + iClone.Namespace = canary.Namespace + iClone.OwnerReferences = []metav1.OwnerReference{ + *metav1.NewControllerRef(canary, schema.GroupVersionKind{ + Group: flaggerv1.SchemeGroupVersion.Group, + Version: flaggerv1.SchemeGroupVersion.Version, + Kind: flaggerv1.CanaryKind, + }), + } + + // search for existence + canaryIngress, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Get( + context.TODO(), canaryIngressName, metav1.GetOptions{}) + + // new ingress + if errors.IsNotFound(err) { + // Let K8s set this. Otherwise K8s API complains with "resourceVersion should not be set on objects to be created" + iClone.ObjectMeta.ResourceVersion = "" + _, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Create(context.TODO(), iClone, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("ingress %s.%s create error: %w", iClone.Name, iClone.Namespace, err) + } + skp.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). + Infof("Ingress %s.%s created", iClone.GetName(), canary.Namespace) + return nil + } else if err != nil { + return fmt.Errorf("ingress %s.%s query error: %w", canaryIngressName, canary.Namespace, err) + } + + // existant, updating + if cmp.Diff(iClone.Spec, canaryIngress.Spec) != "" { + ingressClone := canaryIngress.DeepCopy() + ingressClone.Spec = iClone.Spec + ingressClone.Annotations = iClone.Annotations + + _, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Update(context.TODO(), ingressClone, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("ingress %s.%s update error: %w", canaryIngressName, ingressClone.Namespace, err) + } + skp.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). + Infof("Ingress %s updated", canaryIngressName) + } + return nil +} + +func (skp *SkipperRouter) GetRoutes(canary *flaggerv1.Canary) (primaryWeight, canaryWeight int, mirrored bool, err error) { + _, primarySvcName, canarySvcName := canary.GetServiceNames() + + _, canaryIngressName := skp.getIngressNames(canary.Spec.IngressRef.Name) + canaryIngress, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Get(context.TODO(), canaryIngressName, metav1.GetOptions{}) + if err != nil { + err = fmt.Errorf("ingress %s.%s get query error: %w", canaryIngressName, canary.Namespace, err) + return + } + + weights, err := skp.backendWeights(canaryIngress.Annotations) + if err != nil { + err = fmt.Errorf("ingress %s.%s get backendWeights error: %w", canaryIngressName, canary.Namespace, err) + return + } + var ok bool + primaryWeight, ok = weights[primarySvcName] + if !ok { + err = fmt.Errorf("ingress %s.%s could not get weights[primarySvcName]", canaryIngressName, canary.Namespace) + return + } + canaryWeight, ok = weights[canarySvcName] + if !ok { + err = fmt.Errorf("ingress %s.%s could not get weights[canarySvcName]", canaryIngressName, canary.Namespace) + return + } + mirrored = false + skp.logger.With("GetRoutes", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). + Debugf("GetRoutes primaryWeight: %d, canaryWeight: %d", primaryWeight, canaryWeight) + return +} + +func (skp *SkipperRouter) SetRoutes(canary *flaggerv1.Canary, primaryWeight, canaryWeight int, _ bool) (err error) { + _, primarySvcName, canarySvcName := canary.GetServiceNames() + _, canaryIngressName := skp.getIngressNames(canary.Spec.IngressRef.Name) + canaryIngress, err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Get(context.TODO(), canaryIngressName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("ingress %s.%s get query error: %w", canaryIngressName, canary.Namespace, err) + } + + iClone := canaryIngress.DeepCopy() + + // TODO: A/B testing + + // Canary + iClone.Annotations = skp.makeAnnotations(iClone.Annotations, map[string]int{ + primarySvcName: primaryWeight, + canarySvcName: canaryWeight, + }) + + // Disable the canary-ingress route after the canary process + if canaryWeight == 0 { + iClone.Annotations[skipperpredicateAnnotationKey] = canaryRouteDisable + } + + _, err = skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Update( + context.TODO(), iClone, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("ingress %s.%s update error %w", iClone.Name, iClone.Namespace, err) + } + skp.logger.With("SetRoutes", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). + Debugf("primaryWeight: %d, canaryWeight: %d", primaryWeight, canaryWeight) + + return err +} + +func (skp *SkipperRouter) Finalize(canary *flaggerv1.Canary) error { + gracePeriodSeconds := int64(2) + _, canaryIngressName := skp.getIngressNames(canary.Spec.IngressRef.Name) + skp.logger.With("deleteCanaryIngress", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). + Debugf("Deleting Canary Ingress: %s", canaryIngressName) + + err := skp.kubeClient.NetworkingV1beta1().Ingresses(canary.Namespace).Delete( + context.TODO(), canaryIngressName, metav1.DeleteOptions{GracePeriodSeconds: &gracePeriodSeconds}) + if err != nil { + return fmt.Errorf("ingress %s.%s unable to remove canary ingress: %w", canaryIngressName, canary.Namespace, err) + } + return nil +} + +func (skp *SkipperRouter) makeAnnotations(annotations map[string]string, backendWeights map[string]int) map[string]string { + b, err := json.Marshal(backendWeights) + if err != nil { + skp.logger.Errorf("Skipper:makeAnnotations: unable to marshal backendWeights %w", err) + return annotations + } + annotations[skipperBackendWeightsAnnotationKey] = string(b) + // adding more weight to canary route solves traffic bypassing through apexIngress + annotations[skipperpredicateAnnotationKey] = canaryRouteWeight + + return annotations +} + +// parse backend-weights annotation if it exists +func (skp *SkipperRouter) backendWeights(annotation map[string]string) (backendWeights map[string]int, err error) { + backends, ok := annotation[skipperBackendWeightsAnnotationKey] + if ok { + err = json.Unmarshal([]byte(backends), &backendWeights) + } else { + err = errors.NewNotFound(schema.GroupResource{Group: "Skipper Canary Ingress", Resource: "Annotation"}, + skipperBackendWeightsAnnotationKey) + } + return +} + +// getIngressNames returns the primary and canary Kubernetes Ingress names +func (skp *SkipperRouter) getIngressNames(name string) (apexName, canaryName string) { + return name, fmt.Sprintf(canaryPatternf, name) +} diff --git a/pkg/router/skipper_test.go b/pkg/router/skipper_test.go new file mode 100644 index 00000000..e782f38e --- /dev/null +++ b/pkg/router/skipper_test.go @@ -0,0 +1,107 @@ +package router + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestSkipperRouter_Reconcile(t *testing.T) { + assert := assert.New(t) + mocks := newFixture(nil) + + for _, tt := range []struct { + name string + mocks func() fixture + wantErr bool + }{ + { + "creating new canary ingress w/ default settings", + func() fixture { return mocks }, + false, + }, { + "updating existing canary ingress", + func() fixture { + ti := newTestIngress() + ti.Annotations["something"] = "changed" + _, err := mocks.kubeClient.NetworkingV1beta1().Ingresses("default").Update( + context.TODO(), ti, metav1.UpdateOptions{}) + assert.NoError(err) + return mocks + }, + false, + }, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + mocks := tt.mocks() + router := &SkipperRouter{ + kubeClient: mocks.kubeClient, + logger: mocks.logger, + } + assert.NoError(router.Reconcile(mocks.ingressCanary)) + canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name) + inCanary, err := router.kubeClient.NetworkingV1beta1().Ingresses("default").Get( + context.TODO(), canaryName, metav1.GetOptions{}) + assert.NoError(err) + // test initialisation + assert.JSONEq(`{ "podinfo-primary": 100, "podinfo-canary": 0 }`, inCanary.Annotations["zalando.org/backend-weights"]) + assert.Equal("podinfo-primary", inCanary.Spec.Rules[0].HTTP.Paths[0].Backend.ServiceName, "backend flipped over") + assert.Equal("podinfo-canary", inCanary.Spec.Rules[0].HTTP.Paths[1].Backend.ServiceName, "backend flipped over") + assert.Len(inCanary.Spec.Rules[0].HTTP.Paths, 2) + inApex, err := router.kubeClient.NetworkingV1beta1().Ingresses("default").Get( + context.TODO(), mocks.ingressCanary.Spec.IngressRef.Name, metav1.GetOptions{}) + assert.NoError(err) + assert.Equal(inCanary.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort, + inApex.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort, "canary backend not cloned") + assert.Equal(inCanary.Spec.Rules[0].HTTP.Paths[0].Backend.ServicePort, + inCanary.Spec.Rules[0].HTTP.Paths[1].Backend.ServicePort, "canary backend not cloned") + }) + } +} + +func TestSkipperRouter_GetSetRoutes(t *testing.T) { + assert := assert.New(t) + mocks := newFixture(nil) + + router := &SkipperRouter{logger: mocks.logger, kubeClient: mocks.kubeClient} + assert.NoError(router.Reconcile(mocks.ingressCanary)) + + p, c, m, err := router.GetRoutes(mocks.ingressCanary) + assert.NoError(err) + assert.Equal(100, p) + assert.Equal(0, c) + assert.Equal(false, m) + + tests := []struct { + name string + primary, canary int + }{ + {name: "0%", primary: 100, canary: 0}, + {name: "10%", primary: 90, canary: 10}, + {name: "20%", primary: 80, canary: 20}, + {name: "30%", primary: 70, canary: 30}, + {name: "85%", primary: 15, canary: 85}, + {name: "100%", primary: 0, canary: 100}, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + assert.NoError(router.SetRoutes(mocks.ingressCanary, tt.primary, tt.canary, false)) + inCanary, err := router.kubeClient.NetworkingV1beta1().Ingresses("default").Get( + context.TODO(), fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name), metav1.GetOptions{}) + assert.NoError(err) + assert.JSONEq(fmt.Sprintf(`{"podinfo-primary": %d,"podinfo-canary": %d}`, tt.primary, tt.canary), + inCanary.Annotations["zalando.org/backend-weights"]) + p, c, m, err = router.GetRoutes(mocks.ingressCanary) + assert.NoError(err) + assert.Equal(tt.primary, p) + assert.Equal(tt.canary, c) + assert.Equal(false, m) + }) + } + +}