From 48467eb8b35f1917fa90cedaf62912bb1ff7449e Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 14 May 2020 12:06:56 +0300 Subject: [PATCH 1/6] Add ingress class support for Contour Add `-ingress-class` command flag. When set, the specified class is used to annotate the generated HTTPProxy objects. --- cmd/flagger/main.go | 8 +++++--- pkg/controller/scheduler_daemonset_fixture_test.go | 2 +- pkg/controller/scheduler_deployment_fixture_test.go | 2 +- pkg/router/contour.go | 9 +++++++++ pkg/router/contour_test.go | 2 ++ pkg/router/factory.go | 3 +++ 6 files changed, 21 insertions(+), 5 deletions(-) diff --git a/cmd/flagger/main.go b/cmd/flagger/main.go index cfb53443..540179a8 100644 --- a/cmd/flagger/main.go +++ b/cmd/flagger/main.go @@ -54,6 +54,7 @@ var ( meshProvider string selectorLabels string ingressAnnotationsPrefix string + ingressClass string enableLeaderElection bool leaderElectionNamespace string enableConfigTracking bool @@ -77,9 +78,10 @@ func init() { flag.BoolVar(&zapReplaceGlobals, "zap-replace-globals", false, "Whether to change the logging level of the global zap logger.") flag.StringVar(&zapEncoding, "zap-encoding", "json", "Zap logger encoding.") flag.StringVar(&namespace, "namespace", "", "Namespace that flagger would watch canary object.") - flag.StringVar(&meshProvider, "mesh-provider", "istio", "Service mesh provider, can be istio, linkerd, appmesh, supergloo, nginx or smi.") + flag.StringVar(&meshProvider, "mesh-provider", "istio", "Service mesh provider, can be istio, linkerd, appmesh, contour, gloo or nginx.") flag.StringVar(&selectorLabels, "selector-labels", "app,name,app.kubernetes.io/name", "List of pod labels that Flagger uses to create pod selectors.") - flag.StringVar(&ingressAnnotationsPrefix, "ingress-annotations-prefix", "nginx.ingress.kubernetes.io", "Annotations prefix for ingresses.") + flag.StringVar(&ingressAnnotationsPrefix, "ingress-annotations-prefix", "nginx.ingress.kubernetes.io", "Annotations prefix for NGINX ingresses.") + flag.StringVar(&ingressClass, "ingress-class", "", "Ingress class used for annotating HTTPProxy objects.") flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election.") flag.StringVar(&leaderElectionNamespace, "leader-election-namespace", "kube-system", "Namespace used to create the leader election config map.") flag.BoolVar(&enableConfigTracking, "enable-config-tracking", true, "Enable secrets and configmaps tracking.") @@ -169,7 +171,7 @@ func main() { // start HTTP server go server.ListenAndServe(port, 3*time.Second, logger, stopCh) - routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, ingressAnnotationsPrefix, logger, meshClient) + routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, ingressAnnotationsPrefix, ingressClass, logger, meshClient) var configTracker canary.Tracker if enableConfigTracking { diff --git a/pkg/controller/scheduler_daemonset_fixture_test.go b/pkg/controller/scheduler_daemonset_fixture_test.go index 02d8778c..ac620814 100644 --- a/pkg/controller/scheduler_daemonset_fixture_test.go +++ b/pkg/controller/scheduler_daemonset_fixture_test.go @@ -76,7 +76,7 @@ func newDaemonSetFixture(c *flaggerv1.Canary) daemonSetFixture { } // init router - rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", logger, flaggerClient) + rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", "", logger, flaggerClient) // init observer observerFactory, _ := observers.NewFactory("fake") diff --git a/pkg/controller/scheduler_deployment_fixture_test.go b/pkg/controller/scheduler_deployment_fixture_test.go index fadf8366..d1bfb3ba 100644 --- a/pkg/controller/scheduler_deployment_fixture_test.go +++ b/pkg/controller/scheduler_deployment_fixture_test.go @@ -104,7 +104,7 @@ func newDeploymentFixture(c *flaggerv1.Canary) fixture { } // init router - rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", logger, flaggerClient) + rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", "", logger, flaggerClient) // init observer observerFactory, _ := observers.NewFactory("fake") diff --git a/pkg/router/contour.go b/pkg/router/contour.go index bf3079af..a2e7b6b0 100644 --- a/pkg/router/contour.go +++ b/pkg/router/contour.go @@ -23,10 +23,13 @@ type ContourRouter struct { contourClient clientset.Interface flaggerClient clientset.Interface logger *zap.SugaredLogger + ingressClass string } // Reconcile creates or updates the HTTP proxy func (cr *ContourRouter) Reconcile(canary *flaggerv1.Canary) error { + annotation := "projectcontour.io/ingress.class" + apexName, primaryName, canaryName := canary.GetServiceNames() newSpec := contourv1.HTTPProxySpec{ @@ -151,6 +154,12 @@ func (cr *ContourRouter) Reconcile(canary *flaggerv1.Canary) error { }, } + if cr.ingressClass != "" { + proxy.Annotations = map[string]string{ + annotation: cr.ingressClass, + } + } + _, err = cr.contourClient.ProjectcontourV1().HTTPProxies(canary.Namespace).Create(context.TODO(), proxy, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("HTTPProxy %s.%s create error: %w", apexName, canary.Namespace, err) diff --git a/pkg/router/contour_test.go b/pkg/router/contour_test.go index e54e70b7..abb6562d 100644 --- a/pkg/router/contour_test.go +++ b/pkg/router/contour_test.go @@ -17,6 +17,7 @@ func TestContourRouter_Reconcile(t *testing.T) { flaggerClient: mocks.flaggerClient, contourClient: mocks.meshClient, kubeClient: mocks.kubeClient, + ingressClass: "contour", } // init @@ -31,6 +32,7 @@ func TestContourRouter_Reconcile(t *testing.T) { require.Len(t, services, 2) assert.Equal(t, uint32(100), services[0].Weight) assert.Equal(t, uint32(0), services[1].Weight) + assert.Equal(t, "contour", proxy.Annotations["projectcontour.io/ingress.class"]) // test update cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) diff --git a/pkg/router/factory.go b/pkg/router/factory.go index 9efcf8a6..fed25506 100644 --- a/pkg/router/factory.go +++ b/pkg/router/factory.go @@ -17,12 +17,14 @@ type Factory struct { meshClient clientset.Interface flaggerClient clientset.Interface ingressAnnotationsPrefix string + ingressClass string logger *zap.SugaredLogger } func NewFactory(kubeConfig *restclient.Config, kubeClient kubernetes.Interface, flaggerClient clientset.Interface, ingressAnnotationsPrefix string, + ingressClass string, logger *zap.SugaredLogger, meshClient clientset.Interface) *Factory { return &Factory{ @@ -31,6 +33,7 @@ func NewFactory(kubeConfig *restclient.Config, kubeClient kubernetes.Interface, kubeClient: kubeClient, flaggerClient: flaggerClient, ingressAnnotationsPrefix: ingressAnnotationsPrefix, + ingressClass: ingressClass, logger: logger, } } From 5c7fd5d4db18cb5ba719689ebf13e277b2839476 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 14 May 2020 12:16:24 +0300 Subject: [PATCH 2/6] Add ingress class option to Helm chart --- charts/flagger/README.md | 3 +++ charts/flagger/templates/deployment.yaml | 3 +++ charts/flagger/values.yaml | 6 ++++++ 3 files changed, 12 insertions(+) diff --git a/charts/flagger/README.md b/charts/flagger/README.md index 6ff665ae..631ed472 100644 --- a/charts/flagger/README.md +++ b/charts/flagger/README.md @@ -79,6 +79,7 @@ To install Flagger and Prometheus for **Contour**: $ helm upgrade -i flagger flagger/flagger \ --namespace=projectcontour \ --set meshProvider=contour \ + --set ingressClass=contour \ --set prometheus.install=true ``` @@ -135,6 +136,8 @@ Parameter | Description | Default `tolerations` | List of node taints to tolerate | `[]` `istio.kubeconfig.secretName` | The name of the Kubernetes secret containing the Istio shared control plane kubeconfig | None `istio.kubeconfig.key` | The name of Kubernetes secret data key that contains the Istio control plane kubeconfig | `kubeconfig` +`ingressAnnotationsPrefix` | Annotations prefix for NGINX ingresses | None +`ingressClass` | Ingress class used for annotating HTTPProxy objects, e.g. `contour` | None Specify each parameter using the `--set key=value[,key=value]` argument to `helm upgrade`. For example, diff --git a/charts/flagger/templates/deployment.yaml b/charts/flagger/templates/deployment.yaml index f4934857..9666a15f 100644 --- a/charts/flagger/templates/deployment.yaml +++ b/charts/flagger/templates/deployment.yaml @@ -103,6 +103,9 @@ spec: {{- if .Values.ingressAnnotationsPrefix }} - -ingress-annotations-prefix={{ .Values.ingressAnnotationsPrefix }} {{- end }} + {{- if .Values.ingressClass }} + - -ingress-class={{ .Values.ingressClass }} + {{- end }} {{- if .Values.eventWebhook }} - -event-webhook={{ .Values.eventWebhook }} {{- end }} diff --git a/charts/flagger/values.yaml b/charts/flagger/values.yaml index 9ff73da0..5b8fa00f 100644 --- a/charts/flagger/values.yaml +++ b/charts/flagger/values.yaml @@ -30,6 +30,12 @@ selectorLabels: "" configTracking: enabled: true +# annotations prefix for NGINX ingresses +ingressAnnotationsPrefix: "" + +# ingress class used for annotating HTTPProxy objects +ingressClass: "" + # when enabled, it will add a security context for the flagger pod. You may # need to disable this if you are running flagger on OpenShift securityContext: From ff94e14d5a056fbda2625ccc4ef09b5f5e0aa146 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 14 May 2020 12:21:22 +0300 Subject: [PATCH 3/6] Update Contour e2e to v1.4 --- test/e2e-contour.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/e2e-contour.sh b/test/e2e-contour.sh index 676461ae..740ad3b6 100755 --- a/test/e2e-contour.sh +++ b/test/e2e-contour.sh @@ -4,7 +4,7 @@ set -o errexit REPO_ROOT=$(git rev-parse --show-toplevel) -CONTOUR_VER="release-1.3" +CONTOUR_VER="release-1.4" echo '>>> Installing Contour' kubectl apply -f https://raw.githubusercontent.com/projectcontour/contour/${CONTOUR_VER}/examples/render/contour.yaml @@ -19,7 +19,8 @@ echo '>>> Installing Flagger' helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ --namespace projectcontour \ --set prometheus.install=true \ ---set meshProvider=contour +--set meshProvider=contour \ +--set ingressClass=contour kubectl -n projectcontour set image deployment/flagger flagger=test/flagger:latest From fbaf8fedc71b16a9121c88a52f129353f074d96e Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 14 May 2020 12:27:11 +0300 Subject: [PATCH 4/6] Set ingress class in factory --- pkg/router/factory.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/router/factory.go b/pkg/router/factory.go index fed25506..15df37fe 100644 --- a/pkg/router/factory.go +++ b/pkg/router/factory.go @@ -102,6 +102,7 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf flaggerClient: factory.flaggerClient, kubeClient: factory.kubeClient, contourClient: factory.meshClient, + ingressClass: factory.ingressClass, } case strings.HasPrefix(provider, flaggerv1.GlooProvider): upstreamDiscoveryNs := flaggerv1.GlooProvider + "-system" From 68ccbc4817e0e244fd5f1e60d78a6ca2b6b2faa1 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 14 May 2020 12:29:54 +0300 Subject: [PATCH 5/6] Add ingress class e2e test --- test/e2e-contour-tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/e2e-contour-tests.sh b/test/e2e-contour-tests.sh index 90f9c84b..8c03498f 100755 --- a/test/e2e-contour-tests.sh +++ b/test/e2e-contour-tests.sh @@ -100,6 +100,8 @@ until ${ok}; do fi done +kubectl -n test get httpproxy podinfo -oyaml | grep 'projectcontour.io/ingress.class: contour' + echo '✔ Canary initialization test passed' echo '>>> Triggering canary deployment' From 3e19ef0f01b0badf1737324437e1b09416543a5b Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Thu, 14 May 2020 12:48:12 +0300 Subject: [PATCH 6/6] Make Contour annotation const Co-authored-by: Takeshi Yoneda --- pkg/router/contour.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/router/contour.go b/pkg/router/contour.go index a2e7b6b0..3b2752ea 100644 --- a/pkg/router/contour.go +++ b/pkg/router/contour.go @@ -28,7 +28,7 @@ type ContourRouter struct { // Reconcile creates or updates the HTTP proxy func (cr *ContourRouter) Reconcile(canary *flaggerv1.Canary) error { - annotation := "projectcontour.io/ingress.class" + const annotation = "projectcontour.io/ingress.class" apexName, primaryName, canaryName := canary.GetServiceNames()