Merge pull request #323 from weaveworks/prep-0.18.6

Release v0.18.6

.circleci/config.yml

@@ -3,7 +3,7 @@ jobs:
 
   build-binary:
     docker:
-      - image: circleci/golang:1.12
+      - image: circleci/golang:1.13
     working_directory: ~/build
     steps:
       - checkout
@@ -44,7 +44,7 @@ jobs:
 
   push-container:
     docker:
-      - image: circleci/golang:1.12
+      - image: circleci/golang:1.13
     steps:
       - checkout
       - setup_remote_docker:
@@ -56,7 +56,7 @@ jobs:
 
   push-binary:
     docker:
-      - image: circleci/golang:1.12
+      - image: circleci/golang:1.13
     working_directory: ~/build
     steps:
       - checkout
@@ -132,6 +132,9 @@ jobs:
       - run: test/e2e-kind.sh
       - run: test/e2e-nginx.sh
       - run: test/e2e-nginx-tests.sh
+      - run: test/e2e-nginx-cleanup.sh
+      - run: test/e2e-nginx-custom-annotations.sh
+      - run: test/e2e-nginx-tests.sh
 
   e2e-linkerd-testing:
     machine: true
@@ -146,7 +149,7 @@ jobs:
 
   push-helm-charts:
     docker:
-      - image: circleci/golang:1.12
+      - image: circleci/golang:1.13
     steps:
       - checkout
       - run:

CHANGELOG.md

@@ -2,6 +2,71 @@
 
 All notable changes to this project are documented in this file.
 
+## 0.18.6 (2019-10-03) 
+
+Adds support for App Mesh conformance tests and latency metric checks
+
+#### Improvements 
+
+- Add support for acceptance testing when using App Mesh [#322](https://github.com/weaveworks/flagger/pull/322)
+- Add Kustomize installer for App Mesh [#310](https://github.com/weaveworks/flagger/pull/310)
+- Update Linkerd to v2.5.0 and Prometheus to v2.12.0 [#323](https://github.com/weaveworks/flagger/pull/323)
+
+#### Fixes
+
+- Fix slack/teams notification fields mapping [#318](https://github.com/weaveworks/flagger/pull/318)
+
+## 0.18.5 (2019-10-02) 
+
+Adds support for [confirm-promotion](https://docs.flagger.app/how-it-works#webhooks) webhooks and blue/green deployments when using a service mesh
+
+#### Features
+
+- Implement confirm-promotion hook [#307](https://github.com/weaveworks/flagger/pull/307)
+- Implement B/G for service mesh providers [#305](https://github.com/weaveworks/flagger/pull/305)
+
+#### Improvements 
+
+- Canary promotion improvements to avoid dropping in-flight requests [#310](https://github.com/weaveworks/flagger/pull/310)
+- Update end-to-end tests to Kubernetes v1.15.3 and Istio 1.3.0 [#306](https://github.com/weaveworks/flagger/pull/306) 
+
+#### Fixes
+
+- Skip primary check for App Mesh [#315](https://github.com/weaveworks/flagger/pull/315)
+
+## 0.18.4 (2019-09-08) 
+
+Adds support for NGINX custom annotations and Helm v3 acceptance testing
+
+#### Features
+
+- Add annotations prefix for NGINX ingresses [#293](https://github.com/weaveworks/flagger/pull/293)
+- Add wide columns in CRD [#289](https://github.com/weaveworks/flagger/pull/289)
+- loadtester: implement Helm v3 test command [#296](https://github.com/weaveworks/flagger/pull/296)
+- loadtester: add gPRC health check to load tester image [#295](https://github.com/weaveworks/flagger/pull/295)
+
+#### Fixes
+
+- loadtester: fix tests error logging [#286](https://github.com/weaveworks/flagger/pull/286)
+
+## 0.18.3 (2019-08-22) 
+
+Adds support for tillerless helm tests and protobuf health checking
+
+#### Features
+
+- loadtester: add support for tillerless helm [#280](https://github.com/weaveworks/flagger/pull/280)
+- loadtester: add support for protobuf health checking [#280](https://github.com/weaveworks/flagger/pull/280)
+
+#### Improvements 
+
+- Set HTTP listeners for AppMesh virtual routers [#272](https://github.com/weaveworks/flagger/pull/272)
+
+#### Fixes
+
+- Add missing fields to CRD validation spec [#271](https://github.com/weaveworks/flagger/pull/271)
+- Fix App Mesh backends validation in CRD [#281](https://github.com/weaveworks/flagger/pull/281)
+
 ## 0.18.2 (2019-08-05) 
 
 Fixes multi-port support for Istio

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.9
+FROM alpine:3.10
 
 RUN addgroup -S flagger \
     && adduser -S -g flagger flagger \

Dockerfile.loadtester

@@ -9,13 +9,24 @@ WORKDIR /home/app
 RUN curl -sSLo hey "https://storage.googleapis.com/jblabs/dist/hey_linux_v0.1.2" && \
 chmod +x hey && mv hey /usr/local/bin/hey
 
-RUN curl -sSL "https://get.helm.sh/helm-v2.12.3-linux-amd64.tar.gz" | tar xvz && \
+RUN curl -sSL "https://get.helm.sh/helm-v2.14.3-linux-amd64.tar.gz" | tar xvz && \
 chmod +x linux-amd64/helm && mv linux-amd64/helm /usr/local/bin/helm && \
+chmod +x linux-amd64/tiller && mv linux-amd64/tiller /usr/local/bin/tiller && \
 rm -rf linux-amd64
 
+RUN curl -sSL "https://get.helm.sh/helm-v3.0.0-beta.3-linux-amd64.tar.gz" | tar xvz && \
+chmod +x linux-amd64/helm && mv linux-amd64/helm /usr/local/bin/helmv3 && \
+rm -rf linux-amd64
+
+RUN GRPC_HEALTH_PROBE_VERSION=v0.3.0 && \
+wget -qO /usr/local/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
+chmod +x /usr/local/bin/grpc_health_probe
+
 RUN curl -sSL "https://github.com/bojand/ghz/releases/download/v0.39.0/ghz_0.39.0_Linux_x86_64.tar.gz" | tar xz -C /tmp && \
 mv /tmp/ghz /usr/local/bin && chmod +x /usr/local/bin/ghz && rm -rf /tmp/ghz-web
 
+ADD https://raw.githubusercontent.com/grpc/grpc-proto/master/grpc/health/v1/health.proto /tmp/ghz/health.proto
+
 RUN ls /tmp
 
 COPY ./bin/loadtester .
@@ -24,4 +35,7 @@ RUN chown -R app:app ./
 
 USER app
 
+RUN curl -sSL "https://github.com/rimusz/helm-tiller/archive/v0.8.3.tar.gz" | tar xvz && \
+helm init --client-only && helm plugin install helm-tiller-0.8.3 && helm plugin list
+
 ENTRYPOINT ["./loadtester"]

Makefile

@@ -7,15 +7,8 @@ LT_VERSION?=$(shell grep 'VERSION' cmd/loadtester/main.go | awk '{ print $$4 }'
 TS=$(shell date +%Y-%m-%d_%H-%M-%S)
 
 run:
-	GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=istio -namespace=test \
-	-metrics-server=https://prometheus.istio.weavedx.com \
-	-enable-leader-election=true
-
-run2:
-	GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=istio -namespace=test \
-	-metrics-server=https://prometheus.istio.weavedx.com \
-	-enable-leader-election=true \
-	-port=9092
+	GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=istio -namespace=test-istio \
+	-metrics-server=https://prometheus.istio.flagger.dev
 
 run-appmesh:
 	GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=appmesh \
@@ -38,8 +31,8 @@ run-nop:
 	-metrics-server=https://prometheus.istio.weavedx.com
 
 run-linkerd:
-	GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=smi:linkerd -namespace=demo \
-	-metrics-server=https://linkerd-prometheus.istio.weavedx.com
+	GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=linkerd -namespace=dev \
+	-metrics-server=https://prometheus.linkerd.flagger.dev
 
 build:
 	GIT_COMMIT=$$(git rev-list -1 HEAD) && GO111MODULE=on CGO_ENABLED=0 GOOS=linux go build  -ldflags "-s -w -X github.com/weaveworks/flagger/pkg/version.REVISION=$${GIT_COMMIT}" -a -installsuffix cgo -o ./bin/flagger ./cmd/flagger/*

README.md

@@ -163,7 +163,7 @@ For more details on how the canary analysis and promotion works please [read the
 | A/B testing (headers and cookies filters)    | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_minus_sign: |
 | Webhooks (acceptance/load testing)           | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
 | Request success rate check (L7 metric)       | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
-| Request duration check (L7 metric)           | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_check_mark: |
+| Request duration check (L7 metric)           | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
 | Custom promql checks                         | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
 | Traffic policy, CORS, retries and timeouts   | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_minus_sign: |
 

artifacts/appmesh/canary.yaml

@@ -41,8 +41,20 @@ spec:
       # percentage (0-100)
       threshold: 99
       interval: 1m
-    # external checks (optional)
+    - name: request-duration
+      # maximum req duration P99
+      # milliseconds
+      threshold: 500
+      interval: 30s
+    # testing (optional)
     webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 30s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token"
       - name: load-test
         url: http://flagger-loadtester.test/
         timeout: 5s

artifacts/appmesh/deployment.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.7.0
+        image: stefanprodan/podinfo:3.1.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/appmesh/ingress.yaml

@@ -13,7 +13,7 @@ data:
         - address:
             socket_address:
               address: 0.0.0.0
-              port_value: 80
+              port_value: 8080
           filter_chains:
             - filters:
                 - name: envoy.http_connection_manager
@@ -48,11 +48,15 @@ data:
           connect_timeout: 0.30s
           type: strict_dns
           lb_policy: round_robin
-          http2_protocol_options: {}
-          hosts:
-            - socket_address:
-                address: podinfo.test
-                port_value: 9898
+          load_assignment:
+            cluster_name: podinfo
+            endpoints:
+              - lb_endpoints:
+                  - endpoint:
+                      address:
+                        socket_address:
+                          address: podinfo.test
+                          port_value: 9898
     admin:
       access_log_path: /dev/null
       address:
@@ -91,7 +95,7 @@ spec:
       terminationGracePeriodSeconds: 30
       containers:
         - name: ingress
-          image: "envoyproxy/envoy-alpine:d920944aed67425f91fc203774aebce9609e5d9a"
+          image: "envoyproxy/envoy-alpine:v1.11.1"
           securityContext:
             capabilities:
               drop:
@@ -99,25 +103,20 @@ spec:
               add:
                 - NET_BIND_SERVICE
           command:
-            - /usr/bin/dumb-init
-            - --
-          args:
             - /usr/local/bin/envoy
-            - --base-id 30
-            - --v2-config-only
+          args:
             - -l
             - $loglevel
             - -c
             - /config/envoy.yaml
+            - --base-id
+            - "1234"
           ports:
             - name: admin
               containerPort: 9999
               protocol: TCP
             - name: http
-              containerPort: 80
-              protocol: TCP
-            - name: https
-              containerPort: 443
+              containerPort: 8080
               protocol: TCP
           livenessProbe:
             initialDelaySeconds: 5
@@ -151,11 +150,7 @@ spec:
     - protocol: TCP
       name: http
       port: 80
-      targetPort: 80
-    - protocol: TCP
-      name: https
-      port: 443
-      targetPort: 443
+      targetPort: http
   type: LoadBalancer
 ---
 apiVersion: appmesh.k8s.aws/v1beta1

artifacts/flagger/crd.yaml

@@ -33,6 +33,22 @@ spec:
     - name: Weight
       type: string
       JSONPath: .status.canaryWeight
+    - name: FailedChecks
+      type: string
+      JSONPath: .status.failedChecks
+      priority: 1
+    - name: Interval
+      type: string
+      JSONPath: .spec.canaryAnalysis.interval
+      priority: 1
+    - name: StepWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.stepWeight
+      priority: 1
+    - name: MaxWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.maxWeight
+      priority: 1
     - name: LastTransitionTime
       type: string
       JSONPath: .status.lastTransitionTime
@@ -54,7 +70,7 @@ spec:
             targetRef:
               description: Deployment selector
               type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -67,7 +83,7 @@ spec:
               anyOf:
                 - type: string
                 - type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -80,7 +96,7 @@ spec:
               anyOf:
                 - type: string
                 - type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -90,7 +106,7 @@ spec:
                   type: string
             service:
               type: object
-              required: ['port']
+              required: ["port"]
               properties:
                 port:
                   description: Container port number
@@ -108,7 +124,7 @@ spec:
                   description: AppMesh backend array
                   anyOf:
                     - type: string
-                    - type: object
+                    - type: array
                 timeout:
                   description: Istio HTTP or gRPC request timeout
                   type: string
@@ -178,7 +194,7 @@ spec:
                   properties:
                     items:
                       type: object
-                      required: ['name', 'threshold']
+                      required: ["name", "threshold"]
                       properties:
                         name:
                           description: Name of the Prometheus metric
@@ -199,7 +215,7 @@ spec:
                   properties:
                     items:
                       type: object
-                      required: ['name', 'url', 'timeout']
+                      required: ["name", "url"]
                       properties:
                         name:
                           description: Name of the webhook
@@ -212,6 +228,7 @@ spec:
                             - confirm-rollout
                             - pre-rollout
                             - rollout
+                            - confirm-promotion
                             - post-rollout
                         url:
                           description: URL address of this webhook
@@ -237,6 +254,7 @@ spec:
                 - Initialized
                 - Waiting
                 - Progressing
+                - Promoting
                 - Finalising
                 - Succeeded
                 - Failed
@@ -262,7 +280,7 @@ spec:
               properties:
                 items:
                   type: object
-                  required: ['type', 'status', 'reason']
+                  required: ["type", "status", "reason"]
                   properties:
                     lastTransitionTime:
                       description: LastTransitionTime of this condition

artifacts/flagger/deployment.yaml

@@ -22,7 +22,7 @@ spec:
       serviceAccountName: flagger
       containers:
       - name: flagger
-        image: weaveworks/flagger:0.18.2
+        image: weaveworks/flagger:0.18.6
         imagePullPolicy: IfNotPresent
         ports:
         - name: http

artifacts/helmtester/deployment.yaml

@@ -19,7 +19,7 @@ spec:
       serviceAccountName: tiller
       containers:
         - name: helmtester
-          image: weaveworks/flagger-loadtester:0.4.0
+          image: weaveworks/flagger-loadtester:0.8.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

artifacts/loadtester/deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: loadtester
-          image: weaveworks/flagger-loadtester:0.6.1
+          image: weaveworks/flagger-loadtester:0.8.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

charts/flagger/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 name: flagger
-version: 0.18.2
-appVersion: 0.18.2
+version: 0.18.6
+appVersion: 0.18.6
 kubeVersion: ">=1.11.0-0"
 engine: gotpl
 description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio, Linkerd, App Mesh, Gloo or NGINX routing for traffic shifting and Prometheus metrics for canary analysis.

charts/flagger/README.md

@@ -74,6 +74,7 @@ Parameter | Description | Default
 `msteams.url` | Microsoft Teams incoming webhook | None
 `leaderElection.enabled` | leader election must be enabled when running more than one replica | `false`
 `leaderElection.replicaCount` | number of replicas | `1`
+`ingressAnnotationsPrefix` | annotations prefix for ingresses | `custom.ingress.kubernetes.io`
 `rbac.create` | if `true`, create and use RBAC resources | `true`
 `rbac.pspEnabled` | If `true`, create and use a restricted pod security policy | `false`
 `crd.create` | if `true`, create Flagger's CRDs | `true`

charts/flagger/templates/crd.yaml

@@ -34,6 +34,22 @@ spec:
     - name: Weight
       type: string
       JSONPath: .status.canaryWeight
+    - name: FailedChecks
+      type: string
+      JSONPath: .status.failedChecks
+      priority: 1
+    - name: Interval
+      type: string
+      JSONPath: .spec.canaryAnalysis.interval
+      priority: 1
+    - name: StepWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.stepWeight
+      priority: 1
+    - name: MaxWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.maxWeight
+      priority: 1
     - name: LastTransitionTime
       type: string
       JSONPath: .status.lastTransitionTime
@@ -109,7 +125,7 @@ spec:
                   description: AppMesh backend array
                   anyOf:
                     - type: string
-                    - type: object
+                    - type: array
                 timeout:
                   description: Istio HTTP or gRPC request timeout
                   type: string
@@ -200,7 +216,7 @@ spec:
                   properties:
                     items:
                       type: object
-                      required: ['name', 'url', 'timeout']
+                      required: ["name", "url"]
                       properties:
                         name:
                           description: Name of the webhook
@@ -213,6 +229,7 @@ spec:
                             - confirm-rollout
                             - pre-rollout
                             - rollout
+                            - confirm-promotion
                             - post-rollout
                         url:
                           description: URL address of this webhook
@@ -238,6 +255,7 @@ spec:
                 - Initialized
                 - Waiting
                 - Progressing
+                - Promoting
                 - Finalising
                 - Succeeded
                 - Failed

charts/flagger/templates/deployment.yaml

@@ -20,6 +20,10 @@ spec:
       labels:
         app.kubernetes.io/name: {{ template "flagger.name" . }}
         app.kubernetes.io/instance: {{ .Release.Name }}
+      annotations:
+        {{- if .Values.podAnnotations }}
+{{ toYaml .Values.podAnnotations | indent 8 }}
+        {{- end }}
     spec:
       serviceAccountName: {{ template "flagger.serviceAccountName" . }}
       affinity:
@@ -72,6 +76,9 @@ spec:
           - -enable-leader-election=true
           - -leader-election-namespace={{ .Release.Namespace }}
           {{- end }}
+          {{- if .Values.ingressAnnotationsPrefix }}
+          - -ingress-annotations-prefix={{ .Values.ingressAnnotationsPrefix }}
+          {{- end }}
           livenessProbe:
             exec:
               command:

charts/flagger/templates/prometheus.yaml

@@ -238,7 +238,7 @@ spec:
       serviceAccountName: {{ template "flagger.serviceAccountName" . }}-prometheus
       containers:
         - name: prometheus
-          image: "docker.io/prom/prometheus:v2.10.0"
+          image: "docker.io/prom/prometheus:v2.12.0"
           imagePullPolicy: IfNotPresent
           args:
             - '--storage.tsdb.retention=2h'

charts/flagger/values.yaml

@@ -2,10 +2,12 @@
 
 image:
   repository: weaveworks/flagger
-  tag: 0.18.2
+  tag: 0.18.6
   pullPolicy: IfNotPresent
   pullSecret:
 
+podAnnotations: {}
+
 metricsServer: "http://prometheus:9090"
 
 # accepted values are istio, appmesh, nginx or supergloo:mesh.namespace (defaults to istio)

charts/grafana/templates/deployment.yaml

@@ -20,6 +20,9 @@ spec:
         release: {{ .Release.Name }}
       annotations:
         prometheus.io/scrape: 'false'
+        {{- if .Values.podAnnotations }}
+{{ toYaml .Values.podAnnotations | indent 8 }}
+        {{- end }}
     spec:
       containers:
         - name: {{ .Chart.Name }}

charts/grafana/values.yaml

@@ -9,6 +9,8 @@ image:
   tag: 6.2.5
   pullPolicy: IfNotPresent
 
+podAnnotations: {}
+
 service:
   type: ClusterIP
   port: 80

charts/loadtester/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 name: loadtester
-version: 0.6.0
-appVersion: 0.6.1
+version: 0.8.0
+appVersion: 0.8.0
 kubeVersion: ">=1.11.0-0"
 engine: gotpl
 description: Flagger's load testing services based on rakyll/hey and bojand/ghz that generates traffic during canary analysis when configured as a webhook.

charts/loadtester/templates/deployment.yaml

@@ -18,9 +18,14 @@ spec:
         app: {{ include "loadtester.name" . }}
       annotations:
         appmesh.k8s.aws/ports: "444"
+        {{- if .Values.podAnnotations }}
+{{ toYaml .Values.podAnnotations | indent 8 }}
+        {{- end }}
     spec:
       {{- if .Values.serviceAccountName }}
       serviceAccountName: {{ .Values.serviceAccountName }}
+      {{- else if .Values.rbac.create }}
+      serviceAccountName: {{ include "loadtester.fullname" . }}
       {{- end }}
       containers:
         - name: {{ .Chart.Name }}

charts/loadtester/templates/rbac.yaml

@@ -0,0 +1,54 @@
+---
+{{- if .Values.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+{{- if eq .Values.rbac.scope "cluster" }}
+kind: ClusterRole
+{{- else }}
+kind: Role
+{{- end }}
+metadata:
+  name: {{ template "loadtester.fullname" . }}
+  labels:
+    helm.sh/chart: {{ template "loadtester.chart" . }}
+    app.kubernetes.io/name: {{ template "loadtester.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+rules:
+{{ toYaml .Values.rbac.rules | indent 2 }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+{{- if eq .Values.rbac.scope "cluster" }}
+kind: ClusterRoleBinding
+{{- else }}
+kind: RoleBinding
+{{- end }}
+metadata:
+  name: {{ template "loadtester.fullname" . }}
+  labels:
+    helm.sh/chart: {{ template "loadtester.chart" . }}
+    app.kubernetes.io/name: {{ template "loadtester.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  {{- if eq .Values.rbac.scope "cluster" }}
+  kind: ClusterRole
+  {{- else }}
+  kind: Role
+  {{- end }}
+  name: {{ template "loadtester.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "loadtester.fullname" . }}
+    namespace: {{ .Release.Namespace }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "loadtester.fullname" . }}
+  labels:
+    helm.sh/chart: {{ template "loadtester.chart" . }}
+    app.kubernetes.io/name: {{ template "loadtester.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

charts/loadtester/values.yaml

@@ -2,9 +2,11 @@ replicaCount: 1
 
 image:
   repository: weaveworks/flagger-loadtester
-  tag: 0.6.1
+  tag: 0.8.0
   pullPolicy: IfNotPresent
 
+podAnnotations: {}
+
 logLevel: info
 cmd:
   timeout: 1h
@@ -27,6 +29,20 @@ tolerations: []
 
 affinity: {}
 
+rbac:
+  # rbac.create: `true` if rbac resources should be created
+  create: false
+  # rbac.scope: `cluster` to create cluster-scope rbac resources (ClusterRole/ClusterRoleBinding)
+  # otherwise, namespace-scope rbac resources will be created (Role/RoleBinding)
+  scope:
+  # rbac.rules: array of rules to apply to the role. example:
+  # rules:
+  # - apiGroups: [""]
+  #   resources: ["pods"]
+  #   verbs: ["list", "get"]
+  rules: []
+
+# name of an existing service account to use - if not creating rbac resources
 serviceAccountName: ""
 
 # App Mesh virtual node settings

charts/podinfo/Chart.yaml

@@ -1,10 +1,10 @@
 apiVersion: v1
-version: 3.0.0
-appVersion: 2.0.0
+version: 3.1.0
+appVersion: 3.1.0
 name: podinfo
 engine: gotpl
 description: Flagger canary deployment demo chart
-home: https://github.com/weaveworks/flagger
+home: https://flagger.app
 maintainers:
 - email: stefanprodan@users.noreply.github.com
   name: stefanprodan

charts/podinfo/templates/deployment.yaml

@@ -21,6 +21,9 @@ spec:
         app: {{ template "podinfo.fullname" . }}
       annotations:
         prometheus.io/scrape: 'true'
+        {{- if .Values.podAnnotations }}
+{{ toYaml .Values.podAnnotations | indent 8 }}
+        {{- end }}
     spec:
       terminationGracePeriodSeconds: 30
       containers:
@@ -34,6 +37,9 @@ spec:
             - --random-delay={{ .Values.faults.delay }}
             - --random-error={{ .Values.faults.error }}
             - --config-path=/podinfo/config
+            {{- range .Values.backends }}
+            - --backend-url={{ . }}
+            {{- end }}
           env:
           {{- if .Values.message }}
           - name: PODINFO_UI_MESSAGE

charts/podinfo/templates/tests/jwt.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ template "podinfo.fullname" . }}-jwt-test-{{ randAlphaNum 5 | lower }}
+  labels:
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    app: {{ template "podinfo.name" . }}
+  annotations:
+    "helm.sh/hook": test-success
+    sidecar.istio.io/inject: "false"
+    linkerd.io/inject: disabled
+    appmesh.k8s.aws/sidecarInjectorWebhook: disabled
+spec:
+  containers:
+    - name: tools
+      image: giantswarm/tiny-tools
+      command:
+        - sh
+        - -c
+        - |
+          TOKEN=$(curl -sd 'test' ${PODINFO_SVC}/token | jq -r .token) &&
+          curl -H "Authorization: Bearer ${TOKEN}" ${PODINFO_SVC}/token/validate | grep test
+      env:
+      - name: PODINFO_SVC
+        value: {{ template "podinfo.fullname" . }}:{{ .Values.service.port }}
+  restartPolicy: Never
+

charts/podinfo/templates/tests/test-config.yaml

@@ -1,22 +0,0 @@
-{{- $url := printf "%s%s.%s:%v" (include "podinfo.fullname" .) (include "podinfo.suffix" .) .Release.Namespace .Values.service.port -}}
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name:  {{ template "podinfo.fullname" . }}-tests
-  labels:
-    heritage: {{ .Release.Service }}
-    release: {{ .Release.Name }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    app: {{ template "podinfo.name" . }}
-data:
-  run.sh: |-
-    @test "HTTP POST /echo" {
-      run curl --retry 3 --connect-timeout 2 -sSX POST -d 'test' {{ $url }}/echo
-      [ $output = "test" ]
-    }
-    @test "HTTP POST /store" {
-      curl --retry 3 --connect-timeout 2 -sSX POST -d 'test' {{ $url }}/store
-    }
-    @test "HTTP GET /" {
-      curl --retry 3 --connect-timeout 2 -sS {{ $url }} | grep hostname
-    }

charts/podinfo/templates/tests/test-pod.yaml

@@ -1,43 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: {{ template "podinfo.fullname" . }}-tests-{{ randAlphaNum 5 | lower }}
-  annotations:
-    "helm.sh/hook": test-success
-    sidecar.istio.io/inject: "false"
-  labels:
-    heritage: {{ .Release.Service }}
-    release: {{ .Release.Name }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    app: {{ template "podinfo.name" . }}
-spec:
-  initContainers:
-    - name: "test-framework"
-      image: "dduportal/bats:0.4.0"
-      command:
-      - "bash"
-      - "-c"
-      - |
-        set -ex
-        # copy bats to tools dir
-        cp -R /usr/local/libexec/ /tools/bats/
-      volumeMounts:
-      - mountPath: /tools
-        name: tools
-  containers:
-    - name: {{ .Release.Name }}-ui-test
-      image: dduportal/bats:0.4.0
-      command: ["/tools/bats/bats", "-t", "/tests/run.sh"]
-      volumeMounts:
-      - mountPath: /tests
-        name: tests
-        readOnly: true
-      - mountPath: /tools
-        name: tools
-  volumes:
-  - name: tests
-    configMap:
-      name: {{ template "podinfo.fullname" . }}-tests
-  - name: tools
-    emptyDir: {}
-  restartPolicy: Never

charts/podinfo/values.yaml

@@ -1,22 +1,25 @@
 # Default values for podinfo.
 image:
   repository: stefanprodan/podinfo
-  tag: 2.0.0
+  tag: 3.1.0
   pullPolicy: IfNotPresent
 
+podAnnotations: {}
+
 service:
+  enabled: false
   type: ClusterIP
   port: 9898
 
 hpa:
   enabled: true
   minReplicas: 2
-  maxReplicas: 2
+  maxReplicas: 4
   cpu: 80
   memory: 512Mi
 
 canary:
-  enabled: true
+  enabled: false
   # Istio traffic policy tls can be DISABLE or ISTIO_MUTUAL
   istioTLS: DISABLE
   istioIngress:
@@ -69,6 +72,7 @@ fullnameOverride: ""
 
 logLevel: info
 backend: #http://backend-podinfo:9898/echo
+backends: []
 message: #UI greetings
 
 faults:

cmd/flagger/main.go

@@ -33,25 +33,26 @@ import (
 )
 
 var (
-	masterURL               string
-	kubeconfig              string
-	metricsServer           string
-	controlLoopInterval     time.Duration
-	logLevel                string
-	port                    string
-	msteamsURL              string
-	slackURL                string
-	slackUser               string
-	slackChannel            string
-	threadiness             int
-	zapReplaceGlobals       bool
-	zapEncoding             string
-	namespace               string
-	meshProvider            string
-	selectorLabels          string
-	enableLeaderElection    bool
-	leaderElectionNamespace string
-	ver                     bool
+	masterURL                string
+	kubeconfig               string
+	metricsServer            string
+	controlLoopInterval      time.Duration
+	logLevel                 string
+	port                     string
+	msteamsURL               string
+	slackURL                 string
+	slackUser                string
+	slackChannel             string
+	threadiness              int
+	zapReplaceGlobals        bool
+	zapEncoding              string
+	namespace                string
+	meshProvider             string
+	selectorLabels           string
+	ingressAnnotationsPrefix string
+	enableLeaderElection     bool
+	leaderElectionNamespace  string
+	ver                      bool
 )
 
 func init() {
@@ -71,6 +72,7 @@ func init() {
 	flag.StringVar(&namespace, "namespace", "", "Namespace that flagger would watch canary object.")
 	flag.StringVar(&meshProvider, "mesh-provider", "istio", "Service mesh provider, can be istio, linkerd, appmesh, supergloo, nginx or smi.")
 	flag.StringVar(&selectorLabels, "selector-labels", "app,name,app.kubernetes.io/name", "List of pod labels that Flagger uses to create pod selectors.")
+	flag.StringVar(&ingressAnnotationsPrefix, "ingress-annotations-prefix", "nginx.ingress.kubernetes.io", "Annotations prefix for ingresses.")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election.")
 	flag.StringVar(&leaderElectionNamespace, "leader-election-namespace", "kube-system", "Namespace used to create the leader election config map.")
 	flag.BoolVar(&ver, "version", false, "Print version")
@@ -175,7 +177,7 @@ func main() {
 	// start HTTP server
 	go server.ListenAndServe(port, 3*time.Second, logger, stopCh)
 
-	routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, logger, meshClient)
+	routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, ingressAnnotationsPrefix, logger, meshClient)
 
 	c := controller.NewController(
 		kubeClient,

cmd/loadtester/main.go

@@ -10,7 +10,7 @@ import (
 	"time"
 )
 
-var VERSION = "0.6.1"
+var VERSION = "0.8.0"
 var (
 	logLevel          string
 	port              string

docs/gitbook/how-it-works.md

@@ -143,7 +143,7 @@ status:
 ```
 
 The `Promoted` status condition can have one of the following reasons:
-Initialized, Waiting, Progressing, Finalising, Succeeded or Failed.
+Initialized, Waiting, Progressing, Promoting, Finalising, Succeeded or Failed.
 A failed canary will have the promoted status set to `false`,
 the reason to `failed` and the last applied spec will be different to the last promoted one.
 
@@ -153,6 +153,26 @@ Wait for a successful rollout:
 kubectl wait canary/podinfo --for=condition=promoted
 ```
 
+CI example:
+
+```bash
+# update the container image
+kubectl set image deployment/podinfo podinfod=stefanprodan/podinfo:3.0.1
+
+# wait for Flagger to detect the change
+ok=false
+until ${ok}; do
+    kubectl get canary/podinfo | grep 'Progressing' && ok=true || ok=false
+    sleep 5
+done
+
+# wait for the canary analysis to finish
+kubectl wait canary/podinfo --for=condition=promoted --timeout=5m
+
+# check if the deployment was successful 
+kubectl get canary/podinfo | grep Succeeded
+```
+
 ### Istio routing
 
 Flagger creates an Istio Virtual Service and Destination Rules based on the Canary service spec. 
@@ -344,12 +364,13 @@ A canary deployment is triggered by changes in any of the following objects:
 Gated canary promotion stages:
 
 * scan for canary deployments
-* check Istio virtual service routes are mapped to primary and canary ClusterIP services
-* check primary and canary deployments status
+* check primary and canary deployment status
     * halt advancement if a rolling update is underway
     * halt advancement if pods are unhealthy
-* call pre-rollout webhooks are check results
-    * halt advancement if any hook returned a non HTTP 2xx result
+* call confirm-rollout webhooks and check results
+    * halt advancement if any hook returns a non HTTP 2xx result
+* call pre-rollout webhooks and check results
+    * halt advancement if any hook returns a non HTTP 2xx result
     * increment the failed checks counter
 * increase canary traffic weight percentage from 0% to 5% (step weight)
 * call rollout webhooks and check results
@@ -366,8 +387,11 @@ Gated canary promotion stages:
     * halt advancement if any webhook call fails
     * halt advancement while canary request success rate is under the threshold
     * halt advancement while canary request duration P99 is over the threshold
+    * halt advancement while any custom metric check fails
     * halt advancement if the primary or canary deployment becomes unhealthy 
     * halt advancement while canary deployment is being scaled up/down by HPA
+* call confirm-promotion webhooks and check results
+    * halt advancement if any hook returns a non HTTP 2xx result
 * promote canary to primary
     * copy ConfigMaps and Secrets from canary to primary
     * copy canary deployment spec template over primary
@@ -377,7 +401,7 @@ Gated canary promotion stages:
 * scale to zero the canary deployment
 * mark rollout as finished
 * call post-rollout webhooks
-* post the analysis result to Slack
+* post the analysis result to Slack or MS Teams
 * wait for the canary deployment to be updated and start over
 
 ### Canary Analysis
@@ -663,6 +687,9 @@ The canary advancement is paused if a pre-rollout hook fails and if the number o
 threshold the canary will be rollback.
 * Rollout hooks are executed during the analysis on each iteration before the metric checks. 
 If a rollout hook call fails the canary advancement is paused and eventfully rolled back.
+* Confirm-promotion hooks are executed before the promotion step.
+The canary promotion is paused until the hooks return HTTP 200.
+While the promotion is paused, Flagger will continue to run the metrics checks and rollout hooks.
 * Post-rollout hooks are executed after the canary has been promoted or rolled back. 
 If a post rollout hook fails the error is logged.
 
@@ -687,6 +714,9 @@ Spec:
         timeout: 15s
         metadata:
           cmd: "hey -z 1m -q 5 -c 2 http://podinfo-canary.test:9898/"
+      - name: "promotion gate"
+        type: confirm-promotion
+        url: http://flagger-loadtester.test/gate/approve
       - name: "notify"
         type: post-rollout
         url: http://telegram.bot:8080/
@@ -798,6 +828,18 @@ webhooks:
       cmd: "ghz -z 1m -q 10 -c 2 --insecure podinfo.test:9898"
 ```
 
+`ghz` uses reflection to identify which gRPC method to call. If you do not wish to enable reflection for your gRPC service you can implement a standardized health check from the [grpc-proto](https://github.com/grpc/grpc-proto) library. To use this [health check schema](https://github.com/grpc/grpc-proto/blob/master/grpc/health/v1/health.proto) without reflection you can pass a parameter to `ghz` like this
+
+```yaml
+webhooks:
+  - name: grpc-load-test-no-reflection
+    url: http://flagger-loadtester.test/
+    timeout: 5s
+    metadata:
+      type: cmd
+      cmd: "ghz --insecure --proto=/tmp/ghz/health.proto --call=grpc.health.v1.Health/Check podinfo.test:9898"
+```
+
 The load tester can run arbitrary commands as long as the binary is present in the container image.
 For example if you you want to replace `hey` with another CLI, you can create your own Docker image:
 
@@ -870,6 +912,20 @@ Now you can add pre-rollout webhooks to the canary analysis spec:
 When the canary analysis starts, Flagger will call the pre-rollout webhooks before routing traffic to the canary.
 If the helm test fails, Flagger will retry until the analysis threshold is reached and the canary is rolled back.
 
+If you are using Helm v3, you'll have to create a dedicated service account and add the release namespace to the test command:
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "smoke test"
+        type: pre-rollout
+        url: http://flagger-helmtester.kube-system/
+        timeout: 3m
+        metadata:
+          type: "helmv3"
+          cmd: "test run {{ .Release.Name }} --cleanup -n {{ .Release.Namespace }}"
+```
+
 As an alternative to Helm you can use the [Bash Automated Testing System](https://github.com/bats-core/bats-core) to run your tests. 
 
 ```yaml
@@ -888,8 +944,8 @@ Note that you should create a ConfigMap with your Bats tests and mount it inside
 
 ### Manual Gating
 
-For manual approval of a canary deployment you can use the `confirm-rollout` webhook. 
-The confirmation hooks are executed before the pre-rollout hooks. 
+For manual approval of a canary deployment you can use the `confirm-rollout` and `confirm-promotion` webhooks. 
+The confirmation rollout hooks are executed before the pre-rollout hooks. 
 Flagger will halt the canary traffic shifting and analysis until the confirm webhook returns HTTP status 200.
 
 Manual gating with Flagger's tester:
@@ -948,3 +1004,16 @@ kubectl get canary/podinfo
 NAME      STATUS        WEIGHT
 podinfo   Waiting       0
 ```
+
+The `confirm-promotion` hook type can be used to manually approve the canary promotion.
+While the promotion is paused, Flagger will continue to run the metrics checks and load tests.
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "promotion gate"
+        type: confirm-promotion
+        url: http://flagger-loadtester.test/gate/halt
+```
+
+If you have notifications enabled, Flagger will post a message to Slack or MS Teams if a canary promotion is waiting for approval.

docs/gitbook/install/flagger-install-on-eks-appmesh.md

@@ -14,14 +14,6 @@ The App Mesh integration with EKS is made out of the following components:
 * Admission controller - injects the Envoy sidecar and assigns Kubernetes pods to App Mesh virtual nodes
 * Metrics server - Prometheus instance that collects and stores Envoy's metrics
 
-Prerequisites:
-
-* jq
-* homebrew
-* openssl
-* kubectl
-* AWS CLI (default region us-west-2)
-
 ### Create a Kubernetes cluster
 
 In order to create an EKS cluster you can use [eksctl](https://eksctl.io).
@@ -40,6 +32,8 @@ Create an EKS cluster:
 ```bash
 eksctl create cluster --name=appmesh \
 --region=us-west-2 \
+--nodes 3 \
+--node-volume-size=120 \
 --appmesh-access
 ```
 
@@ -98,21 +92,39 @@ kubectl -n kube-system top pods
 
 ### Install the App Mesh components
 
-Run the App Mesh installer:
+Create the `appmesh-system` namespace:
 
-```bash
-curl -fsSL https://git.io/get-app-mesh-eks.sh | bash -
+```sh
+kubectl create ns appmesh-system
 ```
 
-The installer does the following:
+Apply the App Mesh CRDs:
 
-* creates the `appmesh-system` namespace
-* generates a certificate signed by Kubernetes CA
-* registers the App Mesh mutating webhook
-* deploys the App Mesh webhook in `appmesh-system` namespace
-* deploys the App Mesh CRDs
-* deploys the App Mesh controller in `appmesh-system` namespace
-* creates a mesh called `global`
+```sh
+kubectl apply -f https://raw.githubusercontent.com/aws/eks-charts/master/stable/appmesh-controller/crds/crds.yaml
+```
+
+Add the EKS repository to Helm:
+
+```sh
+helm repo add eks https://aws.github.io/eks-charts
+```
+
+Install the App Mesh CRD controller:
+
+```sh
+helm upgrade -i appmesh-controller eks/appmesh-controller \
+--wait --namespace appmesh-system
+```
+
+Install the App Mesh admission controller:
+
+```sh
+helm upgrade -i appmesh-inject eks/appmesh-inject \
+--wait --namespace appmesh-system \
+--set mesh.create=true \
+--set mesh.name=global
+```
 
 Verify that the global mesh is active:
 
@@ -125,7 +137,7 @@ Status:
     Type:                  MeshActive
 ```
 
-### Install Flagger and Grafana
+### Install Flagger, Prometheus and Grafana
 
 Add Flagger Helm repository:
 
@@ -156,10 +168,8 @@ You can enable **Slack** notifications with:
 
 ```bash
 helm upgrade -i flagger flagger/flagger \
+--reuse-values \
 --namespace=appmesh-system \
---set crd.create=false \
---set meshProvider=appmesh \
---set metricsServer=http://prometheus.appmesh:9090 \
 --set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \
 --set slack.channel=general \
 --set slack.user=flagger

docs/gitbook/install/flagger-install-on-google-cloud.md

@@ -186,7 +186,7 @@ Install cert-manager's CRDs:
 ```bash
 CERT_REPO=https://raw.githubusercontent.com/jetstack/cert-manager
 
-kubectl apply -f ${CERT_REPO}/release-0.7/deploy/manifests/00-crds.yaml
+kubectl apply -f ${CERT_REPO}/release-0.10/deploy/manifests/00-crds.yaml
 ```
 
 Create the cert-manager namespace and disable resource validation:
@@ -204,7 +204,7 @@ helm repo add jetstack https://charts.jetstack.io && \
 helm repo update && \
 helm upgrade -i cert-manager \
 --namespace cert-manager \
---version v0.7.0 \
+--version v0.10.0 \
 jetstack/cert-manager
 ```
 
@@ -339,7 +339,7 @@ Find the GKE Istio version with:
 kubectl -n istio-system get deploy istio-pilot -oyaml | grep image:
 ```
 
-Install Prometheus in istio-system namespace (replace `1.0.6-gke.3` with your version):
+Install Prometheus in istio-system namespace:
 
 ```bash
 kubectl -n istio-system apply -f \

docs/gitbook/install/flagger-install-on-kubernetes.md

@@ -153,6 +153,14 @@ Note that you'll need kubectl 1.14 to run the above the command or you can downl
 kustomize build github.com/weaveworks/flagger//kustomize/istio | kubectl apply -f -
 ```
 
+Install Flagger for AWS App Mesh:
+
+```bash
+kubectl apply -k github.com/weaveworks/flagger//kustomize/appmesh
+```
+
+This deploys Flagger and Prometheus (configured to scrape the App Mesh Envoy sidecars) in the `appmesh-system` namespace.
+
 Install Flagger for Linkerd:
 
 ```bash

docs/gitbook/tutorials/canary-helm-gitops.md

@@ -51,7 +51,7 @@ helm upgrade -i frontend flagger/podinfo \
 --namespace test \
 --set nameOverride=frontend \
 --set backend=http://backend.test:9898/echo \
---set canary.loadtest.enabled=true \
+--set canary.enabled=true \
 --set canary.istioIngress.enabled=true \
 --set canary.istioIngress.gateway=public-gateway.istio-system.svc.cluster.local \
 --set canary.istioIngress.host=frontend.istio.example.com
@@ -91,7 +91,7 @@ Now let's install the `backend` release without exposing it outside the mesh:
 helm upgrade -i backend flagger/podinfo \
 --namespace test \
 --set nameOverride=backend \
---set canary.loadtest.enabled=true \
+--set canary.enabled=true \
 --set canary.istioIngress.enabled=false
 ```
 
@@ -138,7 +138,7 @@ helm upgrade -i frontend flagger/podinfo/ \
 --reuse-values \
 --set canary.loadtest.enabled=true \
 --set canary.helmtest.enabled=true \
---set image.tag=2.0.1
+--set image.tag=3.1.1
 ```
 
 Flagger detects that the deployment revision changed and starts the canary analysis:
@@ -177,6 +177,7 @@ Now trigger a canary deployment for the `backend` app, but this time you'll chan
 helm upgrade -i backend flagger/podinfo/ \
 --namespace test \
 --reuse-values \
+--set canary.loadtest.enabled=true \
 --set canary.helmtest.enabled=true \
 --set httpServer.timeout=25s
 ```
@@ -283,7 +284,7 @@ metadata:
   namespace: test
   annotations:
     flux.weave.works/automated: "true"
-    flux.weave.works/tag.chart-image: semver:~2.0
+    flux.weave.works/tag.chart-image: semver:~3.1
 spec:
   releaseName: frontend
   chart:
@@ -293,7 +294,7 @@ spec:
   values:
     image:
       repository: stefanprodan/podinfo
-      tag: 2.0.0
+      tag: 3.1.0
       backend: http://backend-podinfo:9898/echo
       canary:
         enabled: true
@@ -311,7 +312,7 @@ In the `chart` section I've defined the release source by specifying the Helm re
 In the `values` section I've overwritten the defaults set in values.yaml.
 
 With the `flux.weave.works` annotations I instruct Flux to automate this release. 
-When an image tag in the sem ver range of `2.0.0 - 2.0.99` is pushed to Quay, 
+When an image tag in the sem ver range of `3.1.0 - 3.1.99` is pushed to Docker Hub, 
 Flux will upgrade the Helm release and from there Flagger will pick up the change and start a canary deployment.
 
 Install [Weave Flux](https://github.com/weaveworks/flux) and its Helm Operator by specifying your Git repo URL:
@@ -344,9 +345,9 @@ launch the `frontend` and `backend` apps.
 
 A CI/CD pipeline for the `frontend` release could look like this:
 
-* cut a release from the master branch of the podinfo code repo with the git tag `2.0.1`
-* CI builds the image and pushes the `podinfo:2.0.1` image to the container registry
-* Flux scans the registry and updates the Helm release `image.tag` to `2.0.1`
+* cut a release from the master branch of the podinfo code repo with the git tag `3.1.1`
+* CI builds the image and pushes the `podinfo:3.1.1` image to the container registry
+* Flux scans the registry and updates the Helm release `image.tag` to `3.1.1`
 * Flux commits and push the change to the cluster repo
 * Flux applies the updated Helm release on the cluster
 * Flux Helm Operator picks up the change and calls Tiller to upgrade the release
@@ -354,9 +355,9 @@ A CI/CD pipeline for the `frontend` release could look like this:
 * Flagger runs the helm test before routing traffic to the canary service
 * Flagger starts the load test and runs the canary analysis 
 * Based on the analysis result the canary deployment is promoted to production or rolled back
-* Flagger sends a Slack notification with the canary result
+* Flagger sends a Slack or MS Teams notification with the canary result
 
-If the canary fails, fix the bug, do another patch release eg `2.0.2` and the whole process will run again.
+If the canary fails, fix the bug, do another patch release eg `3.1.2` and the whole process will run again.
 
 A canary deployment can fail due to any of the following reasons:
 

docs/gitbook/usage/appmesh-progressive-delivery.md

@@ -39,8 +39,7 @@ helm upgrade -i flagger-loadtester flagger/loadtester \
 --namespace=test \
 --set meshName=global \
 --set "backends[0]=podinfo.test" \
---set "backends[1]=podinfo-canary.test" \
---set "backends[2]=podinfo-primary.test"
+--set "backends[1]=podinfo-canary.test"
 ```
 
 Create a canary custom resource:
@@ -92,8 +91,20 @@ spec:
       # percentage (0-100)
       threshold: 99
       interval: 1m
-    # external checks (optional)
+    - name: request-duration
+      # maximum req duration P99
+      # milliseconds
+      threshold: 500
+      interval: 30s
+    # testing (optional)
     webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 30s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token"
       - name: load-test
         url: http://flagger-loadtester.test/
         timeout: 5s
@@ -127,14 +138,18 @@ virtualnode.appmesh.k8s.aws/podinfo
 virtualnode.appmesh.k8s.aws/podinfo-canary
 virtualnode.appmesh.k8s.aws/podinfo-primary
 virtualservice.appmesh.k8s.aws/podinfo.test
+virtualservice.appmesh.k8s.aws/podinfo-canary.test
 ```
 
+After the boostrap, the podinfo deployment will be scaled to zero and the traffic to `podinfo.test` will be routed
+to the primary pods. During the canary analysis, the `podinfo-canary.test` address can be used to target directly the canary pods.
+
 The App Mesh specific settings are:
 
 ```yaml
   service:
     port: 9898
-    meshName: global.appmesh-system
+    meshName: global
     backends:
       - backend1.test
       - backend2.test
@@ -178,7 +193,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=stefanprodan/podinfo:2.0.1
+podinfod=stefanprodan/podinfo:3.1.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -191,28 +206,34 @@ Status:
   Failed Checks:         0
   Phase:                 Succeeded
 Events:
-  Type     Reason  Age   From     Message
-  ----     ------  ----  ----     -------
-  Normal   Synced  3m    flagger  New revision detected podinfo.test
-  Normal   Synced  3m    flagger  Scaling up podinfo.test
-  Warning  Synced  3m    flagger  Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 5
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 10
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 15
-  Normal   Synced  2m    flagger  Advance podinfo.test canary weight 20
-  Normal   Synced  2m    flagger  Advance podinfo.test canary weight 25
-  Normal   Synced  1m    flagger  Advance podinfo.test canary weight 30
-  Normal   Synced  1m    flagger  Advance podinfo.test canary weight 35
-  Normal   Synced  55s   flagger  Advance podinfo.test canary weight 40
-  Normal   Synced  45s   flagger  Advance podinfo.test canary weight 45
-  Normal   Synced  35s   flagger  Advance podinfo.test canary weight 50
-  Normal   Synced  25s   flagger  Copying podinfo.test template spec to podinfo-primary.test
-  Warning  Synced  15s   flagger  Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available
-  Normal   Synced  5s    flagger  Promotion completed! Scaling down podinfo.test
+ New revision detected! Scaling up podinfo.test
+ Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available
+ Pre-rollout check acceptance-test passed
+ Advance podinfo.test canary weight 5
+ Advance podinfo.test canary weight 10
+ Advance podinfo.test canary weight 15
+ Advance podinfo.test canary weight 20
+ Advance podinfo.test canary weight 25
+ Advance podinfo.test canary weight 30
+ Advance podinfo.test canary weight 35
+ Advance podinfo.test canary weight 40
+ Advance podinfo.test canary weight 45
+ Advance podinfo.test canary weight 50
+ Copying podinfo.test template spec to podinfo-primary.test
+ Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available
+ Routing all traffic to primary
+ Promotion completed! Scaling down podinfo.test
 ```
 
+When the canary analysis starts, Flagger will call the pre-rollout webhooks before routing traffic to the canary.
+
 **Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis.
 
+A canary deployment is triggered by changes in any of the following objects:
+* Deployment PodSpec (container image, command, ports, env, resources, etc)
+* ConfigMaps mounted as volumes or mapped to environment variables
+* Secrets mounted as volumes or mapped to environment variables
+
 During the analysis the canary’s progress can be monitored with Grafana. The App Mesh dashboard URL is 
 http://localhost:3000/d/flagger-appmesh/appmesh-canary?refresh=10s&orgId=1&var-namespace=test&var-primary=podinfo-primary&var-canary=podinfo
 
@@ -224,9 +245,9 @@ You can monitor all canaries with:
 watch kubectl get canaries --all-namespaces
 
 NAMESPACE   NAME      STATUS        WEIGHT   LASTTRANSITIONTIME
-test        podinfo   Progressing   15       2019-03-16T14:05:07Z
-prod        frontend  Succeeded     0        2019-03-15T16:15:07Z
-prod        backend   Failed        0        2019-03-14T17:05:07Z
+test        podinfo   Progressing   15       2019-10-02T14:05:07Z
+prod        frontend  Succeeded     0        2019-10-02T16:15:07Z
+prod        backend   Failed        0        2019-10-02T17:05:07Z
 ```
 
 If you’ve enabled the Slack notifications, you should receive the following messages:
@@ -241,19 +262,25 @@ Trigger a canary deployment:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=stefanprodan/podinfo:2.0.2
+podinfod=stefanprodan/podinfo:3.1.2
 ```
 
 Exec into the load tester pod with:
 
 ```bash
-kubectl -n test exec -it flagger-loadtester-xx-xx sh
+kubectl -n test exec -it deploy/flagger-loadtester bash
 ```
 
 Generate HTTP 500 errors:
 
 ```bash
-hey -z 1m -c 5 -q 5 http://podinfo.test:9898/status/500
+hey -z 1m -c 5 -q 5 http://podinfo-canary.test:9898/status/500
+```
+
+Generate latency:
+
+```bash
+watch -n 1 curl http://podinfo-canary.test:9898/delay/1
 ```
 
 When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, 
@@ -264,22 +291,21 @@ kubectl -n test describe canary/podinfo
 
 Status:
   Canary Weight:         0
-  Failed Checks:         10
+  Failed Checks:         5
   Phase:                 Failed
 Events:
-  Type     Reason  Age   From     Message
-  ----     ------  ----  ----     -------
-  Normal   Synced  3m    flagger  Starting canary deployment for podinfo.test
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 5
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 10
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 15
-  Normal   Synced  3m    flagger  Halt podinfo.test advancement success rate 69.17% < 99%
-  Normal   Synced  2m    flagger  Halt podinfo.test advancement success rate 61.39% < 99%
-  Normal   Synced  2m    flagger  Halt podinfo.test advancement success rate 55.06% < 99%
-  Normal   Synced  2m    flagger  Halt podinfo.test advancement success rate 47.00% < 99%
-  Normal   Synced  2m    flagger  (combined from similar events): Halt podinfo.test advancement success rate 38.08% < 99%
-  Warning  Synced  1m    flagger  Rolling back podinfo.test failed checks threshold reached 10
-  Warning  Synced  1m    flagger  Canary failed! Scaling down podinfo.test
+ Starting canary analysis for podinfo.test
+ Pre-rollout check acceptance-test passed
+ Advance podinfo.test canary weight 5
+ Advance podinfo.test canary weight 10
+ Advance podinfo.test canary weight 15
+ Halt podinfo.test advancement success rate 69.17% < 99%
+ Halt podinfo.test advancement success rate 61.39% < 99%
+ Halt podinfo.test advancement success rate 55.06% < 99%
+ Halt podinfo.test advancement request duration 1.20s > 0.5s
+ Halt podinfo.test advancement request duration 1.45s > 0.5s
+ Rolling back podinfo.test failed checks threshold reached 5
+ Canary failed! Scaling down podinfo.test
 ```
 
 If you’ve enabled the Slack notifications, you’ll receive a message if the progress deadline is exceeded, 

go.mod

@@ -1,6 +1,6 @@
 module github.com/weaveworks/flagger
 
-go 1.12
+go 1.13
 
 require (
 	cloud.google.com/go v0.37.4 // indirect

kustomize/README.md

@@ -19,6 +19,14 @@ Note that you'll need kubectl 1.14 to run the above the command or you can downl
 kustomize build github.com/weaveworks/flagger//kustomize/istio | kubectl apply -f -
 ```
 
+Install Flagger for AWS App Mesh:
+
+```bash
+kubectl apply -k github.com/weaveworks/flagger//kustomize/appmesh
+```
+
+This deploys Flagger and Prometheus (configured to scrape the App Mesh Envoy sidecars) in the `appmesh-system` namespace.
+
 Install Flagger for Linkerd:
 
 ```bash

kustomize/appmesh/kustomization.yaml

@@ -0,0 +1,6 @@
+namespace: appmesh-system
+bases:
+  - ../base/flagger
+  - ../base/prometheus
+patchesStrategicMerge:
+  - patch.yaml

kustomize/appmesh/patch.yaml

@@ -0,0 +1,16 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flagger
+spec:
+  template:
+    spec:
+      containers:
+        - name: flagger
+          args:
+            - -log-level=info
+            - -mesh-provider=appmesh
+            - -metrics-server=http://flagger-prometheus:9090
+            - -slack-user=flagger
+            - -slack-channel=
+            - -slack-url=

kustomize/base/flagger/crd.yaml

@@ -33,6 +33,22 @@ spec:
     - name: Weight
       type: string
       JSONPath: .status.canaryWeight
+    - name: FailedChecks
+      type: string
+      JSONPath: .status.failedChecks
+      priority: 1
+    - name: Interval
+      type: string
+      JSONPath: .spec.canaryAnalysis.interval
+      priority: 1
+    - name: StepWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.stepWeight
+      priority: 1
+    - name: MaxWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.maxWeight
+      priority: 1
     - name: LastTransitionTime
       type: string
       JSONPath: .status.lastTransitionTime
@@ -54,7 +70,7 @@ spec:
             targetRef:
               description: Deployment selector
               type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -67,7 +83,7 @@ spec:
               anyOf:
                 - type: string
                 - type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -80,7 +96,7 @@ spec:
               anyOf:
                 - type: string
                 - type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -90,7 +106,7 @@ spec:
                   type: string
             service:
               type: object
-              required: ['port']
+              required: ["port"]
               properties:
                 port:
                   description: Container port number
@@ -108,7 +124,7 @@ spec:
                   description: AppMesh backend array
                   anyOf:
                     - type: string
-                    - type: object
+                    - type: array
                 timeout:
                   description: Istio HTTP or gRPC request timeout
                   type: string
@@ -178,7 +194,7 @@ spec:
                   properties:
                     items:
                       type: object
-                      required: ['name', 'threshold']
+                      required: ["name", "threshold"]
                       properties:
                         name:
                           description: Name of the Prometheus metric
@@ -199,7 +215,7 @@ spec:
                   properties:
                     items:
                       type: object
-                      required: ['name', 'url', 'timeout']
+                      required: ["name", "url"]
                       properties:
                         name:
                           description: Name of the webhook
@@ -212,6 +228,7 @@ spec:
                             - confirm-rollout
                             - pre-rollout
                             - rollout
+                            - confirm-promotion
                             - post-rollout
                         url:
                           description: URL address of this webhook
@@ -237,6 +254,7 @@ spec:
                 - Initialized
                 - Waiting
                 - Progressing
+                - Promoting
                 - Finalising
                 - Succeeded
                 - Failed
@@ -262,7 +280,7 @@ spec:
               properties:
                 items:
                   type: object
-                  required: ['type', 'status', 'reason']
+                  required: ["type", "status", "reason"]
                   properties:
                     lastTransitionTime:
                       description: LastTransitionTime of this condition

kustomize/base/flagger/kustomization.yaml

@@ -8,4 +8,4 @@ resources:
   - deployment.yaml
 images:
   - name: weaveworks/flagger
-    newTag: 0.18.2
+    newTag: 0.18.6

kustomize/base/prometheus/deployment.yaml

@@ -19,7 +19,7 @@ spec:
       serviceAccountName: flagger-prometheus
       containers:
           - name: prometheus
-            image: prom/prometheus:v2.10.0
+            image: prom/prometheus:v2.12.0
             imagePullPolicy: IfNotPresent
             args:
               - '--storage.tsdb.retention=2h'

kustomize/podinfo/deployment.yaml

@@ -10,7 +10,7 @@ spec:
   progressDeadlineSeconds: 60
   strategy:
     rollingUpdate:
-      maxUnavailable: 0
+      maxUnavailable: 1
     type: RollingUpdate
   selector:
     matchLabels:
@@ -24,21 +24,27 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: stefanprodan/podinfo:2.0.0
+        image: stefanprodan/podinfo:3.1.0
         imagePullPolicy: IfNotPresent
         ports:
-        - containerPort: 9898
-          name: http
+        - name: http
+          containerPort: 9898
+          protocol: TCP
+        - name: http-metrics
+          containerPort: 9797
+          protocol: TCP
+        - name: grpc
+          containerPort: 9999
           protocol: TCP
         command:
         - ./podinfo
         - --port=9898
+        - --port-metrics=9797
+        - --grpc-port=9999
+        - --grpc-service-name=podinfo
         - --level=info
         - --random-delay=false
         - --random-error=false
-        env:
-        - name: PODINFO_UI_COLOR
-          value: blue
         livenessProbe:
           exec:
             command:

kustomize/tester/deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: loadtester
-          image: weaveworks/flagger-loadtester:0.6.1
+          image: weaveworks/flagger-loadtester:0.8.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

pkg/apis/appmesh/v1beta1/types.go

@@ -302,7 +302,7 @@ type VirtualNodeCondition struct {
 	Reason *string `json:"reason,omitempty"`
 	// A human readable message indicating details about the transition.
 	// +optional
-	Message *string `json:"reason,omitempty"`
+	Message *string `json:"message,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/apis/flagger/v1alpha3/status.go

@@ -47,7 +47,9 @@ const (
 	CanaryPhaseWaiting CanaryPhase = "Waiting"
 	// CanaryPhaseProgressing means the canary analysis is underway
 	CanaryPhaseProgressing CanaryPhase = "Progressing"
-	// CanaryPhaseProgressing means the canary analysis is finished and traffic has been routed back to primary
+	// CanaryPhasePromoting means the canary analysis is finished and the primary spec has been updated
+	CanaryPhasePromoting CanaryPhase = "Promoting"
+	// CanaryPhaseProgressing means the canary promotion is finished and traffic has been routed back to primary
 	CanaryPhaseFinalising CanaryPhase = "Finalising"
 	// CanaryPhaseSucceeded means the canary analysis has been successful
 	// and the canary deployment has been promoted

pkg/apis/flagger/v1alpha3/types.go

@@ -139,6 +139,8 @@ const (
 	PostRolloutHook HookType = "post-rollout"
 	// ConfirmRolloutHook halt canary analysis until webhook returns HTTP 200
 	ConfirmRolloutHook HookType = "confirm-rollout"
+	// ConfirmPromotionHook halt canary promotion until webhook returns HTTP 200
+	ConfirmPromotionHook HookType = "confirm-promotion"
 )
 
 // CanaryWebhook holds the reference to external checks used for canary analysis

pkg/canary/status.go

@@ -211,6 +211,9 @@ func (c *Deployer) MakeStatusConditions(canaryStatus flaggerv1.CanaryStatus,
 	case flaggerv1.CanaryPhaseProgressing:
 		status = corev1.ConditionUnknown
 		message = "New revision detected, starting canary analysis."
+	case flaggerv1.CanaryPhasePromoting:
+		status = corev1.ConditionUnknown
+		message = "Canary analysis completed, starting primary rolling update."
 	case flaggerv1.CanaryPhaseFinalising:
 		status = corev1.ConditionUnknown
 		message = "Canary analysis completed, routing all traffic to primary."

pkg/controller/controller_test.go

@@ -82,7 +82,7 @@ func SetupMocks(abtest bool) Mocks {
 	flaggerInformer := flaggerInformerFactory.Flagger().V1alpha3().Canaries()
 
 	// init router
-	rf := router.NewFactory(nil, kubeClient, flaggerClient, logger, flaggerClient)
+	rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", logger, flaggerClient)
 
 	// init observer
 	observerFactory, _ := metrics.NewFactory("fake", "istio", 5*time.Second)

pkg/controller/scheduler.go

@@ -99,7 +99,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 	// create primary deployment and hpa if needed
 	// skip primary check for Istio since the deployment will become ready after the ClusterIP are created
 	skipPrimaryCheck := false
-	if skipLivenessChecks || strings.Contains(provider, "istio") {
+	if skipLivenessChecks || strings.Contains(provider, "istio") || strings.Contains(provider, "appmesh") {
 		skipPrimaryCheck = true
 	}
 	label, ports, err := c.deployer.Initialize(cd, skipPrimaryCheck)
@@ -214,7 +214,27 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		return
 	}
 
-	// scale canary to zero if analysis has succeeded
+	// route all traffic to primary if analysis has succeeded
+	if cd.Status.Phase == flaggerv1.CanaryPhasePromoting {
+		if provider != "kubernetes" {
+			c.recordEventInfof(cd, "Routing all traffic to primary")
+			if err := meshRouter.SetRoutes(cd, 100, 0); err != nil {
+				c.recordEventWarningf(cd, "%v", err)
+				return
+			}
+			c.recorder.SetWeight(cd, 100, 0)
+		}
+
+		// update status phase
+		if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseFinalising); err != nil {
+			c.recordEventWarningf(cd, "%v", err)
+			return
+		}
+
+		return
+	}
+
+	// scale canary to zero if promotion has finished
 	if cd.Status.Phase == flaggerv1.CanaryPhaseFinalising {
 		if err := c.deployer.Scale(cd, 0); err != nil {
 			c.recordEventWarningf(cd, "%v", err)
@@ -283,7 +303,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 
 	// check if the canary success rate is above the threshold
 	// skip check if no traffic is routed to canary
-	if canaryWeight == 0 {
+	if canaryWeight == 0 && cd.Status.Iterations == 0 {
 		c.recordEventInfof(cd, "Starting canary analysis for %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)
 
 		// run pre-rollout web hooks
@@ -304,8 +324,8 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		}
 	}
 
-	// canary fix routing: A/B testing
-	if len(cd.Spec.CanaryAnalysis.Match) > 0 || cd.Spec.CanaryAnalysis.Iterations > 0 {
+	// strategy: A/B testing
+	if len(cd.Spec.CanaryAnalysis.Match) > 0 && cd.Spec.CanaryAnalysis.Iterations > 0 {
 		// route traffic to canary and increment iterations
 		if cd.Spec.CanaryAnalysis.Iterations > cd.Status.Iterations {
 			if err := meshRouter.SetRoutes(cd, 0, 100); err != nil {
@@ -323,6 +343,11 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 			return
 		}
 
+		// check promotion gate
+		if promote := c.runConfirmPromotionHooks(cd); !promote {
+			return
+		}
+
 		// promote canary - max iterations reached
 		if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations {
 			c.recordEventInfof(cd, "Copying %s.%s template spec to %s.%s",
@@ -331,6 +356,47 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 				c.recordEventWarningf(cd, "%v", err)
 				return
 			}
+
+			// update status phase
+			if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhasePromoting); err != nil {
+				c.recordEventWarningf(cd, "%v", err)
+				return
+			}
+			return
+		}
+
+		return
+	}
+
+	// strategy: Blue/Green
+	if cd.Spec.CanaryAnalysis.Iterations > 0 {
+		// increment iterations
+		if cd.Spec.CanaryAnalysis.Iterations > cd.Status.Iterations {
+			if err := c.deployer.SetStatusIterations(cd, cd.Status.Iterations+1); err != nil {
+				c.recordEventWarningf(cd, "%v", err)
+				return
+			}
+			c.recordEventInfof(cd, "Advance %s.%s canary iteration %v/%v",
+				cd.Name, cd.Namespace, cd.Status.Iterations+1, cd.Spec.CanaryAnalysis.Iterations)
+			return
+		}
+
+		// check promotion gate
+		if promote := c.runConfirmPromotionHooks(cd); !promote {
+			return
+		}
+
+		// route all traffic to canary - max iterations reached
+		if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations {
+			if provider != "kubernetes" {
+				c.recordEventInfof(cd, "Routing all traffic to canary")
+				if err := meshRouter.SetRoutes(cd, 0, 100); err != nil {
+					c.recordEventWarningf(cd, "%v", err)
+					return
+				}
+				c.recorder.SetWeight(cd, 0, 100)
+			}
+
 			// increment iterations
 			if err := c.deployer.SetStatusIterations(cd, cd.Status.Iterations+1); err != nil {
 				c.recordEventWarningf(cd, "%v", err)
@@ -339,82 +405,80 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 			return
 		}
 
-		// route all traffic to primary
+		// promote canary - max iterations reached
 		if cd.Spec.CanaryAnalysis.Iterations < cd.Status.Iterations {
-			primaryWeight = 100
-			canaryWeight = 0
-			if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
-				c.recordEventWarningf(cd, "%v", err)
-				return
-			}
-			c.recorder.SetWeight(cd, primaryWeight, canaryWeight)
-
-			// update status phase
-			if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseFinalising); err != nil {
-				c.recordEventWarningf(cd, "%v", err)
-				return
-			}
-
-			c.recordEventInfof(cd, "Routing all traffic to primary")
-			return
-		}
-
-		return
-	}
-
-	// canary incremental traffic weight
-	if canaryWeight < maxWeight {
-		primaryWeight -= cd.Spec.CanaryAnalysis.StepWeight
-		if primaryWeight < 0 {
-			primaryWeight = 0
-		}
-		canaryWeight += cd.Spec.CanaryAnalysis.StepWeight
-		if primaryWeight > 100 {
-			primaryWeight = 100
-		}
-
-		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
-			c.recordEventWarningf(cd, "%v", err)
-			return
-		}
-
-		// update weight status
-		if err := c.deployer.SetStatusWeight(cd, canaryWeight); err != nil {
-			c.recordEventWarningf(cd, "%v", err)
-			return
-		}
-
-		c.recorder.SetWeight(cd, primaryWeight, canaryWeight)
-		c.recordEventInfof(cd, "Advance %s.%s canary weight %v", cd.Name, cd.Namespace, canaryWeight)
-
-		// promote canary
-		if canaryWeight >= maxWeight {
 			c.recordEventInfof(cd, "Copying %s.%s template spec to %s.%s",
 				cd.Spec.TargetRef.Name, cd.Namespace, primaryName, cd.Namespace)
 			if err := c.deployer.Promote(cd); err != nil {
 				c.recordEventWarningf(cd, "%v", err)
 				return
 			}
-		}
-	} else {
-		// route all traffic to primary
-		primaryWeight = 100
-		canaryWeight = 0
-		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
-			c.recordEventWarningf(cd, "%v", err)
-			return
-		}
-		c.recorder.SetWeight(cd, primaryWeight, canaryWeight)
 
-		// update status phase
-		if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseFinalising); err != nil {
-			c.recordEventWarningf(cd, "%v", err)
+			// update status phase
+			if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhasePromoting); err != nil {
+				c.recordEventWarningf(cd, "%v", err)
+				return
+			}
 			return
 		}
 
-		c.recordEventInfof(cd, "Routing all traffic to primary")
 		return
 	}
+
+	// strategy: Canary progressive traffic increase
+	if cd.Spec.CanaryAnalysis.StepWeight > 0 {
+		// increase traffic weight
+		if canaryWeight < maxWeight {
+			primaryWeight -= cd.Spec.CanaryAnalysis.StepWeight
+			if primaryWeight < 0 {
+				primaryWeight = 0
+			}
+			canaryWeight += cd.Spec.CanaryAnalysis.StepWeight
+			if canaryWeight > 100 {
+				canaryWeight = 100
+			}
+
+			if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
+				c.recordEventWarningf(cd, "%v", err)
+				return
+			}
+
+			if err := c.deployer.SetStatusWeight(cd, canaryWeight); err != nil {
+				c.recordEventWarningf(cd, "%v", err)
+				return
+			}
+
+			c.recorder.SetWeight(cd, primaryWeight, canaryWeight)
+			c.recordEventInfof(cd, "Advance %s.%s canary weight %v", cd.Name, cd.Namespace, canaryWeight)
+			return
+		}
+
+		// promote canary - max weight reached
+		if canaryWeight >= maxWeight {
+			// check promotion gate
+			if promote := c.runConfirmPromotionHooks(cd); !promote {
+				return
+			}
+
+			// update primary spec
+			c.recordEventInfof(cd, "Copying %s.%s template spec to %s.%s",
+				cd.Spec.TargetRef.Name, cd.Namespace, primaryName, cd.Namespace)
+			if err := c.deployer.Promote(cd); err != nil {
+				c.recordEventWarningf(cd, "%v", err)
+				return
+			}
+
+			// update status phase
+			if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhasePromoting); err != nil {
+				c.recordEventWarningf(cd, "%v", err)
+				return
+			}
+
+			return
+		}
+
+	}
+
 }
 
 func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, meshRouter router.Interface, primaryWeight int, canaryWeight int) bool {
@@ -466,6 +530,7 @@ func (c *Controller) shouldAdvance(cd *flaggerv1.Canary) (bool, error) {
 		cd.Status.Phase == flaggerv1.CanaryPhaseInitializing ||
 		cd.Status.Phase == flaggerv1.CanaryPhaseProgressing ||
 		cd.Status.Phase == flaggerv1.CanaryPhaseWaiting ||
+		cd.Status.Phase == flaggerv1.CanaryPhasePromoting ||
 		cd.Status.Phase == flaggerv1.CanaryPhaseFinalising {
 		return true, nil
 	}
@@ -490,6 +555,7 @@ func (c *Controller) shouldAdvance(cd *flaggerv1.Canary) (bool, error) {
 func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, shouldAdvance bool) bool {
 	c.recorder.SetStatus(cd, cd.Status.Phase)
 	if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing ||
+		cd.Status.Phase == flaggerv1.CanaryPhasePromoting ||
 		cd.Status.Phase == flaggerv1.CanaryPhaseFinalising {
 		return true
 	}
@@ -565,6 +631,23 @@ func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary) bool {
 	return true
 }
 
+func (c *Controller) runConfirmPromotionHooks(canary *flaggerv1.Canary) bool {
+	for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
+		if webhook.Type == flaggerv1.ConfirmPromotionHook {
+			err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
+			if err != nil {
+				c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for promotion approval %s",
+					canary.Name, canary.Namespace, webhook.Name)
+				c.sendNotification(canary, "Canary promotion is waiting for approval.", false, false)
+				return false
+			} else {
+				c.recordEventInfof(canary, "Confirm-promotion check %s passed", webhook.Name)
+			}
+		}
+	}
+	return true
+}
+
 func (c *Controller) runPreRolloutHooks(canary *flaggerv1.Canary) bool {
 	for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
 		if webhook.Type == flaggerv1.PreRolloutHook {

pkg/controller/scheduler_test.go

@@ -162,12 +162,23 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
 
 func TestScheduler_Promotion(t *testing.T) {
 	mocks := SetupMocks(false)
+
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
+	// check initialized status
+	c, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if c.Status.Phase != v1alpha3.CanaryPhaseInitialized {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseInitialized)
+	}
+
 	// update
 	dep2 := newTestDeploymentV2()
-	_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
+	_, err = mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -205,9 +216,32 @@ func TestScheduler_Promotion(t *testing.T) {
 	// advance
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
+	// check progressing status
+	c, err = mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if c.Status.Phase != v1alpha3.CanaryPhaseProgressing {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseProgressing)
+	}
+
 	// promote
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
+	// check promoting status
+	c, err = mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if c.Status.Phase != v1alpha3.CanaryPhasePromoting {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhasePromoting)
+	}
+
+	// finalise
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
 	primaryWeight, canaryWeight, err = mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -251,11 +285,15 @@ func TestScheduler_Promotion(t *testing.T) {
 	}
 
 	// check finalising status
-	c, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	c, err = mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
+	if c.Status.Phase != v1alpha3.CanaryPhaseFinalising {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseFinalising)
+	}
+
 	// scale canary to zero
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 

pkg/loadtester/bash.go

@@ -24,7 +24,7 @@ func (task *BashTask) Run(ctx context.Context) (bool, error) {
 
 	if err != nil {
 		task.logger.With("canary", task.canary).Errorf("command failed %s %v %s", task.command, err, out)
-		return false, fmt.Errorf(" %v %v", err, out)
+		return false, fmt.Errorf(" %v %s", err, out)
 	} else {
 		if task.logCmdOutput {
 			fmt.Printf("%s\n", out)

pkg/loadtester/helm.go

@@ -20,14 +20,14 @@ func (task *HelmTask) Hash() string {
 }
 
 func (task *HelmTask) Run(ctx context.Context) (bool, error) {
-	helmCmd := fmt.Sprintf("helm %s", task.command)
+	helmCmd := fmt.Sprintf("%s %s", TaskTypeHelm, task.command)
 	task.logger.With("canary", task.canary).Infof("running command %v", helmCmd)
 
-	cmd := exec.CommandContext(ctx, "helm", strings.Fields(task.command)...)
+	cmd := exec.CommandContext(ctx, TaskTypeHelm, strings.Fields(task.command)...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		task.logger.With("canary", task.canary).Errorf("command failed %s %v %s", task.command, err, out)
-		return false, fmt.Errorf(" %v %v", err, out)
+		return false, fmt.Errorf(" %v %s", err, out)
 	} else {
 		if task.logCmdOutput {
 			fmt.Printf("%s\n", out)

pkg/loadtester/helmv3.go

@@ -0,0 +1,42 @@
+package loadtester
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+const TaskTypeHelmv3 = "helmv3"
+
+type HelmTaskv3 struct {
+	TaskBase
+	command      string
+	logCmdOutput bool
+}
+
+func (task *HelmTaskv3) Hash() string {
+	return hash(task.canary + task.command)
+}
+
+func (task *HelmTaskv3) Run(ctx context.Context) (bool, error) {
+	helmCmd := fmt.Sprintf("%s %s", TaskTypeHelmv3, task.command)
+	task.logger.With("canary", task.canary).Infof("running command %v", helmCmd)
+
+	cmd := exec.CommandContext(ctx, TaskTypeHelmv3, strings.Fields(task.command)...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		task.logger.With("canary", task.canary).Errorf("command failed %s %v %s", task.command, err, out)
+		return false, fmt.Errorf(" %v %s", err, out)
+	} else {
+		if task.logCmdOutput {
+			fmt.Printf("%s\n", out)
+		}
+		task.logger.With("canary", task.canary).Infof("command finished %v", helmCmd)
+	}
+	return true, nil
+}
+
+func (task *HelmTaskv3) String() string {
+	return task.command
+}

pkg/loadtester/server.go

@@ -185,6 +185,31 @@ func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogge
 				return
 			}
 
+			// run helmv3 command (blocking task)
+			if typ == TaskTypeHelmv3 {
+				helm := HelmTaskv3{
+					command:      payload.Metadata["cmd"],
+					logCmdOutput: true,
+					TaskBase: TaskBase{
+						canary: fmt.Sprintf("%s.%s", payload.Name, payload.Namespace),
+						logger: logger,
+					},
+				}
+
+				ctx, cancel := context.WithTimeout(context.Background(), taskRunner.timeout)
+				defer cancel()
+
+				ok, err := helm.Run(ctx)
+				if !ok {
+					w.WriteHeader(http.StatusInternalServerError)
+					w.Write([]byte(err.Error()))
+					return
+				}
+
+				w.WriteHeader(http.StatusOK)
+				return
+			}
+
 			taskFactory, ok := GetTaskFactory(typ)
 			if !ok {
 				w.WriteHeader(http.StatusBadRequest)

pkg/notifier/slack.go

@@ -74,7 +74,7 @@ func (s *Slack) Post(workload string, namespace string, message string, fields [
 		color = "danger"
 	}
 
-	sfields := make([]SlackField, len(fields))
+	sfields := make([]SlackField, 0, len(fields))
 	for _, f := range fields {
 		sfields = append(sfields, SlackField{f.Name, f.Value, false})
 	}

pkg/notifier/slack_test.go

@@ -9,6 +9,11 @@ import (
 )
 
 func TestSlack_Post(t *testing.T) {
+	fields := []Field{
+		{Name: "name1", Value: "value1"},
+		{Name: "name2", Value: "value2"},
+	}
+
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		b, err := ioutil.ReadAll(r.Body)
 		if err != nil {
@@ -20,6 +25,10 @@ func TestSlack_Post(t *testing.T) {
 		if payload.Attachments[0].AuthorName != "podinfo.test" {
 			t.Fatal("wrong author name")
 		}
+
+		if len(payload.Attachments[0].Fields) != len(fields) {
+			t.Fatal("wrong facts")
+		}
 	}))
 	defer ts.Close()
 
@@ -28,7 +37,7 @@ func TestSlack_Post(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	err = slack.Post("podinfo", "test", "test", nil, true)
+	err = slack.Post("podinfo", "test", "test", fields, true)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/notifier/teams.go

@@ -45,7 +45,7 @@ func NewMSTeams(hookURL string) (*MSTeams, error) {
 
 // Post MS Teams message
 func (s *MSTeams) Post(workload string, namespace string, message string, fields []Field, warn bool) error {
-	facts := make([]MSTeamsField, len(fields))
+	facts := make([]MSTeamsField, 0, len(fields))
 	for _, f := range fields {
 		facts = append(facts, MSTeamsField{f.Name, f.Value})
 	}

pkg/notifier/teams_test.go

@@ -9,6 +9,12 @@ import (
 )
 
 func TestTeams_Post(t *testing.T) {
+
+	fields := []Field{
+		{Name: "name1", Value: "value1"},
+		{Name: "name2", Value: "value2"},
+	}
+
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		b, err := ioutil.ReadAll(r.Body)
 		if err != nil {
@@ -20,6 +26,9 @@ func TestTeams_Post(t *testing.T) {
 		if payload.Sections[0].ActivitySubtitle != "podinfo.test" {
 			t.Fatal("wrong activity subtitle")
 		}
+		if len(payload.Sections[0].Facts) != len(fields) {
+			t.Fatal("wrong facts")
+		}
 	}))
 	defer ts.Close()
 
@@ -28,7 +37,7 @@ func TestTeams_Post(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	err = teams.Post("podinfo", "test", "test", nil, true)
+	err = teams.Post("podinfo", "test", "test", fields, true)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/router/appmesh.go

@@ -56,9 +56,16 @@ func (ar *AppMeshRouter) Reconcile(canary *flaggerv1.Canary) error {
 		return err
 	}
 
-	// sync virtual service e.g. app.namespace
+	// sync main virtual service
 	// DNS app.namespace
-	err = ar.reconcileVirtualService(canary, targetHost)
+	err = ar.reconcileVirtualService(canary, targetHost, 0)
+	if err != nil {
+		return err
+	}
+
+	// sync canary virtual service
+	// DNS app-canary.namespace
+	err = ar.reconcileVirtualService(canary, fmt.Sprintf("%s.%s", canaryName, canary.Namespace), 100)
 	if err != nil {
 		return err
 	}
@@ -148,7 +155,7 @@ func (ar *AppMeshRouter) reconcileVirtualNode(canary *flaggerv1.Canary, name str
 }
 
 // reconcileVirtualService creates or updates a virtual service
-func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name string) error {
+func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name string, canaryWeight int64) error {
 	targetName := canary.Spec.TargetRef.Name
 	canaryVirtualNode := fmt.Sprintf("%s-canary", targetName)
 	primaryVirtualNode := fmt.Sprintf("%s-primary", targetName)
@@ -185,11 +192,11 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name
 						WeightedTargets: []AppmeshV1beta1.WeightedTarget{
 							{
 								VirtualNodeName: canaryVirtualNode,
-								Weight:          0,
+								Weight:          canaryWeight,
 							},
 							{
 								VirtualNodeName: primaryVirtualNode,
-								Weight:          100,
+								Weight:          100 - canaryWeight,
 							},
 						},
 					},

pkg/router/appmesh_test.go

@@ -37,6 +37,23 @@ func TestAppmeshRouter_Reconcile(t *testing.T) {
 		t.Errorf("Got routes %v wanted %v", targetsCount, 2)
 	}
 
+	// check canary virtual service
+	vsCanaryName := fmt.Sprintf("%s-canary.%s", mocks.appmeshCanary.Spec.TargetRef.Name, mocks.appmeshCanary.Namespace)
+	vsCanary, err := router.appmeshClient.AppmeshV1beta1().VirtualServices("default").Get(vsCanaryName, metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	// check if the canary virtual service routes all traffic to the canary virtual node
+	target := vsCanary.Spec.Routes[0].Http.Action.WeightedTargets[0]
+	canaryVirtualNodeName := fmt.Sprintf("%s-canary", mocks.appmeshCanary.Spec.TargetRef.Name)
+	if target.VirtualNodeName != canaryVirtualNodeName {
+		t.Errorf("Got VirtualNodeName %v wanted %v", target.VirtualNodeName, canaryVirtualNodeName)
+	}
+	if target.Weight != 100 {
+		t.Errorf("Got weight %v wanted %v", target.Weight, 100)
+	}
+
 	// check virtual node
 	vnName := mocks.appmeshCanary.Spec.TargetRef.Name
 	vn, err := router.appmeshClient.AppmeshV1beta1().VirtualNodes("default").Get(vnName, metav1.GetOptions{})
@@ -103,7 +120,7 @@ func TestAppmeshRouter_Reconcile(t *testing.T) {
 
 	weight := vs.Spec.Routes[0].Http.Action.WeightedTargets[0].Weight
 	if weight != 50 {
-		t.Errorf("Got weight %v wanted %v", weight, 502)
+		t.Errorf("Got weight %v wanted %v", weight, 50)
 	}
 
 	// test URI update

pkg/router/factory.go

@@ -11,23 +11,26 @@ import (
 )
 
 type Factory struct {
-	kubeConfig    *restclient.Config
-	kubeClient    kubernetes.Interface
-	meshClient    clientset.Interface
-	flaggerClient clientset.Interface
-	logger        *zap.SugaredLogger
+	kubeConfig               *restclient.Config
+	kubeClient               kubernetes.Interface
+	meshClient               clientset.Interface
+	flaggerClient            clientset.Interface
+	ingressAnnotationsPrefix string
+	logger                   *zap.SugaredLogger
 }
 
 func NewFactory(kubeConfig *restclient.Config, kubeClient kubernetes.Interface,
 	flaggerClient clientset.Interface,
+	ingressAnnotationsPrefix string,
 	logger *zap.SugaredLogger,
 	meshClient clientset.Interface) *Factory {
 	return &Factory{
-		kubeConfig:    kubeConfig,
-		meshClient:    meshClient,
-		kubeClient:    kubeClient,
-		flaggerClient: flaggerClient,
-		logger:        logger,
+		kubeConfig:               kubeConfig,
+		meshClient:               meshClient,
+		kubeClient:               kubeClient,
+		flaggerClient:            flaggerClient,
+		ingressAnnotationsPrefix: ingressAnnotationsPrefix,
+		logger:                   logger,
 	}
 }
 
@@ -51,8 +54,9 @@ func (factory *Factory) MeshRouter(provider string) Interface {
 		return &NopRouter{}
 	case provider == "nginx":
 		return &IngressRouter{
-			logger:     factory.logger,
-			kubeClient: factory.kubeClient,
+			logger:            factory.logger,
+			kubeClient:        factory.kubeClient,
+			annotationsPrefix: factory.ingressAnnotationsPrefix,
 		}
 	case provider == "appmesh":
 		return &AppMeshRouter{

pkg/router/ingress.go

@@ -2,6 +2,9 @@ package router
 
 import (
 	"fmt"
+	"strconv"
+	"strings"
+
 	"github.com/google/go-cmp/cmp"
 	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 	"go.uber.org/zap"
@@ -10,13 +13,12 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes"
-	"strconv"
-	"strings"
 )
 
 type IngressRouter struct {
-	kubeClient kubernetes.Interface
-	logger     *zap.SugaredLogger
+	kubeClient        kubernetes.Interface
+	annotationsPrefix string
+	logger            *zap.SugaredLogger
 }
 
 func (i *IngressRouter) Reconcile(canary *flaggerv1.Canary) error {
@@ -115,7 +117,7 @@ func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) (
 	// A/B testing
 	if len(canary.Spec.CanaryAnalysis.Match) > 0 {
 		for k := range canaryIngress.Annotations {
-			if k == "nginx.ingress.kubernetes.io/canary-by-cookie" || k == "nginx.ingress.kubernetes.io/canary-by-header" {
+			if k == i.GetAnnotationWithPrefix("canary-by-cookie") || k == i.GetAnnotationWithPrefix("canary-by-header") {
 				return 0, 100, nil
 			}
 		}
@@ -123,7 +125,7 @@ func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) (
 
 	// Canary
 	for k, v := range canaryIngress.Annotations {
-		if k == "nginx.ingress.kubernetes.io/canary-weight" {
+		if k == i.GetAnnotationWithPrefix("canary-weight") {
 			val, err := strconv.Atoi(v)
 			if err != nil {
 				return 0, 0, err
@@ -170,12 +172,12 @@ func (i *IngressRouter) SetRoutes(
 		iClone.Annotations = i.makeHeaderAnnotations(iClone.Annotations, header, headerValue, cookie)
 	} else {
 		// canary
-		iClone.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = fmt.Sprintf("%v", canaryWeight)
+		iClone.Annotations[i.GetAnnotationWithPrefix("canary-weight")] = fmt.Sprintf("%v", canaryWeight)
 	}
 
 	// toggle canary
 	if canaryWeight > 0 {
-		iClone.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
+		iClone.Annotations[i.GetAnnotationWithPrefix("canary")] = "true"
 	} else {
 		iClone.Annotations = i.makeAnnotations(iClone.Annotations)
 	}
@@ -191,14 +193,14 @@ func (i *IngressRouter) SetRoutes(
 func (i *IngressRouter) makeAnnotations(annotations map[string]string) map[string]string {
 	res := make(map[string]string)
 	for k, v := range annotations {
-		if !strings.Contains(k, "nginx.ingress.kubernetes.io/canary") &&
+		if !strings.Contains(k, i.GetAnnotationWithPrefix("canary")) &&
 			!strings.Contains(k, "kubectl.kubernetes.io/last-applied-configuration") {
 			res[k] = v
 		}
 	}
 
-	res["nginx.ingress.kubernetes.io/canary"] = "false"
-	res["nginx.ingress.kubernetes.io/canary-weight"] = "0"
+	res[i.GetAnnotationWithPrefix("canary")] = "false"
+	res[i.GetAnnotationWithPrefix("canary-weight")] = "0"
 
 	return res
 }
@@ -207,25 +209,29 @@ func (i *IngressRouter) makeHeaderAnnotations(annotations map[string]string,
 	header string, headerValue string, cookie string) map[string]string {
 	res := make(map[string]string)
 	for k, v := range annotations {
-		if !strings.Contains(v, "nginx.ingress.kubernetes.io/canary") {
+		if !strings.Contains(v, i.GetAnnotationWithPrefix("canary")) {
 			res[k] = v
 		}
 	}
 
-	res["nginx.ingress.kubernetes.io/canary"] = "true"
-	res["nginx.ingress.kubernetes.io/canary-weight"] = "0"
+	res[i.GetAnnotationWithPrefix("canary")] = "true"
+	res[i.GetAnnotationWithPrefix("canary-weight")] = "0"
 
 	if cookie != "" {
-		res["nginx.ingress.kubernetes.io/canary-by-cookie"] = cookie
+		res[i.GetAnnotationWithPrefix("canary-by-cookie")] = cookie
 	}
 
 	if header != "" {
-		res["nginx.ingress.kubernetes.io/canary-by-header"] = header
+		res[i.GetAnnotationWithPrefix("canary-by-header")] = header
 	}
 
 	if headerValue != "" {
-		res["nginx.ingress.kubernetes.io/canary-by-header-value"] = headerValue
+		res[i.GetAnnotationWithPrefix("canary-by-header-value")] = headerValue
 	}
 
 	return res
 }
+
+func (i *IngressRouter) GetAnnotationWithPrefix(suffix string) string {
+	return fmt.Sprintf("%v/%v", i.annotationsPrefix, suffix)
+}

pkg/router/ingress_test.go

@@ -2,15 +2,17 @@ package router
 
 import (
 	"fmt"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func TestIngressRouter_Reconcile(t *testing.T) {
 	mocks := setupfakeClients()
 	router := &IngressRouter{
-		logger:     mocks.logger,
-		kubeClient: mocks.kubeClient,
+		logger:            mocks.logger,
+		kubeClient:        mocks.kubeClient,
+		annotationsPrefix: "custom.ingress.kubernetes.io",
 	}
 
 	err := router.Reconcile(mocks.ingressCanary)
@@ -18,8 +20,8 @@ func TestIngressRouter_Reconcile(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	canaryAn := "nginx.ingress.kubernetes.io/canary"
-	canaryWeightAn := "nginx.ingress.kubernetes.io/canary-weight"
+	canaryAn := "custom.ingress.kubernetes.io/canary"
+	canaryWeightAn := "custom.ingress.kubernetes.io/canary-weight"
 
 	canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name)
 	inCanary, err := router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{})
@@ -44,8 +46,9 @@ func TestIngressRouter_Reconcile(t *testing.T) {
 func TestIngressRouter_GetSetRoutes(t *testing.T) {
 	mocks := setupfakeClients()
 	router := &IngressRouter{
-		logger:     mocks.logger,
-		kubeClient: mocks.kubeClient,
+		logger:            mocks.logger,
+		kubeClient:        mocks.kubeClient,
+		annotationsPrefix: "prefix1.nginx.ingress.kubernetes.io",
 	}
 
 	err := router.Reconcile(mocks.ingressCanary)
@@ -66,8 +69,8 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	canaryAn := "nginx.ingress.kubernetes.io/canary"
-	canaryWeightAn := "nginx.ingress.kubernetes.io/canary-weight"
+	canaryAn := "prefix1.nginx.ingress.kubernetes.io/canary"
+	canaryWeightAn := "prefix1.nginx.ingress.kubernetes.io/canary-weight"
 
 	canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name)
 	inCanary, err := router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{})

pkg/version/version.go

@@ -1,4 +1,4 @@
 package version
 
-var VERSION = "0.18.2"
+var VERSION = "0.18.6"
 var REVISION = "unknown"

test/README.md

@@ -25,14 +25,17 @@ The e2e testing infrastructure is powered by CircleCI and [Kubernetes Kind](http
 * install latest stable kubectl [e2e-kind.sh](e2e-kind.sh)
 * install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh)
 * create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh)
-* install latest stable Helm CLI [e2e-nginx.sh](e2e-istio.sh)
-* deploy Tiller on the local cluster [e2e-nginx.sh](e2e-istio.sh)
-* install NGINX ingress with Helm [e2e-nginx.sh](e2e-istio.sh)
+* install latest stable Helm CLI [e2e-nginx.sh](e2e-nginx.sh)
+* deploy Tiller on the local cluster [e2e-nginx.sh](e2e-nginx.sh)
+* install NGINX ingress with Helm [e2e-nginx.sh](e2e-nginx.sh)
 * load Flagger image onto the local cluster [e2e-nginx.sh](e2e-nginx.sh)
 * install Flagger and Prometheus in the ingress-nginx namespace [e2e-nginx.sh](e2e-nginx.sh)
-* create a test namespace [e2e-nginx-tests.sh](e2e-tests.sh)
-* deploy the load tester in the test namespace [e2e-nginx-tests.sh](e2e-tests.sh)
-* deploy the demo workload (podinfo) and ingress in the test namespace [e2e-nginx-tests.sh](e2e-tests.sh)
-* test the canary initialization [e2e-nginx-tests.sh](e2e-tests.sh)
-* test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-nginx-tests.sh](e2e-tests.sh)
-* test the A/B testing analysis and promotion using header filters and pre/post rollout webhooks [e2e-nginx-tests.sh](e2e-tests.sh)
+* create a test namespace [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
+* deploy the load tester in the test namespace [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
+* deploy the demo workload (podinfo) and ingress in the test namespace [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
+* test the canary initialization [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
+* test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
+* test the A/B testing analysis and promotion using header filters and pre/post rollout webhooks [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
+* cleanup test environment [e2e-nginx-cleanup.sh](e2e-nginx-cleanup.sh)
+* install NGINX Ingress and Flagger with custom ingress annotations prefix [e2e-nginx-custom-annotations.sh](e2e-nginx-custom-annotations.sh)
+* repeat the canary and A/B testing workflow [e2e-nginx-tests.sh](e2e-nginx-tests.sh)

test/e2e-istio.sh

@@ -2,7 +2,7 @@
 
 set -o errexit
 
-ISTIO_VER="1.2.3"
+ISTIO_VER="1.3.0"
 REPO_ROOT=$(git rev-parse --show-toplevel)
 export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
 
@@ -13,9 +13,9 @@ echo '>>> Installing Istio CRDs'
 helm upgrade -i istio-init istio.io/istio-init --wait --namespace istio-system
 
 echo '>>> Waiting for Istio CRDs to be ready'
-kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-10
-kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-11
-kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-12
+kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-10-${ISTIO_VER}
+kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-11-${ISTIO_VER}
+kubectl -n istio-system wait --for=condition=complete job/istio-init-crd-12-${ISTIO_VER}
 
 echo '>>> Installing Istio control plane'
 helm upgrade -i istio istio.io/istio --wait --namespace istio-system -f ${REPO_ROOT}/test/e2e-istio-values.yaml

test/e2e-kind.sh

@@ -3,7 +3,7 @@
 set -o errexit
 
 REPO_ROOT=$(git rev-parse --show-toplevel)
-KIND_VERSION=v0.4.0
+KIND_VERSION=v0.5.1
 
 if [[ "$1" ]]; then
   KIND_VERSION=$1

test/e2e-linkerd.sh

@@ -2,7 +2,7 @@
 
 set -o errexit
 
-LINKERD_VER="stable-2.4.0"
+LINKERD_VER="stable-2.5.0"
 REPO_ROOT=$(git rev-parse --show-toplevel)
 export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
 

test/e2e-nginx-cleanup.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+
+echo '>>> Deleting NGINX Ingress'
+helm delete --purge nginx-ingress
+
+echo '>>> Deleting Flagger'
+helm delete --purge flagger
+
+echo '>>> Cleanup test namespace'
+kubectl delete namespace test --ignore-not-found=true

test/e2e-nginx-custom-annotations.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+export KUBECONFIG="$(kind get kubeconfig-path --name="kind")"
+NGINX_VERSION=1.12.1
+
+echo '>>> Installing NGINX Ingress'
+helm upgrade -i nginx-ingress stable/nginx-ingress --version=${NGINX_VERSION} \
+--wait \
+--namespace ingress-nginx \
+--set controller.stats.enabled=true \
+--set controller.metrics.enabled=true \
+--set controller.podAnnotations."prometheus\.io/scrape"=true \
+--set controller.podAnnotations."prometheus\.io/port"=10254 \
+--set controller.service.type=NodePort
+
+kubectl -n ingress-nginx patch deployment/nginx-ingress-controller \
+--type='json' -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--annotations-prefix=custom.ingress.kubernetes.io"}]'
+
+kubectl -n ingress-nginx rollout status deployment/nginx-ingress-controller
+kubectl -n ingress-nginx get all
+
+echo '>>> Loading Flagger image'
+kind load docker-image test/flagger:latest
+
+echo '>>> Installing Flagger'
+helm install ${REPO_ROOT}/charts/flagger \
+--name flagger \
+--namespace ingress-nginx \
+--set prometheus.install=true \
+--set ingressAnnotationsPrefix="custom.ingress.kubernetes.io" \
+--set meshProvider=nginx \
+--set crd.create=false
+
+kubectl -n ingress-nginx set image deployment/flagger flagger=test/flagger:latest
+
+kubectl -n ingress-nginx rollout status deployment/flagger
+kubectl -n ingress-nginx rollout status deployment/flagger-prometheus
+

test/e2e-nginx-tests.sh

@@ -43,9 +43,31 @@ spec:
     maxWeight: 30
     stepWeight: 10
     metrics:
-    - name: request-success-rate
+    - name: "http-request-success-rate"
       threshold: 99
       interval: 1m
+      query: |
+        100 - sum(
+                rate(
+                    http_request_duration_seconds_count{
+                        kubernetes_namespace="test",
+                        kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
+                        path="root",
+                        status!~"5.*"
+                    }[1m]
+                )
+            )
+            /
+            sum(
+                rate(
+                    http_request_duration_seconds_count{
+                        kubernetes_namespace="test",
+                        kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
+                        path="root"
+                    }[1m]
+                )
+            )
+            * 100
     - name: "latency"
       threshold: 0.5
       interval: 1m
@@ -55,7 +77,8 @@ spec:
             rate(
               http_request_duration_seconds_bucket{
                 kubernetes_namespace="test",
-                kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"
+                kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
+                path="root"
               }[1m]
             )
           ) by (le)
@@ -94,7 +117,14 @@ echo '>>> Waiting for canary promotion'
 retries=50
 count=0
 ok=false
+failed=false
 until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Failed' && failed=true || failed=false
+    if ${failed}; then
+        kubectl -n ingress-nginx logs deployment/test-flagger
+        echo "Canary failed!"
+        exit 1
+    fi
     kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false
     sleep 10
     kubectl -n ingress-nginx logs deployment/flagger --tail 1
@@ -144,9 +174,31 @@ spec:
           cookie:
             exact: "canary"
     metrics:
-    - name: request-success-rate
+    - name: "http-request-success-rate"
       threshold: 99
       interval: 1m
+      query: |
+        100 - sum(
+                rate(
+                    http_request_duration_seconds_count{
+                        kubernetes_namespace="test",
+                        kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
+                        path="root",
+                        status!~"5.*"
+                    }[1m]
+                )
+            )
+            /
+            sum(
+                rate(
+                    http_request_duration_seconds_count{
+                        kubernetes_namespace="test",
+                        kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
+                        path="root"
+                    }[1m]
+                )
+            )
+            * 100
     webhooks:
       - name: pre
         type: pre-rollout

test/e2e-tests.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This script runs e2e tests for Canary initialization, analysis and promotion
+# This script runs e2e tests for Canary, B/G and A/B initialization, analysis and promotion
 # Prerequisites: Kubernetes Kind, Helm and Istio
 
 set -o errexit
@@ -124,12 +124,100 @@ until ${ok}; do
     fi
 done
 
+echo '>>> Waiting for canary finalization'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
 echo '✔ Canary promotion test passed'
 
 if [[ "$1" = "canary" ]]; then
   exit 0
 fi
 
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1alpha3
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    portDiscovery: true
+    port: 9898
+  canaryAnalysis:
+    interval: 10s
+    threshold: 5
+    iterations: 5
+    metrics:
+    - name: request-success-rate
+      threshold: 99
+      interval: 1m
+    - name: request-duration
+      threshold: 500
+      interval: 30s
+    webhooks:
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          type: cmd
+          cmd: "hey -z 5m -q 10 -c 2 http://podinfo.test:9898/"
+EOF
+
+echo '>>> Triggering B/G deployment'
+kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.2
+
+echo '>>> Waiting for B/G promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '1.4.2' && ok=true || ok=false
+    sleep 10
+    kubectl -n istio-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n test describe deployment/podinfo
+        kubectl -n test describe deployment/podinfo-primary
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '>>> Waiting for B/G finalization'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ B/G promotion test passed'
+
 cat <<EOF | kubectl apply -f -
 apiVersion: flagger.app/v1alpha3
 kind: Canary
@@ -169,6 +257,9 @@ spec:
           type: cmd
           cmd: "hey -z 10m -q 10 -c 2 -H 'Cookie: type=insider' http://podinfo-canary.test:9898/"
           logCmdOutput: "true"
+      - name: promote-gate
+        type: confirm-promotion
+        url: http://flagger-loadtester.test/gate/approve
       - name: post
         type: post-rollout
         url: http://flagger-loadtester.test/
@@ -180,14 +271,14 @@ spec:
 EOF
 
 echo '>>> Triggering A/B testing'
-kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.2
+kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.3
 
 echo '>>> Waiting for A/B testing promotion'
 retries=50
 count=0
 ok=false
 until ${ok}; do
-    kubectl -n test describe deployment/podinfo-primary | grep '1.4.2' && ok=true || ok=false
+    kubectl -n test describe deployment/podinfo-primary | grep '1.4.3' && ok=true || ok=false
     sleep 10
     kubectl -n istio-system logs deployment/flagger --tail 1
     count=$(($count + 1))