Merge pull request #331 from weaveworks/prep-v0.19.0

Release v0.19.0

.circleci/config.yml

@@ -3,7 +3,7 @@ jobs:
 
   build-binary:
     docker:
-      - image: circleci/golang:1.12
+      - image: circleci/golang:1.13
     working_directory: ~/build
     steps:
       - checkout
@@ -44,7 +44,7 @@ jobs:
 
   push-container:
     docker:
-      - image: circleci/golang:1.12
+      - image: circleci/golang:1.13
     steps:
       - checkout
       - setup_remote_docker:
@@ -56,7 +56,7 @@ jobs:
 
   push-binary:
     docker:
-      - image: circleci/golang:1.12
+      - image: circleci/golang:1.13
     working_directory: ~/build
     steps:
       - checkout
@@ -132,6 +132,9 @@ jobs:
       - run: test/e2e-kind.sh
       - run: test/e2e-nginx.sh
       - run: test/e2e-nginx-tests.sh
+      - run: test/e2e-nginx-cleanup.sh
+      - run: test/e2e-nginx-custom-annotations.sh
+      - run: test/e2e-nginx-tests.sh
 
   e2e-linkerd-testing:
     machine: true
@@ -146,7 +149,7 @@ jobs:
 
   push-helm-charts:
     docker:
-      - image: circleci/golang:1.12
+      - image: circleci/golang:1.13
     steps:
       - checkout
       - run:

CHANGELOG.md

@@ -2,6 +2,90 @@
 
 All notable changes to this project are documented in this file.
 
+## 0.19.0 (2019-10-08) 
+
+Adds support for canary and blue/green [traffic mirroring](https://docs.flagger.app/usage/progressive-delivery#traffic-mirroring)
+
+#### Features
+
+- Add traffic mirroring for Istio service mesh [#311](https://github.com/weaveworks/flagger/pull/311)
+- Implement canary service target port [#327](https://github.com/weaveworks/flagger/pull/327)
+
+#### Improvements 
+
+- Allow gPRC protocol for App Mesh [#325](https://github.com/weaveworks/flagger/pull/325)
+- Enforce blue/green when using Kubernetes networking [#326](https://github.com/weaveworks/flagger/pull/326)
+
+#### Fixes
+
+- Fix port discovery diff [#324](https://github.com/weaveworks/flagger/pull/324)
+- Helm chart: Enable Prometheus scraping of Flagger metrics [#2141d88](https://github.com/weaveworks/flagger/commit/2141d88ce1cc6be220dab34171c215a334ecde24)
+
+## 0.18.6 (2019-10-03) 
+
+Adds support for App Mesh conformance tests and latency metric checks
+
+#### Improvements 
+
+- Add support for acceptance testing when using App Mesh [#322](https://github.com/weaveworks/flagger/pull/322)
+- Add Kustomize installer for App Mesh [#310](https://github.com/weaveworks/flagger/pull/310)
+- Update Linkerd to v2.5.0 and Prometheus to v2.12.0 [#323](https://github.com/weaveworks/flagger/pull/323)
+
+#### Fixes
+
+- Fix slack/teams notification fields mapping [#318](https://github.com/weaveworks/flagger/pull/318)
+
+## 0.18.5 (2019-10-02) 
+
+Adds support for [confirm-promotion](https://docs.flagger.app/how-it-works#webhooks) webhooks and blue/green deployments when using a service mesh
+
+#### Features
+
+- Implement confirm-promotion hook [#307](https://github.com/weaveworks/flagger/pull/307)
+- Implement B/G for service mesh providers [#305](https://github.com/weaveworks/flagger/pull/305)
+
+#### Improvements 
+
+- Canary promotion improvements to avoid dropping in-flight requests [#310](https://github.com/weaveworks/flagger/pull/310)
+- Update end-to-end tests to Kubernetes v1.15.3 and Istio 1.3.0 [#306](https://github.com/weaveworks/flagger/pull/306) 
+
+#### Fixes
+
+- Skip primary check for App Mesh [#315](https://github.com/weaveworks/flagger/pull/315)
+
+## 0.18.4 (2019-09-08) 
+
+Adds support for NGINX custom annotations and Helm v3 acceptance testing
+
+#### Features
+
+- Add annotations prefix for NGINX ingresses [#293](https://github.com/weaveworks/flagger/pull/293)
+- Add wide columns in CRD [#289](https://github.com/weaveworks/flagger/pull/289)
+- loadtester: implement Helm v3 test command [#296](https://github.com/weaveworks/flagger/pull/296)
+- loadtester: add gPRC health check to load tester image [#295](https://github.com/weaveworks/flagger/pull/295)
+
+#### Fixes
+
+- loadtester: fix tests error logging [#286](https://github.com/weaveworks/flagger/pull/286)
+
+## 0.18.3 (2019-08-22) 
+
+Adds support for tillerless helm tests and protobuf health checking
+
+#### Features
+
+- loadtester: add support for tillerless helm [#280](https://github.com/weaveworks/flagger/pull/280)
+- loadtester: add support for protobuf health checking [#280](https://github.com/weaveworks/flagger/pull/280)
+
+#### Improvements 
+
+- Set HTTP listeners for AppMesh virtual routers [#272](https://github.com/weaveworks/flagger/pull/272)
+
+#### Fixes
+
+- Add missing fields to CRD validation spec [#271](https://github.com/weaveworks/flagger/pull/271)
+- Fix App Mesh backends validation in CRD [#281](https://github.com/weaveworks/flagger/pull/281)
+
 ## 0.18.2 (2019-08-05) 
 
 Fixes multi-port support for Istio

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.9
+FROM alpine:3.10
 
 RUN addgroup -S flagger \
     && adduser -S -g flagger flagger \

Dockerfile.loadtester

@@ -9,13 +9,24 @@ WORKDIR /home/app
 RUN curl -sSLo hey "https://storage.googleapis.com/jblabs/dist/hey_linux_v0.1.2" && \
 chmod +x hey && mv hey /usr/local/bin/hey
 
-RUN curl -sSL "https://get.helm.sh/helm-v2.12.3-linux-amd64.tar.gz" | tar xvz && \
+RUN curl -sSL "https://get.helm.sh/helm-v2.14.3-linux-amd64.tar.gz" | tar xvz && \
 chmod +x linux-amd64/helm && mv linux-amd64/helm /usr/local/bin/helm && \
+chmod +x linux-amd64/tiller && mv linux-amd64/tiller /usr/local/bin/tiller && \
 rm -rf linux-amd64
 
+RUN curl -sSL "https://get.helm.sh/helm-v3.0.0-beta.3-linux-amd64.tar.gz" | tar xvz && \
+chmod +x linux-amd64/helm && mv linux-amd64/helm /usr/local/bin/helmv3 && \
+rm -rf linux-amd64
+
+RUN GRPC_HEALTH_PROBE_VERSION=v0.3.0 && \
+wget -qO /usr/local/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
+chmod +x /usr/local/bin/grpc_health_probe
+
 RUN curl -sSL "https://github.com/bojand/ghz/releases/download/v0.39.0/ghz_0.39.0_Linux_x86_64.tar.gz" | tar xz -C /tmp && \
 mv /tmp/ghz /usr/local/bin && chmod +x /usr/local/bin/ghz && rm -rf /tmp/ghz-web
 
+ADD https://raw.githubusercontent.com/grpc/grpc-proto/master/grpc/health/v1/health.proto /tmp/ghz/health.proto
+
 RUN ls /tmp
 
 COPY ./bin/loadtester .
@@ -24,4 +35,7 @@ RUN chown -R app:app ./
 
 USER app
 
+RUN curl -sSL "https://github.com/rimusz/helm-tiller/archive/v0.8.3.tar.gz" | tar xvz && \
+helm init --client-only && helm plugin install helm-tiller-0.8.3 && helm plugin list
+
 ENTRYPOINT ["./loadtester"]

Makefile

@@ -7,15 +7,8 @@ LT_VERSION?=$(shell grep 'VERSION' cmd/loadtester/main.go | awk '{ print $$4 }'
 TS=$(shell date +%Y-%m-%d_%H-%M-%S)
 
 run:
-	GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=istio -namespace=test \
-	-metrics-server=https://prometheus.istio.weavedx.com \
-	-enable-leader-election=true
-
-run2:
-	GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=istio -namespace=test \
-	-metrics-server=https://prometheus.istio.weavedx.com \
-	-enable-leader-election=true \
-	-port=9092
+	GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=istio -namespace=test-istio \
+	-metrics-server=https://prometheus.istio.flagger.dev
 
 run-appmesh:
 	GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=appmesh \
@@ -38,8 +31,8 @@ run-nop:
 	-metrics-server=https://prometheus.istio.weavedx.com
 
 run-linkerd:
-	GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=smi:linkerd -namespace=demo \
-	-metrics-server=https://linkerd-prometheus.istio.weavedx.com
+	GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=linkerd -namespace=dev \
+	-metrics-server=https://prometheus.linkerd.flagger.dev
 
 build:
 	GIT_COMMIT=$$(git rev-list -1 HEAD) && GO111MODULE=on CGO_ENABLED=0 GOOS=linux go build  -ldflags "-s -w -X github.com/weaveworks/flagger/pkg/version.REVISION=$${GIT_COMMIT}" -a -installsuffix cgo -o ./bin/flagger ./cmd/flagger/*

README.md

@@ -70,7 +70,6 @@ metadata:
 spec:
   # service mesh provider (optional)
   # can be: kubernetes, istio, linkerd, appmesh, nginx, gloo, supergloo
-  # use the kubernetes provider for Blue/Green style deployments
   provider: istio
   # deployment reference
   targetRef:
@@ -86,14 +85,12 @@ spec:
     kind: HorizontalPodAutoscaler
     name: podinfo
   service:
-    # container port
+    # ClusterIP port number
     port: 9898
-    # Istio gateways (optional)
-    gateways:
-    - public-gateway.istio-system.svc.cluster.local
-    # Istio virtual service host names (optional)
-    hosts:
-    - podinfo.example.com
+    # container port name or number (optional)
+    targetPort: 9898
+    # port name can be http or grpc (default http)
+    portName: http
     # HTTP match conditions (optional)
     match:
       - uri:
@@ -101,10 +98,6 @@ spec:
     # HTTP rewrite (optional)
     rewrite:
       uri: /
-    # cross-origin resource sharing policy (optional)
-    corsPolicy:
-      allowOrigin:
-        - example.com
     # request timeout (optional)
     timeout: 5s
   # promote the canary without analysing it (default false)
@@ -144,7 +137,7 @@ spec:
             topic="podinfo"
           }[1m]
         )
-    # external checks (optional)
+    # testing (optional)
     webhooks:
       - name: load-test
         url: http://flagger-loadtester.test/
@@ -157,15 +150,17 @@ For more details on how the canary analysis and promotion works please [read the
 
 ## Features
 
-| Feature                                      | Istio              | Linkerd            | App Mesh           | NGINX              | Gloo               |
-| -------------------------------------------- | ------------------ | ------------------ |------------------  |------------------  |------------------  |
-| Canary deployments (weighted traffic)        | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
-| A/B testing (headers and cookies filters)    | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_minus_sign: |
-| Webhooks (acceptance/load testing)           | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
-| Request success rate check (L7 metric)       | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
-| Request duration check (L7 metric)           | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_check_mark: |
-| Custom promql checks                         | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
-| Traffic policy, CORS, retries and timeouts   | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_minus_sign: |
+| Feature                                      | Istio              | Linkerd            | App Mesh           | NGINX              | Gloo               | Kubernetes CNI     |
+| -------------------------------------------- | ------------------ | ------------------ |------------------  |------------------  |------------------  |------------------  |
+| Canary deployments (weighted traffic)        | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: |
+| A/B testing (headers and cookies routing)    | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: |
+| Blue/Green deployments (traffic switch)      | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
+| Webhooks (acceptance/load testing)           | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
+| Manual gating (approve/pause/resume)         | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
+| Request success rate check (L7 metric)       | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: |
+| Request duration check (L7 metric)           | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_minus_sign: |
+| Custom promql checks                         | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
+| Traffic policy, CORS, retries and timeouts   | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_minus_sign: |
 
 ## Roadmap
 

artifacts/appmesh/canary.yaml

@@ -20,6 +20,9 @@ spec:
   service:
     # container port
     port: 9898
+    # container port name (optional)
+    # can be http or grpc
+    portName: http
     # App Mesh reference
     meshName: global
   # define the canary analysis timing and KPIs
@@ -41,8 +44,20 @@ spec:
       # percentage (0-100)
       threshold: 99
       interval: 1m
-    # external checks (optional)
+    - name: request-duration
+      # maximum req duration P99
+      # milliseconds
+      threshold: 500
+      interval: 30s
+    # testing (optional)
     webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 30s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token"
       - name: load-test
         url: http://flagger-loadtester.test/
         timeout: 5s

artifacts/appmesh/deployment.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.7.0
+        image: stefanprodan/podinfo:3.1.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/appmesh/ingress.yaml

@@ -13,7 +13,7 @@ data:
         - address:
             socket_address:
               address: 0.0.0.0
-              port_value: 80
+              port_value: 8080
           filter_chains:
             - filters:
                 - name: envoy.http_connection_manager
@@ -48,11 +48,15 @@ data:
           connect_timeout: 0.30s
           type: strict_dns
           lb_policy: round_robin
-          http2_protocol_options: {}
-          hosts:
-            - socket_address:
-                address: podinfo.test
-                port_value: 9898
+          load_assignment:
+            cluster_name: podinfo
+            endpoints:
+              - lb_endpoints:
+                  - endpoint:
+                      address:
+                        socket_address:
+                          address: podinfo.test
+                          port_value: 9898
     admin:
       access_log_path: /dev/null
       address:
@@ -91,7 +95,7 @@ spec:
       terminationGracePeriodSeconds: 30
       containers:
         - name: ingress
-          image: "envoyproxy/envoy-alpine:d920944aed67425f91fc203774aebce9609e5d9a"
+          image: "envoyproxy/envoy-alpine:v1.11.1"
           securityContext:
             capabilities:
               drop:
@@ -99,25 +103,20 @@ spec:
               add:
                 - NET_BIND_SERVICE
           command:
-            - /usr/bin/dumb-init
-            - --
-          args:
             - /usr/local/bin/envoy
-            - --base-id 30
-            - --v2-config-only
+          args:
             - -l
             - $loglevel
             - -c
             - /config/envoy.yaml
+            - --base-id
+            - "1234"
           ports:
             - name: admin
               containerPort: 9999
               protocol: TCP
             - name: http
-              containerPort: 80
-              protocol: TCP
-            - name: https
-              containerPort: 443
+              containerPort: 8080
               protocol: TCP
           livenessProbe:
             initialDelaySeconds: 5
@@ -151,11 +150,7 @@ spec:
     - protocol: TCP
       name: http
       port: 80
-      targetPort: 80
-    - protocol: TCP
-      name: https
-      port: 443
-      targetPort: 443
+      targetPort: http
   type: LoadBalancer
 ---
 apiVersion: appmesh.k8s.aws/v1beta1

artifacts/canaries/deployment.yaml

@@ -20,12 +20,13 @@ spec:
     metadata:
       annotations:
         prometheus.io/scrape: "true"
+        prometheus.io/port: "9898"
       labels:
         app: podinfo
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.7.0
+        image: stefanprodan/podinfo:3.1.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/flagger/crd.yaml

@@ -33,6 +33,26 @@ spec:
     - name: Weight
       type: string
       JSONPath: .status.canaryWeight
+    - name: FailedChecks
+      type: string
+      JSONPath: .status.failedChecks
+      priority: 1
+    - name: Interval
+      type: string
+      JSONPath: .spec.canaryAnalysis.interval
+      priority: 1
+    - name: Mirror
+      type: boolean
+      JSONPath: .spec.canaryAnalysis.mirror
+      priority: 1
+    - name: StepWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.stepWeight
+      priority: 1
+    - name: MaxWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.maxWeight
+      priority: 1
     - name: LastTransitionTime
       type: string
       JSONPath: .status.lastTransitionTime
@@ -46,12 +66,15 @@ spec:
             - canaryAnalysis
           properties:
             provider:
+              description: Traffic managent provider
               type: string
             progressDeadlineSeconds:
+              description: Deployment progress deadline
               type: number
             targetRef:
+              description: Deployment selector
               type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -60,10 +83,11 @@ spec:
                 name:
                   type: string
             autoscalerRef:
+              description: HPA selector
               anyOf:
                 - type: string
                 - type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -72,10 +96,11 @@ spec:
                 name:
                   type: string
             ingressRef:
+              description: NGINX ingress selector
               anyOf:
                 - type: string
                 - type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -85,18 +110,68 @@ spec:
                   type: string
             service:
               type: object
-              required: ['port']
+              required: ["port"]
               properties:
                 port:
+                  description: Container port number
                   type: number
                 portName:
+                  description: Container port name
                   type: string
+                targetPort:
+                  description: Container target port name
+                  anyOf:
+                    - type: string
+                    - type: number
                 portDiscovery:
+                  description: Enable port dicovery
                   type: boolean
                 meshName:
+                  description: AppMesh mesh name
                   type: string
+                backends:
+                  description: AppMesh backend array
+                  anyOf:
+                    - type: string
+                    - type: array
                 timeout:
+                  description: Istio HTTP or gRPC request timeout
                   type: string
+                trafficPolicy:
+                  description: Istio traffic policy
+                  anyOf:
+                    - type: string
+                    - type: object
+                match:
+                  description: Istio URL match conditions
+                  anyOf:
+                    - type: string
+                    - type: array
+                rewrite:
+                  description: Istio URL rewrite
+                  anyOf:
+                    - type: string
+                    - type: object
+                headers:
+                  description: Istio headers operations
+                  anyOf:
+                    - type: string
+                    - type: object
+                corsPolicy:
+                  description: Istio CORS policy
+                  anyOf:
+                    - type: string
+                    - type: object
+                gateways:
+                  description: Istio gateways list
+                  anyOf:
+                    - type: string
+                    - type: array
+                hosts:
+                  description: Istio hosts list
+                  anyOf:
+                    - type: string
+                    - type: array
             skipAnalysis:
               type: boolean
             canaryAnalysis:
@@ -117,13 +192,21 @@ spec:
                 stepWeight:
                   description: Canary incremental traffic percentage step
                   type: number
+                mirror:
+                  description: Mirror traffic to canary before shifting
+                  type: boolean
+                match:
+                  description: A/B testing match conditions
+                  anyOf:
+                    - type: string
+                    - type: array
                 metrics:
                   description: Prometheus query list for this canary
                   type: array
                   properties:
                     items:
                       type: object
-                      required: ['name', 'threshold']
+                      required: ["name", "threshold"]
                       properties:
                         name:
                           description: Name of the Prometheus metric
@@ -144,7 +227,7 @@ spec:
                   properties:
                     items:
                       type: object
-                      required: ['name', 'url', 'timeout']
+                      required: ["name", "url"]
                       properties:
                         name:
                           description: Name of the webhook
@@ -154,8 +237,10 @@ spec:
                           type: string
                           enum:
                             - ""
+                            - confirm-rollout
                             - pre-rollout
                             - rollout
+                            - confirm-promotion
                             - post-rollout
                         url:
                           description: URL address of this webhook
@@ -165,6 +250,11 @@ spec:
                           description: Request timeout for this webhook
                           type: string
                           pattern: "^[0-9]+(m|s)"
+                        metadata:
+                          description: Metadata (key-value pairs) for this webhook
+                          anyOf:
+                            - type: string
+                            - type: object
         status:
           properties:
             phase:
@@ -176,6 +266,7 @@ spec:
                 - Initialized
                 - Waiting
                 - Progressing
+                - Promoting
                 - Finalising
                 - Succeeded
                 - Failed
@@ -201,7 +292,7 @@ spec:
               properties:
                 items:
                   type: object
-                  required: ['type', 'status', 'reason']
+                  required: ["type", "status", "reason"]
                   properties:
                     lastTransitionTime:
                       description: LastTransitionTime of this condition

artifacts/flagger/deployment.yaml

@@ -22,7 +22,7 @@ spec:
       serviceAccountName: flagger
       containers:
       - name: flagger
-        image: weaveworks/flagger:0.18.2
+        image: weaveworks/flagger:0.19.0
         imagePullPolicy: IfNotPresent
         ports:
         - name: http

artifacts/helmtester/deployment.yaml

@@ -19,7 +19,7 @@ spec:
       serviceAccountName: tiller
       containers:
         - name: helmtester
-          image: weaveworks/flagger-loadtester:0.4.0
+          image: weaveworks/flagger-loadtester:0.8.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

artifacts/loadtester/deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: loadtester
-          image: weaveworks/flagger-loadtester:0.6.1
+          image: weaveworks/flagger-loadtester:0.9.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

charts/flagger/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 name: flagger
-version: 0.18.2
-appVersion: 0.18.2
+version: 0.19.0
+appVersion: 0.19.0
 kubeVersion: ">=1.11.0-0"
 engine: gotpl
 description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio, Linkerd, App Mesh, Gloo or NGINX routing for traffic shifting and Prometheus metrics for canary analysis.

charts/flagger/README.md

@@ -74,6 +74,7 @@ Parameter | Description | Default
 `msteams.url` | Microsoft Teams incoming webhook | None
 `leaderElection.enabled` | leader election must be enabled when running more than one replica | `false`
 `leaderElection.replicaCount` | number of replicas | `1`
+`ingressAnnotationsPrefix` | annotations prefix for ingresses | `custom.ingress.kubernetes.io`
 `rbac.create` | if `true`, create and use RBAC resources | `true`
 `rbac.pspEnabled` | If `true`, create and use a restricted pod security policy | `false`
 `crd.create` | if `true`, create Flagger's CRDs | `true`

charts/flagger/templates/crd.yaml

@@ -34,6 +34,26 @@ spec:
     - name: Weight
       type: string
       JSONPath: .status.canaryWeight
+    - name: FailedChecks
+      type: string
+      JSONPath: .status.failedChecks
+      priority: 1
+    - name: Interval
+      type: string
+      JSONPath: .spec.canaryAnalysis.interval
+      priority: 1
+    - name: Mirror
+      type: boolean
+      JSONPath: .spec.canaryAnalysis.mirror
+      priority: 1
+    - name: StepWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.stepWeight
+      priority: 1
+    - name: MaxWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.maxWeight
+      priority: 1
     - name: LastTransitionTime
       type: string
       JSONPath: .status.lastTransitionTime
@@ -47,10 +67,13 @@ spec:
             - canaryAnalysis
           properties:
             provider:
+              description: Traffic managent provider
               type: string
             progressDeadlineSeconds:
+              description: Deployment progress deadline
               type: number
             targetRef:
+              description: Deployment selector
               type: object
               required: ['apiVersion', 'kind', 'name']
               properties:
@@ -61,6 +84,7 @@ spec:
                 name:
                   type: string
             autoscalerRef:
+              description: HPA selector
               anyOf:
                 - type: string
                 - type: object
@@ -73,6 +97,7 @@ spec:
                 name:
                   type: string
             ingressRef:
+              description: NGINX ingress selector
               anyOf:
                 - type: string
                 - type: object
@@ -89,15 +114,65 @@ spec:
               required: ['port']
               properties:
                 port:
+                  description: Container port number
                   type: number
                 portName:
+                  description: Container port name
                   type: string
+                targetPort:
+                  description: Container target port name
+                  anyOf:
+                    - type: string
+                    - type: number
                 portDiscovery:
+                  description: Enable port dicovery
                   type: boolean
                 meshName:
+                  description: AppMesh mesh name
                   type: string
+                backends:
+                  description: AppMesh backend array
+                  anyOf:
+                    - type: string
+                    - type: array
                 timeout:
+                  description: Istio HTTP or gRPC request timeout
                   type: string
+                trafficPolicy:
+                  description: Istio traffic policy
+                  anyOf:
+                    - type: string
+                    - type: object
+                match:
+                  description: Istio URL match conditions
+                  anyOf:
+                    - type: string
+                    - type: array
+                rewrite:
+                  description: Istio URL rewrite
+                  anyOf:
+                    - type: string
+                    - type: object
+                headers:
+                  description: Istio headers operations
+                  anyOf:
+                    - type: string
+                    - type: object
+                corsPolicy:
+                  description: Istio CORS policy
+                  anyOf:
+                    - type: string
+                    - type: object
+                gateways:
+                  description: Istio gateways list
+                  anyOf:
+                    - type: string
+                    - type: array
+                hosts:
+                  description: Istio hosts list
+                  anyOf:
+                    - type: string
+                    - type: array
             skipAnalysis:
               type: boolean
             canaryAnalysis:
@@ -118,6 +193,14 @@ spec:
                 stepWeight:
                   description: Canary incremental traffic percentage step
                   type: number
+                mirror:
+                  description: Mirror traffic to canary before shifting
+                  type: boolean
+                match:
+                  description: A/B testing match conditions
+                  anyOf:
+                    - type: string
+                    - type: array
                 metrics:
                   description: Prometheus query list for this canary
                   type: array
@@ -145,7 +228,7 @@ spec:
                   properties:
                     items:
                       type: object
-                      required: ['name', 'url', 'timeout']
+                      required: ["name", "url"]
                       properties:
                         name:
                           description: Name of the webhook
@@ -155,8 +238,10 @@ spec:
                           type: string
                           enum:
                             - ""
+                            - confirm-rollout
                             - pre-rollout
                             - rollout
+                            - confirm-promotion
                             - post-rollout
                         url:
                           description: URL address of this webhook
@@ -166,6 +251,11 @@ spec:
                           description: Request timeout for this webhook
                           type: string
                           pattern: "^[0-9]+(m|s)"
+                        metadata:
+                          description: Metadata (key-value pairs) for this webhook
+                          anyOf:
+                            - type: string
+                            - type: object
         status:
           properties:
             phase:
@@ -177,6 +267,7 @@ spec:
                 - Initialized
                 - Waiting
                 - Progressing
+                - Promoting
                 - Finalising
                 - Succeeded
                 - Failed

charts/flagger/templates/deployment.yaml

@@ -20,6 +20,10 @@ spec:
       labels:
         app.kubernetes.io/name: {{ template "flagger.name" . }}
         app.kubernetes.io/instance: {{ .Release.Name }}
+      annotations:
+        {{- if .Values.podAnnotations }}
+{{ toYaml .Values.podAnnotations | indent 8 }}
+        {{- end }}
     spec:
       serviceAccountName: {{ template "flagger.serviceAccountName" . }}
       affinity:
@@ -72,6 +76,9 @@ spec:
           - -enable-leader-election=true
           - -leader-election-namespace={{ .Release.Namespace }}
           {{- end }}
+          {{- if .Values.ingressAnnotationsPrefix }}
+          - -ingress-annotations-prefix={{ .Values.ingressAnnotationsPrefix }}
+          {{- end }}
           livenessProbe:
             exec:
               command:

charts/flagger/templates/prometheus.yaml

@@ -238,7 +238,7 @@ spec:
       serviceAccountName: {{ template "flagger.serviceAccountName" . }}-prometheus
       containers:
         - name: prometheus
-          image: "docker.io/prom/prometheus:v2.10.0"
+          image: "docker.io/prom/prometheus:v2.12.0"
           imagePullPolicy: IfNotPresent
           args:
             - '--storage.tsdb.retention=2h'

charts/flagger/values.yaml

@@ -2,10 +2,14 @@
 
 image:
   repository: weaveworks/flagger
-  tag: 0.18.2
+  tag: 0.19.0
   pullPolicy: IfNotPresent
   pullSecret:
 
+podAnnotations:
+  prometheus.io/scrape: "true"
+  prometheus.io/port: "8080"
+
 metricsServer: "http://prometheus:9090"
 
 # accepted values are istio, appmesh, nginx or supergloo:mesh.namespace (defaults to istio)

charts/grafana/templates/deployment.yaml

@@ -20,6 +20,9 @@ spec:
         release: {{ .Release.Name }}
       annotations:
         prometheus.io/scrape: 'false'
+        {{- if .Values.podAnnotations }}
+{{ toYaml .Values.podAnnotations | indent 8 }}
+        {{- end }}
     spec:
       containers:
         - name: {{ .Chart.Name }}

charts/grafana/values.yaml

@@ -9,6 +9,8 @@ image:
   tag: 6.2.5
   pullPolicy: IfNotPresent
 
+podAnnotations: {}
+
 service:
   type: ClusterIP
   port: 80

charts/loadtester/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 name: loadtester
-version: 0.6.0
-appVersion: 0.6.1
+version: 0.9.0
+appVersion: 0.9.0
 kubeVersion: ">=1.11.0-0"
 engine: gotpl
 description: Flagger's load testing services based on rakyll/hey and bojand/ghz that generates traffic during canary analysis when configured as a webhook.

charts/loadtester/templates/deployment.yaml

@@ -18,9 +18,14 @@ spec:
         app: {{ include "loadtester.name" . }}
       annotations:
         appmesh.k8s.aws/ports: "444"
+        {{- if .Values.podAnnotations }}
+{{ toYaml .Values.podAnnotations | indent 8 }}
+        {{- end }}
     spec:
       {{- if .Values.serviceAccountName }}
       serviceAccountName: {{ .Values.serviceAccountName }}
+      {{- else if .Values.rbac.create }}
+      serviceAccountName: {{ include "loadtester.fullname" . }}
       {{- end }}
       containers:
         - name: {{ .Chart.Name }}

charts/loadtester/templates/rbac.yaml

@@ -0,0 +1,54 @@
+---
+{{- if .Values.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+{{- if eq .Values.rbac.scope "cluster" }}
+kind: ClusterRole
+{{- else }}
+kind: Role
+{{- end }}
+metadata:
+  name: {{ template "loadtester.fullname" . }}
+  labels:
+    helm.sh/chart: {{ template "loadtester.chart" . }}
+    app.kubernetes.io/name: {{ template "loadtester.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+rules:
+{{ toYaml .Values.rbac.rules | indent 2 }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+{{- if eq .Values.rbac.scope "cluster" }}
+kind: ClusterRoleBinding
+{{- else }}
+kind: RoleBinding
+{{- end }}
+metadata:
+  name: {{ template "loadtester.fullname" . }}
+  labels:
+    helm.sh/chart: {{ template "loadtester.chart" . }}
+    app.kubernetes.io/name: {{ template "loadtester.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  {{- if eq .Values.rbac.scope "cluster" }}
+  kind: ClusterRole
+  {{- else }}
+  kind: Role
+  {{- end }}
+  name: {{ template "loadtester.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "loadtester.fullname" . }}
+    namespace: {{ .Release.Namespace }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "loadtester.fullname" . }}
+  labels:
+    helm.sh/chart: {{ template "loadtester.chart" . }}
+    app.kubernetes.io/name: {{ template "loadtester.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

charts/loadtester/values.yaml

@@ -2,9 +2,13 @@ replicaCount: 1
 
 image:
   repository: weaveworks/flagger-loadtester
-  tag: 0.6.1
+  tag: 0.9.0
   pullPolicy: IfNotPresent
 
+podAnnotations:
+  prometheus.io/scrape: "true"
+  prometheus.io/port: "8080"
+
 logLevel: info
 cmd:
   timeout: 1h
@@ -27,6 +31,20 @@ tolerations: []
 
 affinity: {}
 
+rbac:
+  # rbac.create: `true` if rbac resources should be created
+  create: false
+  # rbac.scope: `cluster` to create cluster-scope rbac resources (ClusterRole/ClusterRoleBinding)
+  # otherwise, namespace-scope rbac resources will be created (Role/RoleBinding)
+  scope:
+  # rbac.rules: array of rules to apply to the role. example:
+  # rules:
+  # - apiGroups: [""]
+  #   resources: ["pods"]
+  #   verbs: ["list", "get"]
+  rules: []
+
+# name of an existing service account to use - if not creating rbac resources
 serviceAccountName: ""
 
 # App Mesh virtual node settings

charts/podinfo/Chart.yaml

@@ -1,10 +1,10 @@
 apiVersion: v1
-version: 2.3.0
-appVersion: 1.7.0
+version: 3.1.0
+appVersion: 3.1.0
 name: podinfo
 engine: gotpl
 description: Flagger canary deployment demo chart
-home: https://github.com/weaveworks/flagger
+home: https://flagger.app
 maintainers:
 - email: stefanprodan@users.noreply.github.com
   name: stefanprodan

charts/podinfo/templates/deployment.yaml

@@ -21,10 +21,13 @@ spec:
         app: {{ template "podinfo.fullname" . }}
       annotations:
         prometheus.io/scrape: 'true'
+        {{- if .Values.podAnnotations }}
+{{ toYaml .Values.podAnnotations | indent 8 }}
+        {{- end }}
     spec:
       terminationGracePeriodSeconds: 30
       containers:
-        - name: {{ .Chart.Name }}
+        - name: podinfo
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           command:
@@ -34,6 +37,9 @@ spec:
             - --random-delay={{ .Values.faults.delay }}
             - --random-error={{ .Values.faults.error }}
             - --config-path=/podinfo/config
+            {{- range .Values.backends }}
+            - --backend-url={{ . }}
+            {{- end }}
           env:
           {{- if .Values.message }}
           - name: PODINFO_UI_MESSAGE

charts/podinfo/templates/tests/jwt.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ template "podinfo.fullname" . }}-jwt-test-{{ randAlphaNum 5 | lower }}
+  labels:
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    app: {{ template "podinfo.name" . }}
+  annotations:
+    "helm.sh/hook": test-success
+    sidecar.istio.io/inject: "false"
+    linkerd.io/inject: disabled
+    appmesh.k8s.aws/sidecarInjectorWebhook: disabled
+spec:
+  containers:
+    - name: tools
+      image: giantswarm/tiny-tools
+      command:
+        - sh
+        - -c
+        - |
+          TOKEN=$(curl -sd 'test' ${PODINFO_SVC}/token | jq -r .token) &&
+          curl -H "Authorization: Bearer ${TOKEN}" ${PODINFO_SVC}/token/validate | grep test
+      env:
+      - name: PODINFO_SVC
+        value: {{ template "podinfo.fullname" . }}:{{ .Values.service.port }}
+  restartPolicy: Never
+

charts/podinfo/templates/tests/test-config.yaml

@@ -1,22 +0,0 @@
-{{- $url := printf "%s%s.%s:%v" (include "podinfo.fullname" .) (include "podinfo.suffix" .) .Release.Namespace .Values.service.port -}}
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name:  {{ template "podinfo.fullname" . }}-tests
-  labels:
-    heritage: {{ .Release.Service }}
-    release: {{ .Release.Name }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    app: {{ template "podinfo.name" . }}
-data:
-  run.sh: |-
-    @test "HTTP POST /echo" {
-      run curl --retry 3 --connect-timeout 2 -sSX POST -d 'test' {{ $url }}/echo
-      [ $output = "test" ]
-    }
-    @test "HTTP POST /store" {
-      curl --retry 3 --connect-timeout 2 -sSX POST -d 'test' {{ $url }}/store
-    }
-    @test "HTTP GET /" {
-      curl --retry 3 --connect-timeout 2 -sS {{ $url }} | grep hostname
-    }

charts/podinfo/templates/tests/test-pod.yaml

@@ -1,43 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: {{ template "podinfo.fullname" . }}-tests-{{ randAlphaNum 5 | lower }}
-  annotations:
-    "helm.sh/hook": test-success
-    sidecar.istio.io/inject: "false"
-  labels:
-    heritage: {{ .Release.Service }}
-    release: {{ .Release.Name }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    app: {{ template "podinfo.name" . }}
-spec:
-  initContainers:
-    - name: "test-framework"
-      image: "dduportal/bats:0.4.0"
-      command:
-      - "bash"
-      - "-c"
-      - |
-        set -ex
-        # copy bats to tools dir
-        cp -R /usr/local/libexec/ /tools/bats/
-      volumeMounts:
-      - mountPath: /tools
-        name: tools
-  containers:
-    - name: {{ .Release.Name }}-ui-test
-      image: dduportal/bats:0.4.0
-      command: ["/tools/bats/bats", "-t", "/tests/run.sh"]
-      volumeMounts:
-      - mountPath: /tests
-        name: tests
-        readOnly: true
-      - mountPath: /tools
-        name: tools
-  volumes:
-  - name: tests
-    configMap:
-      name: {{ template "podinfo.fullname" . }}-tests
-  - name: tools
-    emptyDir: {}
-  restartPolicy: Never

charts/podinfo/values.yaml

@@ -1,22 +1,25 @@
 # Default values for podinfo.
 image:
-  repository: quay.io/stefanprodan/podinfo
-  tag: 1.7.0
+  repository: stefanprodan/podinfo
+  tag: 3.1.0
   pullPolicy: IfNotPresent
 
+podAnnotations: {}
+
 service:
+  enabled: false
   type: ClusterIP
   port: 9898
 
 hpa:
   enabled: true
   minReplicas: 2
-  maxReplicas: 2
+  maxReplicas: 4
   cpu: 80
   memory: 512Mi
 
 canary:
-  enabled: true
+  enabled: false
   # Istio traffic policy tls can be DISABLE or ISTIO_MUTUAL
   istioTLS: DISABLE
   istioIngress:
@@ -69,6 +72,7 @@ fullnameOverride: ""
 
 logLevel: info
 backend: #http://backend-podinfo:9898/echo
+backends: []
 message: #UI greetings
 
 faults:

cmd/flagger/main.go

@@ -33,25 +33,26 @@ import (
 )
 
 var (
-	masterURL               string
-	kubeconfig              string
-	metricsServer           string
-	controlLoopInterval     time.Duration
-	logLevel                string
-	port                    string
-	msteamsURL              string
-	slackURL                string
-	slackUser               string
-	slackChannel            string
-	threadiness             int
-	zapReplaceGlobals       bool
-	zapEncoding             string
-	namespace               string
-	meshProvider            string
-	selectorLabels          string
-	enableLeaderElection    bool
-	leaderElectionNamespace string
-	ver                     bool
+	masterURL                string
+	kubeconfig               string
+	metricsServer            string
+	controlLoopInterval      time.Duration
+	logLevel                 string
+	port                     string
+	msteamsURL               string
+	slackURL                 string
+	slackUser                string
+	slackChannel             string
+	threadiness              int
+	zapReplaceGlobals        bool
+	zapEncoding              string
+	namespace                string
+	meshProvider             string
+	selectorLabels           string
+	ingressAnnotationsPrefix string
+	enableLeaderElection     bool
+	leaderElectionNamespace  string
+	ver                      bool
 )
 
 func init() {
@@ -71,6 +72,7 @@ func init() {
 	flag.StringVar(&namespace, "namespace", "", "Namespace that flagger would watch canary object.")
 	flag.StringVar(&meshProvider, "mesh-provider", "istio", "Service mesh provider, can be istio, linkerd, appmesh, supergloo, nginx or smi.")
 	flag.StringVar(&selectorLabels, "selector-labels", "app,name,app.kubernetes.io/name", "List of pod labels that Flagger uses to create pod selectors.")
+	flag.StringVar(&ingressAnnotationsPrefix, "ingress-annotations-prefix", "nginx.ingress.kubernetes.io", "Annotations prefix for ingresses.")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election.")
 	flag.StringVar(&leaderElectionNamespace, "leader-election-namespace", "kube-system", "Namespace used to create the leader election config map.")
 	flag.BoolVar(&ver, "version", false, "Print version")
@@ -175,7 +177,7 @@ func main() {
 	// start HTTP server
 	go server.ListenAndServe(port, 3*time.Second, logger, stopCh)
 
-	routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, logger, meshClient)
+	routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, ingressAnnotationsPrefix, logger, meshClient)
 
 	c := controller.NewController(
 		kubeClient,

cmd/loadtester/main.go

@@ -10,7 +10,7 @@ import (
 	"time"
 )
 
-var VERSION = "0.6.1"
+var VERSION = "0.9.0"
 var (
 	logLevel          string
 	port              string

docs/gitbook/faq.md

@@ -7,10 +7,12 @@
 Flagger can run automated application analysis, promotion and rollback for the following deployment strategies:
 * Canary (progressive traffic shifting)
     * Istio, Linkerd, App Mesh, NGINX, Gloo
+* Canary (traffic mirroring)
+    * Istio
 * A/B Testing (HTTP headers and cookies traffic routing)
     * Istio, NGINX
 * Blue/Green (traffic switch)
-    * Kubernetes CNI
+    * Kubernetes CNI, Istio, Linkerd, App Mesh, NGINX, Gloo
 
 For Canary deployments and A/B testing you'll need a Layer 7 traffic management solution like a service mesh or an ingress controller.
 For Blue/Green deployments no service mesh or ingress controller is required.
@@ -102,6 +104,42 @@ The above configuration will run an analysis for five minutes.
 Flagger starts the load test for the canary service (green version) and checks the Prometheus metrics every 30 seconds.
 If the analysis result is positive, Flagger will promote the canary (green version) to primary (blue version).
 
+**When can I use traffic mirroring?**
+
+Traffic Mirroring is a pre-stage in a Canary (progressive traffic shifting) or
+Blue/Green deployment strategy. Traffic mirroring will copy each incoming
+request, sending one request to the primary and one to the canary service.
+The response from the primary is sent back to the user. The response from the canary
+is discarded.  Metrics are collected on both requests so that the deployment will
+only proceed if the canary metrics are healthy.
+
+Mirroring is supported by Istio only.
+
+In Istio, mirrored requests have `-shadow` appended to the `Host` (HTTP) or
+`Authority` (HTTP/2) header; for example requests to `podinfo.test` that are
+mirrored will be reported in telemetry with a destination host `podinfo.test-shadow`.
+
+Mirroring must only be used for requests that are **idempotent** or capable of
+being processed twice (once by the primary and once by the canary). Reads are
+idempotent. Before using mirroring on requests that may be writes, you should
+consider what will happen if a write is duplicated and handled by the primary
+and canary.
+
+To use mirroring, set `spec.canaryAnalysis.mirror` to `true`. Example for
+traffic shifting:
+
+```yaml
+apiVersion: flagger.app/v1alpha3
+kind: Canary
+spec:
+  provider: istio
+  canaryAnalysis:
+    mirror: true
+    interval: 30s
+    stepWeight: 20
+    maxWeight: 50
+```
+
 ### Kubernetes services
 
 **How is an application exposed inside the cluster?**
@@ -120,8 +158,10 @@ spec:
     kind: Deployment
     name: podinfo
   service:
-    # container port (required)
+    # ClusterIP port number (required)
     port: 9898
+    # container port name or number
+    targetPort: http
     # port name can be http or grpc (default http)
     portName: http
 ```
@@ -291,6 +331,190 @@ spec:
               topologyKey: kubernetes.io/hostname
 ```
 
+### Istio routing
+
+**How does Flagger interact with Istio?**
+
+Flagger creates an Istio Virtual Service and Destination Rules based on the Canary service spec. 
+The service configuration lets you expose an app inside or outside the mesh.
+You can also define traffic policies, HTTP match conditions, URI rewrite rules, CORS policies, timeout and retries.
+
+The following spec exposes the `frontend` workload inside the mesh on `frontend.test.svc.cluster.local:9898` 
+and outside the mesh on `frontend.example.com`. You'll have to specify an Istio ingress gateway for external hosts.
+
+```yaml
+apiVersion: flagger.app/v1alpha3
+kind: Canary
+metadata:
+  name: frontend
+  namespace: test
+spec:
+  service:
+    # container port
+    port: 9898
+    # service port name (optional, will default to "http")
+    portName: http-frontend
+    # Istio gateways (optional)
+    gateways:
+    - public-gateway.istio-system.svc.cluster.local
+    - mesh
+    # Istio virtual service host names (optional)
+    hosts:
+    - frontend.example.com
+    # Istio traffic policy
+    trafficPolicy:
+      tls:
+        # use ISTIO_MUTUAL when mTLS is enabled
+        mode: DISABLE
+    # HTTP match conditions (optional)
+    match:
+      - uri:
+          prefix: /
+    # HTTP rewrite (optional)
+    rewrite:
+      uri: /
+    # Envoy timeout and retry policy (optional)
+    headers:
+      request:
+        add:
+          x-envoy-upstream-rq-timeout-ms: "15000"
+          x-envoy-max-retries: "10"
+          x-envoy-retry-on: "gateway-error,connect-failure,refused-stream"
+    # cross-origin resource sharing policy (optional)
+    corsPolicy:
+      allowOrigin:
+        - example.com
+      allowMethods:
+        - GET
+      allowCredentials: false
+      allowHeaders:
+        - x-some-header
+      maxAge: 24h
+```
+
+For the above spec Flagger will generate the following virtual service:
+
+```yaml
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: frontend
+  namespace: test
+  ownerReferences:
+    - apiVersion: flagger.app/v1alpha3
+      blockOwnerDeletion: true
+      controller: true
+      kind: Canary
+      name: podinfo
+      uid: 3a4a40dd-3875-11e9-8e1d-42010a9c0fd1
+spec:
+  gateways:
+    - public-gateway.istio-system.svc.cluster.local
+    - mesh
+  hosts:
+    - frontend.example.com
+    - frontend
+  http:
+  - appendHeaders:
+      x-envoy-max-retries: "10"
+      x-envoy-retry-on: gateway-error,connect-failure,refused-stream
+      x-envoy-upstream-rq-timeout-ms: "15000"
+    corsPolicy:
+      allowHeaders:
+      - x-some-header
+      allowMethods:
+      - GET
+      allowOrigin:
+      - example.com
+      maxAge: 24h
+    match:
+    - uri:
+        prefix: /
+    rewrite:
+      uri: /
+    route:
+    - destination:
+        host: podinfo-primary
+      weight: 100
+    - destination:
+        host: podinfo-canary
+      weight: 0
+```
+
+For each destination in the virtual service a rule is generated:
+
+```yaml
+apiVersion: networking.istio.io/v1alpha3
+kind: DestinationRule
+metadata:
+  name: frontend-primary
+  namespace: test
+spec:
+  host: frontend-primary
+  trafficPolicy:
+    tls:
+      mode: DISABLE
+---
+apiVersion: networking.istio.io/v1alpha3
+kind: DestinationRule
+metadata:
+  name: frontend-canary
+  namespace: test
+spec:
+  host: frontend-canary
+  trafficPolicy:
+    tls:
+      mode: DISABLE
+```
+
+Flagger keeps in sync the virtual service and destination rules with the canary service spec.
+Any direct modification to the virtual service spec will be overwritten.
+
+To expose a workload inside the mesh on `http://backend.test.svc.cluster.local:9898`,
+the service spec can contain only the container port and the traffic policy:
+
+```yaml
+apiVersion: flagger.app/v1alpha3
+kind: Canary
+metadata:
+  name: backend
+  namespace: test
+spec:
+  service:
+    port: 9898
+    trafficPolicy:
+      tls:
+        mode: DISABLE
+```
+
+Based on the above spec, Flagger will create several ClusterIP services like:
+
+```yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: backend-primary
+  ownerReferences:
+  - apiVersion: flagger.app/v1alpha3
+    blockOwnerDeletion: true
+    controller: true
+    kind: Canary
+    name: backend
+    uid: 2ca1a9c7-2ef6-11e9-bd01-42010a9c0145
+spec:
+  type: ClusterIP
+  ports:
+  - name: http
+    port: 9898
+    protocol: TCP
+    targetPort: 9898
+  selector:
+    app: backend-primary
+```
+
+Flagger works for user facing apps exposed outside the cluster via an ingress gateway
+and for backend HTTP APIs that are accessible only from inside the mesh.
+
 ### Istio Ingress Gateway
 
 **How can I expose multiple canaries on the same external domain?**

docs/gitbook/how-it-works.md

@@ -4,8 +4,6 @@
 a horizontal pod autoscaler \(HPA\) and creates a series of objects 
 \(Kubernetes deployments, ClusterIP services, virtual service, traffic split or ingress\) to drive the canary analysis and promotion. 
 
-![Flagger Canary Process](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-hpa.png)
-
 ### Canary Custom Resource
 
 For a deployment named _podinfo_, a canary promotion can be defined using Flagger's custom resource:
@@ -19,8 +17,7 @@ metadata:
 spec:
   # service mesh provider (optional)
   # can be: kubernetes, istio, linkerd, appmesh, nginx, gloo, supergloo
-  # use the kubernetes provider for Blue/Green style deployments
-  provider: istio
+  provider: linkerd
   # deployment reference
   targetRef:
     apiVersion: apps/v1
@@ -35,16 +32,15 @@ spec:
     kind: HorizontalPodAutoscaler
     name: podinfo
   service:
-    # container port
+    # ClusterIP port number
     port: 9898
-    # service port name (optional, will default to "http")
-    portName: http-podinfo
-    # Istio gateways (optional)
-    gateways:
-    - public-gateway.istio-system.svc.cluster.local
-    # Istio virtual service host names (optional)
-    hosts:
-    - podinfo.example.com
+    # ClusterIP port name can be http or grpc (default http)
+    portName: http
+    # container port number or name (optional)
+    targetPort: 9898
+    # add all the other container ports
+    # to the ClusterIP services (default false)
+    portDiscovery: false
   # promote the canary without analysing it (default false)
   skipAnalysis: false
   # define the canary analysis timing and KPIs
@@ -71,15 +67,13 @@ spec:
       # milliseconds
       threshold: 500
       interval: 30s
-    # external checks (optional)
+    # testing (optional)
     webhooks:
-      - name: integration-tests
-        url: http://podinfo.test:9898/echo
-        timeout: 1m
-        # key-value pairs (optional)
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
         metadata:
-          test: "all"
-          token: "16688eb5e9f289f1991c"
+          cmd: "hey -z 1m -q 10 -c 2 http://podinfo.test:9898/"
 ```
 
 **Note** that the target deployment must have a single label selector in the format `app: <DEPLOYMENT-NAME>`:
@@ -102,8 +96,8 @@ spec:
 Besides `app` Flagger supports `name` and `app.kubernetes.io/name` selectors. If you use a different 
 convention you can specify your label with the `-selector-labels` flag.
 
-The target deployment should expose a TCP port that will be used by Flagger to create the ClusterIP Service and 
-the Istio Virtual Service. The container port from the target deployment should match the `service.port` value.
+The target deployment should expose a TCP port that will be used by Flagger to create the ClusterIP Services.
+The container port from the target deployment should match the `service.port` or `service.targetPort`.
 
 ### Canary status
 
@@ -143,7 +137,7 @@ status:
 ```
 
 The `Promoted` status condition can have one of the following reasons:
-Initialized, Waiting, Progressing, Finalising, Succeeded or Failed.
+Initialized, Waiting, Progressing, Promoting, Finalising, Succeeded or Failed.
 A failed canary will have the promoted status set to `false`,
 the reason to `failed` and the last applied spec will be different to the last promoted one.
 
@@ -153,184 +147,26 @@ Wait for a successful rollout:
 kubectl wait canary/podinfo --for=condition=promoted
 ```
 
-### Istio routing
+CI example:
 
-Flagger creates an Istio Virtual Service and Destination Rules based on the Canary service spec. 
-The service configuration lets you expose an app inside or outside the mesh.
-You can also define traffic policies, HTTP match conditions, URI rewrite rules, CORS policies, timeout and retries.
+```bash
+# update the container image
+kubectl set image deployment/podinfo podinfod=stefanprodan/podinfo:3.0.1
 
-The following spec exposes the `frontend` workload inside the mesh on `frontend.test.svc.cluster.local:9898` 
-and outside the mesh on `frontend.example.com`. You'll have to specify an Istio ingress gateway for external hosts.
+# wait for Flagger to detect the change
+ok=false
+until ${ok}; do
+    kubectl get canary/podinfo | grep 'Progressing' && ok=true || ok=false
+    sleep 5
+done
 
-```yaml
-apiVersion: flagger.app/v1alpha3
-kind: Canary
-metadata:
-  name: frontend
-  namespace: test
-spec:
-  service:
-    # container port
-    port: 9898
-    # service port name (optional, will default to "http")
-    portName: http-frontend
-    # Istio gateways (optional)
-    gateways:
-    - public-gateway.istio-system.svc.cluster.local
-    - mesh
-    # Istio virtual service host names (optional)
-    hosts:
-    - frontend.example.com
-    # Istio traffic policy (optional)
-    trafficPolicy:
-      loadBalancer:
-        simple: LEAST_CONN
-    # HTTP match conditions (optional)
-    match:
-      - uri:
-          prefix: /
-    # HTTP rewrite (optional)
-    rewrite:
-      uri: /
-    # Envoy timeout and retry policy (optional)
-    headers:
-      request:
-        add:
-          x-envoy-upstream-rq-timeout-ms: "15000"
-          x-envoy-max-retries: "10"
-          x-envoy-retry-on: "gateway-error,connect-failure,refused-stream"
-    # cross-origin resource sharing policy (optional)
-    corsPolicy:
-      allowOrigin:
-        - example.com
-      allowMethods:
-        - GET
-      allowCredentials: false
-      allowHeaders:
-        - x-some-header
-      maxAge: 24h
+# wait for the canary analysis to finish
+kubectl wait canary/podinfo --for=condition=promoted --timeout=5m
+
+# check if the deployment was successful 
+kubectl get canary/podinfo | grep Succeeded
 ```
 
-For the above spec Flagger will generate the following virtual service:
-
-```yaml
-apiVersion: networking.istio.io/v1alpha3
-kind: VirtualService
-metadata:
-  name: frontend
-  namespace: test
-  ownerReferences:
-    - apiVersion: flagger.app/v1alpha3
-      blockOwnerDeletion: true
-      controller: true
-      kind: Canary
-      name: podinfo
-      uid: 3a4a40dd-3875-11e9-8e1d-42010a9c0fd1
-spec:
-  gateways:
-    - public-gateway.istio-system.svc.cluster.local
-    - mesh
-  hosts:
-    - frontend.example.com
-    - frontend
-  http:
-  - appendHeaders:
-      x-envoy-max-retries: "10"
-      x-envoy-retry-on: gateway-error,connect-failure,refused-stream
-      x-envoy-upstream-rq-timeout-ms: "15000"
-    corsPolicy:
-      allowHeaders:
-      - x-some-header
-      allowMethods:
-      - GET
-      allowOrigin:
-      - example.com
-      maxAge: 24h
-    match:
-    - uri:
-        prefix: /
-    rewrite:
-      uri: /
-    route:
-    - destination:
-        host: podinfo-primary
-      weight: 100
-    - destination:
-        host: podinfo-canary
-      weight: 0
-```
-
-For each destination in the virtual service a rule is generated:
-
-```yaml
-apiVersion: networking.istio.io/v1alpha3
-kind: DestinationRule
-metadata:
-  name: frontend-primary
-  namespace: test
-spec:
-  host: frontend-primary
-  trafficPolicy:
-    loadBalancer:
-      simple: LEAST_CONN
----
-apiVersion: networking.istio.io/v1alpha3
-kind: DestinationRule
-metadata:
-  name: frontend-canary
-  namespace: test
-spec:
-  host: frontend-canary
-  trafficPolicy:
-    loadBalancer:
-      simple: LEAST_CONN
-```
-
-Flagger keeps in sync the virtual service and destination rules with the canary service spec.
-Any direct modification to the virtual service spec will be overwritten.
-
-To expose a workload inside the mesh on `http://backend.test.svc.cluster.local:9898`,
-the service spec can contain only the container port:
-
-```yaml
-apiVersion: flagger.app/v1alpha3
-kind: Canary
-metadata:
-  name: backend
-  namespace: test
-spec:
-  service:
-    port: 9898
-```
-
-Based on the above spec, Flagger will create several ClusterIP services like:
-
-```yaml
-apiVersion: v1
-kind: Service
-metadata:
-  name: backend-primary
-  ownerReferences:
-  - apiVersion: flagger.app/v1alpha3
-    blockOwnerDeletion: true
-    controller: true
-    kind: Canary
-    name: backend
-    uid: 2ca1a9c7-2ef6-11e9-bd01-42010a9c0145
-spec:
-  type: ClusterIP
-  ports:
-  - name: http
-    port: 9898
-    protocol: TCP
-    targetPort: 9898
-  selector:
-    app: backend-primary
-```
-
-Flagger works for user facing apps exposed outside the cluster via an ingress gateway
-and for backend HTTP APIs that are accessible only from inside the mesh.
-
 ### Canary Stages
 
 ![Flagger Canary Stages](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-steps.png)
@@ -344,12 +180,13 @@ A canary deployment is triggered by changes in any of the following objects:
 Gated canary promotion stages:
 
 * scan for canary deployments
-* check Istio virtual service routes are mapped to primary and canary ClusterIP services
-* check primary and canary deployments status
+* check primary and canary deployment status
     * halt advancement if a rolling update is underway
     * halt advancement if pods are unhealthy
-* call pre-rollout webhooks are check results
-    * halt advancement if any hook returned a non HTTP 2xx result
+* call confirm-rollout webhooks and check results
+    * halt advancement if any hook returns a non HTTP 2xx result
+* call pre-rollout webhooks and check results
+    * halt advancement if any hook returns a non HTTP 2xx result
     * increment the failed checks counter
 * increase canary traffic weight percentage from 0% to 5% (step weight)
 * call rollout webhooks and check results
@@ -366,8 +203,11 @@ Gated canary promotion stages:
     * halt advancement if any webhook call fails
     * halt advancement while canary request success rate is under the threshold
     * halt advancement while canary request duration P99 is over the threshold
+    * halt advancement while any custom metric check fails
     * halt advancement if the primary or canary deployment becomes unhealthy 
     * halt advancement while canary deployment is being scaled up/down by HPA
+* call confirm-promotion webhooks and check results
+    * halt advancement if any hook returns a non HTTP 2xx result
 * promote canary to primary
     * copy ConfigMaps and Secrets from canary to primary
     * copy canary deployment spec template over primary
@@ -377,7 +217,7 @@ Gated canary promotion stages:
 * scale to zero the canary deployment
 * mark rollout as finished
 * call post-rollout webhooks
-* post the analysis result to Slack
+* post the analysis result to Slack or MS Teams
 * wait for the canary deployment to be updated and start over
 
 ### Canary Analysis
@@ -464,6 +304,29 @@ interval * threshold
 
 Make sure that the analysis threshold is lower than the number of iterations.
 
+### Blue/Green deployments
+
+For applications that are not deployed on a service mesh, Flagger can orchestrate blue/green style deployments 
+with Kubernetes L4 networking. When using Istio you have the option to mirror traffic between blue and green.
+
+You can use the blue/green deployment strategy by replacing `stepWeight/maxWeight` with `iterations` in the `canaryAnalysis` spec:
+
+```yaml
+  canaryAnalysis:
+    # schedule interval (default 60s)
+    interval: 1m
+    # total number of iterations
+    iterations: 10
+    # max number of failed iterations before rollback
+    threshold: 2
+    # Traffic shadowing (compatible with Istio only)
+    mirror: true
+```
+
+With the above configuration Flagger will run conformance and load tests on the canary pods for ten minutes. 
+If the metrics analysis succeeds, live traffic will be switched from the old version to the new one when the
+canary is promoted.
+
 ### HTTP Metrics
 
 The canary analysis is using the following Prometheus queries:
@@ -507,7 +370,7 @@ sum(
 )
 ```
 
-App Mesh query:
+Envoy query (App Mesh or Gloo):
 
 ```javascript
 sum(
@@ -515,7 +378,7 @@ sum(
         envoy_cluster_upstream_rq{
           kubernetes_namespace="$namespace",
           kubernetes_pod_name=~"$workload",
-          response_code!~"5.*"
+          envoy_response_code!~"5.*"
         }[$interval]
     )
 ) 
@@ -560,7 +423,7 @@ histogram_quantile(0.99,
 )
 ```
 
-App Mesh query:
+Envoy query (App Mesh or Gloo):
 
 ```javascript
 histogram_quantile(0.99, 
@@ -663,6 +526,9 @@ The canary advancement is paused if a pre-rollout hook fails and if the number o
 threshold the canary will be rollback.
 * Rollout hooks are executed during the analysis on each iteration before the metric checks. 
 If a rollout hook call fails the canary advancement is paused and eventfully rolled back.
+* Confirm-promotion hooks are executed before the promotion step.
+The canary promotion is paused until the hooks return HTTP 200.
+While the promotion is paused, Flagger will continue to run the metrics checks and rollout hooks.
 * Post-rollout hooks are executed after the canary has been promoted or rolled back. 
 If a post rollout hook fails the error is logged.
 
@@ -687,6 +553,9 @@ Spec:
         timeout: 15s
         metadata:
           cmd: "hey -z 1m -q 5 -c 2 http://podinfo-canary.test:9898/"
+      - name: "promotion gate"
+        type: confirm-promotion
+        url: http://flagger-loadtester.test/gate/approve
       - name: "notify"
         type: post-rollout
         url: http://telegram.bot:8080/
@@ -798,6 +667,18 @@ webhooks:
       cmd: "ghz -z 1m -q 10 -c 2 --insecure podinfo.test:9898"
 ```
 
+`ghz` uses reflection to identify which gRPC method to call. If you do not wish to enable reflection for your gRPC service you can implement a standardized health check from the [grpc-proto](https://github.com/grpc/grpc-proto) library. To use this [health check schema](https://github.com/grpc/grpc-proto/blob/master/grpc/health/v1/health.proto) without reflection you can pass a parameter to `ghz` like this
+
+```yaml
+webhooks:
+  - name: grpc-load-test-no-reflection
+    url: http://flagger-loadtester.test/
+    timeout: 5s
+    metadata:
+      type: cmd
+      cmd: "ghz --insecure --proto=/tmp/ghz/health.proto --call=grpc.health.v1.Health/Check podinfo.test:9898"
+```
+
 The load tester can run arbitrary commands as long as the binary is present in the container image.
 For example if you you want to replace `hey` with another CLI, you can create your own Docker image:
 
@@ -870,6 +751,20 @@ Now you can add pre-rollout webhooks to the canary analysis spec:
 When the canary analysis starts, Flagger will call the pre-rollout webhooks before routing traffic to the canary.
 If the helm test fails, Flagger will retry until the analysis threshold is reached and the canary is rolled back.
 
+If you are using Helm v3, you'll have to create a dedicated service account and add the release namespace to the test command:
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "smoke test"
+        type: pre-rollout
+        url: http://flagger-helmtester.kube-system/
+        timeout: 3m
+        metadata:
+          type: "helmv3"
+          cmd: "test run {{ .Release.Name }} --cleanup -n {{ .Release.Namespace }}"
+```
+
 As an alternative to Helm you can use the [Bash Automated Testing System](https://github.com/bats-core/bats-core) to run your tests. 
 
 ```yaml
@@ -888,8 +783,8 @@ Note that you should create a ConfigMap with your Bats tests and mount it inside
 
 ### Manual Gating
 
-For manual approval of a canary deployment you can use the `confirm-rollout` webhook. 
-The confirmation hooks are executed before the pre-rollout hooks. 
+For manual approval of a canary deployment you can use the `confirm-rollout` and `confirm-promotion` webhooks. 
+The confirmation rollout hooks are executed before the pre-rollout hooks. 
 Flagger will halt the canary traffic shifting and analysis until the confirm webhook returns HTTP status 200.
 
 Manual gating with Flagger's tester:
@@ -948,3 +843,16 @@ kubectl get canary/podinfo
 NAME      STATUS        WEIGHT
 podinfo   Waiting       0
 ```
+
+The `confirm-promotion` hook type can be used to manually approve the canary promotion.
+While the promotion is paused, Flagger will continue to run the metrics checks and load tests.
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "promotion gate"
+        type: confirm-promotion
+        url: http://flagger-loadtester.test/gate/halt
+```
+
+If you have notifications enabled, Flagger will post a message to Slack or MS Teams if a canary promotion is waiting for approval.

docs/gitbook/install/flagger-install-on-eks-appmesh.md

@@ -14,19 +14,10 @@ The App Mesh integration with EKS is made out of the following components:
 * Admission controller - injects the Envoy sidecar and assigns Kubernetes pods to App Mesh virtual nodes
 * Metrics server - Prometheus instance that collects and stores Envoy's metrics
 
-Prerequisites:
-
-* jq
-* homebrew
-* openssl
-* kubectl
-* AWS CLI (default region us-west-2)
-
 ### Create a Kubernetes cluster
 
 In order to create an EKS cluster you can use [eksctl](https://eksctl.io).
-Eksctl is an open source command-line utility made by Weaveworks in collaboration with Amazon, 
-it’s a Kubernetes-native tool written in Go.
+Eksctl is an open source command-line utility made by Weaveworks in collaboration with Amazon.
 
 On MacOS you can install eksctl with Homebrew:
 
@@ -40,6 +31,8 @@ Create an EKS cluster:
 ```bash
 eksctl create cluster --name=appmesh \
 --region=us-west-2 \
+--nodes 3 \
+--node-volume-size=120 \
 --appmesh-access
 ```
 
@@ -98,21 +91,39 @@ kubectl -n kube-system top pods
 
 ### Install the App Mesh components
 
-Run the App Mesh installer:
+Create the `appmesh-system` namespace:
 
-```bash
-curl -fsSL https://git.io/get-app-mesh-eks.sh | bash -
+```sh
+kubectl create ns appmesh-system
 ```
 
-The installer does the following:
+Apply the App Mesh CRDs:
 
-* creates the `appmesh-system` namespace
-* generates a certificate signed by Kubernetes CA
-* registers the App Mesh mutating webhook
-* deploys the App Mesh webhook in `appmesh-system` namespace
-* deploys the App Mesh CRDs
-* deploys the App Mesh controller in `appmesh-system` namespace
-* creates a mesh called `global`
+```sh
+kubectl apply -f https://raw.githubusercontent.com/aws/eks-charts/master/stable/appmesh-controller/crds/crds.yaml
+```
+
+Add the EKS repository to Helm:
+
+```sh
+helm repo add eks https://aws.github.io/eks-charts
+```
+
+Install the App Mesh CRD controller:
+
+```sh
+helm upgrade -i appmesh-controller eks/appmesh-controller \
+--wait --namespace appmesh-system
+```
+
+Install the App Mesh admission controller:
+
+```sh
+helm upgrade -i appmesh-inject eks/appmesh-inject \
+--wait --namespace appmesh-system \
+--set mesh.create=true \
+--set mesh.name=global
+```
 
 Verify that the global mesh is active:
 
@@ -125,6 +136,16 @@ Status:
     Type:                  MeshActive
 ```
 
+In order to collect the App Mesh metrics that Flagger needs to run the canary analysis, 
+you'll need to setup a Prometheus instance to scrape the Envoy sidecars.
+
+Install the App Mesh Prometheus:
+
+```sh
+helm upgrade -i appmesh-prometheus eks/appmesh-prometheus \
+--wait --namespace appmesh-system
+```
+
 ### Install Flagger and Grafana
 
 Add Flagger Helm repository:
@@ -139,27 +160,22 @@ Install Flagger's Canary CRD:
 kubectl apply -f https://raw.githubusercontent.com/weaveworks/flagger/master/artifacts/flagger/crd.yaml
 ```
 
-Deploy Flagger and Prometheus in the _**appmesh-system**_ namespace:
+Deploy Flagger in the _**appmesh-system**_ namespace:
 
 ```bash
 helm upgrade -i flagger flagger/flagger \
 --namespace=appmesh-system \
 --set crd.create=false \
 --set meshProvider=appmesh \
---set prometheus.install=true
+--set metricsServer=appmesh-prometheus:9090
 ```
 
-In order to collect the App Mesh metrics that Flagger needs to run the canary analysis, 
-you'll need to setup a Prometheus instance to scrape the Envoy sidecars.
-
-You can enable **Slack** notifications with:
+You can enable Slack or MS Teams notifications with:
 
 ```bash
 helm upgrade -i flagger flagger/flagger \
+--reuse-values \
 --namespace=appmesh-system \
---set crd.create=false \
---set meshProvider=appmesh \
---set metricsServer=http://prometheus.appmesh:9090 \
 --set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \
 --set slack.channel=general \
 --set slack.user=flagger
@@ -171,7 +187,7 @@ Deploy Grafana in the _**appmesh-system**_ namespace:
 ```bash
 helm upgrade -i flagger-grafana flagger/grafana \
 --namespace=appmesh-system \
---set url=http://flagger-prometheus.appmesh-system:9090
+--set url=http://appmesh-prometheus:9090
 ```
 
 You can access Grafana using port forwarding:

docs/gitbook/install/flagger-install-on-google-cloud.md

@@ -186,7 +186,7 @@ Install cert-manager's CRDs:
 ```bash
 CERT_REPO=https://raw.githubusercontent.com/jetstack/cert-manager
 
-kubectl apply -f ${CERT_REPO}/release-0.7/deploy/manifests/00-crds.yaml
+kubectl apply -f ${CERT_REPO}/release-0.10/deploy/manifests/00-crds.yaml
 ```
 
 Create the cert-manager namespace and disable resource validation:
@@ -204,7 +204,7 @@ helm repo add jetstack https://charts.jetstack.io && \
 helm repo update && \
 helm upgrade -i cert-manager \
 --namespace cert-manager \
---version v0.7.0 \
+--version v0.10.0 \
 jetstack/cert-manager
 ```
 
@@ -339,7 +339,7 @@ Find the GKE Istio version with:
 kubectl -n istio-system get deploy istio-pilot -oyaml | grep image:
 ```
 
-Install Prometheus in istio-system namespace (replace `1.0.6-gke.3` with your version):
+Install Prometheus in istio-system namespace:
 
 ```bash
 kubectl -n istio-system apply -f \

docs/gitbook/install/flagger-install-on-kubernetes.md

@@ -153,6 +153,14 @@ Note that you'll need kubectl 1.14 to run the above the command or you can downl
 kustomize build github.com/weaveworks/flagger//kustomize/istio | kubectl apply -f -
 ```
 
+Install Flagger for AWS App Mesh:
+
+```bash
+kubectl apply -k github.com/weaveworks/flagger//kustomize/appmesh
+```
+
+This deploys Flagger and Prometheus (configured to scrape the App Mesh Envoy sidecars) in the `appmesh-system` namespace.
+
 Install Flagger for Linkerd:
 
 ```bash

docs/gitbook/tutorials/canary-helm-gitops.md

@@ -51,7 +51,7 @@ helm upgrade -i frontend flagger/podinfo \
 --namespace test \
 --set nameOverride=frontend \
 --set backend=http://backend.test:9898/echo \
---set canary.loadtest.enabled=true \
+--set canary.enabled=true \
 --set canary.istioIngress.enabled=true \
 --set canary.istioIngress.gateway=public-gateway.istio-system.svc.cluster.local \
 --set canary.istioIngress.host=frontend.istio.example.com
@@ -91,7 +91,7 @@ Now let's install the `backend` release without exposing it outside the mesh:
 helm upgrade -i backend flagger/podinfo \
 --namespace test \
 --set nameOverride=backend \
---set canary.loadtest.enabled=true \
+--set canary.enabled=true \
 --set canary.istioIngress.enabled=false
 ```
 
@@ -138,7 +138,7 @@ helm upgrade -i frontend flagger/podinfo/ \
 --reuse-values \
 --set canary.loadtest.enabled=true \
 --set canary.helmtest.enabled=true \
---set image.tag=1.7.1
+--set image.tag=3.1.1
 ```
 
 Flagger detects that the deployment revision changed and starts the canary analysis:
@@ -177,6 +177,7 @@ Now trigger a canary deployment for the `backend` app, but this time you'll chan
 helm upgrade -i backend flagger/podinfo/ \
 --namespace test \
 --reuse-values \
+--set canary.loadtest.enabled=true \
 --set canary.helmtest.enabled=true \
 --set httpServer.timeout=25s
 ```
@@ -283,17 +284,17 @@ metadata:
   namespace: test
   annotations:
     flux.weave.works/automated: "true"
-    flux.weave.works/tag.chart-image: semver:~1.7
+    flux.weave.works/tag.chart-image: semver:~3.1
 spec:
   releaseName: frontend
   chart:
-    repository: https://stefanprodan.github.io/flagger/
-    name: podinfo
-    version: 2.3.0
+    git: https://github.com/weaveowrks/flagger
+    ref: master
+    path: charts/podinfo
   values:
     image:
-      repository: quay.io/stefanprodan/podinfo
-      tag: 1.7.0
+      repository: stefanprodan/podinfo
+      tag: 3.1.0
       backend: http://backend-podinfo:9898/echo
       canary:
         enabled: true
@@ -311,26 +312,26 @@ In the `chart` section I've defined the release source by specifying the Helm re
 In the `values` section I've overwritten the defaults set in values.yaml.
 
 With the `flux.weave.works` annotations I instruct Flux to automate this release. 
-When an image tag in the sem ver range of `1.7.0 - 1.7.99` is pushed to Quay, 
+When an image tag in the sem ver range of `3.1.0 - 3.1.99` is pushed to Docker Hub, 
 Flux will upgrade the Helm release and from there Flagger will pick up the change and start a canary deployment.
 
 Install [Weave Flux](https://github.com/weaveworks/flux) and its Helm Operator by specifying your Git repo URL:
 
 ```bash
-helm repo add weaveworks https://weaveworks.github.io/flux
+helm repo add fluxcd https://charts.fluxcd.io
 
 helm install --name flux \
 --set helmOperator.create=true \
 --set helmOperator.createCRD=true \
 --set git.url=git@github.com:<USERNAME>/<REPOSITORY> \
---namespace flux \
-weaveworks/flux
+--namespace fluxcd \
+fluxcd/flux
 ```
 
 At startup Flux generates a SSH key and logs the public key. Find the SSH public key with:
 
 ```bash
-kubectl -n flux logs deployment/flux | grep identity.pub | cut -d '"' -f2
+kubectl -n fluxcd logs deployment/flux | grep identity.pub | cut -d '"' -f2
 ```
 
 In order to sync your cluster state with Git you need to copy the public key and create a 
@@ -344,9 +345,9 @@ launch the `frontend` and `backend` apps.
 
 A CI/CD pipeline for the `frontend` release could look like this:
 
-* cut a release from the master branch of the podinfo code repo with the git tag `1.7.1`
-* CI builds the image and pushes the `podinfo:1.7.1` image to the container registry
-* Flux scans the registry and updates the Helm release `image.tag` to `1.7.1`
+* cut a release from the master branch of the podinfo code repo with the git tag `3.1.1`
+* CI builds the image and pushes the `podinfo:3.1.1` image to the container registry
+* Flux scans the registry and updates the Helm release `image.tag` to `3.1.1`
 * Flux commits and push the change to the cluster repo
 * Flux applies the updated Helm release on the cluster
 * Flux Helm Operator picks up the change and calls Tiller to upgrade the release
@@ -354,9 +355,9 @@ A CI/CD pipeline for the `frontend` release could look like this:
 * Flagger runs the helm test before routing traffic to the canary service
 * Flagger starts the load test and runs the canary analysis 
 * Based on the analysis result the canary deployment is promoted to production or rolled back
-* Flagger sends a Slack notification with the canary result
+* Flagger sends a Slack or MS Teams notification with the canary result
 
-If the canary fails, fix the bug, do another patch release eg `1.7.2` and the whole process will run again.
+If the canary fails, fix the bug, do another patch release eg `3.1.2` and the whole process will run again.
 
 A canary deployment can fail due to any of the following reasons:
 

docs/gitbook/usage/ab-testing.md

@@ -131,7 +131,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/abtest \
-podinfod=quay.io/stefanprodan/podinfo:1.7.1
+podinfod=stefanprodan/podinfo:3.1.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

docs/gitbook/usage/appmesh-progressive-delivery.md

@@ -39,8 +39,7 @@ helm upgrade -i flagger-loadtester flagger/loadtester \
 --namespace=test \
 --set meshName=global \
 --set "backends[0]=podinfo.test" \
---set "backends[1]=podinfo-canary.test" \
---set "backends[2]=podinfo-primary.test"
+--set "backends[1]=podinfo-canary.test"
 ```
 
 Create a canary custom resource:
@@ -68,6 +67,9 @@ spec:
   service:
     # container port
     port: 9898
+    # container port name (optional)
+    # can be http or grpc
+    portName: http
     # App Mesh reference
     meshName: global
     # App Mesh egress (optional) 
@@ -92,8 +94,20 @@ spec:
       # percentage (0-100)
       threshold: 99
       interval: 1m
-    # external checks (optional)
+    - name: request-duration
+      # maximum req duration P99
+      # milliseconds
+      threshold: 500
+      interval: 30s
+    # testing (optional)
     webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 30s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token"
       - name: load-test
         url: http://flagger-loadtester.test/
         timeout: 5s
@@ -127,14 +141,18 @@ virtualnode.appmesh.k8s.aws/podinfo
 virtualnode.appmesh.k8s.aws/podinfo-canary
 virtualnode.appmesh.k8s.aws/podinfo-primary
 virtualservice.appmesh.k8s.aws/podinfo.test
+virtualservice.appmesh.k8s.aws/podinfo-canary.test
 ```
 
+After the boostrap, the podinfo deployment will be scaled to zero and the traffic to `podinfo.test` will be routed
+to the primary pods. During the canary analysis, the `podinfo-canary.test` address can be used to target directly the canary pods.
+
 The App Mesh specific settings are:
 
 ```yaml
   service:
     port: 9898
-    meshName: global.appmesh-system
+    meshName: global
     backends:
       - backend1.test
       - backend2.test
@@ -178,7 +196,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.7.1
+podinfod=stefanprodan/podinfo:3.1.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -191,28 +209,34 @@ Status:
   Failed Checks:         0
   Phase:                 Succeeded
 Events:
-  Type     Reason  Age   From     Message
-  ----     ------  ----  ----     -------
-  Normal   Synced  3m    flagger  New revision detected podinfo.test
-  Normal   Synced  3m    flagger  Scaling up podinfo.test
-  Warning  Synced  3m    flagger  Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 5
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 10
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 15
-  Normal   Synced  2m    flagger  Advance podinfo.test canary weight 20
-  Normal   Synced  2m    flagger  Advance podinfo.test canary weight 25
-  Normal   Synced  1m    flagger  Advance podinfo.test canary weight 30
-  Normal   Synced  1m    flagger  Advance podinfo.test canary weight 35
-  Normal   Synced  55s   flagger  Advance podinfo.test canary weight 40
-  Normal   Synced  45s   flagger  Advance podinfo.test canary weight 45
-  Normal   Synced  35s   flagger  Advance podinfo.test canary weight 50
-  Normal   Synced  25s   flagger  Copying podinfo.test template spec to podinfo-primary.test
-  Warning  Synced  15s   flagger  Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available
-  Normal   Synced  5s    flagger  Promotion completed! Scaling down podinfo.test
+ New revision detected! Scaling up podinfo.test
+ Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available
+ Pre-rollout check acceptance-test passed
+ Advance podinfo.test canary weight 5
+ Advance podinfo.test canary weight 10
+ Advance podinfo.test canary weight 15
+ Advance podinfo.test canary weight 20
+ Advance podinfo.test canary weight 25
+ Advance podinfo.test canary weight 30
+ Advance podinfo.test canary weight 35
+ Advance podinfo.test canary weight 40
+ Advance podinfo.test canary weight 45
+ Advance podinfo.test canary weight 50
+ Copying podinfo.test template spec to podinfo-primary.test
+ Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available
+ Routing all traffic to primary
+ Promotion completed! Scaling down podinfo.test
 ```
 
+When the canary analysis starts, Flagger will call the pre-rollout webhooks before routing traffic to the canary.
+
 **Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis.
 
+A canary deployment is triggered by changes in any of the following objects:
+* Deployment PodSpec (container image, command, ports, env, resources, etc)
+* ConfigMaps mounted as volumes or mapped to environment variables
+* Secrets mounted as volumes or mapped to environment variables
+
 During the analysis the canary’s progress can be monitored with Grafana. The App Mesh dashboard URL is 
 http://localhost:3000/d/flagger-appmesh/appmesh-canary?refresh=10s&orgId=1&var-namespace=test&var-primary=podinfo-primary&var-canary=podinfo
 
@@ -224,9 +248,9 @@ You can monitor all canaries with:
 watch kubectl get canaries --all-namespaces
 
 NAMESPACE   NAME      STATUS        WEIGHT   LASTTRANSITIONTIME
-test        podinfo   Progressing   15       2019-03-16T14:05:07Z
-prod        frontend  Succeeded     0        2019-03-15T16:15:07Z
-prod        backend   Failed        0        2019-03-14T17:05:07Z
+test        podinfo   Progressing   15       2019-10-02T14:05:07Z
+prod        frontend  Succeeded     0        2019-10-02T16:15:07Z
+prod        backend   Failed        0        2019-10-02T17:05:07Z
 ```
 
 If you’ve enabled the Slack notifications, you should receive the following messages:
@@ -241,19 +265,25 @@ Trigger a canary deployment:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.2
+podinfod=stefanprodan/podinfo:3.1.2
 ```
 
 Exec into the load tester pod with:
 
 ```bash
-kubectl -n test exec -it flagger-loadtester-xx-xx sh
+kubectl -n test exec -it deploy/flagger-loadtester bash
 ```
 
 Generate HTTP 500 errors:
 
 ```bash
-hey -z 1m -c 5 -q 5 http://podinfo.test:9898/status/500
+hey -z 1m -c 5 -q 5 http://podinfo-canary.test:9898/status/500
+```
+
+Generate latency:
+
+```bash
+watch -n 1 curl http://podinfo-canary.test:9898/delay/1
 ```
 
 When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, 
@@ -264,22 +294,21 @@ kubectl -n test describe canary/podinfo
 
 Status:
   Canary Weight:         0
-  Failed Checks:         10
+  Failed Checks:         5
   Phase:                 Failed
 Events:
-  Type     Reason  Age   From     Message
-  ----     ------  ----  ----     -------
-  Normal   Synced  3m    flagger  Starting canary deployment for podinfo.test
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 5
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 10
-  Normal   Synced  3m    flagger  Advance podinfo.test canary weight 15
-  Normal   Synced  3m    flagger  Halt podinfo.test advancement success rate 69.17% < 99%
-  Normal   Synced  2m    flagger  Halt podinfo.test advancement success rate 61.39% < 99%
-  Normal   Synced  2m    flagger  Halt podinfo.test advancement success rate 55.06% < 99%
-  Normal   Synced  2m    flagger  Halt podinfo.test advancement success rate 47.00% < 99%
-  Normal   Synced  2m    flagger  (combined from similar events): Halt podinfo.test advancement success rate 38.08% < 99%
-  Warning  Synced  1m    flagger  Rolling back podinfo.test failed checks threshold reached 10
-  Warning  Synced  1m    flagger  Canary failed! Scaling down podinfo.test
+ Starting canary analysis for podinfo.test
+ Pre-rollout check acceptance-test passed
+ Advance podinfo.test canary weight 5
+ Advance podinfo.test canary weight 10
+ Advance podinfo.test canary weight 15
+ Halt podinfo.test advancement success rate 69.17% < 99%
+ Halt podinfo.test advancement success rate 61.39% < 99%
+ Halt podinfo.test advancement success rate 55.06% < 99%
+ Halt podinfo.test advancement request duration 1.20s > 0.5s
+ Halt podinfo.test advancement request duration 1.45s > 0.5s
+ Rolling back podinfo.test failed checks threshold reached 5
+ Canary failed! Scaling down podinfo.test
 ```
 
 If you’ve enabled the Slack notifications, you’ll receive a message if the progress deadline is exceeded, 

docs/gitbook/usage/blue-green.md

@@ -80,7 +80,6 @@ metadata:
   namespace: test
 spec:
   # service mesh provider can be: kubernetes, istio, appmesh, nginx, gloo
-  # use the kubernetes provider for Blue/Green style deployments
   provider: kubernetes
   # deployment reference
   targetRef:
@@ -96,7 +95,6 @@ spec:
     kind: HorizontalPodAutoscaler
     name: podinfo
   service:
-    # container port
     port: 9898
     portDiscovery: true
   canaryAnalysis:
@@ -172,7 +170,7 @@ Trigger a deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.7.1
+podinfod=stefanprodan/podinfo:3.1.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -297,7 +295,7 @@ Trigger a deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.7.3
+podinfod=stefanprodan/podinfo:3.1.3
 ```
 
 Generate 404s:

docs/gitbook/usage/gloo-progressive-delivery.md

@@ -103,6 +103,7 @@ metadata:
   name: podinfo
   namespace: test
 spec:
+  provider: gloo
   # deployment reference
   targetRef:
     apiVersion: apps/v1
@@ -117,8 +118,10 @@ spec:
   # to make progress before it is rollback (default 600s)
   progressDeadlineSeconds: 60
   service:
-    # container port
+    # ClusterIP port number
     port: 9898
+    # container port number or name (optional)
+    targetPort: 9898
   canaryAnalysis:
     # schedule interval (default 60s)
     interval: 10s
@@ -197,7 +200,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.7.1
+podinfod=stefanprodan/podinfo:3.1.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -251,7 +254,7 @@ Trigger another canary deployment:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.2
+podinfod=stefanprodan/podinfo:3.1.2
 ```
 
 Generate HTTP 500 errors:
@@ -334,7 +337,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.3
+podinfod=stefanprodan/podinfo:3.1.3
 ```
 
 Generate 404s:
@@ -362,5 +365,3 @@ Canary failed! Scaling down podinfo.test
 ```
 
 If you have Slack configured, Flagger will send a notification with the reason why the canary failed.
-
-

docs/gitbook/usage/linkerd-progressive-delivery.md

@@ -67,8 +67,10 @@ spec:
   # to make progress before it is rollback (default 600s)
   progressDeadlineSeconds: 60
   service:
-    # container port
+    # ClusterIP port number
     port: 9898
+    # container port number or name (optional)
+    targetPort: 9898
   canaryAnalysis:
     # schedule interval (default 60s)
     interval: 30s
@@ -150,7 +152,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.7.1
+podinfod=stefanprodan/podinfo:3.1.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -208,7 +210,7 @@ Trigger another canary deployment:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.7.2
+podinfod=stefanprodan/podinfo:3.1.2
 ```
 
 Exec into the load tester pod with:
@@ -297,7 +299,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.7.3
+podinfod=stefanprodan/podinfo:3.1.3
 ```
 
 Generate 404s:
@@ -444,7 +446,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.7.4
+podinfod=stefanprodan/podinfo:3.1.4
 ```
 
 Flagger detects that the deployment revision changed and starts the A/B testing:

docs/gitbook/usage/nginx-progressive-delivery.md

@@ -86,7 +86,7 @@ spec:
         paths:
           - backend:
               serviceName: podinfo
-              servicePort: 9898
+              servicePort: 80
 ```
 
 Save the above resource as podinfo-ingress.yaml and then apply it:
@@ -104,6 +104,7 @@ metadata:
   name: podinfo
   namespace: test
 spec:
+  provider: nginx
   # deployment reference
   targetRef:
     apiVersion: apps/v1
@@ -123,8 +124,10 @@ spec:
   # to make progress before it is rollback (default 600s)
   progressDeadlineSeconds: 60
   service:
-    # container port
-    port: 9898
+    # ClusterIP port number
+    port: 80
+    # container port number or name
+    targetPort: 9898
   canaryAnalysis:
     # schedule interval (default 60s)
     interval: 10s
@@ -143,8 +146,15 @@ spec:
       # percentage (0-100)
       threshold: 99
       interval: 1m
-    # load testing (optional)
+    # testing (optional)
     webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 30s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary/token | grep token"
       - name: load-test
         url: http://flagger-loadtester.test/
         timeout: 5s
@@ -189,7 +199,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.7.1
+podinfod=stefanprodan/podinfo:3.1.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -243,7 +253,7 @@ Trigger another canary deployment:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.2
+podinfod=stefanprodan/podinfo:3.1.2
 ```
 
 Generate HTTP 500 errors:
@@ -313,7 +323,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.3
+podinfod=stefanprodan/podinfo:3.1.3
 ```
 
 Generate high response latency:
@@ -387,7 +397,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.5.0
+podinfod=stefanprodan/podinfo:3.1.4
 ```
 
 Flagger detects that the deployment revision changed and starts the A/B testing:
@@ -418,4 +428,3 @@ Events:
   Warning  Synced  15s   flagger  Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available
   Normal   Synced  5s    flagger  Promotion completed! Scaling down podinfo.test
 ```
-

docs/gitbook/usage/progressive-delivery.md

@@ -2,8 +2,14 @@
 
 This guide shows you how to use Istio and Flagger to automate canary deployments.
 
+![Flagger Canary Stages](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-steps.png)
+
 ### Bootstrap
 
+Flagger takes a Kubernetes deployment and optionally a horizontal pod autoscaler (HPA),
+then creates a series of objects (Kubernetes deployments, ClusterIP services, Istio destination rules and virtual services).
+These objects expose the application inside the mesh and drive the canary analysis and promotion.
+
 Create a test namespace with Istio sidecar injection enabled:
 
 ```bash
@@ -49,14 +55,21 @@ spec:
     kind: HorizontalPodAutoscaler
     name: podinfo
   service:
-    # container port
+    # service port number
     port: 9898
+    # container port number or name (optional)
+    targetPort: 9898
     # Istio gateways (optional)
     gateways:
     - public-gateway.istio-system.svc.cluster.local
     # Istio virtual service host names (optional)
     hosts:
     - app.example.com
+    # Istio traffic policy (optional)
+    trafficPolicy:
+      tls:
+        # use ISTIO_MUTUAL when mTLS is enabled
+        mode: DISABLE
   canaryAnalysis:
     # schedule interval (default 60s)
     interval: 1m
@@ -79,8 +92,15 @@ spec:
       # milliseconds
       threshold: 500
       interval: 30s
-    # generate traffic during analysis
+    # testing (optional)
     webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 30s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary:9898/token | grep token"
       - name: load-test
         url: http://flagger-loadtester.test/
         timeout: 5s
@@ -94,6 +114,11 @@ Save the above resource as podinfo-canary.yaml and then apply it:
 kubectl apply -f ./podinfo-canary.yaml
 ```
 
+When the canary analysis starts, Flagger will call the pre-rollout webhooks before routing traffic to the canary.
+The canary analysis will run for five minutes while validating the HTTP metrics and rollout hooks every minute.
+
+![Flagger Canary Process](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-hpa.png)
+
 After a couple of seconds Flagger will create the canary objects:
 
 ```bash
@@ -119,7 +144,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.7.1
+podinfod=stefanprodan/podinfo:3.1.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -154,6 +179,11 @@ Events:
 
 **Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis.
 
+A canary deployment is triggered by changes in any of the following objects:
+* Deployment PodSpec (container image, command, ports, env, resources, etc)
+* ConfigMaps mounted as volumes or mapped to environment variables
+* Secrets mounted as volumes or mapped to environment variables
+
 You can monitor all canaries with:
 
 ```bash
@@ -169,14 +199,17 @@ prod        backend   Failed        0        2019-01-14T17:05:07Z
 
 During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses the rollout.
 
-Create a tester pod and exec into it:
+Trigger another canary deployment:
 
 ```bash
-kubectl -n test run tester \
---image=quay.io/stefanprodan/podinfo:1.2.1 \
--- ./podinfo --port=9898
+kubectl -n test set image deployment/podinfo \
+podinfod=stefanprodan/podinfo:3.1.2
+```
 
-kubectl -n test exec -it tester-xx-xx sh
+Exec into the load tester pod with:
+
+```bash
+kubectl -n test exec -it flagger-loadtester-xx-xx sh
 ```
 
 Generate HTTP 500 errors:
@@ -217,3 +250,81 @@ Events:
   Warning  Synced  1m    flagger  Canary failed! Scaling down podinfo.test
 ```
 
+### Traffic mirroring
+
+![Flagger Canary Traffic Shadowing](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-traffic-mirroring.png)
+
+For applications that perform read operations, Flagger can be configured to drive canary releases with traffic mirroring.
+Istio traffic mirroring will copy each incoming request, sending one request to the primary and one to the canary service.
+The response from the primary is sent back to the user and the response from the canary is discarded.
+Metrics are collected on both requests so that the deployment will only proceed if the canary metrics are within the threshold values.
+
+Note that mirroring should be used for requests that are **idempotent** or capable of being processed twice
+(once by the primary and once by the canary).
+
+You can enable mirroring by replacing `stepWeight/maxWeight` with `iterations` and
+by setting `canaryAnalysis.mirror` to `true`:
+
+```yaml
+apiVersion: flagger.app/v1alpha3
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  canaryAnalysis:
+    # schedule interval
+    interval: 1m
+    # max number of failed metric checks before rollback
+    threshold: 5
+    # total number of iterations
+    iterations: 10
+    # enable traffic shadowing 
+    mirror: true
+    metrics:
+    - name: request-success-rate
+      threshold: 99
+      interval: 1m
+    - name: request-duration
+      threshold: 500
+      interval: 1m
+    webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 30s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary:9898/token | grep token"
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          cmd: "hey -z 1m -q 10 -c 2 http://podinfo.test:9898/"
+```
+
+With the above configuration, Flagger will run a canary release with the following steps:
+* detect new revision (deployment spec, secrets or configmaps changes)
+* scale from zero the canary deployment
+* wait for the HPA to set the canary minimum replicas
+* check canary pods health
+* run the acceptance tests
+* abort the canary release if tests fail
+* start the load tests
+* mirror traffic from primary to canary
+* check request success rate and request duration every minute
+* abort the canary release if the metrics check failure threshold is reached
+* stop traffic mirroring after the number of iterations is reached
+* route live traffic to the canary pods
+* promote the canary (update the primary secrets, configmaps and deployment spec)
+* wait for the primary deployment rollout to finish
+* wait for the HPA to set the primary minimum replicas
+* check primary pods health
+* switch live traffic back to primary
+* scale to zero the canary
+* send notification with the canary analysis result
+
+The above procedure can be extended with [custom metrics](https://docs.flagger.app/how-it-works#custom-metrics) checks,
+[webhooks](https://docs.flagger.app/how-it-works#webhooks),
+[manual promotion](https://docs.flagger.app/how-it-works#manual-gating) approval and
+[Slack or MS Teams](https://docs.flagger.app/usage/alerting) notifications.

go.mod

@@ -1,6 +1,6 @@
 module github.com/weaveworks/flagger
 
-go 1.12
+go 1.13
 
 require (
 	cloud.google.com/go v0.37.4 // indirect

hack/update-codegen.sh

@@ -1,23 +1,38 @@
 #!/usr/bin/env bash
 
 set -o errexit
+set -o nounset
 set -o pipefail
 
-REPO_ROOT=$(git rev-parse --show-toplevel)
-CODEGEN_VERSION="@v0.0.0-20190620073620-d55040311883"
-CODEGEN_PKG=${CODEGEN_PKG:-$(echo `go env GOPATH`"/pkg/mod/k8s.io/code-generator${CODEGEN_VERSION}")}
+SCRIPT_ROOT=$(realpath $(dirname ${BASH_SOURCE})/..)
 
-if [[ "$CIRCLECI" ]]; then
-    ls $(echo `go env GOPATH`'/pkg/mod/k8s.io/');
-    mkdir -p ${REPO_ROOT}/bin;
-    cp -r ${CODEGEN_PKG} ${REPO_ROOT}/bin/;
-    CODEGEN_PKG=${REPO_ROOT}/bin/code-generator${CODEGEN_VERSION};
-    echo ">> $CODEGEN_PKG";
-fi
+# Grab code-generator version from go.sum.
+CODEGEN_VERSION=$(grep 'k8s.io/code-generator' go.sum | awk '{print $2}' | head -1)
+CODEGEN_PKG=$(echo `go env GOPATH`"/pkg/mod/k8s.io/code-generator@${CODEGEN_VERSION}")
 
+echo ">> Using ${CODEGEN_PKG}"
+
+# code-generator does work with go.mod but makes assumptions about
+# the project living in `$GOPATH/src`. To work around this and support
+# any location; create a temporary directory, use this as an output
+# base, and copy everything back once generated.
+TEMP_DIR=$(mktemp -d)
+cleanup() {
+    echo ">> Removing ${TEMP_DIR}"
+    rm -rf ${TEMP_DIR}
+}
+trap "cleanup" EXIT SIGINT
+
+echo ">> Temporary output directory ${TEMP_DIR}"
+
+# Ensure we can execute.
 chmod +x ${CODEGEN_PKG}/generate-groups.sh
 
 ${CODEGEN_PKG}/generate-groups.sh all \
     github.com/weaveworks/flagger/pkg/client github.com/weaveworks/flagger/pkg/apis \
     "appmesh:v1beta1 istio:v1alpha3 flagger:v1alpha3 smi:v1alpha1" \
-    --go-header-file ${REPO_ROOT}/hack/boilerplate.go.txt
+    --output-base "${TEMP_DIR}" \
+    --go-header-file ${SCRIPT_ROOT}/hack/boilerplate.go.txt
+
+# Copy everything back.
+cp -r "${TEMP_DIR}/github.com/weaveworks/flagger/." "${SCRIPT_ROOT}/"

kustomize/README.md

@@ -19,6 +19,14 @@ Note that you'll need kubectl 1.14 to run the above the command or you can downl
 kustomize build github.com/weaveworks/flagger//kustomize/istio | kubectl apply -f -
 ```
 
+Install Flagger for AWS App Mesh:
+
+```bash
+kubectl apply -k github.com/weaveworks/flagger//kustomize/appmesh
+```
+
+This deploys Flagger and Prometheus (configured to scrape the App Mesh Envoy sidecars) in the `appmesh-system` namespace.
+
 Install Flagger for Linkerd:
 
 ```bash

kustomize/appmesh/kustomization.yaml

@@ -0,0 +1,6 @@
+namespace: appmesh-system
+bases:
+  - ../base/flagger
+  - ../base/prometheus
+patchesStrategicMerge:
+  - patch.yaml

kustomize/appmesh/patch.yaml

@@ -0,0 +1,16 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flagger
+spec:
+  template:
+    spec:
+      containers:
+        - name: flagger
+          args:
+            - -log-level=info
+            - -mesh-provider=appmesh
+            - -metrics-server=http://flagger-prometheus:9090
+            - -slack-user=flagger
+            - -slack-channel=
+            - -slack-url=

kustomize/base/flagger/crd.yaml

@@ -33,6 +33,26 @@ spec:
     - name: Weight
       type: string
       JSONPath: .status.canaryWeight
+    - name: FailedChecks
+      type: string
+      JSONPath: .status.failedChecks
+      priority: 1
+    - name: Interval
+      type: string
+      JSONPath: .spec.canaryAnalysis.interval
+      priority: 1
+    - name: Mirror
+      type: boolean
+      JSONPath: .spec.canaryAnalysis.mirror
+      priority: 1
+    - name: StepWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.stepWeight
+      priority: 1
+    - name: MaxWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.maxWeight
+      priority: 1
     - name: LastTransitionTime
       type: string
       JSONPath: .status.lastTransitionTime
@@ -46,12 +66,15 @@ spec:
             - canaryAnalysis
           properties:
             provider:
+              description: Traffic managent provider
               type: string
             progressDeadlineSeconds:
+              description: Deployment progress deadline
               type: number
             targetRef:
+              description: Deployment selector
               type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -60,10 +83,11 @@ spec:
                 name:
                   type: string
             autoscalerRef:
+              description: HPA selector
               anyOf:
                 - type: string
                 - type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -72,10 +96,11 @@ spec:
                 name:
                   type: string
             ingressRef:
+              description: NGINX ingress selector
               anyOf:
                 - type: string
                 - type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -85,18 +110,68 @@ spec:
                   type: string
             service:
               type: object
-              required: ['port']
+              required: ["port"]
               properties:
                 port:
+                  description: Container port number
                   type: number
                 portName:
+                  description: Container port name
                   type: string
+                targetPort:
+                  description: Container target port name
+                  anyOf:
+                    - type: string
+                    - type: number
                 portDiscovery:
+                  description: Enable port dicovery
                   type: boolean
                 meshName:
+                  description: AppMesh mesh name
                   type: string
+                backends:
+                  description: AppMesh backend array
+                  anyOf:
+                    - type: string
+                    - type: array
                 timeout:
+                  description: Istio HTTP or gRPC request timeout
                   type: string
+                trafficPolicy:
+                  description: Istio traffic policy
+                  anyOf:
+                    - type: string
+                    - type: object
+                match:
+                  description: Istio URL match conditions
+                  anyOf:
+                    - type: string
+                    - type: array
+                rewrite:
+                  description: Istio URL rewrite
+                  anyOf:
+                    - type: string
+                    - type: object
+                headers:
+                  description: Istio headers operations
+                  anyOf:
+                    - type: string
+                    - type: object
+                corsPolicy:
+                  description: Istio CORS policy
+                  anyOf:
+                    - type: string
+                    - type: object
+                gateways:
+                  description: Istio gateways list
+                  anyOf:
+                    - type: string
+                    - type: array
+                hosts:
+                  description: Istio hosts list
+                  anyOf:
+                    - type: string
+                    - type: array
             skipAnalysis:
               type: boolean
             canaryAnalysis:
@@ -117,13 +192,21 @@ spec:
                 stepWeight:
                   description: Canary incremental traffic percentage step
                   type: number
+                mirror:
+                  description: Mirror traffic to canary before shifting
+                  type: boolean
+                match:
+                  description: A/B testing match conditions
+                  anyOf:
+                    - type: string
+                    - type: array
                 metrics:
                   description: Prometheus query list for this canary
                   type: array
                   properties:
                     items:
                       type: object
-                      required: ['name', 'threshold']
+                      required: ["name", "threshold"]
                       properties:
                         name:
                           description: Name of the Prometheus metric
@@ -144,7 +227,7 @@ spec:
                   properties:
                     items:
                       type: object
-                      required: ['name', 'url', 'timeout']
+                      required: ["name", "url"]
                       properties:
                         name:
                           description: Name of the webhook
@@ -154,8 +237,10 @@ spec:
                           type: string
                           enum:
                             - ""
+                            - confirm-rollout
                             - pre-rollout
                             - rollout
+                            - confirm-promotion
                             - post-rollout
                         url:
                           description: URL address of this webhook
@@ -165,6 +250,11 @@ spec:
                           description: Request timeout for this webhook
                           type: string
                           pattern: "^[0-9]+(m|s)"
+                        metadata:
+                          description: Metadata (key-value pairs) for this webhook
+                          anyOf:
+                            - type: string
+                            - type: object
         status:
           properties:
             phase:
@@ -176,6 +266,7 @@ spec:
                 - Initialized
                 - Waiting
                 - Progressing
+                - Promoting
                 - Finalising
                 - Succeeded
                 - Failed
@@ -201,7 +292,7 @@ spec:
               properties:
                 items:
                   type: object
-                  required: ['type', 'status', 'reason']
+                  required: ["type", "status", "reason"]
                   properties:
                     lastTransitionTime:
                       description: LastTransitionTime of this condition

kustomize/base/flagger/deployment.yaml

@@ -15,6 +15,7 @@ spec:
         app: flagger
       annotations:
         prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
     spec:
       serviceAccountName: flagger
       containers:

kustomize/base/flagger/kustomization.yaml

@@ -8,4 +8,4 @@ resources:
   - deployment.yaml
 images:
   - name: weaveworks/flagger
-    newTag: 0.18.2
+    newTag: 0.19.0

kustomize/base/prometheus/deployment.yaml

@@ -19,7 +19,7 @@ spec:
       serviceAccountName: flagger-prometheus
       containers:
           - name: prometheus
-            image: prom/prometheus:v2.10.0
+            image: prom/prometheus:v2.12.0
             imagePullPolicy: IfNotPresent
             args:
               - '--storage.tsdb.retention=2h'

kustomize/podinfo/deployment.yaml

@@ -10,7 +10,7 @@ spec:
   progressDeadlineSeconds: 60
   strategy:
     rollingUpdate:
-      maxUnavailable: 0
+      maxUnavailable: 1
     type: RollingUpdate
   selector:
     matchLabels:
@@ -19,26 +19,33 @@ spec:
     metadata:
       annotations:
         prometheus.io/scrape: "true"
+        prometheus.io/port: "9797"
       labels:
         app: podinfo
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.7.0
+        image: stefanprodan/podinfo:3.1.0
         imagePullPolicy: IfNotPresent
         ports:
-        - containerPort: 9898
-          name: http
+        - name: http
+          containerPort: 9898
+          protocol: TCP
+        - name: http-metrics
+          containerPort: 9797
+          protocol: TCP
+        - name: grpc
+          containerPort: 9999
           protocol: TCP
         command:
         - ./podinfo
         - --port=9898
+        - --port-metrics=9797
+        - --grpc-port=9999
+        - --grpc-service-name=podinfo
         - --level=info
         - --random-delay=false
         - --random-error=false
-        env:
-        - name: PODINFO_UI_COLOR
-          value: blue
         livenessProbe:
           exec:
             command:

kustomize/tester/deployment.yaml

@@ -14,10 +14,11 @@ spec:
         app: flagger-loadtester
       annotations:
         prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
     spec:
       containers:
         - name: loadtester
-          image: weaveworks/flagger-loadtester:0.6.1
+          image: weaveworks/flagger-loadtester:0.9.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

kustomize/tester/kustomization.yaml

@@ -2,6 +2,4 @@ namespace: test
 resources:
   - service.yaml
   - deployment.yaml
-images:
-  - name: weaveworks/flagger-loadtester
-    newTag: 0.6.1
+

pkg/apis/appmesh/v1beta1/types.go

@@ -102,13 +102,18 @@ type VirtualServiceSpec struct {
 	Routes []Route `json:"routes,omitempty"`
 }
 
+// VirtualRouter is the spec for a VirtualRouter resource
 type VirtualRouter struct {
-	Name string `json:"name"`
+	Name      string     `json:"name"`
+	Listeners []Listener `json:"listeners,omitempty"`
 }
 
 type Route struct {
-	Name string    `json:"name"`
-	Http HttpRoute `json:"http"`
+	Name string `json:"name"`
+	// +optional
+	Http *HttpRoute `json:"http,omitempty"`
+	// +optional
+	Tcp *TcpRoute `json:"tcp,omitempty"`
 }
 
 type HttpRoute struct {
@@ -124,6 +129,14 @@ type HttpRouteAction struct {
 	WeightedTargets []WeightedTarget `json:"weightedTargets"`
 }
 
+type TcpRoute struct {
+	Action TcpRouteAction `json:"action"`
+}
+
+type TcpRouteAction struct {
+	WeightedTargets []WeightedTarget `json:"weightedTargets"`
+}
+
 type WeightedTarget struct {
 	VirtualNodeName string `json:"virtualNodeName"`
 	Weight          int64  `json:"weight"`
@@ -203,6 +216,8 @@ type VirtualNodeSpec struct {
 	ServiceDiscovery *ServiceDiscovery `json:"serviceDiscovery,omitempty"`
 	// +optional
 	Backends []Backend `json:"backends,omitempty"`
+	// +optional
+	Logging *Logging `json:"logging,omitempty"`
 }
 
 type Listener struct {
@@ -237,6 +252,21 @@ type VirtualServiceBackend struct {
 	VirtualServiceName string `json:"virtualServiceName"`
 }
 
+// Logging refers to https://docs.aws.amazon.com/app-mesh/latest/APIReference/API_Logging.html
+type Logging struct {
+	AccessLog *AccessLog `json:"accessLog"`
+}
+
+// AccessLog refers to https://docs.aws.amazon.com/app-mesh/latest/APIReference/API_AccessLog.html
+type AccessLog struct {
+	File *FileAccessLog `json:"file"`
+}
+
+// FileAccessLog refers to https://docs.aws.amazon.com/app-mesh/latest/APIReference/API_FileAccessLog.html
+type FileAccessLog struct {
+	Path string `json:"path"`
+}
+
 // VirtualNodeStatus is the status for a VirtualNode resource
 type VirtualNodeStatus struct {
 	MeshArn *string `json:"meshArn,omitempty"`
@@ -272,7 +302,7 @@ type VirtualNodeCondition struct {
 	Reason *string `json:"reason,omitempty"`
 	// A human readable message indicating details about the transition.
 	// +optional
-	Message *string `json:"reason,omitempty"`
+	Message *string `json:"message,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

pkg/apis/appmesh/v1beta1/zz_generated.deepcopy.go

@@ -24,6 +24,27 @@ import (
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AccessLog) DeepCopyInto(out *AccessLog) {
+	*out = *in
+	if in.File != nil {
+		in, out := &in.File, &out.File
+		*out = new(FileAccessLog)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AccessLog.
+func (in *AccessLog) DeepCopy() *AccessLog {
+	if in == nil {
+		return nil
+	}
+	out := new(AccessLog)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Backend) DeepCopyInto(out *Backend) {
 	*out = *in
@@ -73,6 +94,22 @@ func (in *DnsServiceDiscovery) DeepCopy() *DnsServiceDiscovery {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FileAccessLog) DeepCopyInto(out *FileAccessLog) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileAccessLog.
+func (in *FileAccessLog) DeepCopy() *FileAccessLog {
+	if in == nil {
+		return nil
+	}
+	out := new(FileAccessLog)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *HttpRoute) DeepCopyInto(out *HttpRoute) {
 	*out = *in
@@ -145,6 +182,27 @@ func (in *Listener) DeepCopy() *Listener {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Logging) DeepCopyInto(out *Logging) {
+	*out = *in
+	if in.AccessLog != nil {
+		in, out := &in.AccessLog, &out.AccessLog
+		*out = new(AccessLog)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Logging.
+func (in *Logging) DeepCopy() *Logging {
+	if in == nil {
+		return nil
+	}
+	out := new(Logging)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Mesh) DeepCopyInto(out *Mesh) {
 	*out = *in
@@ -304,7 +362,16 @@ func (in *PortMapping) DeepCopy() *PortMapping {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Route) DeepCopyInto(out *Route) {
 	*out = *in
-	in.Http.DeepCopyInto(&out.Http)
+	if in.Http != nil {
+		in, out := &in.Http, &out.Http
+		*out = new(HttpRoute)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Tcp != nil {
+		in, out := &in.Tcp, &out.Tcp
+		*out = new(TcpRoute)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -344,6 +411,44 @@ func (in *ServiceDiscovery) DeepCopy() *ServiceDiscovery {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TcpRoute) DeepCopyInto(out *TcpRoute) {
+	*out = *in
+	in.Action.DeepCopyInto(&out.Action)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TcpRoute.
+func (in *TcpRoute) DeepCopy() *TcpRoute {
+	if in == nil {
+		return nil
+	}
+	out := new(TcpRoute)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TcpRouteAction) DeepCopyInto(out *TcpRouteAction) {
+	*out = *in
+	if in.WeightedTargets != nil {
+		in, out := &in.WeightedTargets, &out.WeightedTargets
+		*out = make([]WeightedTarget, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TcpRouteAction.
+func (in *TcpRouteAction) DeepCopy() *TcpRouteAction {
+	if in == nil {
+		return nil
+	}
+	out := new(TcpRouteAction)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *VirtualNode) DeepCopyInto(out *VirtualNode) {
 	*out = *in
@@ -453,6 +558,11 @@ func (in *VirtualNodeSpec) DeepCopyInto(out *VirtualNodeSpec) {
 		*out = make([]Backend, len(*in))
 		copy(*out, *in)
 	}
+	if in.Logging != nil {
+		in, out := &in.Logging, &out.Logging
+		*out = new(Logging)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -514,6 +624,11 @@ func (in *VirtualNodeStatus) DeepCopy() *VirtualNodeStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *VirtualRouter) DeepCopyInto(out *VirtualRouter) {
 	*out = *in
+	if in.Listeners != nil {
+		in, out := &in.Listeners, &out.Listeners
+		*out = make([]Listener, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -640,7 +755,7 @@ func (in *VirtualServiceSpec) DeepCopyInto(out *VirtualServiceSpec) {
 	if in.VirtualRouter != nil {
 		in, out := &in.VirtualRouter, &out.VirtualRouter
 		*out = new(VirtualRouter)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 	if in.Routes != nil {
 		in, out := &in.Routes, &out.Routes

pkg/apis/flagger/v1alpha3/status.go

@@ -47,7 +47,9 @@ const (
 	CanaryPhaseWaiting CanaryPhase = "Waiting"
 	// CanaryPhaseProgressing means the canary analysis is underway
 	CanaryPhaseProgressing CanaryPhase = "Progressing"
-	// CanaryPhaseProgressing means the canary analysis is finished and traffic has been routed back to primary
+	// CanaryPhasePromoting means the canary analysis is finished and the primary spec has been updated
+	CanaryPhasePromoting CanaryPhase = "Promoting"
+	// CanaryPhaseProgressing means the canary promotion is finished and traffic has been routed back to primary
 	CanaryPhaseFinalising CanaryPhase = "Finalising"
 	// CanaryPhaseSucceeded means the canary analysis has been successful
 	// and the canary deployment has been promoted

pkg/apis/flagger/v1alpha3/types.go

@@ -19,9 +19,11 @@ package v1alpha3
 import (
 	"time"
 
-	istiov1alpha3 "github.com/weaveworks/flagger/pkg/apis/istio/v1alpha3"
 	hpav1 "k8s.io/api/autoscaling/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+
+	istiov1alpha3 "github.com/weaveworks/flagger/pkg/apis/istio/v1alpha3"
 )
 
 const (
@@ -88,10 +90,11 @@ type CanaryList struct {
 // CanaryService is used to create ClusterIP services
 // and Istio Virtual Service
 type CanaryService struct {
-	Port          int32  `json:"port"`
-	PortName      string `json:"portName,omitempty"`
-	PortDiscovery bool   `json:"portDiscovery"`
-	Timeout       string `json:"timeout,omitempty"`
+	Port          int32              `json:"port"`
+	PortName      string             `json:"portName,omitempty"`
+	TargetPort    intstr.IntOrString `json:"targetPort,omitempty"`
+	PortDiscovery bool               `json:"portDiscovery"`
+	Timeout       string             `json:"timeout,omitempty"`
 	// Istio
 	Gateways      []string                         `json:"gateways,omitempty"`
 	Hosts         []string                         `json:"hosts,omitempty"`
@@ -111,6 +114,7 @@ type CanaryAnalysis struct {
 	Interval   string                           `json:"interval"`
 	Threshold  int                              `json:"threshold"`
 	MaxWeight  int                              `json:"maxWeight"`
+	Mirror     bool                             `json:"mirror,omitempty"`
 	StepWeight int                              `json:"stepWeight"`
 	Metrics    []CanaryMetric                   `json:"metrics"`
 	Webhooks   []CanaryWebhook                  `json:"webhooks,omitempty"`
@@ -139,6 +143,8 @@ const (
 	PostRolloutHook HookType = "post-rollout"
 	// ConfirmRolloutHook halt canary analysis until webhook returns HTTP 200
 	ConfirmRolloutHook HookType = "confirm-rollout"
+	// ConfirmPromotionHook halt canary promotion until webhook returns HTTP 200
+	ConfirmPromotionHook HookType = "confirm-promotion"
 )
 
 // CanaryWebhook holds the reference to external checks used for canary analysis

pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go

@@ -159,6 +159,7 @@ func (in *CanaryMetric) DeepCopy() *CanaryMetric {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *CanaryService) DeepCopyInto(out *CanaryService) {
 	*out = *in
+	out.TargetPort = in.TargetPort
 	if in.Gateways != nil {
 		in, out := &in.Gateways, &out.Gateways
 		*out = make([]string, len(*in))

pkg/canary/deployer.go

@@ -7,8 +7,6 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/mitchellh/hashstructure"
-	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
-	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
 	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
 	hpav1 "k8s.io/api/autoscaling/v2beta1"
@@ -16,7 +14,11 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/kubernetes"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
+	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
 )
 
 // Deployer is managing the operations for Kubernetes deployment kind
@@ -203,7 +205,7 @@ func (c *Deployer) createPrimaryDeployment(cd *flaggerv1.Canary) (string, *map[s
 
 	var ports *map[string]int32
 	if cd.Spec.Service.PortDiscovery {
-		p, err := c.getPorts(canaryDep, cd.Spec.Service.Port)
+		p, err := c.getPorts(cd, canaryDep)
 		if err != nil {
 			return "", nil, fmt.Errorf("port discovery failed with error: %v", err)
 		}
@@ -392,7 +394,7 @@ var sidecars = map[string]bool{
 }
 
 // getPorts returns a list of all container ports
-func (c *Deployer) getPorts(deployment *appsv1.Deployment, canaryPort int32) (map[string]int32, error) {
+func (c *Deployer) getPorts(cd *flaggerv1.Canary, deployment *appsv1.Deployment) (map[string]int32, error) {
 	ports := make(map[string]int32)
 
 	for _, container := range deployment.Spec.Template.Spec.Containers {
@@ -401,9 +403,22 @@ func (c *Deployer) getPorts(deployment *appsv1.Deployment, canaryPort int32) (ma
 			continue
 		}
 		for i, p := range container.Ports {
-			// exclude canary.service.port
-			if p.ContainerPort == canaryPort {
-				continue
+			// exclude canary.service.port or canary.service.targetPort
+			if cd.Spec.Service.TargetPort.String() == "0" {
+				if p.ContainerPort == cd.Spec.Service.Port {
+					continue
+				}
+			} else {
+				if cd.Spec.Service.TargetPort.Type == intstr.Int {
+					if p.ContainerPort == cd.Spec.Service.TargetPort.IntVal {
+						continue
+					}
+				}
+				if cd.Spec.Service.TargetPort.Type == intstr.String {
+					if p.Name == cd.Spec.Service.TargetPort.StrVal {
+						continue
+					}
+				}
 			}
 			name := fmt.Sprintf("tcp-%s-%v", container.Name, i)
 			if p.Name != "" {

pkg/canary/deployer_test.go

@@ -3,8 +3,9 @@ package canary
 import (
 	"testing"
 
-	"github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 )
 
 func TestCanaryDeployer_Sync(t *testing.T) {
@@ -229,7 +230,7 @@ func TestCanaryDeployer_SetState(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	err = mocks.deployer.SetStatusPhase(mocks.canary, v1alpha3.CanaryPhaseProgressing)
+	err = mocks.deployer.SetStatusPhase(mocks.canary, flaggerv1.CanaryPhaseProgressing)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -239,8 +240,8 @@ func TestCanaryDeployer_SetState(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	if res.Status.Phase != v1alpha3.CanaryPhaseProgressing {
-		t.Errorf("Got %v wanted %v", res.Status.Phase, v1alpha3.CanaryPhaseProgressing)
+	if res.Status.Phase != flaggerv1.CanaryPhaseProgressing {
+		t.Errorf("Got %v wanted %v", res.Status.Phase, flaggerv1.CanaryPhaseProgressing)
 	}
 }
 
@@ -251,8 +252,8 @@ func TestCanaryDeployer_SyncStatus(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	status := v1alpha3.CanaryStatus{
-		Phase:        v1alpha3.CanaryPhaseProgressing,
+	status := flaggerv1.CanaryStatus{
+		Phase:        flaggerv1.CanaryPhaseProgressing,
 		FailedChecks: 2,
 	}
 	err = mocks.deployer.SyncStatus(mocks.canary, status)

pkg/canary/mock.go

@@ -1,9 +1,6 @@
 package canary
 
 import (
-	"github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
-	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
-	fakeFlagger "github.com/weaveworks/flagger/pkg/client/clientset/versioned/fake"
 	"github.com/weaveworks/flagger/pkg/logger"
 	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
@@ -13,10 +10,14 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/fake"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
+	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
+	fakeFlagger "github.com/weaveworks/flagger/pkg/client/clientset/versioned/fake"
 )
 
 type Mocks struct {
-	canary        *v1alpha3.Canary
+	canary        *flaggerv1.Canary
 	kubeClient    kubernetes.Interface
 	flaggerClient clientset.Interface
 	deployer      Deployer
@@ -172,14 +173,14 @@ func NewTestSecretVol() *corev1.Secret {
 	}
 }
 
-func newTestCanary() *v1alpha3.Canary {
-	cd := &v1alpha3.Canary{
-		TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()},
+func newTestCanary() *flaggerv1.Canary {
+	cd := &flaggerv1.Canary{
+		TypeMeta: metav1.TypeMeta{APIVersion: flaggerv1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: "default",
 			Name:      "podinfo",
 		},
-		Spec: v1alpha3.CanarySpec{
+		Spec: flaggerv1.CanarySpec{
 			TargetRef: hpav1.CrossVersionObjectReference{
 				Name:       "podinfo",
 				APIVersion: "apps/v1",
@@ -189,13 +190,13 @@ func newTestCanary() *v1alpha3.Canary {
 				Name:       "podinfo",
 				APIVersion: "autoscaling/v2beta1",
 				Kind:       "HorizontalPodAutoscaler",
-			}, Service: v1alpha3.CanaryService{
+			}, Service: flaggerv1.CanaryService{
 				Port: 9898,
-			}, CanaryAnalysis: v1alpha3.CanaryAnalysis{
+			}, CanaryAnalysis: flaggerv1.CanaryAnalysis{
 				Threshold:  10,
 				StepWeight: 10,
 				MaxWeight:  50,
-				Metrics: []v1alpha3.CanaryMetric{
+				Metrics: []flaggerv1.CanaryMetric{
 					{
 						Name:      "istio_requests_total",
 						Threshold: 99,

pkg/canary/ready.go

@@ -4,10 +4,11 @@ import (
 	"fmt"
 	"time"
 
-	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 )
 
 // IsPrimaryReady checks the primary deployment status and returns an error if

pkg/canary/status.go

@@ -2,14 +2,15 @@ package canary
 
 import (
 	"fmt"
-	"k8s.io/client-go/util/retry"
 
 	"github.com/mitchellh/hashstructure"
 	ex "github.com/pkg/errors"
-	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/retry"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 )
 
 // SyncStatus encodes the canary pod spec and updates the canary status
@@ -211,6 +212,9 @@ func (c *Deployer) MakeStatusConditions(canaryStatus flaggerv1.CanaryStatus,
 	case flaggerv1.CanaryPhaseProgressing:
 		status = corev1.ConditionUnknown
 		message = "New revision detected, starting canary analysis."
+	case flaggerv1.CanaryPhasePromoting:
+		status = corev1.ConditionUnknown
+		message = "Canary analysis completed, starting primary rolling update."
 	case flaggerv1.CanaryPhaseFinalising:
 		status = corev1.ConditionUnknown
 		message = "Canary analysis completed, routing all traffic to primary."

pkg/canary/tracker.go

@@ -4,14 +4,16 @@ import (
 	"crypto/sha256"
 	"encoding/json"
 	"fmt"
-	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
-	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
+
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
+	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
 )
 
 // ConfigTracker is managing the operations for Kubernetes ConfigMaps and Secrets

pkg/controller/controller.go

@@ -5,16 +5,6 @@ import (
 	"sync"
 	"time"
 
-	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
-	"github.com/weaveworks/flagger/pkg/canary"
-	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
-	flaggerscheme "github.com/weaveworks/flagger/pkg/client/clientset/versioned/scheme"
-	flaggerinformers "github.com/weaveworks/flagger/pkg/client/informers/externalversions/flagger/v1alpha3"
-	flaggerlisters "github.com/weaveworks/flagger/pkg/client/listers/flagger/v1alpha3"
-	"github.com/weaveworks/flagger/pkg/metrics"
-	"github.com/weaveworks/flagger/pkg/notifier"
-	"github.com/weaveworks/flagger/pkg/router"
-
 	"github.com/google/go-cmp/cmp"
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
@@ -28,6 +18,16 @@ import (
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
+	"github.com/weaveworks/flagger/pkg/canary"
+	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
+	flaggerscheme "github.com/weaveworks/flagger/pkg/client/clientset/versioned/scheme"
+	flaggerinformers "github.com/weaveworks/flagger/pkg/client/informers/externalversions/flagger/v1alpha3"
+	flaggerlisters "github.com/weaveworks/flagger/pkg/client/listers/flagger/v1alpha3"
+	"github.com/weaveworks/flagger/pkg/metrics"
+	"github.com/weaveworks/flagger/pkg/notifier"
+	"github.com/weaveworks/flagger/pkg/router"
 )
 
 const controllerAgentName = "flagger"

pkg/controller/controller_test.go

@@ -4,16 +4,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
-	istiov1alpha1 "github.com/weaveworks/flagger/pkg/apis/istio/common/v1alpha1"
-	istiov1alpha3 "github.com/weaveworks/flagger/pkg/apis/istio/v1alpha3"
-	"github.com/weaveworks/flagger/pkg/canary"
-	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
-	fakeFlagger "github.com/weaveworks/flagger/pkg/client/clientset/versioned/fake"
-	informers "github.com/weaveworks/flagger/pkg/client/informers/externalversions"
-	"github.com/weaveworks/flagger/pkg/logger"
-	"github.com/weaveworks/flagger/pkg/metrics"
-	"github.com/weaveworks/flagger/pkg/router"
 	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
 	hpav1 "k8s.io/api/autoscaling/v1"
@@ -24,6 +14,17 @@ import (
 	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
+	istiov1alpha1 "github.com/weaveworks/flagger/pkg/apis/istio/common/v1alpha1"
+	istiov1alpha3 "github.com/weaveworks/flagger/pkg/apis/istio/v1alpha3"
+	"github.com/weaveworks/flagger/pkg/canary"
+	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
+	fakeFlagger "github.com/weaveworks/flagger/pkg/client/clientset/versioned/fake"
+	informers "github.com/weaveworks/flagger/pkg/client/informers/externalversions"
+	"github.com/weaveworks/flagger/pkg/logger"
+	"github.com/weaveworks/flagger/pkg/metrics"
+	"github.com/weaveworks/flagger/pkg/router"
 )
 
 var (
@@ -32,7 +33,7 @@ var (
 )
 
 type Mocks struct {
-	canary        *v1alpha3.Canary
+	canary        *flaggerv1.Canary
 	kubeClient    kubernetes.Interface
 	meshClient    clientset.Interface
 	flaggerClient clientset.Interface
@@ -42,11 +43,9 @@ type Mocks struct {
 	router        router.Interface
 }
 
-func SetupMocks(abtest bool) Mocks {
-	// init canary
-	c := newTestCanary()
-	if abtest {
-		c = newTestCanaryAB()
+func SetupMocks(c *flaggerv1.Canary) Mocks {
+	if c == nil {
+		c = newTestCanary()
 	}
 	flaggerClient := fakeFlagger.NewSimpleClientset(c)
 
@@ -82,7 +81,7 @@ func SetupMocks(abtest bool) Mocks {
 	flaggerInformer := flaggerInformerFactory.Flagger().V1alpha3().Canaries()
 
 	// init router
-	rf := router.NewFactory(nil, kubeClient, flaggerClient, logger, flaggerClient)
+	rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", logger, flaggerClient)
 
 	// init observer
 	observerFactory, _ := metrics.NewFactory("fake", "istio", 5*time.Second)
@@ -228,14 +227,14 @@ func NewTestSecretVol() *corev1.Secret {
 	}
 }
 
-func newTestCanary() *v1alpha3.Canary {
-	cd := &v1alpha3.Canary{
-		TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()},
+func newTestCanary() *flaggerv1.Canary {
+	cd := &flaggerv1.Canary{
+		TypeMeta: metav1.TypeMeta{APIVersion: flaggerv1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: "default",
 			Name:      "podinfo",
 		},
-		Spec: v1alpha3.CanarySpec{
+		Spec: flaggerv1.CanarySpec{
 			TargetRef: hpav1.CrossVersionObjectReference{
 				Name:       "podinfo",
 				APIVersion: "apps/v1",
@@ -245,13 +244,13 @@ func newTestCanary() *v1alpha3.Canary {
 				Name:       "podinfo",
 				APIVersion: "autoscaling/v2beta1",
 				Kind:       "HorizontalPodAutoscaler",
-			}, Service: v1alpha3.CanaryService{
+			}, Service: flaggerv1.CanaryService{
 				Port: 9898,
-			}, CanaryAnalysis: v1alpha3.CanaryAnalysis{
+			}, CanaryAnalysis: flaggerv1.CanaryAnalysis{
 				Threshold:  10,
 				StepWeight: 10,
 				MaxWeight:  50,
-				Metrics: []v1alpha3.CanaryMetric{
+				Metrics: []flaggerv1.CanaryMetric{
 					{
 						Name:      "istio_requests_total",
 						Threshold: 99,
@@ -269,14 +268,20 @@ func newTestCanary() *v1alpha3.Canary {
 	return cd
 }
 
-func newTestCanaryAB() *v1alpha3.Canary {
-	cd := &v1alpha3.Canary{
-		TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()},
+func newTestCanaryMirror() *flaggerv1.Canary {
+	cd := newTestCanary()
+	cd.Spec.CanaryAnalysis.Mirror = true
+	return cd
+}
+
+func newTestCanaryAB() *flaggerv1.Canary {
+	cd := &flaggerv1.Canary{
+		TypeMeta: metav1.TypeMeta{APIVersion: flaggerv1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: "default",
 			Name:      "podinfo",
 		},
-		Spec: v1alpha3.CanarySpec{
+		Spec: flaggerv1.CanarySpec{
 			TargetRef: hpav1.CrossVersionObjectReference{
 				Name:       "podinfo",
 				APIVersion: "apps/v1",
@@ -286,9 +291,9 @@ func newTestCanaryAB() *v1alpha3.Canary {
 				Name:       "podinfo",
 				APIVersion: "autoscaling/v2beta1",
 				Kind:       "HorizontalPodAutoscaler",
-			}, Service: v1alpha3.CanaryService{
+			}, Service: flaggerv1.CanaryService{
 				Port: 9898,
-			}, CanaryAnalysis: v1alpha3.CanaryAnalysis{
+			}, CanaryAnalysis: flaggerv1.CanaryAnalysis{
 				Threshold:  10,
 				Iterations: 10,
 				Match: []istiov1alpha3.HTTPMatchRequest{
@@ -300,7 +305,7 @@ func newTestCanaryAB() *v1alpha3.Canary {
 						},
 					},
 				},
-				Metrics: []v1alpha3.CanaryMetric{
+				Metrics: []flaggerv1.CanaryMetric{
 					{
 						Name:      "istio_requests_total",
 						Threshold: 99,

pkg/controller/scheduler.go

@@ -5,10 +5,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/weaveworks/flagger/pkg/router"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"github.com/weaveworks/flagger/pkg/router"
 )
 
 // scheduleCanaries synchronises the canary map with the jobs map,
@@ -81,7 +81,7 @@ func (c *Controller) scheduleCanaries() {
 func (c *Controller) advanceCanary(name string, namespace string, skipLivenessChecks bool) {
 	begin := time.Now()
 	// check if the canary exists
-	cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(namespace).Get(name, v1.GetOptions{})
+	cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(namespace).Get(name, metav1.GetOptions{})
 	if err != nil {
 		c.logger.With("canary", fmt.Sprintf("%s.%s", name, namespace)).
 			Errorf("Canary %s.%s not found", name, namespace)
@@ -99,7 +99,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 	// create primary deployment and hpa if needed
 	// skip primary check for Istio since the deployment will become ready after the ClusterIP are created
 	skipPrimaryCheck := false
-	if skipLivenessChecks || strings.Contains(provider, "istio") {
+	if skipLivenessChecks || strings.Contains(provider, "istio") || strings.Contains(provider, "appmesh") {
 		skipPrimaryCheck = true
 	}
 	label, ports, err := c.deployer.Initialize(cd, skipPrimaryCheck)
@@ -155,7 +155,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 
 	// check if virtual service exists
 	// and if it contains weighted destination routes to the primary and canary services
-	primaryWeight, canaryWeight, err := meshRouter.GetRoutes(cd)
+	primaryWeight, canaryWeight, mirrored, err := meshRouter.GetRoutes(cd)
 	if err != nil {
 		c.recordEventWarningf(cd, "%v", err)
 		return
@@ -176,7 +176,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		// route all traffic back to primary
 		primaryWeight = 100
 		canaryWeight = 0
-		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
+		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, false); err != nil {
 			c.recordEventWarningf(cd, "%v", err)
 			return
 		}
@@ -214,7 +214,27 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		return
 	}
 
-	// scale canary to zero if analysis has succeeded
+	// route all traffic to primary if analysis has succeeded
+	if cd.Status.Phase == flaggerv1.CanaryPhasePromoting {
+		if provider != "kubernetes" {
+			c.recordEventInfof(cd, "Routing all traffic to primary")
+			if err := meshRouter.SetRoutes(cd, 100, 0, false); err != nil {
+				c.recordEventWarningf(cd, "%v", err)
+				return
+			}
+			c.recorder.SetWeight(cd, 100, 0)
+		}
+
+		// update status phase
+		if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseFinalising); err != nil {
+			c.recordEventWarningf(cd, "%v", err)
+			return
+		}
+
+		return
+	}
+
+	// scale canary to zero if promotion has finished
 	if cd.Status.Phase == flaggerv1.CanaryPhaseFinalising {
 		if err := c.deployer.Scale(cd, 0); err != nil {
 			c.recordEventWarningf(cd, "%v", err)
@@ -255,7 +275,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		// route all traffic back to primary
 		primaryWeight = 100
 		canaryWeight = 0
-		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
+		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, false); err != nil {
 			c.recordEventWarningf(cd, "%v", err)
 			return
 		}
@@ -282,8 +302,9 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 	}
 
 	// check if the canary success rate is above the threshold
-	// skip check if no traffic is routed to canary
-	if canaryWeight == 0 {
+	// skip check if no traffic is routed or mirrored to canary
+	if canaryWeight == 0 && cd.Status.Iterations == 0 &&
+		(cd.Spec.CanaryAnalysis.Mirror == false || mirrored == false) {
 		c.recordEventInfof(cd, "Starting canary analysis for %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)
 
 		// run pre-rollout web hooks
@@ -304,11 +325,24 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		}
 	}
 
-	// canary fix routing: A/B testing
-	if len(cd.Spec.CanaryAnalysis.Match) > 0 || cd.Spec.CanaryAnalysis.Iterations > 0 {
+	// use blue/green strategy for kubernetes provider
+	if provider == "kubernetes" {
+		if len(cd.Spec.CanaryAnalysis.Match) > 0 {
+			c.recordEventWarningf(cd, "A/B testing is not supported when using the kubernetes provider")
+			cd.Spec.CanaryAnalysis.Match = nil
+		}
+		if cd.Spec.CanaryAnalysis.Iterations < 1 {
+			c.recordEventWarningf(cd, "Progressive traffic is not supported when using the kubernetes provider")
+			c.recordEventWarningf(cd, "Setting canaryAnalysis.iterations: 10")
+			cd.Spec.CanaryAnalysis.Iterations = 10
+		}
+	}
+
+	// strategy: A/B testing
+	if len(cd.Spec.CanaryAnalysis.Match) > 0 && cd.Spec.CanaryAnalysis.Iterations > 0 {
 		// route traffic to canary and increment iterations
 		if cd.Spec.CanaryAnalysis.Iterations > cd.Status.Iterations {
-			if err := meshRouter.SetRoutes(cd, 0, 100); err != nil {
+			if err := meshRouter.SetRoutes(cd, 0, 100, false); err != nil {
 				c.recordEventWarningf(cd, "%v", err)
 				return
 			}
@@ -323,6 +357,11 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 			return
 		}
 
+		// check promotion gate
+		if promote := c.runConfirmPromotionHooks(cd); !promote {
+			return
+		}
+
 		// promote canary - max iterations reached
 		if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations {
 			c.recordEventInfof(cd, "Copying %s.%s template spec to %s.%s",
@@ -331,6 +370,60 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 				c.recordEventWarningf(cd, "%v", err)
 				return
 			}
+
+			// update status phase
+			if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhasePromoting); err != nil {
+				c.recordEventWarningf(cd, "%v", err)
+				return
+			}
+			return
+		}
+
+		return
+	}
+
+	// strategy: Blue/Green
+	if cd.Spec.CanaryAnalysis.Iterations > 0 {
+		// increment iterations
+		if cd.Spec.CanaryAnalysis.Iterations > cd.Status.Iterations {
+			// If in "mirror" mode, mirror requests during the entire B/G canary test
+			if provider != "kubernetes" &&
+				cd.Spec.CanaryAnalysis.Mirror == true && mirrored == false {
+				if err := meshRouter.SetRoutes(cd, 100, 0, true); err != nil {
+					c.recordEventWarningf(cd, "%v", err)
+				}
+				c.logger.With("canary", fmt.Sprintf("%s.%s", name, namespace)).
+					Infof("Start traffic mirroring")
+			}
+			if err := c.deployer.SetStatusIterations(cd, cd.Status.Iterations+1); err != nil {
+				c.recordEventWarningf(cd, "%v", err)
+				return
+			}
+			c.recordEventInfof(cd, "Advance %s.%s canary iteration %v/%v",
+				cd.Name, cd.Namespace, cd.Status.Iterations+1, cd.Spec.CanaryAnalysis.Iterations)
+			return
+		}
+
+		// check promotion gate
+		if promote := c.runConfirmPromotionHooks(cd); !promote {
+			return
+		}
+
+		// route all traffic to canary - max iterations reached
+		if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations {
+			if provider != "kubernetes" {
+				if cd.Spec.CanaryAnalysis.Mirror {
+					c.recordEventInfof(cd, "Stop traffic mirroring and route all traffic to canary")
+				} else {
+					c.recordEventInfof(cd, "Routing all traffic to canary")
+				}
+				if err := meshRouter.SetRoutes(cd, 0, 100, false); err != nil {
+					c.recordEventWarningf(cd, "%v", err)
+					return
+				}
+				c.recorder.SetWeight(cd, 0, 100)
+			}
+
 			// increment iterations
 			if err := c.deployer.SetStatusIterations(cd, cd.Status.Iterations+1); err != nil {
 				c.recordEventWarningf(cd, "%v", err)
@@ -339,82 +432,98 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 			return
 		}
 
-		// route all traffic to primary
+		// promote canary - max iterations reached
 		if cd.Spec.CanaryAnalysis.Iterations < cd.Status.Iterations {
-			primaryWeight = 100
-			canaryWeight = 0
-			if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
-				c.recordEventWarningf(cd, "%v", err)
-				return
-			}
-			c.recorder.SetWeight(cd, primaryWeight, canaryWeight)
-
-			// update status phase
-			if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseFinalising); err != nil {
-				c.recordEventWarningf(cd, "%v", err)
-				return
-			}
-
-			c.recordEventInfof(cd, "Routing all traffic to primary")
-			return
-		}
-
-		return
-	}
-
-	// canary incremental traffic weight
-	if canaryWeight < maxWeight {
-		primaryWeight -= cd.Spec.CanaryAnalysis.StepWeight
-		if primaryWeight < 0 {
-			primaryWeight = 0
-		}
-		canaryWeight += cd.Spec.CanaryAnalysis.StepWeight
-		if primaryWeight > 100 {
-			primaryWeight = 100
-		}
-
-		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
-			c.recordEventWarningf(cd, "%v", err)
-			return
-		}
-
-		// update weight status
-		if err := c.deployer.SetStatusWeight(cd, canaryWeight); err != nil {
-			c.recordEventWarningf(cd, "%v", err)
-			return
-		}
-
-		c.recorder.SetWeight(cd, primaryWeight, canaryWeight)
-		c.recordEventInfof(cd, "Advance %s.%s canary weight %v", cd.Name, cd.Namespace, canaryWeight)
-
-		// promote canary
-		if canaryWeight >= maxWeight {
 			c.recordEventInfof(cd, "Copying %s.%s template spec to %s.%s",
 				cd.Spec.TargetRef.Name, cd.Namespace, primaryName, cd.Namespace)
 			if err := c.deployer.Promote(cd); err != nil {
 				c.recordEventWarningf(cd, "%v", err)
 				return
 			}
-		}
-	} else {
-		// route all traffic to primary
-		primaryWeight = 100
-		canaryWeight = 0
-		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
-			c.recordEventWarningf(cd, "%v", err)
-			return
-		}
-		c.recorder.SetWeight(cd, primaryWeight, canaryWeight)
 
-		// update status phase
-		if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseFinalising); err != nil {
-			c.recordEventWarningf(cd, "%v", err)
+			// update status phase
+			if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhasePromoting); err != nil {
+				c.recordEventWarningf(cd, "%v", err)
+				return
+			}
 			return
 		}
 
-		c.recordEventInfof(cd, "Routing all traffic to primary")
 		return
 	}
+
+	// strategy: Canary progressive traffic increase
+	if cd.Spec.CanaryAnalysis.StepWeight > 0 {
+		// increase traffic weight
+		if canaryWeight < maxWeight {
+			// If in "mirror" mode, do one step of mirroring before shifting traffic to canary.
+			// When mirroring, all requests go to primary and canary, but only responses from
+			// primary go back to the user.
+			if cd.Spec.CanaryAnalysis.Mirror && canaryWeight == 0 {
+				if mirrored == false {
+					mirrored = true
+					primaryWeight = 100
+					canaryWeight = 0
+				} else {
+					mirrored = false
+					primaryWeight = 100 - cd.Spec.CanaryAnalysis.StepWeight
+					canaryWeight = cd.Spec.CanaryAnalysis.StepWeight
+				}
+				c.logger.With("canary", fmt.Sprintf("%s.%s", name, namespace)).
+					Infof("Running mirror step %d/%d/%t", primaryWeight, canaryWeight, mirrored)
+			} else {
+
+				primaryWeight -= cd.Spec.CanaryAnalysis.StepWeight
+				if primaryWeight < 0 {
+					primaryWeight = 0
+				}
+				canaryWeight += cd.Spec.CanaryAnalysis.StepWeight
+				if canaryWeight > 100 {
+					canaryWeight = 100
+				}
+			}
+
+			if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, mirrored); err != nil {
+				c.recordEventWarningf(cd, "%v", err)
+				return
+			}
+
+			if err := c.deployer.SetStatusWeight(cd, canaryWeight); err != nil {
+				c.recordEventWarningf(cd, "%v", err)
+				return
+			}
+
+			c.recorder.SetWeight(cd, primaryWeight, canaryWeight)
+			c.recordEventInfof(cd, "Advance %s.%s canary weight %v", cd.Name, cd.Namespace, canaryWeight)
+			return
+		}
+
+		// promote canary - max weight reached
+		if canaryWeight >= maxWeight {
+			// check promotion gate
+			if promote := c.runConfirmPromotionHooks(cd); !promote {
+				return
+			}
+
+			// update primary spec
+			c.recordEventInfof(cd, "Copying %s.%s template spec to %s.%s",
+				cd.Spec.TargetRef.Name, cd.Namespace, primaryName, cd.Namespace)
+			if err := c.deployer.Promote(cd); err != nil {
+				c.recordEventWarningf(cd, "%v", err)
+				return
+			}
+
+			// update status phase
+			if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhasePromoting); err != nil {
+				c.recordEventWarningf(cd, "%v", err)
+				return
+			}
+
+			return
+		}
+
+	}
+
 }
 
 func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, meshRouter router.Interface, primaryWeight int, canaryWeight int) bool {
@@ -425,7 +534,7 @@ func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, meshRouter router.
 	// route all traffic to primary
 	primaryWeight = 100
 	canaryWeight = 0
-	if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
+	if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight, false); err != nil {
 		c.recordEventWarningf(cd, "%v", err)
 		return false
 	}
@@ -466,6 +575,7 @@ func (c *Controller) shouldAdvance(cd *flaggerv1.Canary) (bool, error) {
 		cd.Status.Phase == flaggerv1.CanaryPhaseInitializing ||
 		cd.Status.Phase == flaggerv1.CanaryPhaseProgressing ||
 		cd.Status.Phase == flaggerv1.CanaryPhaseWaiting ||
+		cd.Status.Phase == flaggerv1.CanaryPhasePromoting ||
 		cd.Status.Phase == flaggerv1.CanaryPhaseFinalising {
 		return true, nil
 	}
@@ -490,6 +600,7 @@ func (c *Controller) shouldAdvance(cd *flaggerv1.Canary) (bool, error) {
 func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, shouldAdvance bool) bool {
 	c.recorder.SetStatus(cd, cd.Status.Phase)
 	if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing ||
+		cd.Status.Phase == flaggerv1.CanaryPhasePromoting ||
 		cd.Status.Phase == flaggerv1.CanaryPhaseFinalising {
 		return true
 	}
@@ -565,6 +676,23 @@ func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary) bool {
 	return true
 }
 
+func (c *Controller) runConfirmPromotionHooks(canary *flaggerv1.Canary) bool {
+	for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
+		if webhook.Type == flaggerv1.ConfirmPromotionHook {
+			err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
+			if err != nil {
+				c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for promotion approval %s",
+					canary.Name, canary.Namespace, webhook.Name)
+				c.sendNotification(canary, "Canary promotion is waiting for approval.", false, false)
+				return false
+			} else {
+				c.recordEventInfof(canary, "Confirm-promotion check %s passed", webhook.Name)
+			}
+		}
+	}
+	return true
+}
+
 func (c *Controller) runPreRolloutHooks(canary *flaggerv1.Canary) bool {
 	for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
 		if webhook.Type == flaggerv1.PreRolloutHook {

pkg/controller/scheduler_test.go

@@ -2,13 +2,16 @@ package controller
 
 import (
 	"fmt"
-	"github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 )
 
 func TestScheduler_Init(t *testing.T) {
-	mocks := SetupMocks(false)
+	mocks := SetupMocks(nil)
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
 	_, err := mocks.kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
@@ -18,7 +21,7 @@ func TestScheduler_Init(t *testing.T) {
 }
 
 func TestScheduler_NewRevision(t *testing.T) {
-	mocks := SetupMocks(false)
+	mocks := SetupMocks(nil)
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
 	// update
@@ -42,12 +45,12 @@ func TestScheduler_NewRevision(t *testing.T) {
 }
 
 func TestScheduler_Rollback(t *testing.T) {
-	mocks := SetupMocks(false)
+	mocks := SetupMocks(nil)
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
 	// update failed checks to max
-	err := mocks.deployer.SyncStatus(mocks.canary, v1alpha3.CanaryStatus{Phase: v1alpha3.CanaryPhaseProgressing, FailedChecks: 11})
+	err := mocks.deployer.SyncStatus(mocks.canary, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseProgressing, FailedChecks: 11})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -60,13 +63,13 @@ func TestScheduler_Rollback(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	if c.Status.Phase != v1alpha3.CanaryPhaseFailed {
-		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseFailed)
+	if c.Status.Phase != flaggerv1.CanaryPhaseFailed {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, flaggerv1.CanaryPhaseFailed)
 	}
 }
 
 func TestScheduler_SkipAnalysis(t *testing.T) {
-	mocks := SetupMocks(false)
+	mocks := SetupMocks(nil)
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
@@ -101,13 +104,13 @@ func TestScheduler_SkipAnalysis(t *testing.T) {
 		t.Errorf("Got skip analysis %v wanted %v", c.Spec.SkipAnalysis, true)
 	}
 
-	if c.Status.Phase != v1alpha3.CanaryPhaseSucceeded {
-		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseSucceeded)
+	if c.Status.Phase != flaggerv1.CanaryPhaseSucceeded {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, flaggerv1.CanaryPhaseSucceeded)
 	}
 }
 
 func TestScheduler_NewRevisionReset(t *testing.T) {
-	mocks := SetupMocks(false)
+	mocks := SetupMocks(nil)
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
@@ -123,7 +126,7 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
 	// advance
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
-	primaryWeight, canaryWeight, err := mocks.router.GetRoutes(mocks.canary)
+	primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -136,6 +139,10 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
 		t.Errorf("Got canary route %v wanted %v", canaryWeight, 10)
 	}
 
+	if mirrored != false {
+		t.Errorf("Got mirrored %v wanted %v", mirrored, false)
+	}
+
 	// second update
 	dep2.Spec.Template.Spec.ServiceAccountName = "test"
 	_, err = mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
@@ -146,7 +153,7 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
 	// detect changes
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
-	primaryWeight, canaryWeight, err = mocks.router.GetRoutes(mocks.canary)
+	primaryWeight, canaryWeight, mirrored, err = mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -158,16 +165,31 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
 	if canaryWeight != 0 {
 		t.Errorf("Got canary route %v wanted %v", canaryWeight, 0)
 	}
+
+	if mirrored != false {
+		t.Errorf("Got mirrored %v wanted %v", mirrored, false)
+	}
 }
 
 func TestScheduler_Promotion(t *testing.T) {
-	mocks := SetupMocks(false)
+	mocks := SetupMocks(nil)
+
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
+	// check initialized status
+	c, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if c.Status.Phase != flaggerv1.CanaryPhaseInitialized {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, flaggerv1.CanaryPhaseInitialized)
+	}
+
 	// update
 	dep2 := newTestDeploymentV2()
-	_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
+	_, err = mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -190,14 +212,14 @@ func TestScheduler_Promotion(t *testing.T) {
 	// detect configs changes
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
-	primaryWeight, canaryWeight, err := mocks.router.GetRoutes(mocks.canary)
+	primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
 	primaryWeight = 60
 	canaryWeight = 40
-	err = mocks.router.SetRoutes(mocks.canary, primaryWeight, canaryWeight)
+	err = mocks.router.SetRoutes(mocks.canary, primaryWeight, canaryWeight, mirrored)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -205,10 +227,33 @@ func TestScheduler_Promotion(t *testing.T) {
 	// advance
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
+	// check progressing status
+	c, err = mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if c.Status.Phase != flaggerv1.CanaryPhaseProgressing {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, flaggerv1.CanaryPhaseProgressing)
+	}
+
 	// promote
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
-	primaryWeight, canaryWeight, err = mocks.router.GetRoutes(mocks.canary)
+	// check promoting status
+	c, err = mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if c.Status.Phase != flaggerv1.CanaryPhasePromoting {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, flaggerv1.CanaryPhasePromoting)
+	}
+
+	// finalise
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	primaryWeight, canaryWeight, mirrored, err = mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -221,6 +266,10 @@ func TestScheduler_Promotion(t *testing.T) {
 		t.Errorf("Got canary route %v wanted %v", canaryWeight, 0)
 	}
 
+	if mirrored != false {
+		t.Errorf("Got mirrored %v wanted %v", mirrored, false)
+	}
+
 	primaryDep, err := mocks.kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
@@ -251,11 +300,15 @@ func TestScheduler_Promotion(t *testing.T) {
 	}
 
 	// check finalising status
-	c, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	c, err = mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
+	if c.Status.Phase != flaggerv1.CanaryPhaseFinalising {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, flaggerv1.CanaryPhaseFinalising)
+	}
+
 	// scale canary to zero
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
@@ -264,13 +317,71 @@ func TestScheduler_Promotion(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	if c.Status.Phase != v1alpha3.CanaryPhaseSucceeded {
-		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseSucceeded)
+	if c.Status.Phase != flaggerv1.CanaryPhaseSucceeded {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, flaggerv1.CanaryPhaseSucceeded)
+	}
+}
+
+func TestScheduler_Mirroring(t *testing.T) {
+	mocks := SetupMocks(newTestCanaryMirror())
+	// init
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	// update
+	dep2 := newTestDeploymentV2()
+	_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	// detect pod spec changes
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	// advance
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	// check if traffic is mirrored to canary
+	primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if primaryWeight != 100 {
+		t.Errorf("Got primary route %v wanted %v", primaryWeight, 100)
+	}
+
+	if canaryWeight != 0 {
+		t.Errorf("Got canary route %v wanted %v", canaryWeight, 0)
+	}
+
+	if mirrored != true {
+		t.Errorf("Got mirrored %v wanted %v", mirrored, true)
+	}
+
+	// advance
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	// check if traffic is mirrored to canary
+	primaryWeight, canaryWeight, mirrored, err = mocks.router.GetRoutes(mocks.canary)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if primaryWeight != 90 {
+		t.Errorf("Got primary route %v wanted %v", primaryWeight, 90)
+	}
+
+	if canaryWeight != 10 {
+		t.Errorf("Got canary route %v wanted %v", canaryWeight, 10)
+	}
+
+	if mirrored != false {
+		t.Errorf("Got mirrored %v wanted %v", mirrored, false)
 	}
 }
 
 func TestScheduler_ABTesting(t *testing.T) {
-	mocks := SetupMocks(true)
+	mocks := SetupMocks(newTestCanaryAB())
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
@@ -288,7 +399,7 @@ func TestScheduler_ABTesting(t *testing.T) {
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
 	// check if traffic is routed to canary
-	primaryWeight, canaryWeight, err := mocks.router.GetRoutes(mocks.canary)
+	primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -301,6 +412,10 @@ func TestScheduler_ABTesting(t *testing.T) {
 		t.Errorf("Got canary route %v wanted %v", canaryWeight, 100)
 	}
 
+	if mirrored != false {
+		t.Errorf("Got mirrored %v wanted %v", mirrored, false)
+	}
+
 	cd, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
@@ -323,8 +438,8 @@ func TestScheduler_ABTesting(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	if c.Status.Phase != v1alpha3.CanaryPhaseFinalising {
-		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseFinalising)
+	if c.Status.Phase != flaggerv1.CanaryPhaseFinalising {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, flaggerv1.CanaryPhaseFinalising)
 	}
 
 	// check if the container image tag was updated
@@ -348,13 +463,13 @@ func TestScheduler_ABTesting(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	if c.Status.Phase != v1alpha3.CanaryPhaseSucceeded {
-		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseSucceeded)
+	if c.Status.Phase != flaggerv1.CanaryPhaseSucceeded {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, flaggerv1.CanaryPhaseSucceeded)
 	}
 }
 
 func TestScheduler_PortDiscovery(t *testing.T) {
-	mocks := SetupMocks(false)
+	mocks := SetupMocks(nil)
 
 	// enable port discovery
 	cd, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
@@ -396,3 +511,93 @@ func TestScheduler_PortDiscovery(t *testing.T) {
 
 	}
 }
+
+func TestScheduler_TargetPortNumber(t *testing.T) {
+	mocks := SetupMocks(nil)
+
+	cd, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	cd.Spec.Service.Port = 80
+	cd.Spec.Service.TargetPort = intstr.FromInt(9898)
+	cd.Spec.Service.PortDiscovery = true
+	_, err = mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Update(cd)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	canarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get("podinfo-canary", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if len(canarySvc.Spec.Ports) != 3 {
+		t.Fatalf("Got svc port count %v wanted %v", len(canarySvc.Spec.Ports), 3)
+	}
+
+	matchPorts := func(lookup string) bool {
+		switch lookup {
+		case
+			"http 80",
+			"http-metrics 8080",
+			"tcp-podinfo-2 8888":
+			return true
+		}
+		return false
+	}
+
+	for _, port := range canarySvc.Spec.Ports {
+		if !matchPorts(fmt.Sprintf("%s %v", port.Name, port.Port)) {
+			t.Fatalf("Got wrong svc port %v", port.Name)
+		}
+
+	}
+}
+
+func TestScheduler_TargetPortName(t *testing.T) {
+	mocks := SetupMocks(nil)
+
+	cd, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	cd.Spec.Service.Port = 8080
+	cd.Spec.Service.TargetPort = intstr.FromString("http")
+	cd.Spec.Service.PortDiscovery = true
+	_, err = mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Update(cd)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	canarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get("podinfo-canary", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if len(canarySvc.Spec.Ports) != 3 {
+		t.Fatalf("Got svc port count %v wanted %v", len(canarySvc.Spec.Ports), 3)
+	}
+
+	matchPorts := func(lookup string) bool {
+		switch lookup {
+		case
+			"http 8080",
+			"http-metrics 8080",
+			"tcp-podinfo-2 8888":
+			return true
+		}
+		return false
+	}
+
+	for _, port := range canarySvc.Spec.Ports {
+		if !matchPorts(fmt.Sprintf("%s %v", port.Name, port.Port)) {
+			t.Fatalf("Got wrong svc port %v", port.Name)
+		}
+
+	}
+}

pkg/controller/webhook.go

@@ -6,11 +6,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"time"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 )
 
 // CallWebhook does a HTTP POST to an external service and

pkg/controller/webhook_test.go

@@ -1,10 +1,11 @@
 package controller
 
 import (
-	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 	"net/http"
 	"net/http/httptest"
 	"testing"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 )
 
 func TestCallWebhook(t *testing.T) {

pkg/loadtester/bash.go

@@ -24,7 +24,7 @@ func (task *BashTask) Run(ctx context.Context) (bool, error) {
 
 	if err != nil {
 		task.logger.With("canary", task.canary).Errorf("command failed %s %v %s", task.command, err, out)
-		return false, fmt.Errorf(" %v %v", err, out)
+		return false, fmt.Errorf(" %v %s", err, out)
 	} else {
 		if task.logCmdOutput {
 			fmt.Printf("%s\n", out)

pkg/loadtester/helm.go

@@ -20,14 +20,14 @@ func (task *HelmTask) Hash() string {
 }
 
 func (task *HelmTask) Run(ctx context.Context) (bool, error) {
-	helmCmd := fmt.Sprintf("helm %s", task.command)
+	helmCmd := fmt.Sprintf("%s %s", TaskTypeHelm, task.command)
 	task.logger.With("canary", task.canary).Infof("running command %v", helmCmd)
 
-	cmd := exec.CommandContext(ctx, "helm", strings.Fields(task.command)...)
+	cmd := exec.CommandContext(ctx, TaskTypeHelm, strings.Fields(task.command)...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		task.logger.With("canary", task.canary).Errorf("command failed %s %v %s", task.command, err, out)
-		return false, fmt.Errorf(" %v %v", err, out)
+		return false, fmt.Errorf(" %v %s", err, out)
 	} else {
 		if task.logCmdOutput {
 			fmt.Printf("%s\n", out)

pkg/loadtester/helmv3.go

@@ -0,0 +1,42 @@
+package loadtester
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+const TaskTypeHelmv3 = "helmv3"
+
+type HelmTaskv3 struct {
+	TaskBase
+	command      string
+	logCmdOutput bool
+}
+
+func (task *HelmTaskv3) Hash() string {
+	return hash(task.canary + task.command)
+}
+
+func (task *HelmTaskv3) Run(ctx context.Context) (bool, error) {
+	helmCmd := fmt.Sprintf("%s %s", TaskTypeHelmv3, task.command)
+	task.logger.With("canary", task.canary).Infof("running command %v", helmCmd)
+
+	cmd := exec.CommandContext(ctx, TaskTypeHelmv3, strings.Fields(task.command)...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		task.logger.With("canary", task.canary).Errorf("command failed %s %v %s", task.command, err, out)
+		return false, fmt.Errorf(" %v %s", err, out)
+	} else {
+		if task.logCmdOutput {
+			fmt.Printf("%s\n", out)
+		}
+		task.logger.With("canary", task.canary).Infof("command finished %v", helmCmd)
+	}
+	return true, nil
+}
+
+func (task *HelmTaskv3) String() string {
+	return task.command
+}

pkg/loadtester/server.go

@@ -185,6 +185,31 @@ func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogge
 				return
 			}
 
+			// run helmv3 command (blocking task)
+			if typ == TaskTypeHelmv3 {
+				helm := HelmTaskv3{
+					command:      payload.Metadata["cmd"],
+					logCmdOutput: true,
+					TaskBase: TaskBase{
+						canary: fmt.Sprintf("%s.%s", payload.Name, payload.Namespace),
+						logger: logger,
+					},
+				}
+
+				ctx, cancel := context.WithTimeout(context.Background(), taskRunner.timeout)
+				defer cancel()
+
+				ok, err := helm.Run(ctx)
+				if !ok {
+					w.WriteHeader(http.StatusInternalServerError)
+					w.Write([]byte(err.Error()))
+					return
+				}
+
+				w.WriteHeader(http.StatusOK)
+				return
+			}
+
 			taskFactory, ok := GetTaskFactory(typ)
 			if !ok {
 				w.WriteHeader(http.StatusBadRequest)

pkg/notifier/client_test.go

@@ -0,0 +1,30 @@
+package notifier
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func Test_postMessage(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var payload = make(map[string]string)
+		err = json.Unmarshal(b, &payload)
+
+		if payload["status"] != "success" {
+			t.Fatal("wrong payload")
+		}
+	}))
+	defer ts.Close()
+
+	err := postMessage(ts.URL, map[string]string{"status": "success"})
+	if err != nil {
+		t.Fatal(err)
+	}
+}

pkg/notifier/slack.go

@@ -74,7 +74,7 @@ func (s *Slack) Post(workload string, namespace string, message string, fields [
 		color = "danger"
 	}
 
-	sfields := make([]SlackField, len(fields))
+	sfields := make([]SlackField, 0, len(fields))
 	for _, f := range fields {
 		sfields = append(sfields, SlackField{f.Name, f.Value, false})
 	}

pkg/notifier/slack_test.go

@@ -0,0 +1,44 @@
+package notifier
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestSlack_Post(t *testing.T) {
+	fields := []Field{
+		{Name: "name1", Value: "value1"},
+		{Name: "name2", Value: "value2"},
+	}
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var payload = SlackPayload{}
+		err = json.Unmarshal(b, &payload)
+
+		if payload.Attachments[0].AuthorName != "podinfo.test" {
+			t.Fatal("wrong author name")
+		}
+
+		if len(payload.Attachments[0].Fields) != len(fields) {
+			t.Fatal("wrong facts")
+		}
+	}))
+	defer ts.Close()
+
+	slack, err := NewSlack(ts.URL, "test", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = slack.Post("podinfo", "test", "test", fields, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

pkg/notifier/teams.go

@@ -45,7 +45,7 @@ func NewMSTeams(hookURL string) (*MSTeams, error) {
 
 // Post MS Teams message
 func (s *MSTeams) Post(workload string, namespace string, message string, fields []Field, warn bool) error {
-	facts := make([]MSTeamsField, len(fields))
+	facts := make([]MSTeamsField, 0, len(fields))
 	for _, f := range fields {
 		facts = append(facts, MSTeamsField{f.Name, f.Value})
 	}

pkg/notifier/teams_test.go

@@ -0,0 +1,44 @@
+package notifier
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestTeams_Post(t *testing.T) {
+
+	fields := []Field{
+		{Name: "name1", Value: "value1"},
+		{Name: "name2", Value: "value2"},
+	}
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		b, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var payload = MSTeamsPayload{}
+		err = json.Unmarshal(b, &payload)
+
+		if payload.Sections[0].ActivitySubtitle != "podinfo.test" {
+			t.Fatal("wrong activity subtitle")
+		}
+		if len(payload.Sections[0].Facts) != len(fields) {
+			t.Fatal("wrong facts")
+		}
+	}))
+	defer ts.Close()
+
+	teams, err := NewMSTeams(ts.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = teams.Post("podinfo", "test", "test", fields, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

pkg/router/appmesh.go

@@ -2,16 +2,19 @@ package router
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
-	AppmeshV1beta1 "github.com/weaveworks/flagger/pkg/apis/appmesh/v1beta1"
-	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
-	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
 	"go.uber.org/zap"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes"
+
+	appmeshv1 "github.com/weaveworks/flagger/pkg/apis/appmesh/v1beta1"
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
+	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
 )
 
 // AppMeshRouter is managing AppMesh virtual services
@@ -56,9 +59,16 @@ func (ar *AppMeshRouter) Reconcile(canary *flaggerv1.Canary) error {
 		return err
 	}
 
-	// sync virtual service e.g. app.namespace
+	// sync main virtual service
 	// DNS app.namespace
-	err = ar.reconcileVirtualService(canary, targetHost)
+	err = ar.reconcileVirtualService(canary, targetHost, 0)
+	if err != nil {
+		return err
+	}
+
+	// sync canary virtual service
+	// DNS app-canary.namespace
+	err = ar.reconcileVirtualService(canary, fmt.Sprintf("%s.%s", canaryName, canary.Namespace), 100)
 	if err != nil {
 		return err
 	}
@@ -69,27 +79,28 @@ func (ar *AppMeshRouter) Reconcile(canary *flaggerv1.Canary) error {
 // reconcileVirtualNode creates or updates a virtual node
 // the virtual node naming format is name-role-namespace
 func (ar *AppMeshRouter) reconcileVirtualNode(canary *flaggerv1.Canary, name string, host string) error {
-	vnSpec := AppmeshV1beta1.VirtualNodeSpec{
+	protocol := getProtocol(canary)
+	vnSpec := appmeshv1.VirtualNodeSpec{
 		MeshName: canary.Spec.Service.MeshName,
-		Listeners: []AppmeshV1beta1.Listener{
+		Listeners: []appmeshv1.Listener{
 			{
-				PortMapping: AppmeshV1beta1.PortMapping{
+				PortMapping: appmeshv1.PortMapping{
 					Port:     int64(canary.Spec.Service.Port),
-					Protocol: "http",
+					Protocol: protocol,
 				},
 			},
 		},
-		ServiceDiscovery: &AppmeshV1beta1.ServiceDiscovery{
-			Dns: &AppmeshV1beta1.DnsServiceDiscovery{
+		ServiceDiscovery: &appmeshv1.ServiceDiscovery{
+			Dns: &appmeshv1.DnsServiceDiscovery{
 				HostName: host,
 			},
 		},
 	}
 
-	backends := []AppmeshV1beta1.Backend{}
+	backends := []appmeshv1.Backend{}
 	for _, b := range canary.Spec.Service.Backends {
-		backend := AppmeshV1beta1.Backend{
-			VirtualService: AppmeshV1beta1.VirtualServiceBackend{
+		backend := appmeshv1.Backend{
+			VirtualService: appmeshv1.VirtualServiceBackend{
 				VirtualServiceName: b,
 			},
 		}
@@ -103,7 +114,7 @@ func (ar *AppMeshRouter) reconcileVirtualNode(canary *flaggerv1.Canary, name str
 
 	// create virtual node
 	if errors.IsNotFound(err) {
-		virtualnode = &AppmeshV1beta1.VirtualNode{
+		virtualnode = &appmeshv1.VirtualNode{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      name,
 				Namespace: canary.Namespace,
@@ -148,10 +159,11 @@ func (ar *AppMeshRouter) reconcileVirtualNode(canary *flaggerv1.Canary, name str
 }
 
 // reconcileVirtualService creates or updates a virtual service
-func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name string) error {
+func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name string, canaryWeight int64) error {
 	targetName := canary.Spec.TargetRef.Name
 	canaryVirtualNode := fmt.Sprintf("%s-canary", targetName)
 	primaryVirtualNode := fmt.Sprintf("%s-primary", targetName)
+	protocol := getProtocol(canary)
 
 	// App Mesh supports only URI prefix
 	routePrefix := "/"
@@ -161,27 +173,35 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name
 		routePrefix = canary.Spec.Service.Match[0].Uri.Prefix
 	}
 
-	vsSpec := AppmeshV1beta1.VirtualServiceSpec{
+	vsSpec := appmeshv1.VirtualServiceSpec{
 		MeshName: canary.Spec.Service.MeshName,
-		VirtualRouter: &AppmeshV1beta1.VirtualRouter{
+		VirtualRouter: &appmeshv1.VirtualRouter{
 			Name: fmt.Sprintf("%s-router", targetName),
+			Listeners: []appmeshv1.Listener{
+				{
+					PortMapping: appmeshv1.PortMapping{
+						Port:     int64(canary.Spec.Service.Port),
+						Protocol: protocol,
+					},
+				},
+			},
 		},
-		Routes: []AppmeshV1beta1.Route{
+		Routes: []appmeshv1.Route{
 			{
 				Name: fmt.Sprintf("%s-route", targetName),
-				Http: AppmeshV1beta1.HttpRoute{
-					Match: AppmeshV1beta1.HttpRouteMatch{
+				Http: &appmeshv1.HttpRoute{
+					Match: appmeshv1.HttpRouteMatch{
 						Prefix: routePrefix,
 					},
-					Action: AppmeshV1beta1.HttpRouteAction{
-						WeightedTargets: []AppmeshV1beta1.WeightedTarget{
+					Action: appmeshv1.HttpRouteAction{
+						WeightedTargets: []appmeshv1.WeightedTarget{
 							{
 								VirtualNodeName: canaryVirtualNode,
-								Weight:          0,
+								Weight:          canaryWeight,
 							},
 							{
 								VirtualNodeName: primaryVirtualNode,
-								Weight:          100,
+								Weight:          100 - canaryWeight,
 							},
 						},
 					},
@@ -194,7 +214,7 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name
 
 	// create virtual service
 	if errors.IsNotFound(err) {
-		virtualService = &AppmeshV1beta1.VirtualService{
+		virtualService = &appmeshv1.VirtualService{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      name,
 				Namespace: canary.Namespace,
@@ -223,7 +243,7 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name
 
 	// update virtual service but keep the original target weights
 	if virtualService != nil {
-		if diff := cmp.Diff(vsSpec, virtualService.Spec, cmpopts.IgnoreTypes(AppmeshV1beta1.WeightedTarget{})); diff != "" {
+		if diff := cmp.Diff(vsSpec, virtualService.Spec, cmpopts.IgnoreTypes(appmeshv1.WeightedTarget{})); diff != "" {
 			vsClone := virtualService.DeepCopy()
 			vsClone.Spec = vsSpec
 			vsClone.Spec.Routes[0].Http.Action = virtualService.Spec.Routes[0].Http.Action
@@ -244,6 +264,7 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name
 func (ar *AppMeshRouter) GetRoutes(canary *flaggerv1.Canary) (
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 	err error,
 ) {
 	targetName := canary.Spec.TargetRef.Name
@@ -278,6 +299,8 @@ func (ar *AppMeshRouter) GetRoutes(canary *flaggerv1.Canary) (
 			vsName, targetName, targetName)
 	}
 
+	mirrored = false
+
 	return
 }
 
@@ -286,6 +309,7 @@ func (ar *AppMeshRouter) SetRoutes(
 	canary *flaggerv1.Canary,
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 ) error {
 	targetName := canary.Spec.TargetRef.Name
 	vsName := fmt.Sprintf("%s.%s", targetName, canary.Namespace)
@@ -298,8 +322,8 @@ func (ar *AppMeshRouter) SetRoutes(
 	}
 
 	vsClone := vs.DeepCopy()
-	vsClone.Spec.Routes[0].Http.Action = AppmeshV1beta1.HttpRouteAction{
-		WeightedTargets: []AppmeshV1beta1.WeightedTarget{
+	vsClone.Spec.Routes[0].Http.Action = appmeshv1.HttpRouteAction{
+		WeightedTargets: []appmeshv1.WeightedTarget{
 			{
 				VirtualNodeName: fmt.Sprintf("%s-canary", targetName),
 				Weight:          int64(canaryWeight),
@@ -318,3 +342,10 @@ func (ar *AppMeshRouter) SetRoutes(
 
 	return nil
 }
+
+func getProtocol(canary *flaggerv1.Canary) string {
+	if strings.Contains(canary.Spec.Service.PortName, "grpc") {
+		return "grpc"
+	}
+	return "http"
+}

pkg/router/appmesh_test.go

@@ -2,8 +2,9 @@ package router
 
 import (
 	"fmt"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func TestAppmeshRouter_Reconcile(t *testing.T) {
@@ -37,6 +38,23 @@ func TestAppmeshRouter_Reconcile(t *testing.T) {
 		t.Errorf("Got routes %v wanted %v", targetsCount, 2)
 	}
 
+	// check canary virtual service
+	vsCanaryName := fmt.Sprintf("%s-canary.%s", mocks.appmeshCanary.Spec.TargetRef.Name, mocks.appmeshCanary.Namespace)
+	vsCanary, err := router.appmeshClient.AppmeshV1beta1().VirtualServices("default").Get(vsCanaryName, metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	// check if the canary virtual service routes all traffic to the canary virtual node
+	target := vsCanary.Spec.Routes[0].Http.Action.WeightedTargets[0]
+	canaryVirtualNodeName := fmt.Sprintf("%s-canary", mocks.appmeshCanary.Spec.TargetRef.Name)
+	if target.VirtualNodeName != canaryVirtualNodeName {
+		t.Errorf("Got VirtualNodeName %v wanted %v", target.VirtualNodeName, canaryVirtualNodeName)
+	}
+	if target.Weight != 100 {
+		t.Errorf("Got weight %v wanted %v", target.Weight, 100)
+	}
+
 	// check virtual node
 	vnName := mocks.appmeshCanary.Spec.TargetRef.Name
 	vn, err := router.appmeshClient.AppmeshV1beta1().VirtualNodes("default").Get(vnName, metav1.GetOptions{})
@@ -103,7 +121,7 @@ func TestAppmeshRouter_Reconcile(t *testing.T) {
 
 	weight := vs.Spec.Routes[0].Http.Action.WeightedTargets[0].Weight
 	if weight != 50 {
-		t.Errorf("Got weight %v wanted %v", weight, 502)
+		t.Errorf("Got weight %v wanted %v", weight, 50)
 	}
 
 	// test URI update
@@ -144,12 +162,12 @@ func TestAppmeshRouter_GetSetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	err = router.SetRoutes(mocks.appmeshCanary, 60, 40)
+	err = router.SetRoutes(mocks.appmeshCanary, 60, 40, false)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	p, c, err := router.GetRoutes(mocks.appmeshCanary)
+	p, c, m, err := router.GetRoutes(mocks.appmeshCanary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -161,4 +179,8 @@ func TestAppmeshRouter_GetSetRoutes(t *testing.T) {
 	if c != 40 {
 		t.Errorf("Got canary weight %v wanted %v", c, 40)
 	}
+
+	if m != false {
+		t.Errorf("Got mirror %v wanted %v", m, false)
+	}
 }

pkg/router/factory.go

@@ -4,30 +4,34 @@ import (
 	"context"
 	"strings"
 
-	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
 	"go.uber.org/zap"
 	"k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
+
+	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
 )
 
 type Factory struct {
-	kubeConfig    *restclient.Config
-	kubeClient    kubernetes.Interface
-	meshClient    clientset.Interface
-	flaggerClient clientset.Interface
-	logger        *zap.SugaredLogger
+	kubeConfig               *restclient.Config
+	kubeClient               kubernetes.Interface
+	meshClient               clientset.Interface
+	flaggerClient            clientset.Interface
+	ingressAnnotationsPrefix string
+	logger                   *zap.SugaredLogger
 }
 
 func NewFactory(kubeConfig *restclient.Config, kubeClient kubernetes.Interface,
 	flaggerClient clientset.Interface,
+	ingressAnnotationsPrefix string,
 	logger *zap.SugaredLogger,
 	meshClient clientset.Interface) *Factory {
 	return &Factory{
-		kubeConfig:    kubeConfig,
-		meshClient:    meshClient,
-		kubeClient:    kubeClient,
-		flaggerClient: flaggerClient,
-		logger:        logger,
+		kubeConfig:               kubeConfig,
+		meshClient:               meshClient,
+		kubeClient:               kubeClient,
+		flaggerClient:            flaggerClient,
+		ingressAnnotationsPrefix: ingressAnnotationsPrefix,
+		logger:                   logger,
 	}
 }
 
@@ -51,8 +55,9 @@ func (factory *Factory) MeshRouter(provider string) Interface {
 		return &NopRouter{}
 	case provider == "nginx":
 		return &IngressRouter{
-			logger:     factory.logger,
-			kubeClient: factory.kubeClient,
+			logger:            factory.logger,
+			kubeClient:        factory.kubeClient,
+			annotationsPrefix: factory.ingressAnnotationsPrefix,
 		}
 	case provider == "appmesh":
 		return &AppMeshRouter{

pkg/router/gloo.go

@@ -5,20 +5,20 @@ import (
 	"fmt"
 	"strings"
 
+	gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1"
 	solokitclients "github.com/solo-io/solo-kit/pkg/api/v1/clients"
 	"github.com/solo-io/solo-kit/pkg/api/v1/clients/factory"
 	"github.com/solo-io/solo-kit/pkg/api/v1/clients/kube"
 	crdv1 "github.com/solo-io/solo-kit/pkg/api/v1/clients/kube/crd/solo.io/v1"
 	solokitcore "github.com/solo-io/solo-kit/pkg/api/v1/resources/core"
 	solokiterror "github.com/solo-io/solo-kit/pkg/errors"
-
-	gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1"
-	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
-	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
 	"go.uber.org/zap"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/rest"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
+	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
 )
 
 // GlooRouter is managing Istio virtual services
@@ -63,11 +63,11 @@ func NewGlooRouterWithClient(ctx context.Context, routingRuleClient gloov1.Upstr
 // Reconcile creates or updates the Istio virtual service
 func (gr *GlooRouter) Reconcile(canary *flaggerv1.Canary) error {
 	// do we have routes already?
-	if _, _, err := gr.GetRoutes(canary); err == nil {
+	if _, _, _, err := gr.GetRoutes(canary); err == nil {
 		// we have routes, no need to do anything else
 		return nil
 	} else if solokiterror.IsNotExist(err) {
-		return gr.SetRoutes(canary, 100, 0)
+		return gr.SetRoutes(canary, 100, 0, false)
 	} else {
 		return err
 	}
@@ -77,6 +77,7 @@ func (gr *GlooRouter) Reconcile(canary *flaggerv1.Canary) error {
 func (gr *GlooRouter) GetRoutes(canary *flaggerv1.Canary) (
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 	err error,
 ) {
 	targetName := canary.Spec.TargetRef.Name
@@ -101,6 +102,8 @@ func (gr *GlooRouter) GetRoutes(canary *flaggerv1.Canary) (
 			targetName, canary.Namespace, targetName, targetName)
 	}
 
+	mirrored = false
+
 	return
 }
 
@@ -109,6 +112,7 @@ func (gr *GlooRouter) SetRoutes(
 	canary *flaggerv1.Canary,
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 ) error {
 	targetName := canary.Spec.TargetRef.Name
 

pkg/router/gloo_test.go

@@ -68,15 +68,16 @@ func TestGlooRouter_SetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	p, c, err := router.GetRoutes(mocks.canary)
+	p, c, m, err := router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
 	p = 50
 	c = 50
+	m = false
 
-	err = router.SetRoutes(mocks.canary, p, c)
+	err = router.SetRoutes(mocks.canary, p, c, m)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -127,7 +128,7 @@ func TestGlooRouter_GetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	p, c, err := router.GetRoutes(mocks.canary)
+	p, c, m, err := router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -139,4 +140,8 @@ func TestGlooRouter_GetRoutes(t *testing.T) {
 	if c != 0 {
 		t.Errorf("Got canary weight %v wanted %v", c, 0)
 	}
+
+	if m != false {
+		t.Errorf("Got mirror %v wanted %v", m, false)
+	}
 }

pkg/router/ingress.go

@@ -2,21 +2,24 @@ package router
 
 import (
 	"fmt"
+	"strconv"
+	"strings"
+
 	"github.com/google/go-cmp/cmp"
-	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 	"go.uber.org/zap"
 	"k8s.io/api/extensions/v1beta1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes"
-	"strconv"
-	"strings"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 )
 
 type IngressRouter struct {
-	kubeClient kubernetes.Interface
-	logger     *zap.SugaredLogger
+	kubeClient        kubernetes.Interface
+	annotationsPrefix string
+	logger            *zap.SugaredLogger
 }
 
 func (i *IngressRouter) Reconcile(canary *flaggerv1.Canary) error {
@@ -104,29 +107,30 @@ func (i *IngressRouter) Reconcile(canary *flaggerv1.Canary) error {
 func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) (
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 	err error,
 ) {
 	canaryIngressName := fmt.Sprintf("%s-canary", canary.Spec.IngressRef.Name)
 	canaryIngress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canaryIngressName, metav1.GetOptions{})
 	if err != nil {
-		return 0, 0, err
+		return 0, 0, false, err
 	}
 
 	// A/B testing
 	if len(canary.Spec.CanaryAnalysis.Match) > 0 {
 		for k := range canaryIngress.Annotations {
-			if k == "nginx.ingress.kubernetes.io/canary-by-cookie" || k == "nginx.ingress.kubernetes.io/canary-by-header" {
-				return 0, 100, nil
+			if k == i.GetAnnotationWithPrefix("canary-by-cookie") || k == i.GetAnnotationWithPrefix("canary-by-header") {
+				return 0, 100, false, nil
 			}
 		}
 	}
 
 	// Canary
 	for k, v := range canaryIngress.Annotations {
-		if k == "nginx.ingress.kubernetes.io/canary-weight" {
+		if k == i.GetAnnotationWithPrefix("canary-weight") {
 			val, err := strconv.Atoi(v)
 			if err != nil {
-				return 0, 0, err
+				return 0, 0, false, err
 			}
 
 			canaryWeight = val
@@ -135,6 +139,7 @@ func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) (
 	}
 
 	primaryWeight = 100 - canaryWeight
+	mirrored = false
 	return
 }
 
@@ -142,6 +147,7 @@ func (i *IngressRouter) SetRoutes(
 	canary *flaggerv1.Canary,
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 ) error {
 	canaryIngressName := fmt.Sprintf("%s-canary", canary.Spec.IngressRef.Name)
 	canaryIngress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canaryIngressName, metav1.GetOptions{})
@@ -170,12 +176,12 @@ func (i *IngressRouter) SetRoutes(
 		iClone.Annotations = i.makeHeaderAnnotations(iClone.Annotations, header, headerValue, cookie)
 	} else {
 		// canary
-		iClone.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = fmt.Sprintf("%v", canaryWeight)
+		iClone.Annotations[i.GetAnnotationWithPrefix("canary-weight")] = fmt.Sprintf("%v", canaryWeight)
 	}
 
 	// toggle canary
 	if canaryWeight > 0 {
-		iClone.Annotations["nginx.ingress.kubernetes.io/canary"] = "true"
+		iClone.Annotations[i.GetAnnotationWithPrefix("canary")] = "true"
 	} else {
 		iClone.Annotations = i.makeAnnotations(iClone.Annotations)
 	}
@@ -191,14 +197,14 @@ func (i *IngressRouter) SetRoutes(
 func (i *IngressRouter) makeAnnotations(annotations map[string]string) map[string]string {
 	res := make(map[string]string)
 	for k, v := range annotations {
-		if !strings.Contains(k, "nginx.ingress.kubernetes.io/canary") &&
+		if !strings.Contains(k, i.GetAnnotationWithPrefix("canary")) &&
 			!strings.Contains(k, "kubectl.kubernetes.io/last-applied-configuration") {
 			res[k] = v
 		}
 	}
 
-	res["nginx.ingress.kubernetes.io/canary"] = "false"
-	res["nginx.ingress.kubernetes.io/canary-weight"] = "0"
+	res[i.GetAnnotationWithPrefix("canary")] = "false"
+	res[i.GetAnnotationWithPrefix("canary-weight")] = "0"
 
 	return res
 }
@@ -207,25 +213,29 @@ func (i *IngressRouter) makeHeaderAnnotations(annotations map[string]string,
 	header string, headerValue string, cookie string) map[string]string {
 	res := make(map[string]string)
 	for k, v := range annotations {
-		if !strings.Contains(v, "nginx.ingress.kubernetes.io/canary") {
+		if !strings.Contains(v, i.GetAnnotationWithPrefix("canary")) {
 			res[k] = v
 		}
 	}
 
-	res["nginx.ingress.kubernetes.io/canary"] = "true"
-	res["nginx.ingress.kubernetes.io/canary-weight"] = "0"
+	res[i.GetAnnotationWithPrefix("canary")] = "true"
+	res[i.GetAnnotationWithPrefix("canary-weight")] = "0"
 
 	if cookie != "" {
-		res["nginx.ingress.kubernetes.io/canary-by-cookie"] = cookie
+		res[i.GetAnnotationWithPrefix("canary-by-cookie")] = cookie
 	}
 
 	if header != "" {
-		res["nginx.ingress.kubernetes.io/canary-by-header"] = header
+		res[i.GetAnnotationWithPrefix("canary-by-header")] = header
 	}
 
 	if headerValue != "" {
-		res["nginx.ingress.kubernetes.io/canary-by-header-value"] = headerValue
+		res[i.GetAnnotationWithPrefix("canary-by-header-value")] = headerValue
 	}
 
 	return res
 }
+
+func (i *IngressRouter) GetAnnotationWithPrefix(suffix string) string {
+	return fmt.Sprintf("%v/%v", i.annotationsPrefix, suffix)
+}

pkg/router/ingress_test.go

@@ -2,15 +2,17 @@ package router
 
 import (
 	"fmt"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func TestIngressRouter_Reconcile(t *testing.T) {
 	mocks := setupfakeClients()
 	router := &IngressRouter{
-		logger:     mocks.logger,
-		kubeClient: mocks.kubeClient,
+		logger:            mocks.logger,
+		kubeClient:        mocks.kubeClient,
+		annotationsPrefix: "custom.ingress.kubernetes.io",
 	}
 
 	err := router.Reconcile(mocks.ingressCanary)
@@ -18,8 +20,8 @@ func TestIngressRouter_Reconcile(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	canaryAn := "nginx.ingress.kubernetes.io/canary"
-	canaryWeightAn := "nginx.ingress.kubernetes.io/canary-weight"
+	canaryAn := "custom.ingress.kubernetes.io/canary"
+	canaryWeightAn := "custom.ingress.kubernetes.io/canary-weight"
 
 	canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name)
 	inCanary, err := router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{})
@@ -44,8 +46,9 @@ func TestIngressRouter_Reconcile(t *testing.T) {
 func TestIngressRouter_GetSetRoutes(t *testing.T) {
 	mocks := setupfakeClients()
 	router := &IngressRouter{
-		logger:     mocks.logger,
-		kubeClient: mocks.kubeClient,
+		logger:            mocks.logger,
+		kubeClient:        mocks.kubeClient,
+		annotationsPrefix: "prefix1.nginx.ingress.kubernetes.io",
 	}
 
 	err := router.Reconcile(mocks.ingressCanary)
@@ -53,21 +56,22 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	p, c, err := router.GetRoutes(mocks.ingressCanary)
+	p, c, m, err := router.GetRoutes(mocks.ingressCanary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
 	p = 50
 	c = 50
+	m = false
 
-	err = router.SetRoutes(mocks.ingressCanary, p, c)
+	err = router.SetRoutes(mocks.ingressCanary, p, c, m)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	canaryAn := "nginx.ingress.kubernetes.io/canary"
-	canaryWeightAn := "nginx.ingress.kubernetes.io/canary-weight"
+	canaryAn := "prefix1.nginx.ingress.kubernetes.io/canary"
+	canaryWeightAn := "prefix1.nginx.ingress.kubernetes.io/canary-weight"
 
 	canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name)
 	inCanary, err := router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{})
@@ -90,8 +94,9 @@ func TestIngressRouter_GetSetRoutes(t *testing.T) {
 
 	p = 100
 	c = 0
+	m = false
 
-	err = router.SetRoutes(mocks.ingressCanary, p, c)
+	err = router.SetRoutes(mocks.ingressCanary, p, c, m)
 	if err != nil {
 		t.Fatal(err.Error())
 	}

pkg/router/istio.go

@@ -5,14 +5,15 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
-	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
-	istiov1alpha3 "github.com/weaveworks/flagger/pkg/apis/istio/v1alpha3"
-	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
 	"go.uber.org/zap"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
+	istiov1alpha3 "github.com/weaveworks/flagger/pkg/apis/istio/v1alpha3"
+	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
 )
 
 // IstioRouter is managing Istio virtual services
@@ -101,8 +102,6 @@ func (ir *IstioRouter) reconcileDestinationRule(canary *flaggerv1.Canary, name s
 
 func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
 	targetName := canary.Spec.TargetRef.Name
-	primaryName := fmt.Sprintf("%s-primary", targetName)
-	canaryName := fmt.Sprintf("%s-canary", targetName)
 
 	// set hosts and add the ClusterIP service host if it doesn't exists
 	hosts := canary.Spec.Service.Hosts
@@ -133,6 +132,8 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
 	}
 
 	// create destinations with primary weight 100% and canary weight 0%
+	primaryName := fmt.Sprintf("%s-primary", targetName)
+	canaryName := fmt.Sprintf("%s-canary", targetName)
 	canaryRoute := []istiov1alpha3.DestinationWeight{
 		makeDestination(canary, primaryName, 100),
 		makeDestination(canary, canaryName, 0),
@@ -210,9 +211,14 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
 		return fmt.Errorf("VirtualService %s.%s query error %v", targetName, canary.Namespace, err)
 	}
 
-	// update service but keep the original destination weights
+	// update service but keep the original destination weights and mirror
 	if virtualService != nil {
-		if diff := cmp.Diff(newSpec, virtualService.Spec, cmpopts.IgnoreFields(istiov1alpha3.DestinationWeight{}, "Weight")); diff != "" {
+		if diff := cmp.Diff(
+			newSpec,
+			virtualService.Spec,
+			cmpopts.IgnoreFields(istiov1alpha3.DestinationWeight{}, "Weight"),
+			cmpopts.IgnoreFields(istiov1alpha3.HTTPRoute{}, "Mirror"),
+		); diff != "" {
 			vtClone := virtualService.DeepCopy()
 			vtClone.Spec = newSpec
 
@@ -232,6 +238,7 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error {
 func (ir *IstioRouter) GetRoutes(canary *flaggerv1.Canary) (
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 	err error,
 ) {
 	targetName := canary.Spec.TargetRef.Name
@@ -264,6 +271,9 @@ func (ir *IstioRouter) GetRoutes(canary *flaggerv1.Canary) (
 			canaryWeight = route.Weight
 		}
 	}
+	if httpRoute.Mirror != nil && httpRoute.Mirror.Host != "" {
+		mirrored = true
+	}
 
 	if primaryWeight == 0 && canaryWeight == 0 {
 		err = fmt.Errorf("VirtualService %s.%s does not contain routes for %s-primary and %s-canary",
@@ -278,6 +288,7 @@ func (ir *IstioRouter) SetRoutes(
 	canary *flaggerv1.Canary,
 	primaryWeight int,
 	canaryWeight int,
+	mirrored bool,
 ) error {
 	targetName := canary.Spec.TargetRef.Name
 	primaryName := fmt.Sprintf("%s-primary", targetName)
@@ -310,6 +321,12 @@ func (ir *IstioRouter) SetRoutes(
 		},
 	}
 
+	if mirrored {
+		vsCopy.Spec.Http[0].Mirror = &istiov1alpha3.Destination{
+			Host: canaryName,
+		}
+	}
+
 	// fix routing (A/B testing)
 	if len(canary.Spec.CanaryAnalysis.Match) > 0 {
 		// merge the common routes with the canary ones

pkg/router/istio_test.go

@@ -2,9 +2,11 @@ package router
 
 import (
 	"fmt"
-	istiov1alpha3 "github.com/weaveworks/flagger/pkg/apis/istio/v1alpha3"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	istiov1alpha3 "github.com/weaveworks/flagger/pkg/apis/istio/v1alpha3"
 )
 
 func TestIstioRouter_Sync(t *testing.T) {
@@ -119,15 +121,16 @@ func TestIstioRouter_SetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	p, c, err := router.GetRoutes(mocks.canary)
+	p, c, m, err := router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	p = 50
-	c = 50
+	p = 60
+	c = 40
+	m = false
 
-	err = router.SetRoutes(mocks.canary, p, c)
+	err = router.SetRoutes(mocks.canary, p, c, m)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -137,16 +140,20 @@ func TestIstioRouter_SetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
+	pHost := fmt.Sprintf("%s-primary", mocks.canary.Spec.TargetRef.Name)
+	cHost := fmt.Sprintf("%s-canary", mocks.canary.Spec.TargetRef.Name)
 	pRoute := istiov1alpha3.DestinationWeight{}
 	cRoute := istiov1alpha3.DestinationWeight{}
+	var mirror *istiov1alpha3.Destination
 
 	for _, http := range vs.Spec.Http {
 		for _, route := range http.Route {
-			if route.Destination.Host == fmt.Sprintf("%s-primary", mocks.canary.Spec.TargetRef.Name) {
+			if route.Destination.Host == pHost {
 				pRoute = route
 			}
-			if route.Destination.Host == fmt.Sprintf("%s-canary", mocks.canary.Spec.TargetRef.Name) {
+			if route.Destination.Host == cHost {
 				cRoute = route
+				mirror = http.Mirror
 			}
 		}
 	}
@@ -158,6 +165,51 @@ func TestIstioRouter_SetRoutes(t *testing.T) {
 	if cRoute.Weight != c {
 		t.Errorf("Got canary weight %v wanted %v", cRoute.Weight, c)
 	}
+
+	if mirror != nil {
+		t.Errorf("Got mirror %v wanted nil", mirror)
+	}
+
+	mirror = nil
+	p = 100
+	c = 0
+	m = true
+
+	err = router.SetRoutes(mocks.canary, p, c, m)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	vs, err = mocks.meshClient.NetworkingV1alpha3().VirtualServices("default").Get("podinfo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	for _, http := range vs.Spec.Http {
+		for _, route := range http.Route {
+			if route.Destination.Host == pHost {
+				pRoute = route
+			}
+			if route.Destination.Host == cHost {
+				cRoute = route
+				mirror = http.Mirror
+			}
+		}
+	}
+
+	if pRoute.Weight != p {
+		t.Errorf("Got primary weight %v wanted %v", pRoute.Weight, p)
+	}
+
+	if cRoute.Weight != c {
+		t.Errorf("Got canary weight %v wanted %v", cRoute.Weight, c)
+	}
+
+	if mirror == nil {
+		t.Errorf("Got mirror nil wanted a mirror")
+	} else if mirror.Host != cHost {
+		t.Errorf("Got mirror host \"%v\" wanted \"%v\"", mirror.Host, cHost)
+	}
 }
 
 func TestIstioRouter_GetRoutes(t *testing.T) {
@@ -174,7 +226,7 @@ func TestIstioRouter_GetRoutes(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	p, c, err := router.GetRoutes(mocks.canary)
+	p, c, m, err := router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -186,6 +238,74 @@ func TestIstioRouter_GetRoutes(t *testing.T) {
 	if c != 0 {
 		t.Errorf("Got canary weight %v wanted %v", c, 0)
 	}
+
+	if m != false {
+		t.Errorf("Got mirror %v wanted %v", m, false)
+	}
+
+	mocks.canary = newMockMirror()
+
+	err = router.Reconcile(mocks.canary)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	p, c, m, err = router.GetRoutes(mocks.canary)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if p != 100 {
+		t.Errorf("Got primary weight %v wanted %v", p, 100)
+	}
+
+	if c != 0 {
+		t.Errorf("Got canary weight %v wanted %v", c, 0)
+	}
+
+	// A Canary resource with mirror on does not automatically create mirroring
+	// in the virtual server (mirroring is activated as a temporary stage).
+	if m != false {
+		t.Errorf("Got mirror %v wanted %v", m, false)
+	}
+
+	// Adjust vs to activate mirroring.
+	vs, err := mocks.meshClient.NetworkingV1alpha3().VirtualServices("default").Get("podinfo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	cHost := fmt.Sprintf("%s-canary", mocks.canary.Spec.TargetRef.Name)
+	for i, http := range vs.Spec.Http {
+		for _, route := range http.Route {
+			if route.Destination.Host == cHost {
+				vs.Spec.Http[i].Mirror = &istiov1alpha3.Destination{
+					Host: cHost,
+				}
+			}
+		}
+	}
+	_, err = mocks.meshClient.NetworkingV1alpha3().VirtualServices(mocks.canary.Namespace).Update(vs)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	p, c, m, err = router.GetRoutes(mocks.canary)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if p != 100 {
+		t.Errorf("Got primary weight %v wanted %v", p, 100)
+	}
+
+	if c != 0 {
+		t.Errorf("Got canary weight %v wanted %v", c, 0)
+	}
+
+	if m != true {
+		t.Errorf("Got mirror %v wanted %v", m, true)
+	}
 }
 
 func TestIstioRouter_HTTPRequestHeaders(t *testing.T) {
@@ -276,8 +396,9 @@ func TestIstioRouter_ABTest(t *testing.T) {
 
 	p := 0
 	c := 100
+	m := false
 
-	err = router.SetRoutes(mocks.abtest, p, c)
+	err = router.SetRoutes(mocks.abtest, p, c, m)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -287,16 +408,20 @@ func TestIstioRouter_ABTest(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
+	pHost := fmt.Sprintf("%s-primary", mocks.abtest.Spec.TargetRef.Name)
+	cHost := fmt.Sprintf("%s-canary", mocks.abtest.Spec.TargetRef.Name)
 	pRoute := istiov1alpha3.DestinationWeight{}
 	cRoute := istiov1alpha3.DestinationWeight{}
+	var mirror *istiov1alpha3.Destination
 
 	for _, http := range vs.Spec.Http {
 		for _, route := range http.Route {
-			if route.Destination.Host == fmt.Sprintf("%s-primary", mocks.abtest.Spec.TargetRef.Name) {
+			if route.Destination.Host == pHost {
 				pRoute = route
 			}
-			if route.Destination.Host == fmt.Sprintf("%s-canary", mocks.abtest.Spec.TargetRef.Name) {
+			if route.Destination.Host == cHost {
 				cRoute = route
+				mirror = http.Mirror
 			}
 		}
 	}
@@ -308,4 +433,8 @@ func TestIstioRouter_ABTest(t *testing.T) {
 	if cRoute.Weight != c {
 		t.Errorf("Got canary weight %v wanted %v", cRoute.Weight, c)
 	}
+
+	if mirror != nil {
+		t.Errorf("Got mirror %v wanted nil", mirror)
+	}
 }

pkg/router/kubernetes.go

@@ -2,9 +2,9 @@ package router
 
 import (
 	"fmt"
+
 	"github.com/google/go-cmp/cmp"
-	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
-	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -12,6 +12,9 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/kubernetes"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
+	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
 )
 
 // KubernetesRouter is managing ClusterIP services
@@ -64,18 +67,24 @@ func (c *KubernetesRouter) reconcileService(canary *flaggerv1.Canary, name strin
 		portName = "http"
 	}
 
+	targetPort := intstr.IntOrString{
+		Type:   intstr.Int,
+		IntVal: canary.Spec.Service.Port,
+	}
+
+	if canary.Spec.Service.TargetPort.String() != "0" {
+		targetPort = canary.Spec.Service.TargetPort
+	}
+
 	svcSpec := corev1.ServiceSpec{
 		Type:     corev1.ServiceTypeClusterIP,
 		Selector: map[string]string{c.label: target},
 		Ports: []corev1.ServicePort{
 			{
-				Name:     portName,
-				Protocol: corev1.ProtocolTCP,
-				Port:     canary.Spec.Service.Port,
-				TargetPort: intstr.IntOrString{
-					Type:   intstr.Int,
-					IntVal: canary.Spec.Service.Port,
-				},
+				Name:       portName,
+				Protocol:   corev1.ProtocolTCP,
+				Port:       canary.Spec.Service.Port,
+				TargetPort: targetPort,
 			},
 		},
 	}
@@ -128,7 +137,10 @@ func (c *KubernetesRouter) reconcileService(canary *flaggerv1.Canary, name strin
 	}
 
 	if svc != nil {
-		portsDiff := cmp.Diff(svcSpec.Ports, svc.Spec.Ports)
+		sortPorts := func(a, b interface{}) bool {
+			return a.(corev1.ServicePort).Port < b.(corev1.ServicePort).Port
+		}
+		portsDiff := cmp.Diff(svcSpec.Ports, svc.Spec.Ports, cmpopts.SortSlices(sortPorts))
 		selectorsDiff := cmp.Diff(svcSpec.Selector, svc.Spec.Selector)
 
 		if portsDiff != "" || selectorsDiff != "" {

pkg/router/kubernetes_test.go

@@ -1,8 +1,9 @@
 package router
 
 import (
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func TestServiceRouter_Create(t *testing.T) {

pkg/router/nop.go

@@ -12,13 +12,13 @@ func (*NopRouter) Reconcile(canary *flaggerv1.Canary) error {
 	return nil
 }
 
-func (*NopRouter) SetRoutes(canary *flaggerv1.Canary, primaryWeight int, canaryWeight int) error {
+func (*NopRouter) SetRoutes(canary *flaggerv1.Canary, primaryWeight int, canaryWeight int, mirror bool) error {
 	return nil
 }
 
-func (*NopRouter) GetRoutes(canary *flaggerv1.Canary) (primaryWeight int, canaryWeight int, err error) {
+func (*NopRouter) GetRoutes(canary *flaggerv1.Canary) (primaryWeight int, canaryWeight int, mirror bool, err error) {
 	if canary.Status.Iterations > 0 {
-		return 0, 100, nil
+		return 0, 100, false, nil
 	}
-	return 100, 0, nil
+	return 100, 0, false, nil
 }

pkg/router/router.go

@@ -4,6 +4,6 @@ import flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 
 type Interface interface {
 	Reconcile(canary *flaggerv1.Canary) error
-	SetRoutes(canary *flaggerv1.Canary, primaryWeight int, canaryWeight int) error
-	GetRoutes(canary *flaggerv1.Canary) (primaryWeight int, canaryWeight int, err error)
+	SetRoutes(canary *flaggerv1.Canary, primaryWeight int, canaryWeight int, mirrored bool) error
+	GetRoutes(canary *flaggerv1.Canary) (primaryWeight int, canaryWeight int, mirrored bool, err error)
 }

pkg/router/router_test.go

@@ -1,12 +1,6 @@
 package router
 
 import (
-	"github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
-	istiov1alpha1 "github.com/weaveworks/flagger/pkg/apis/istio/common/v1alpha1"
-	istiov1alpha3 "github.com/weaveworks/flagger/pkg/apis/istio/v1alpha3"
-	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
-	fakeFlagger "github.com/weaveworks/flagger/pkg/client/clientset/versioned/fake"
-	"github.com/weaveworks/flagger/pkg/logger"
 	"go.uber.org/zap"
 	appsv1 "k8s.io/api/apps/v1"
 	hpav1 "k8s.io/api/autoscaling/v1"
@@ -16,13 +10,20 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/fake"
+
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
+	istiov1alpha1 "github.com/weaveworks/flagger/pkg/apis/istio/common/v1alpha1"
+	istiov1alpha3 "github.com/weaveworks/flagger/pkg/apis/istio/v1alpha3"
+	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
+	fakeFlagger "github.com/weaveworks/flagger/pkg/client/clientset/versioned/fake"
+	"github.com/weaveworks/flagger/pkg/logger"
 )
 
 type fakeClients struct {
-	canary        *v1alpha3.Canary
-	abtest        *v1alpha3.Canary
-	appmeshCanary *v1alpha3.Canary
-	ingressCanary *v1alpha3.Canary
+	canary        *flaggerv1.Canary
+	abtest        *flaggerv1.Canary
+	appmeshCanary *flaggerv1.Canary
+	ingressCanary *flaggerv1.Canary
 	kubeClient    kubernetes.Interface
 	meshClient    clientset.Interface
 	flaggerClient clientset.Interface
@@ -53,28 +54,28 @@ func setupfakeClients() fakeClients {
 	}
 }
 
-func newMockCanaryAppMesh() *v1alpha3.Canary {
-	cd := &v1alpha3.Canary{
-		TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()},
+func newMockCanaryAppMesh() *flaggerv1.Canary {
+	cd := &flaggerv1.Canary{
+		TypeMeta: metav1.TypeMeta{APIVersion: flaggerv1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: "default",
 			Name:      "appmesh",
 		},
-		Spec: v1alpha3.CanarySpec{
+		Spec: flaggerv1.CanarySpec{
 			TargetRef: hpav1.CrossVersionObjectReference{
 				Name:       "podinfo",
 				APIVersion: "apps/v1",
 				Kind:       "Deployment",
 			},
-			Service: v1alpha3.CanaryService{
+			Service: flaggerv1.CanaryService{
 				Port:     9898,
 				MeshName: "global",
 				Backends: []string{"backend.default"},
-			}, CanaryAnalysis: v1alpha3.CanaryAnalysis{
+			}, CanaryAnalysis: flaggerv1.CanaryAnalysis{
 				Threshold:  10,
 				StepWeight: 10,
 				MaxWeight:  50,
-				Metrics: []v1alpha3.CanaryMetric{
+				Metrics: []flaggerv1.CanaryMetric{
 					{
 						Name:      "appmesh_requests_total",
 						Threshold: 99,
@@ -87,20 +88,20 @@ func newMockCanaryAppMesh() *v1alpha3.Canary {
 	return cd
 }
 
-func newMockCanary() *v1alpha3.Canary {
-	cd := &v1alpha3.Canary{
-		TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()},
+func newMockCanary() *flaggerv1.Canary {
+	cd := &flaggerv1.Canary{
+		TypeMeta: metav1.TypeMeta{APIVersion: flaggerv1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: "default",
 			Name:      "podinfo",
 		},
-		Spec: v1alpha3.CanarySpec{
+		Spec: flaggerv1.CanarySpec{
 			TargetRef: hpav1.CrossVersionObjectReference{
 				Name:       "podinfo",
 				APIVersion: "apps/v1",
 				Kind:       "Deployment",
 			},
-			Service: v1alpha3.CanaryService{
+			Service: flaggerv1.CanaryService{
 				Port: 9898,
 				Headers: &istiov1alpha3.Headers{
 					Request: &istiov1alpha3.HeaderOperations{
@@ -115,11 +116,11 @@ func newMockCanary() *v1alpha3.Canary {
 						"POST",
 					},
 				},
-			}, CanaryAnalysis: v1alpha3.CanaryAnalysis{
+			}, CanaryAnalysis: flaggerv1.CanaryAnalysis{
 				Threshold:  10,
 				StepWeight: 10,
 				MaxWeight:  50,
-				Metrics: []v1alpha3.CanaryMetric{
+				Metrics: []flaggerv1.CanaryMetric{
 					{
 						Name:      "istio_requests_total",
 						Threshold: 99,
@@ -137,22 +138,28 @@ func newMockCanary() *v1alpha3.Canary {
 	return cd
 }
 
-func newMockABTest() *v1alpha3.Canary {
-	cd := &v1alpha3.Canary{
-		TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()},
+func newMockMirror() *flaggerv1.Canary {
+	cd := newMockCanary()
+	cd.Spec.CanaryAnalysis.Mirror = true
+	return cd
+}
+
+func newMockABTest() *flaggerv1.Canary {
+	cd := &flaggerv1.Canary{
+		TypeMeta: metav1.TypeMeta{APIVersion: flaggerv1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: "default",
 			Name:      "abtest",
 		},
-		Spec: v1alpha3.CanarySpec{
+		Spec: flaggerv1.CanarySpec{
 			TargetRef: hpav1.CrossVersionObjectReference{
 				Name:       "abtest",
 				APIVersion: "apps/v1",
 				Kind:       "Deployment",
 			},
-			Service: v1alpha3.CanaryService{
+			Service: flaggerv1.CanaryService{
 				Port: 9898,
-			}, CanaryAnalysis: v1alpha3.CanaryAnalysis{
+			}, CanaryAnalysis: flaggerv1.CanaryAnalysis{
 				Threshold:  10,
 				Iterations: 2,
 				Match: []istiov1alpha3.HTTPMatchRequest{
@@ -164,7 +171,7 @@ func newMockABTest() *v1alpha3.Canary {
 						},
 					},
 				},
-				Metrics: []v1alpha3.CanaryMetric{
+				Metrics: []flaggerv1.CanaryMetric{
 					{
 						Name:      "istio_requests_total",
 						Threshold: 99,
@@ -272,14 +279,14 @@ func newMockABTestDeployment() *appsv1.Deployment {
 	return d
 }
 
-func newMockCanaryIngress() *v1alpha3.Canary {
-	cd := &v1alpha3.Canary{
-		TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()},
+func newMockCanaryIngress() *flaggerv1.Canary {
+	cd := &flaggerv1.Canary{
+		TypeMeta: metav1.TypeMeta{APIVersion: flaggerv1.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: "default",
 			Name:      "nginx",
 		},
-		Spec: v1alpha3.CanarySpec{
+		Spec: flaggerv1.CanarySpec{
 			TargetRef: hpav1.CrossVersionObjectReference{
 				Name:       "podinfo",
 				APIVersion: "apps/v1",
@@ -290,13 +297,13 @@ func newMockCanaryIngress() *v1alpha3.Canary {
 				APIVersion: "extensions/v1beta1",
 				Kind:       "Ingress",
 			},
-			Service: v1alpha3.CanaryService{
+			Service: flaggerv1.CanaryService{
 				Port: 9898,
-			}, CanaryAnalysis: v1alpha3.CanaryAnalysis{
+			}, CanaryAnalysis: flaggerv1.CanaryAnalysis{
 				Threshold:  10,
 				StepWeight: 10,
 				MaxWeight:  50,
-				Metrics: []v1alpha3.CanaryMetric{
+				Metrics: []flaggerv1.CanaryMetric{
 					{
 						Name:      "request-success-rate",
 						Threshold: 99,