Merge pull request #297 from weaveworks/prep-0.18.4

Release v0.18.4

.circleci/config.yml

@@ -78,6 +78,17 @@ jobs:
       - run: test/e2e-istio.sh
       - run: test/e2e-tests.sh
 
+  e2e-kubernetes-testing:
+    machine: true
+    steps:
+      - checkout
+      - attach_workspace:
+          at: /tmp/bin
+      - run: test/container-build.sh
+      - run: test/e2e-kind.sh
+      - run: test/e2e-kubernetes.sh
+      - run: test/e2e-kubernetes-tests.sh
+
   e2e-smi-istio-testing:
     machine: true
     steps:
@@ -121,6 +132,9 @@ jobs:
       - run: test/e2e-kind.sh
       - run: test/e2e-nginx.sh
       - run: test/e2e-nginx-tests.sh
+      - run: test/e2e-nginx-cleanup.sh
+      - run: test/e2e-nginx-custom-annotations.sh
+      - run: test/e2e-nginx-tests.sh
 
   e2e-linkerd-testing:
     machine: true
@@ -133,6 +147,47 @@ jobs:
       - run: test/e2e-linkerd.sh
       - run: test/e2e-linkerd-tests.sh
 
+  push-helm-charts:
+    docker:
+      - image: circleci/golang:1.12
+    steps:
+      - checkout
+      - run:
+          name: Install kubectl
+          command: sudo curl -L https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl -o /usr/local/bin/kubectl && sudo chmod +x /usr/local/bin/kubectl
+      - run:
+          name: Install helm
+          command: sudo curl -L https://storage.googleapis.com/kubernetes-helm/helm-v2.14.2-linux-amd64.tar.gz | tar xz && sudo mv linux-amd64/helm /bin/helm && sudo rm -rf linux-amd64
+      - run:
+          name: Initialize helm
+          command:  helm init --client-only --kubeconfig=$HOME/.kube/kubeconfig
+      - run:
+          name: Lint charts
+          command: |
+            helm lint ./charts/*
+      - run:
+          name: Package charts
+          command: |
+            mkdir $HOME/charts
+            helm package ./charts/* --destination $HOME/charts
+      - run:
+          name: Publish charts
+          command: |
+            if echo "${CIRCLE_TAG}" | grep -Eq "[0-9]+(\.[0-9]+)*(-[a-z]+)?$"; then
+              REPOSITORY="https://weaveworksbot:${GITHUB_TOKEN}@github.com/weaveworks/flagger.git"
+              git config user.email weaveworksbot@users.noreply.github.com
+              git config user.name weaveworksbot
+              git remote set-url origin ${REPOSITORY}
+              git checkout gh-pages
+              mv -f $HOME/charts/*.tgz .
+              helm repo index . --url https://flagger.app
+              git add .
+              git commit -m "Publish Helm charts v${CIRCLE_TAG}"
+              git push origin gh-pages
+            else
+              echo "Not a release! Skip charts publish"
+            fi
+
 workflows:
   version: 2
   build-test-push:
@@ -145,12 +200,12 @@ workflows:
       - e2e-istio-testing:
           requires:
             - build-binary
-      - e2e-smi-istio-testing:
-          requires:
-            - build-binary
-      - e2e-supergloo-testing:
+      - e2e-kubernetes-testing:
           requires:
             - build-binary
+#      - e2e-supergloo-testing:
+#          requires:
+#            - build-binary
       - e2e-gloo-testing:
           requires:
             - build-binary
@@ -164,8 +219,8 @@ workflows:
           requires:
             - build-binary
             - e2e-istio-testing
-            - e2e-smi-istio-testing
-            - e2e-supergloo-testing
+            - e2e-kubernetes-testing
+            #- e2e-supergloo-testing
             - e2e-gloo-testing
             - e2e-nginx-testing
             - e2e-linkerd-testing
@@ -187,6 +242,14 @@ workflows:
             tags:
               ignore: /^chart.*/
       - push-binary:
+          requires:
+            - push-container
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              ignore: /^chart.*/
+      - push-helm-charts:
           requires:
             - push-container
           filters:

CHANGELOG.md

@@ -2,6 +2,78 @@
 
 All notable changes to this project are documented in this file.
 
+## 0.18.4 (2019-09-08) 
+
+Adds support for NGINX custom annotations and Helm v3 acceptance testing
+
+#### Features
+
+- Add annotations prefix for NGINX ingresses [#293](https://github.com/weaveworks/flagger/pull/293)
+- Add wide columns in CRD [#289](https://github.com/weaveworks/flagger/pull/289)
+- loadtester: implement Helm v3 test command [#296](https://github.com/weaveworks/flagger/pull/296)
+- loadtester: add gPRC health check to load tester image [#295](https://github.com/weaveworks/flagger/pull/295)
+
+#### Fixes
+
+- loadtester: fix tests error logging [#286](https://github.com/weaveworks/flagger/pull/286)
+
+## 0.18.3 (2019-08-22) 
+
+Adds support for tillerless helm tests and protobuf health checking
+
+#### Features
+
+- loadtester: add support for tillerless helm [#280](https://github.com/weaveworks/flagger/pull/280)
+- loadtester: add support for protobuf health checking [#280](https://github.com/weaveworks/flagger/pull/280)
+
+#### Improvements 
+
+- Set HTTP listeners for AppMesh virtual routers [#272](https://github.com/weaveworks/flagger/pull/272)
+
+#### Fixes
+
+- Add missing fields to CRD validation spec [#271](https://github.com/weaveworks/flagger/pull/271)
+- Fix App Mesh backends validation in CRD [#281](https://github.com/weaveworks/flagger/pull/281)
+
+## 0.18.2 (2019-08-05) 
+
+Fixes multi-port support for Istio
+
+#### Fixes
+
+- Fix port discovery for multiple port services [#267](https://github.com/weaveworks/flagger/pull/267)
+
+#### Improvements 
+
+- Update e2e testing to Istio v1.2.3, Gloo v0.18.8 and NGINX ingress chart v1.12.1 [#268](https://github.com/weaveworks/flagger/pull/268)
+
+## 0.18.1 (2019-07-30) 
+
+Fixes Blue/Green style deployments for Kubernetes and Linkerd providers
+
+#### Fixes
+
+- Fix Blue/Green metrics provider and add e2e tests [#261](https://github.com/weaveworks/flagger/pull/261)
+
+## 0.18.0 (2019-07-29) 
+
+Adds support for [manual gating](https://docs.flagger.app/how-it-works#manual-gating) and pausing/resuming an ongoing analysis
+
+#### Features
+
+- Implement confirm rollout gate, hook and API [#251](https://github.com/weaveworks/flagger/pull/251)
+
+#### Improvements 
+
+- Refactor canary change detection and status [#240](https://github.com/weaveworks/flagger/pull/240)
+- Implement finalising state [#257](https://github.com/weaveworks/flagger/pull/257)
+- Add gRPC load testing tool [#248](https://github.com/weaveworks/flagger/pull/248)
+
+#### Breaking changes
+
+- Due to the status sub-resource changes in [#240](https://github.com/weaveworks/flagger/pull/240), when upgrading Flagger the canaries status phase will be reset to `Initialized`
+- Upgrading Flagger with Helm will fail due to Helm poor support of CRDs, see [workaround](https://github.com/weaveworks/flagger/issues/223)
+
 ## 0.17.0 (2019-07-08) 
 
 Adds support for Linkerd (SMI Traffic Split API), MS Teams notifications and HA mode with leader election

Dockerfile.loadtester

@@ -9,14 +9,33 @@ WORKDIR /home/app
 RUN curl -sSLo hey "https://storage.googleapis.com/jblabs/dist/hey_linux_v0.1.2" && \
 chmod +x hey && mv hey /usr/local/bin/hey
 
-RUN curl -sSL "https://get.helm.sh/helm-v2.12.3-linux-amd64.tar.gz" | tar xvz && \
+RUN curl -sSL "https://get.helm.sh/helm-v2.14.3-linux-amd64.tar.gz" | tar xvz && \
 chmod +x linux-amd64/helm && mv linux-amd64/helm /usr/local/bin/helm && \
+chmod +x linux-amd64/tiller && mv linux-amd64/tiller /usr/local/bin/tiller && \
 rm -rf linux-amd64
 
+RUN curl -sSL "https://get.helm.sh/helm-v3.0.0-beta.3-linux-amd64.tar.gz" | tar xvz && \
+chmod +x linux-amd64/helm && mv linux-amd64/helm /usr/local/bin/helmv3 && \
+rm -rf linux-amd64
+
+RUN GRPC_HEALTH_PROBE_VERSION=v0.3.0 && \
+wget -qO /usr/local/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
+chmod +x /usr/local/bin/grpc_health_probe
+
+RUN curl -sSL "https://github.com/bojand/ghz/releases/download/v0.39.0/ghz_0.39.0_Linux_x86_64.tar.gz" | tar xz -C /tmp && \
+mv /tmp/ghz /usr/local/bin && chmod +x /usr/local/bin/ghz && rm -rf /tmp/ghz-web
+
+ADD https://raw.githubusercontent.com/grpc/grpc-proto/master/grpc/health/v1/health.proto /tmp/ghz/health.proto
+
+RUN ls /tmp
+
 COPY ./bin/loadtester .
 
 RUN chown -R app:app ./
 
 USER app
 
-ENTRYPOINT ["./loadtester"]
+RUN curl -sSL "https://github.com/rimusz/helm-tiller/archive/v0.8.3.tar.gz" | tar xvz && \
+helm init --client-only && helm plugin install helm-tiller-0.8.3 && helm plugin list
+
+ENTRYPOINT ["./loadtester"]

Makefile

@@ -63,8 +63,9 @@ test: test-fmt test-codegen
 
 helm-package:
 	cd charts/ && helm package ./*
-	mv charts/*.tgz docs/
-	helm repo index docs --url https://weaveworks.github.io/flagger --merge ./docs/index.yaml
+	mv charts/*.tgz bin/
+	curl -s https://raw.githubusercontent.com/weaveworks/flagger/gh-pages/index.yaml > ./bin/index.yaml
+	helm repo index bin --url https://flagger.app --merge ./bin/index.yaml
 
 helm-up:
 	helm upgrade --install flagger ./charts/flagger --namespace=istio-system --set crd.create=false

README.md

@@ -35,6 +35,7 @@ Flagger documentation can be found at [docs.flagger.app](https://docs.flagger.ap
   * [Custom metrics](https://docs.flagger.app/how-it-works#custom-metrics)
   * [Webhooks](https://docs.flagger.app/how-it-works#webhooks)
   * [Load testing](https://docs.flagger.app/how-it-works#load-testing)
+  * [Manual gating](https://docs.flagger.app/how-it-works#manual-gating)
   * [FAQ](https://docs.flagger.app/faq)
 * Usage
   * [Istio canary deployments](https://docs.flagger.app/usage/progressive-delivery)

artifacts/ab-testing/deployment.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: quay.io/stefanprodan/podinfo:1.7.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/appmesh/deployment.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: quay.io/stefanprodan/podinfo:2.0.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/canaries/deployment.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: stefanprodan/podinfo:2.0.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/cluster/releases/test/backend.yaml

@@ -5,17 +5,17 @@ metadata:
   namespace: test
   annotations:
     flux.weave.works/automated: "true"
-    flux.weave.works/tag.chart-image: regexp:^1.4.*
+    flux.weave.works/tag.chart-image: regexp:^1.7.*
 spec:
   releaseName: backend
   chart:
     repository: https://flagger.app/
     name: podinfo
-    version: 2.0.0
+    version: 2.2.0
   values:
     image:
       repository: quay.io/stefanprodan/podinfo
-      tag: 1.4.0
+      tag: 1.7.0
       httpServer:
         timeout: 30s
       canary:

artifacts/cluster/releases/test/frontend.yaml

@@ -5,17 +5,17 @@ metadata:
   namespace: test
   annotations:
     flux.weave.works/automated: "true"
-    flux.weave.works/tag.chart-image: semver:~1.4
+    flux.weave.works/tag.chart-image: semver:~1.7
 spec:
   releaseName: frontend
   chart:
     repository: https://flagger.app/
     name: podinfo
-    version: 2.0.0
+    version: 2.2.0
   values:
     image:
       repository: quay.io/stefanprodan/podinfo
-      tag: 1.4.0
+      tag: 1.7.0
       backend: http://backend-podinfo:9898/echo
       canary:
         enabled: true

artifacts/cluster/releases/test/loadtester.yaml

@@ -11,8 +11,8 @@ spec:
   chart:
     repository: https://flagger.app/
     name: loadtester
-    version: 0.1.0
+    version: 0.6.0
   values:
     image:
-      repository: quay.io/stefanprodan/flagger-loadtester
-      tag: 0.1.0
+      repository: weaveworks/flagger-loadtester
+      tag: 0.6.1

artifacts/configs/canary.yaml

@@ -1,59 +0,0 @@
-apiVersion: flagger.app/v1alpha3
-kind: Canary
-metadata:
-  name: podinfo
-  namespace: test
-spec:
-  # deployment reference
-  targetRef:
-    apiVersion: apps/v1
-    kind: Deployment
-    name: podinfo
-  # the maximum time in seconds for the canary deployment
-  # to make progress before it is rollback (default 600s)
-  progressDeadlineSeconds: 60
-  # HPA reference (optional)
-  autoscalerRef:
-    apiVersion: autoscaling/v2beta1
-    kind: HorizontalPodAutoscaler
-    name: podinfo
-  service:
-    # container port
-    port: 9898
-    # Istio gateways (optional)
-    gateways:
-    - public-gateway.istio-system.svc.cluster.local
-    - mesh
-    # Istio virtual service host names (optional)
-    hosts:
-    - app.iowa.weavedx.com
-  canaryAnalysis:
-    # schedule interval (default 60s)
-    interval: 10s
-    # max number of failed metric checks before rollback
-    threshold: 10
-    # max traffic percentage routed to canary
-    # percentage (0-100)
-    maxWeight: 50
-    # canary increment step
-    # percentage (0-100)
-    stepWeight: 5
-    # Istio Prometheus checks
-    metrics:
-    - name: request-success-rate
-      # minimum req success rate (non 5xx responses)
-      # percentage (0-100)
-      threshold: 99
-      interval: 1m
-    - name: request-duration
-      # maximum req duration P99
-      # milliseconds
-      threshold: 500
-      interval: 30s
-    # external checks (optional)
-    webhooks:
-      - name: load-test
-        url: http://flagger-loadtester.test/
-        timeout: 5s
-        metadata:
-          cmd: "hey -z 1m -q 10 -c 2 http://podinfo.test:9898/"

artifacts/configs/configs.yaml

@@ -1,16 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: podinfo-config-env
-  namespace: test
-data:
-  color: blue
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: podinfo-config-vol
-  namespace: test
-data:
-  output: console
-  textmode: "true"

artifacts/configs/deployment.yaml

@@ -1,89 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: podinfo
-  namespace: test
-  labels:
-    app: podinfo
-spec:
-  minReadySeconds: 5
-  revisionHistoryLimit: 5
-  progressDeadlineSeconds: 60
-  strategy:
-    rollingUpdate:
-      maxUnavailable: 0
-    type: RollingUpdate
-  selector:
-    matchLabels:
-      app: podinfo
-  template:
-    metadata:
-      annotations:
-        prometheus.io/scrape: "true"
-      labels:
-        app: podinfo
-    spec:
-      containers:
-      - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.3.0
-        imagePullPolicy: IfNotPresent
-        ports:
-        - containerPort: 9898
-          name: http
-          protocol: TCP
-        command:
-        - ./podinfo
-        - --port=9898
-        - --level=info
-        - --random-delay=false
-        - --random-error=false
-        env:
-        - name: PODINFO_UI_COLOR
-          valueFrom:
-            configMapKeyRef:
-              name: podinfo-config-env
-              key: color
-        - name: SECRET_USER
-          valueFrom:
-            secretKeyRef:
-              name: podinfo-secret-env
-              key: user
-        livenessProbe:
-          exec:
-            command:
-            - podcli
-            - check
-            - http
-            - localhost:9898/healthz
-          initialDelaySeconds: 5
-          timeoutSeconds: 5
-        readinessProbe:
-          exec:
-            command:
-            - podcli
-            - check
-            - http
-            - localhost:9898/readyz
-          initialDelaySeconds: 5
-          timeoutSeconds: 5
-        resources:
-          limits:
-            cpu: 2000m
-            memory: 512Mi
-          requests:
-            cpu: 100m
-            memory: 64Mi
-        volumeMounts:
-          - name: configs
-            mountPath: /etc/podinfo/configs
-            readOnly: true
-          - name: secrets
-            mountPath: /etc/podinfo/secrets
-            readOnly: true
-      volumes:
-        - name: configs
-          configMap:
-            name: podinfo-config-vol
-        - name: secrets
-          secret:
-            secretName: podinfo-secret-vol

artifacts/configs/secrets.yaml

@@ -1,16 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: podinfo-secret-env
-  namespace: test
-data:
-  password: cGFzc3dvcmQ=
-  user: YWRtaW4=
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: podinfo-secret-vol
-  namespace: test
-data:
-  key: cGFzc3dvcmQ=

artifacts/flagger/crd.yaml

@@ -33,6 +33,22 @@ spec:
     - name: Weight
       type: string
       JSONPath: .status.canaryWeight
+    - name: FailedChecks
+      type: string
+      JSONPath: .status.failedChecks
+      priority: 1
+    - name: Interval
+      type: string
+      JSONPath: .spec.canaryAnalysis.interval
+      priority: 1
+    - name: StepWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.stepWeight
+      priority: 1
+    - name: MaxWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.maxWeight
+      priority: 1
     - name: LastTransitionTime
       type: string
       JSONPath: .status.lastTransitionTime
@@ -46,12 +62,15 @@ spec:
             - canaryAnalysis
           properties:
             provider:
+              description: Traffic managent provider
               type: string
             progressDeadlineSeconds:
+              description: Deployment progress deadline
               type: number
             targetRef:
+              description: Deployment selector
               type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -60,10 +79,11 @@ spec:
                 name:
                   type: string
             autoscalerRef:
+              description: HPA selector
               anyOf:
                 - type: string
                 - type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -72,10 +92,11 @@ spec:
                 name:
                   type: string
             ingressRef:
+              description: NGINX ingress selector
               anyOf:
                 - type: string
                 - type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -85,88 +106,197 @@ spec:
                   type: string
             service:
               type: object
-              required: ['port']
+              required: ["port"]
               properties:
                 port:
+                  description: Container port number
                   type: number
                 portName:
+                  description: Container port name
                   type: string
                 portDiscovery:
+                  description: Enable port dicovery
                   type: boolean
                 meshName:
+                  description: AppMesh mesh name
                   type: string
+                backends:
+                  description: AppMesh backend array
+                  anyOf:
+                    - type: string
+                    - type: array
                 timeout:
+                  description: Istio HTTP or gRPC request timeout
                   type: string
+                trafficPolicy:
+                  description: Istio traffic policy
+                  anyOf:
+                    - type: string
+                    - type: object
+                match:
+                  description: Istio URL match conditions
+                  anyOf:
+                    - type: string
+                    - type: array
+                rewrite:
+                  description: Istio URL rewrite
+                  anyOf:
+                    - type: string
+                    - type: object
+                headers:
+                  description: Istio headers operations
+                  anyOf:
+                    - type: string
+                    - type: object
+                corsPolicy:
+                  description: Istio CORS policy
+                  anyOf:
+                    - type: string
+                    - type: object
+                gateways:
+                  description: Istio gateways list
+                  anyOf:
+                    - type: string
+                    - type: array
+                hosts:
+                  description: Istio hosts list
+                  anyOf:
+                    - type: string
+                    - type: array
             skipAnalysis:
               type: boolean
             canaryAnalysis:
               properties:
                 interval:
+                  description: Canary schedule interval
                   type: string
                   pattern: "^[0-9]+(m|s)"
                 iterations:
+                  description: Number of checks to run for A/B Testing and Blue/Green
                   type: number
                 threshold:
+                  description: Max number of failed checks before rollback
                   type: number
                 maxWeight:
+                  description: Max traffic percentage routed to canary
                   type: number
                 stepWeight:
+                  description: Canary incremental traffic percentage step
                   type: number
+                match:
+                  description: A/B testing match conditions
+                  anyOf:
+                    - type: string
+                    - type: array
                 metrics:
+                  description: Prometheus query list for this canary
                   type: array
                   properties:
                     items:
                       type: object
-                      required: ['name', 'threshold']
+                      required: ["name", "threshold"]
                       properties:
                         name:
+                          description: Name of the Prometheus metric
                           type: string
                         interval:
+                          description: Interval of the promql query
                           type: string
                           pattern: "^[0-9]+(m|s)"
                         threshold:
+                          description: Max scalar value accepted for this metric
                           type: number
                         query:
+                          description: Prometheus query
                           type: string
                 webhooks:
+                  description: Webhook list for this canary
                   type: array
                   properties:
                     items:
                       type: object
-                      required: ['name', 'url', 'timeout']
+                      required: ["name", "url", "timeout"]
                       properties:
                         name:
+                          description: Name of the webhook
                           type: string
                         type:
+                          description: Type of the webhook pre, post or during rollout
                           type: string
                           enum:
                             - ""
+                            - confirm-rollout
                             - pre-rollout
                             - rollout
                             - post-rollout
                         url:
+                          description: URL address of this webhook
                           type: string
                           format: url
                         timeout:
+                          description: Request timeout for this webhook
                           type: string
                           pattern: "^[0-9]+(m|s)"
+                        metadata:
+                          description: Metadata (key-value pairs) for this webhook
+                          anyOf:
+                            - type: string
+                            - type: object
         status:
           properties:
             phase:
+              description: Analysis phase of this canary
               type: string
               enum:
                 - ""
+                - Initializing
                 - Initialized
+                - Waiting
                 - Progressing
+                - Finalising
                 - Succeeded
                 - Failed
             canaryWeight:
+              description: Traffic weight percentage routed to canary
               type: number
             failedChecks:
+              description: Failed check count of the current canary analysis
               type: number
             iterations:
+              description: Iteration count of the current canary analysis
               type: number
             lastAppliedSpec:
+              description: LastAppliedSpec of this canary
               type: string
             lastTransitionTime:
+              description: LastTransitionTime of this canary
+              format: date-time
               type: string
+            conditions:
+              description: Status conditions of this canary
+              type: array
+              properties:
+                items:
+                  type: object
+                  required: ["type", "status", "reason"]
+                  properties:
+                    lastTransitionTime:
+                      description: LastTransitionTime of this condition
+                      format: date-time
+                      type: string
+                    lastUpdateTime:
+                      description: LastUpdateTime of this condition
+                      format: date-time
+                      type: string
+                    message:
+                      description: Message associated with this condition
+                      type: string
+                    reason:
+                      description: Reason for the current status of this condition
+                      type: string
+                    status:
+                      description: Status of this condition
+                      type: string
+                    type:
+                      description: Type of this condition
+                      type: string

artifacts/flagger/deployment.yaml

@@ -22,7 +22,7 @@ spec:
       serviceAccountName: flagger
       containers:
       - name: flagger
-        image: weaveworks/flagger:0.17.0
+        image: weaveworks/flagger:0.18.4
         imagePullPolicy: IfNotPresent
         ports:
         - name: http

artifacts/gloo/deployment.yaml

@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: quay.io/stefanprodan/podinfo:1.7.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/helmtester/deployment.yaml

@@ -19,7 +19,7 @@ spec:
       serviceAccountName: tiller
       containers:
         - name: helmtester
-          image: weaveworks/flagger-loadtester:0.4.0
+          image: weaveworks/flagger-loadtester:0.8.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

artifacts/loadtester/deployment.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: loadtester
-          image: weaveworks/flagger-loadtester:0.4.0
+          image: weaveworks/flagger-loadtester:0.8.0
           imagePullPolicy: IfNotPresent
           ports:
             - name: http

artifacts/nginx/deployment.yaml

@@ -23,7 +23,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: quay.io/stefanprodan/podinfo:1.7.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898

artifacts/routing/destination-rule.yaml

@@ -1,45 +0,0 @@
-apiVersion: networking.istio.io/v1alpha3
-kind: VirtualService
-metadata:
-  name: podinfo
-  namespace: test
-spec:
-  gateways:
-    - public-gateway.istio-system.svc.cluster.local
-    - mesh
-  hosts:
-    - podinfo.istio.weavedx.com
-    - podinfo
-  http:
-    - route:
-        - destination:
-            host: podinfo
-            subset: primary
-          weight: 50
-        - destination:
-            host: podinfo
-            subset: canary
-          weight: 50
----
-apiVersion: networking.istio.io/v1alpha3
-kind: DestinationRule
-metadata:
-  name: podinfo-destination
-  namespace: test
-spec:
-  host: podinfo
-  trafficPolicy:
-    loadBalancer:
-      consistentHash:
-        httpCookie:
-          name: istiouser
-          ttl: 30s
-  subsets:
-    - name: primary
-      labels:
-        app: podinfo
-        role: primary
-    - name: canary
-      labels:
-        app: podinfo
-        role: canary

artifacts/routing/match.yaml

@@ -1,43 +0,0 @@
-apiVersion: networking.istio.io/v1alpha3
-kind: VirtualService
-metadata:
-  name: podinfo
-  namespace: test
-spec:
-  gateways:
-  - public-gateway.istio-system.svc.cluster.local
-  - mesh
-  hosts:
-  - app.istio.weavedx.com
-  - podinfo
-  http:
-  - match:
-    - headers:
-        user-agent:
-          regex: ^(?!.*Chrome)(?=.*\bSafari\b).*$
-      uri:
-        prefix: "/version/"
-    rewrite:
-      uri: /api/info
-    route:
-    - destination:
-        host: podinfo-primary
-        port:
-          number: 9898
-      weight: 0
-    - destination:
-        host: podinfo
-        port:
-          number: 9898
-      weight: 100
-  - match:
-      - uri:
-          prefix: "/version/"
-    rewrite:
-      uri: /api/info
-    route:
-    - destination:
-        host: podinfo-primary
-        port:
-          number: 9898
-      weight: 100

artifacts/routing/mirror.yaml

@@ -1,25 +0,0 @@
-apiVersion: networking.istio.io/v1alpha3
-kind: VirtualService
-metadata:
-  name: podinfo
-  namespace: test
-  labels:
-    app: podinfo
-spec:
-  gateways:
-  - public-gateway.istio-system.svc.cluster.local
-  - mesh
-  hosts:
-  - podinfo.iowa.weavedx.com
-  - podinfo
-  http:
-  - route:
-    - destination:
-        host: podinfo-primary
-        port:
-          number: 9898
-      weight: 100
-    mirror:
-      host: podinfo
-      port:
-        number: 9898

artifacts/routing/weight.yaml

@@ -1,26 +0,0 @@
-apiVersion: networking.istio.io/v1alpha3
-kind: VirtualService
-metadata:
-  name: podinfo
-  namespace: test
-  labels:
-    app: podinfo
-spec:
-  gateways:
-  - public-gateway.istio-system.svc.cluster.local
-  - mesh
-  hosts:
-  - podinfo.iowa.weavedx.com
-  - podinfo
-  http:
-  - route:
-    - destination:
-        host: podinfo-primary
-        port:
-          number: 9898
-      weight: 100
-    - destination:
-        host: podinfo
-        port:
-          number: 9898
-      weight: 0

artifacts/workloads/canary-destination.yaml

@@ -1,13 +0,0 @@
-apiVersion: networking.istio.io/v1alpha3
-kind: DestinationRule
-metadata:
-  name: podinfo-canary
-  namespace: test
-spec:
-  host: podinfo-canary
-  trafficPolicy:
-    loadBalancer:
-      consistentHash:
-        httpCookie:
-          name: user
-          ttl: 0s

artifacts/workloads/canary-hpa.yaml

@@ -1,23 +0,0 @@
-apiVersion: autoscaling/v2beta1
-kind: HorizontalPodAutoscaler
-metadata:
-  name: podinfo
-  namespace: test
-spec:
-  scaleTargetRef:
-    apiVersion: apps/v1
-    kind: Deployment
-    name: podinfo
-  minReplicas: 2
-  maxReplicas: 2
-  metrics:
-  - type: Resource
-    resource:
-      name: cpu
-      # scale up if usage is above
-      # 99% of the requested CPU (100m)
-      targetAverageUtilization: 99
-  - type: Resource
-    resource:
-      name: memory
-      targetAverageValue: 200Mi

artifacts/workloads/canary-service.yaml

@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: podinfo-canary
-  namespace: test
-spec:
-  type: ClusterIP
-  selector:
-    app: podinfo
-  ports:
-  - name: http
-    port: 9898
-    protocol: TCP
-    targetPort: http

artifacts/workloads/primary-deployment.yaml

@@ -1,71 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: podinfo-primary
-  namespace: test
-  labels:
-    app: podinfo-primary
-spec:
-  replicas: 1
-  strategy:
-    rollingUpdate:
-      maxUnavailable: 0
-    type: RollingUpdate
-  selector:
-    matchLabels:
-      app: podinfo-primary
-  template:
-    metadata:
-      annotations:
-        prometheus.io/scrape: "true"
-      labels:
-        app: podinfo-primary
-    spec:
-      containers:
-      - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.1
-        imagePullPolicy: IfNotPresent
-        ports:
-        - containerPort: 9898
-          name: http
-          protocol: TCP
-        command:
-        - ./podinfo
-        - --port=9898
-        - --level=info
-        - --random-delay=false
-        - --random-error=false
-        env:
-        - name: PODINFO_UI_COLOR
-          value: blue
-        livenessProbe:
-          exec:
-            command:
-            - podcli
-            - check
-            - http
-            - localhost:9898/healthz
-          initialDelaySeconds: 5
-          failureThreshold: 3
-          periodSeconds: 10
-          successThreshold: 1
-          timeoutSeconds: 1
-        readinessProbe:
-          exec:
-            command:
-            - podcli
-            - check
-            - http
-            - localhost:9898/readyz
-          initialDelaySeconds: 5
-          failureThreshold: 3
-          periodSeconds: 3
-          successThreshold: 1
-          timeoutSeconds: 1
-        resources:
-          limits:
-            cpu: 2000m
-            memory: 512Mi
-          requests:
-            cpu: 10m
-            memory: 64Mi

artifacts/workloads/primary-destination.yaml

@@ -1,13 +0,0 @@
-apiVersion: networking.istio.io/v1alpha3
-kind: DestinationRule
-metadata:
-  name: podinfo-primary
-  namespace: test
-spec:
-  host: podinfo-primary
-  trafficPolicy:
-    loadBalancer:
-      consistentHash:
-        httpCookie:
-          name: user
-          ttl: 0s

artifacts/workloads/primary-hpa.yaml

@@ -1,19 +0,0 @@
-apiVersion: autoscaling/v2beta1
-kind: HorizontalPodAutoscaler
-metadata:
-  name: podinfo-primary
-  namespace: test
-spec:
-  scaleTargetRef:
-    apiVersion: apps/v1
-    kind: Deployment
-    name: podinfo-primary
-  minReplicas: 2
-  maxReplicas: 2
-  metrics:
-  - type: Resource
-    resource:
-      name: cpu
-      # scale up if usage is above
-      # 99% of the requested CPU (100m)
-      targetAverageUtilization: 99

artifacts/workloads/primary-service.yaml

@@ -1,16 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: podinfo-primary
-  namespace: test
-  labels:
-    app: podinfo-primary
-spec:
-  type: ClusterIP
-  selector:
-    app: podinfo-primary
-  ports:
-  - name: http
-    port: 9898
-    protocol: TCP
-    targetPort: http

artifacts/workloads/service.yaml

@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: podinfo
-  namespace: test
-spec:
-  type: ClusterIP
-  selector:
-    app: podinfo-primary
-  ports:
-  - name: http
-    port: 9898
-    protocol: TCP
-    targetPort: http

artifacts/workloads/virtual-service.yaml

@@ -1,23 +0,0 @@
-apiVersion: networking.istio.io/v1alpha3
-kind: VirtualService
-metadata:
-  name: podinfo
-  namespace: test
-  labels:
-    app: podinfo
-spec:
-  gateways:
-  - mesh
-  - public-gateway.istio-system.svc.cluster.local
-  hosts:
-  - podinfo
-  - app.istio.weavedx.com
-  http:
-  - route:
-    - destination:
-        host: podinfo-primary
-      weight: 50
-    - destination:
-        host: podinfo-canary
-      weight: 50
-    timeout: 5s

charts/flagger/Chart.yaml

@@ -1,10 +1,10 @@
 apiVersion: v1
 name: flagger
-version: 0.17.0
-appVersion: 0.17.0
+version: 0.18.4
+appVersion: 0.18.4
 kubeVersion: ">=1.11.0-0"
 engine: gotpl
-description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio, App Mesh or NGINX routing for traffic shifting and Prometheus metrics for canary analysis.
+description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio, Linkerd, App Mesh, Gloo or NGINX routing for traffic shifting and Prometheus metrics for canary analysis.
 home: https://docs.flagger.app
 icon: https://raw.githubusercontent.com/weaveworks/flagger/master/docs/logo/flagger-icon.png
 sources:
@@ -17,4 +17,5 @@ keywords:
 - canary
 - istio
 - appmesh
+- linkerd
 - gitops

charts/flagger/README.md

@@ -1,10 +1,10 @@
 # Flagger
 
 [Flagger](https://github.com/weaveworks/flagger) is a Kubernetes operator that automates the promotion of 
-canary deployments using Istio, App Mesh, NGINX or Gloo routing for traffic shifting and Prometheus metrics for canary analysis. 
+canary deployments using Istio, Linkerd, App Mesh, NGINX or Gloo routing for traffic shifting and Prometheus metrics for canary analysis. 
 Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance indicators
 like HTTP requests success rate, requests average duration and pods health.
-Based on the KPIs analysis a canary is promoted or aborted and the analysis result is published to Slack.
+Based on the KPIs analysis a canary is promoted or aborted and the analysis result is published to Slack or MS Teams.
 
 ## Prerequisites
 
@@ -16,7 +16,13 @@ Based on the KPIs analysis a canary is promoted or aborted and the analysis resu
 Add Flagger Helm repository:
 
 ```console
-helm repo add flagger https://flagger.app
+$ helm repo add flagger https://flagger.app
+```
+
+Install Flagger's custom resource definitions:
+
+```console
+$ kubectl apply -f https://raw.githubusercontent.com/weaveworks/flagger/master/artifacts/flagger/crd.yaml
 ```
 
 To install the chart with the release name `flagger` for Istio:
@@ -24,6 +30,7 @@ To install the chart with the release name `flagger` for Istio:
 ```console
 $ helm upgrade -i flagger flagger/flagger \
     --namespace=istio-system \
+    --set crd.create=false \
     --set meshProvider=istio \
     --set metricsServer=http://prometheus:9090
 ```
@@ -33,6 +40,7 @@ To install the chart with the release name `flagger` for Linkerd:
 ```console
 $ helm upgrade -i flagger flagger/flagger \
     --namespace=linkerd \
+    --set crd.create=false \
     --set meshProvider=linkerd \
     --set metricsServer=http://linkerd-prometheus:9090
 ```
@@ -66,6 +74,7 @@ Parameter | Description | Default
 `msteams.url` | Microsoft Teams incoming webhook | None
 `leaderElection.enabled` | leader election must be enabled when running more than one replica | `false`
 `leaderElection.replicaCount` | number of replicas | `1`
+`ingressAnnotationsPrefix` | annotations prefix for ingresses | `custom.ingress.kubernetes.io`
 `rbac.create` | if `true`, create and use RBAC resources | `true`
 `rbac.pspEnabled` | If `true`, create and use a restricted pod security policy | `false`
 `crd.create` | if `true`, create Flagger's CRDs | `true`
@@ -82,6 +91,7 @@ Specify each parameter using the `--set key=value[,key=value]` argument to `helm
 ```console
 $ helm upgrade -i flagger flagger/flagger \
   --namespace istio-system \
+  --set crd.create=false \
   --set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \
   --set slack.channel=general
 ```

charts/flagger/templates/crd.yaml

@@ -34,6 +34,22 @@ spec:
     - name: Weight
       type: string
       JSONPath: .status.canaryWeight
+    - name: FailedChecks
+      type: string
+      JSONPath: .status.failedChecks
+      priority: 1
+    - name: Interval
+      type: string
+      JSONPath: .spec.canaryAnalysis.interval
+      priority: 1
+    - name: StepWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.stepWeight
+      priority: 1
+    - name: MaxWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.maxWeight
+      priority: 1
     - name: LastTransitionTime
       type: string
       JSONPath: .status.lastTransitionTime
@@ -47,10 +63,13 @@ spec:
             - canaryAnalysis
           properties:
             provider:
+              description: Traffic managent provider
               type: string
             progressDeadlineSeconds:
+              description: Deployment progress deadline
               type: number
             targetRef:
+              description: Deployment selector
               type: object
               required: ['apiVersion', 'kind', 'name']
               properties:
@@ -61,6 +80,7 @@ spec:
                 name:
                   type: string
             autoscalerRef:
+              description: HPA selector
               anyOf:
                 - type: string
                 - type: object
@@ -73,6 +93,7 @@ spec:
                 name:
                   type: string
             ingressRef:
+              description: NGINX ingress selector
               anyOf:
                 - type: string
                 - type: object
@@ -89,31 +110,87 @@ spec:
               required: ['port']
               properties:
                 port:
+                  description: Container port number
                   type: number
                 portName:
+                  description: Container port name
                   type: string
                 portDiscovery:
+                  description: Enable port dicovery
                   type: boolean
                 meshName:
+                  description: AppMesh mesh name
                   type: string
+                backends:
+                  description: AppMesh backend array
+                  anyOf:
+                    - type: string
+                    - type: array
                 timeout:
+                  description: Istio HTTP or gRPC request timeout
                   type: string
+                trafficPolicy:
+                  description: Istio traffic policy
+                  anyOf:
+                    - type: string
+                    - type: object
+                match:
+                  description: Istio URL match conditions
+                  anyOf:
+                    - type: string
+                    - type: array
+                rewrite:
+                  description: Istio URL rewrite
+                  anyOf:
+                    - type: string
+                    - type: object
+                headers:
+                  description: Istio headers operations
+                  anyOf:
+                    - type: string
+                    - type: object
+                corsPolicy:
+                  description: Istio CORS policy
+                  anyOf:
+                    - type: string
+                    - type: object
+                gateways:
+                  description: Istio gateways list
+                  anyOf:
+                    - type: string
+                    - type: array
+                hosts:
+                  description: Istio hosts list
+                  anyOf:
+                    - type: string
+                    - type: array
             skipAnalysis:
               type: boolean
             canaryAnalysis:
               properties:
                 interval:
+                  description: Canary schedule interval
                   type: string
                   pattern: "^[0-9]+(m|s)"
                 iterations:
+                  description: Number of checks to run for A/B Testing and Blue/Green
                   type: number
                 threshold:
+                  description: Max number of failed checks before rollback
                   type: number
                 maxWeight:
+                  description: Max traffic percentage routed to canary
                   type: number
                 stepWeight:
+                  description: Canary incremental traffic percentage step
                   type: number
+                match:
+                  description: A/B testing match conditions
+                  anyOf:
+                    - type: string
+                    - type: array
                 metrics:
+                  description: Prometheus query list for this canary
                   type: array
                   properties:
                     items:
@@ -121,15 +198,20 @@ spec:
                       required: ['name', 'threshold']
                       properties:
                         name:
+                          description: Name of the Prometheus metric
                           type: string
                         interval:
+                          description: Interval of the promql query
                           type: string
                           pattern: "^[0-9]+(m|s)"
                         threshold:
+                          description: Max scalar value accepted for this metric
                           type: number
                         query:
+                          description: Prometheus query
                           type: string
                 webhooks:
+                  description: Webhook list for this canary
                   type: array
                   properties:
                     items:
@@ -137,38 +219,86 @@ spec:
                       required: ['name', 'url', 'timeout']
                       properties:
                         name:
+                          description: Name of the webhook
                           type: string
                         type:
+                          description: Type of the webhook pre, post or during rollout
                           type: string
                           enum:
                             - ""
+                            - confirm-rollout
                             - pre-rollout
                             - rollout
                             - post-rollout
                         url:
+                          description: URL address of this webhook
                           type: string
                           format: url
                         timeout:
+                          description: Request timeout for this webhook
                           type: string
                           pattern: "^[0-9]+(m|s)"
+                        metadata:
+                          description: Metadata (key-value pairs) for this webhook
+                          anyOf:
+                            - type: string
+                            - type: object
         status:
           properties:
             phase:
+              description: Analysis phase of this canary
               type: string
               enum:
                 - ""
+                - Initializing
                 - Initialized
+                - Waiting
                 - Progressing
+                - Finalising
                 - Succeeded
                 - Failed
             canaryWeight:
+              description: Traffic weight percentage routed to canary
               type: number
             failedChecks:
+              description: Failed check count of the current canary analysis
               type: number
             iterations:
+              description: Iteration count of the current canary analysis
               type: number
             lastAppliedSpec:
+              description: LastAppliedSpec of this canary
               type: string
             lastTransitionTime:
+              description: LastTransitionTime of this canary
+              format: date-time
               type: string
+            conditions:
+              description: Status conditions of this canary
+              type: array
+              properties:
+                items:
+                  type: object
+                  required: ['type', 'status', 'reason']
+                  properties:
+                    lastTransitionTime:
+                      description: LastTransitionTime of this condition
+                      format: date-time
+                      type: string
+                    lastUpdateTime:
+                      description: LastUpdateTime of this condition
+                      format: date-time
+                      type: string
+                    message:
+                      description: Message associated with this condition
+                      type: string
+                    reason:
+                      description: Reason for the current status of this condition
+                      type: string
+                    status:
+                      description: Status of this condition
+                      type: string
+                    type:
+                      description: Type of this condition
+                      type: string
 {{- end }}

charts/flagger/templates/deployment.yaml

@@ -32,6 +32,10 @@ spec:
                     app.kubernetes.io/name: {{ template "flagger.name" . }}
                     app.kubernetes.io/instance: {{ .Release.Name }}
                 topologyKey: kubernetes.io/hostname
+      {{- if .Values.image.pullSecret }}
+      imagePullSecrets:
+        - name: {{ .Values.image.pullSecret }}
+      {{- end }}
       containers:
         - name: flagger
           securityContext:
@@ -68,6 +72,9 @@ spec:
           - -enable-leader-election=true
           - -leader-election-namespace={{ .Release.Namespace }}
           {{- end }}
+          {{- if .Values.ingressAnnotationsPrefix }}
+          - -ingress-annotations-prefix={{ .Values.ingressAnnotationsPrefix }}
+          {{- end }}
           livenessProbe:
             exec:
               command:

charts/flagger/values.yaml

@@ -2,8 +2,9 @@
 
 image:
   repository: weaveworks/flagger
-  tag: 0.17.0
+  tag: 0.18.4
   pullPolicy: IfNotPresent
+  pullSecret:
 
 metricsServer: "http://prometheus:9090"
 

charts/loadtester/Chart.yaml

@@ -1,10 +1,10 @@
 apiVersion: v1
 name: loadtester
-version: 0.4.1
-appVersion: 0.4.0
+version: 0.8.0
+appVersion: 0.8.0
 kubeVersion: ">=1.11.0-0"
 engine: gotpl
-description: Flagger's load testing services based on rakyll/hey that generates traffic during canary analysis when configured as a webhook.
+description: Flagger's load testing services based on rakyll/hey and bojand/ghz that generates traffic during canary analysis when configured as a webhook.
 home: https://docs.flagger.app
 icon: https://raw.githubusercontent.com/weaveworks/flagger/master/docs/logo/flagger-icon.png
 sources:

charts/loadtester/templates/deployment.yaml

@@ -21,6 +21,8 @@ spec:
     spec:
       {{- if .Values.serviceAccountName }}
       serviceAccountName: {{ .Values.serviceAccountName }}
+      {{- else if .Values.rbac.create }}
+      serviceAccountName: {{ include "loadtester.fullname" . }}
       {{- end }}
       containers:
         - name: {{ .Chart.Name }}

charts/loadtester/templates/rbac.yaml

@@ -0,0 +1,54 @@
+---
+{{- if .Values.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+{{- if eq .Values.rbac.scope "cluster" }}
+kind: ClusterRole
+{{- else }}
+kind: Role
+{{- end }}
+metadata:
+  name: {{ template "loadtester.fullname" . }}
+  labels:
+    helm.sh/chart: {{ template "loadtester.chart" . }}
+    app.kubernetes.io/name: {{ template "loadtester.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+rules:
+{{ toYaml .Values.rbac.rules | indent 2 }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+{{- if eq .Values.rbac.scope "cluster" }}
+kind: ClusterRoleBinding
+{{- else }}
+kind: RoleBinding
+{{- end }}
+metadata:
+  name: {{ template "loadtester.fullname" . }}
+  labels:
+    helm.sh/chart: {{ template "loadtester.chart" . }}
+    app.kubernetes.io/name: {{ template "loadtester.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  {{- if eq .Values.rbac.scope "cluster" }}
+  kind: ClusterRole
+  {{- else }}
+  kind: Role
+  {{- end }}
+  name: {{ template "loadtester.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "loadtester.fullname" . }}
+    namespace: {{ .Release.Namespace }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "loadtester.fullname" . }}
+  labels:
+    helm.sh/chart: {{ template "loadtester.chart" . }}
+    app.kubernetes.io/name: {{ template "loadtester.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

charts/loadtester/values.yaml

@@ -2,7 +2,7 @@ replicaCount: 1
 
 image:
   repository: weaveworks/flagger-loadtester
-  tag: 0.4.0
+  tag: 0.8.0
   pullPolicy: IfNotPresent
 
 logLevel: info
@@ -27,6 +27,20 @@ tolerations: []
 
 affinity: {}
 
+rbac:
+  # rbac.create: `true` if rbac resources should be created
+  create: false
+  # rbac.scope: `cluster` to create cluster-scope rbac resources (ClusterRole/ClusterRoleBinding)
+  # otherwise, namespace-scope rbac resources will be created (Role/RoleBinding)
+  scope:
+  # rbac.rules: array of rules to apply to the role. example:
+  # rules:
+  # - apiGroups: [""]
+  #   resources: ["pods"]
+  #   verbs: ["list", "get"]
+  rules: []
+
+# name of an existing service account to use - if not creating rbac resources
 serviceAccountName: ""
 
 # App Mesh virtual node settings

charts/podinfo/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
-version: 2.2.0
-appVersion: 1.4.0
+version: 3.0.0
+appVersion: 2.0.0
 name: podinfo
 engine: gotpl
 description: Flagger canary deployment demo chart

charts/podinfo/templates/deployment.yaml

@@ -24,7 +24,7 @@ spec:
     spec:
       terminationGracePeriodSeconds: 30
       containers:
-        - name: {{ .Chart.Name }}
+        - name: podinfo
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           command:

charts/podinfo/values.yaml

@@ -1,7 +1,7 @@
 # Default values for podinfo.
 image:
-  repository: quay.io/stefanprodan/podinfo
-  tag: 1.4.0
+  repository: stefanprodan/podinfo
+  tag: 2.0.0
   pullPolicy: IfNotPresent
 
 service:

cmd/flagger/main.go

@@ -33,25 +33,26 @@ import (
 )
 
 var (
-	masterURL               string
-	kubeconfig              string
-	metricsServer           string
-	controlLoopInterval     time.Duration
-	logLevel                string
-	port                    string
-	msteamsURL              string
-	slackURL                string
-	slackUser               string
-	slackChannel            string
-	threadiness             int
-	zapReplaceGlobals       bool
-	zapEncoding             string
-	namespace               string
-	meshProvider            string
-	selectorLabels          string
-	enableLeaderElection    bool
-	leaderElectionNamespace string
-	ver                     bool
+	masterURL                string
+	kubeconfig               string
+	metricsServer            string
+	controlLoopInterval      time.Duration
+	logLevel                 string
+	port                     string
+	msteamsURL               string
+	slackURL                 string
+	slackUser                string
+	slackChannel             string
+	threadiness              int
+	zapReplaceGlobals        bool
+	zapEncoding              string
+	namespace                string
+	meshProvider             string
+	selectorLabels           string
+	ingressAnnotationsPrefix string
+	enableLeaderElection     bool
+	leaderElectionNamespace  string
+	ver                      bool
 )
 
 func init() {
@@ -71,6 +72,7 @@ func init() {
 	flag.StringVar(&namespace, "namespace", "", "Namespace that flagger would watch canary object.")
 	flag.StringVar(&meshProvider, "mesh-provider", "istio", "Service mesh provider, can be istio, linkerd, appmesh, supergloo, nginx or smi.")
 	flag.StringVar(&selectorLabels, "selector-labels", "app,name,app.kubernetes.io/name", "List of pod labels that Flagger uses to create pod selectors.")
+	flag.StringVar(&ingressAnnotationsPrefix, "ingress-annotations-prefix", "nginx.ingress.kubernetes.io", "Annotations prefix for ingresses.")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election.")
 	flag.StringVar(&leaderElectionNamespace, "leader-election-namespace", "kube-system", "Namespace used to create the leader election config map.")
 	flag.BoolVar(&ver, "version", false, "Print version")
@@ -175,7 +177,7 @@ func main() {
 	// start HTTP server
 	go server.ListenAndServe(port, 3*time.Second, logger, stopCh)
 
-	routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, logger, meshClient)
+	routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, ingressAnnotationsPrefix, logger, meshClient)
 
 	c := controller.NewController(
 		kubeClient,

cmd/loadtester/main.go

@@ -10,7 +10,7 @@ import (
 	"time"
 )
 
-var VERSION = "0.4.0"
+var VERSION = "0.8.0"
 var (
 	logLevel          string
 	port              string
@@ -47,5 +47,7 @@ func main() {
 	go taskRunner.Start(100*time.Millisecond, stopCh)
 
 	logger.Infof("Starting load tester v%s API on port %s", VERSION, port)
-	loadtester.ListenAndServe(port, time.Minute, logger, taskRunner, stopCh)
+
+	gateStorage := loadtester.NewGateStorage("in-memory")
+	loadtester.ListenAndServe(port, time.Minute, logger, taskRunner, gateStorage, stopCh)
 }

docs/gitbook/faq.md

@@ -232,8 +232,7 @@ spec:
         mode: ISTIO_MUTUAL
 ```
 
-Both port `8080` and `9090` will be added to the ClusterIP services but the virtual service
-will point to the port specified in `spec.service.port`.
+Both port `8080` and `9090` will be added to the ClusterIP services.
 
 ### Label selectors
 

docs/gitbook/how-it-works.md

@@ -105,6 +105,54 @@ convention you can specify your label with the `-selector-labels` flag.
 The target deployment should expose a TCP port that will be used by Flagger to create the ClusterIP Service and 
 the Istio Virtual Service. The container port from the target deployment should match the `service.port` value.
 
+### Canary status
+
+Get the current status of canary deployments cluster wide: 
+
+```bash
+kubectl get canaries --all-namespaces
+
+NAMESPACE   NAME      STATUS        WEIGHT   LASTTRANSITIONTIME
+test        podinfo   Progressing   15       2019-06-30T14:05:07Z
+prod        frontend  Succeeded     0        2019-06-30T16:15:07Z
+prod        backend   Failed        0        2019-06-30T17:05:07Z
+```
+
+The status condition reflects the last know state of the canary analysis:
+
+```bash
+kubectl -n test get canary/podinfo -oyaml | awk '/status/,0'
+```
+
+A successful rollout status:
+
+```yaml
+status:
+  canaryWeight: 0
+  failedChecks: 0
+  iterations: 0
+  lastAppliedSpec: "14788816656920327485"
+  lastPromotedSpec: "14788816656920327485"
+  conditions:
+  - lastTransitionTime: "2019-07-10T08:23:18Z"
+    lastUpdateTime: "2019-07-10T08:23:18Z"
+    message: Canary analysis completed successfully, promotion finished.
+    reason: Succeeded
+    status: "True"
+    type: Promoted
+```
+
+The `Promoted` status condition can have one of the following reasons:
+Initialized, Waiting, Progressing, Finalising, Succeeded or Failed.
+A failed canary will have the promoted status set to `false`,
+the reason to `failed` and the last applied spec will be different to the last promoted one.
+
+Wait for a successful rollout:
+
+```bash
+kubectl wait canary/podinfo --for=condition=promoted
+```
+
 ### Istio routing
 
 Flagger creates an Istio Virtual Service and Destination Rules based on the Canary service spec. 
@@ -608,6 +656,8 @@ The canary analysis can be extended with webhooks. Flagger will call each webhoo
 determine from the response status code (HTTP 2xx) if the canary is failing or not.
 
 There are three types of hooks:
+* Confirm-rollout hooks are executed before scaling up the canary deployment and ca be used for manual approval.
+The rollout is paused until the hook returns a successful HTTP status code.
 * Pre-rollout hooks are executed before routing traffic to canary. 
 The canary advancement is paused if a pre-rollout hook fails and if the number of failures reach the 
 threshold the canary will be rollback.
@@ -621,6 +671,9 @@ Spec:
 ```yaml
   canaryAnalysis:
     webhooks:
+      - name: "start gate"
+        type: confirm-rollout
+        url: http://flagger-loadtester.test/gate/approve
       - name: "smoke test"
         type: pre-rollout
         url: http://flagger-helmtester.kube-system/
@@ -677,7 +730,7 @@ that generates traffic during analysis when configured as a webhook.
 
 ![Flagger Load Testing Webhook](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-load-testing.png)
 
-First you need to deploy the load test runner in a namespace with Istio sidecar injection enabled:
+First you need to deploy the load test runner in a namespace with sidecar injection enabled:
 
 ```bash
 export REPO=https://raw.githubusercontent.com/weaveworks/flagger/master
@@ -720,7 +773,7 @@ When the canary analysis starts, Flagger will call the webhooks and the load tes
 in the background, if they are not already running. This will ensure that during the 
 analysis, the `podinfo-canary.test` service will receive a steady stream of GET and POST requests.
 
-If your workload is exposed outside the mesh with the Istio Gateway and TLS you can point `hey` to the 
+If your workload is exposed outside the mesh you can point `hey` to the 
 public URL and use HTTP2.
 
 ```yaml
@@ -733,6 +786,30 @@ webhooks:
       cmd: "hey -z 1m -q 10 -c 2 -h2 https://podinfo.example.com/"
 ```
 
+For gRPC services you can use [bojand/ghz](https://github.com/bojand/ghz) which is a similar tool to Hey but for gPRC:
+
+```yaml
+webhooks:
+  - name: grpc-load-test
+    url: http://flagger-loadtester.test/
+    timeout: 5s
+    metadata:
+      type: cmd
+      cmd: "ghz -z 1m -q 10 -c 2 --insecure podinfo.test:9898"
+```
+
+`ghz` uses reflection to identify which gRPC method to call. If you do not wish to enable reflection for your gRPC service you can implement a standardized health check from the [grpc-proto](https://github.com/grpc/grpc-proto) library. To use this [health check schema](https://github.com/grpc/grpc-proto/blob/master/grpc/health/v1/health.proto) without reflection you can pass a parameter to `ghz` like this
+
+```yaml
+webhooks:
+  - name: grpc-load-test-no-reflection
+    url: http://flagger-loadtester.test/
+    timeout: 5s
+    metadata:
+      type: cmd
+      cmd: "ghz --insecure --proto=/tmp/ghz/health.proto --call=grpc.health.v1.Health/Check podinfo.test:9898"
+```
+
 The load tester can run arbitrary commands as long as the binary is present in the container image.
 For example if you you want to replace `hey` with another CLI, you can create your own Docker image:
 
@@ -805,6 +882,20 @@ Now you can add pre-rollout webhooks to the canary analysis spec:
 When the canary analysis starts, Flagger will call the pre-rollout webhooks before routing traffic to the canary.
 If the helm test fails, Flagger will retry until the analysis threshold is reached and the canary is rolled back.
 
+If you are using Helm v3, you'll have to create a dedicated service account and add the release namespace to the test command:
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "smoke test"
+        type: pre-rollout
+        url: http://flagger-helmtester.kube-system/
+        timeout: 3m
+        metadata:
+          type: "helmv3"
+          cmd: "test run {{ .Release.Name }} --cleanup -n {{ .Release.Namespace }}"
+```
+
 As an alternative to Helm you can use the [Bash Automated Testing System](https://github.com/bats-core/bats-core) to run your tests. 
 
 ```yaml
@@ -820,3 +911,66 @@ As an alternative to Helm you can use the [Bash Automated Testing System](https:
 ```
 
 Note that you should create a ConfigMap with your Bats tests and mount it inside the tester container.
+
+### Manual Gating
+
+For manual approval of a canary deployment you can use the `confirm-rollout` webhook. 
+The confirmation hooks are executed before the pre-rollout hooks. 
+Flagger will halt the canary traffic shifting and analysis until the confirm webhook returns HTTP status 200.
+
+Manual gating with Flagger's tester:
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "gate"
+        type: confirm-rollout
+        url: http://flagger-loadtester.test/gate/halt
+```
+
+The `/gate/halt` returns HTTP 403 thus blocking the rollout. 
+
+If you have notifications enabled, Flagger will post a message to Slack or MS Teams if a canary rollout is waiting for approval.
+
+Change the URL to `/gate/approve` to start the canary analysis:
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "gate"
+        type: confirm-rollout
+        url: http://flagger-loadtester.test/gate/approve
+```
+
+Manual gating can be driven with Flagger's tester API. Set the confirmation URL to `/gate/check`:
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "ask for confirmation"
+        type: confirm-rollout
+        url: http://flagger-loadtester.test/gate/check
+```
+
+By default the gate is closed, you can start or resume the canary rollout with:
+
+```bash
+kubectl -n test exec -it flagger-loadtester-xxxx-xxxx sh
+
+curl -d '{"name": "podinfo","namespace":"test"}' http://localhost:8080/gate/open 
+```
+
+You can pause the rollout at any time with:
+
+```bash
+curl -d '{"name": "podinfo","namespace":"test"}' http://localhost:8080/gate/close 
+```
+
+If a canary analysis is paused the status will change to waiting:
+
+```bash
+kubectl get canary/podinfo
+
+NAME      STATUS        WEIGHT
+podinfo   Waiting       0
+```

docs/gitbook/install/flagger-install-on-eks-appmesh.md

@@ -133,11 +133,18 @@ Add Flagger Helm repository:
 helm repo add flagger https://flagger.app
 ```
 
+Install Flagger's Canary CRD:
+
+```yaml
+kubectl apply -f https://raw.githubusercontent.com/weaveworks/flagger/master/artifacts/flagger/crd.yaml
+```
+
 Deploy Flagger and Prometheus in the _**appmesh-system**_ namespace:
 
 ```bash
 helm upgrade -i flagger flagger/flagger \
 --namespace=appmesh-system \
+--set crd.create=false \
 --set meshProvider=appmesh \
 --set prometheus.install=true
 ```
@@ -150,6 +157,7 @@ You can enable **Slack** notifications with:
 ```bash
 helm upgrade -i flagger flagger/flagger \
 --namespace=appmesh-system \
+--set crd.create=false \
 --set meshProvider=appmesh \
 --set metricsServer=http://prometheus.appmesh:9090 \
 --set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \

docs/gitbook/install/flagger-install-on-google-cloud.md

@@ -186,7 +186,7 @@ Install cert-manager's CRDs:
 ```bash
 CERT_REPO=https://raw.githubusercontent.com/jetstack/cert-manager
 
-kubectl apply -f ${CERT_REPO}/release-0.7/deploy/manifests/00-crds.yaml
+kubectl apply -f ${CERT_REPO}/release-0.10/deploy/manifests/00-crds.yaml
 ```
 
 Create the cert-manager namespace and disable resource validation:
@@ -204,7 +204,7 @@ helm repo add jetstack https://charts.jetstack.io && \
 helm repo update && \
 helm upgrade -i cert-manager \
 --namespace cert-manager \
---version v0.7.0 \
+--version v0.10.0 \
 jetstack/cert-manager
 ```
 
@@ -339,7 +339,7 @@ Find the GKE Istio version with:
 kubectl -n istio-system get deploy istio-pilot -oyaml | grep image:
 ```
 
-Install Prometheus in istio-system namespace (replace `1.0.6-gke.3` with your version):
+Install Prometheus in istio-system namespace:
 
 ```bash
 kubectl -n istio-system apply -f \
@@ -354,11 +354,18 @@ Add Flagger Helm repository:
 helm repo add flagger https://flagger.app
 ```
 
+Install Flagger's Canary CRD:
+
+```yaml
+kubectl apply -f https://raw.githubusercontent.com/weaveworks/flagger/master/artifacts/flagger/crd.yaml
+```
+
 Deploy Flagger in the `istio-system` namespace with Slack notifications enabled:
 
 ```bash
 helm upgrade -i flagger flagger/flagger \
 --namespace=istio-system \
+--set crd.create=false \
 --set metricsServer=http://prometheus.istio-system:9090 \
 --set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \
 --set slack.channel=general \

docs/gitbook/install/flagger-install-on-kubernetes.md

@@ -17,11 +17,18 @@ Add Flagger Helm repository:
 helm repo add flagger https://flagger.app
 ```
 
+Install Flagger's Canary CRD:
+
+```yaml
+kubectl apply -f https://raw.githubusercontent.com/weaveworks/flagger/master/artifacts/flagger/crd.yaml
+```
+
 Deploy Flagger for Istio:
 
 ```bash
 helm upgrade -i flagger flagger/flagger \
 --namespace=istio-system \
+--set crd.create=false \
 --set meshProvider=istio \
 --set metricsServer=http://prometheus:9090
 ```
@@ -31,6 +38,7 @@ Deploy Flagger for Linkerd:
 ```bash
 helm upgrade -i flagger flagger/flagger \
 --namespace=linkerd \
+--set crd.create=false \
 --set meshProvider=linkerd \
 --set metricsServer=http://linkerd-prometheus:9090
 ```
@@ -42,6 +50,7 @@ Enable **Slack** notifications:
 ```bash
 helm upgrade -i flagger flagger/flagger \
 --namespace=istio-system \
+--set crd.create=false \
 --set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \
 --set slack.channel=general \
 --set slack.user=flagger
@@ -52,6 +61,7 @@ Enable **Microsoft Teams** notifications:
 ```bash
 helm upgrade -i flagger flagger/flagger \
 --namespace=istio-system \
+--set crd.create=false \
 --set msteams.url=https://outlook.office.com/webhook/YOUR/TEAMS/WEBHOOK
 ```
 
@@ -131,25 +141,38 @@ As an alternative to Helm, Flagger can be installed with Kustomize.
 Install Flagger for Istio:
 
 ```bash
-kubectl apply -k github.com/weaveworks/flagger/kustomize/istio
+kubectl apply -k github.com/weaveworks/flagger//kustomize/istio
 ```
 
-This deploys Flagger in the `istio-system` namespace and sets the metrics server URL to `http://prometheus.istio-system:9090`.
+This deploys Flagger in the `istio-system` namespace and sets the metrics server URL to Istio's Prometheus instance.
+
+Note that you'll need kubectl 1.14 to run the above the command or you can download the
+[kustomize binary](https://github.com/kubernetes-sigs/kustomize/releases) and run:
+
+```bash
+kustomize build github.com/weaveworks/flagger//kustomize/istio | kubectl apply -f -
+```
 
 Install Flagger for Linkerd:
 
 ```bash
-kubectl apply -k github.com/weaveworks/flagger/kustomize/linkerd
+kubectl apply -k github.com/weaveworks/flagger//kustomize/linkerd
 ```
 
-This deploys Flagger in the `linkerd` namespace and sets the metrics server URL to `http://linkerd-prometheus.linkerd:9090`.
+This deploys Flagger in the `linkerd` namespace and sets the metrics server URL to Linkerd's Prometheus instance.
+
+If you want to install a specific Flagger release, add the version number to the URL:
+
+```bash
+kubectl apply -k github.com/weaveworks/flagger//kustomize/linkerd?ref=0.18.0
+```
 
 **Generic installer**
 
 Install Flagger and Prometheus:
 
 ```bash
-kubectl apply -k github.com/weaveworks/flagger/kustomize/kubernetes
+kubectl apply -k github.com/weaveworks/flagger//kustomize/kubernetes
 ```
 
 This deploys Flagger and Prometheus in the `flagger-system` namespace,
@@ -158,7 +181,7 @@ sets the metrics server URL to `http://flagger-prometheus.flagger-system:9090` a
 The Prometheus instance has a two hours data retention and is configured to scrape all pods in your cluster that
 have the `prometheus.io/scrape: "true"` annotation.
 
-To target a specific provider you need to specify it in the canary custom resource:
+To target a different provider you can specify it in the canary custom resource:
 
 ```yaml
 apiVersion: flagger.app/v1alpha3
@@ -167,7 +190,7 @@ metadata:
   name: app
   namespace: test
 spec:
-  # can be: kubernetes, istio, appmesh, linkerd, smi, nginx, gloo, supergloo
+  # can be: kubernetes, istio, linkerd, appmesh, nginx, gloo
   # use the kubernetes provider for Blue/Green style deployments
   provider: nginx
 ```
@@ -213,3 +236,5 @@ Install Flagger with Slack:
 ```bash
 kubectl apply -k .
 ```
+
+If you want to use MS Teams instead of Slack, replace `-slack-url` with `-msteams-url` and set the webhook address to `https://outlook.office.com/webhook/YOUR/TEAMS/WEBHOOK`.

docs/gitbook/install/flagger-install-with-supergloo.md

@@ -115,11 +115,18 @@ Add Flagger Helm repository:
 helm repo add flagger https://flagger.app
 ```
 
+Install Flagger's Canary CRD:
+
+```yaml
+kubectl apply -f https://raw.githubusercontent.com/weaveworks/flagger/master/artifacts/flagger/crd.yaml
+```
+
 Deploy Flagger in the _**istio-system**_ namespace and set the service mesh provider to SuperGloo:
 
 ```bash
 helm upgrade -i flagger flagger/flagger \
 --namespace=istio-system \
+--set crd.create=false \
 --set metricsServer=http://prometheus.istio-system:9090 \
 --set meshProvider=supergloo:istio.supergloo-system
 ```

docs/gitbook/tutorials/canary-helm-gitops.md

@@ -138,7 +138,7 @@ helm upgrade -i frontend flagger/podinfo/ \
 --reuse-values \
 --set canary.loadtest.enabled=true \
 --set canary.helmtest.enabled=true \
---set image.tag=1.4.1
+--set image.tag=2.0.1
 ```
 
 Flagger detects that the deployment revision changed and starts the canary analysis:
@@ -283,17 +283,17 @@ metadata:
   namespace: test
   annotations:
     flux.weave.works/automated: "true"
-    flux.weave.works/tag.chart-image: semver:~1.4
+    flux.weave.works/tag.chart-image: semver:~2.0
 spec:
   releaseName: frontend
   chart:
-    repository: https://stefanprodan.github.io/flagger/
-    name: podinfo
-    version: 2.0.0
+    git: https://github.com/weaveowrks/flagger
+    ref: master
+    path: charts/podinfo
   values:
     image:
-      repository: quay.io/stefanprodan/podinfo
-      tag: 1.4.0
+      repository: stefanprodan/podinfo
+      tag: 2.0.0
       backend: http://backend-podinfo:9898/echo
       canary:
         enabled: true
@@ -311,25 +311,26 @@ In the `chart` section I've defined the release source by specifying the Helm re
 In the `values` section I've overwritten the defaults set in values.yaml.
 
 With the `flux.weave.works` annotations I instruct Flux to automate this release. 
-When an image tag in the sem ver range of `1.4.0 - 1.4.99` is pushed to Quay, 
+When an image tag in the sem ver range of `2.0.0 - 2.0.99` is pushed to Quay, 
 Flux will upgrade the Helm release and from there Flagger will pick up the change and start a canary deployment.
 
 Install [Weave Flux](https://github.com/weaveworks/flux) and its Helm Operator by specifying your Git repo URL:
 
 ```bash
-helm repo add weaveworks https://weaveworks.github.io/flux
+helm repo add fluxcd https://charts.fluxcd.io
 
 helm install --name flux \
 --set helmOperator.create=true \
+--set helmOperator.createCRD=true \
 --set git.url=git@github.com:<USERNAME>/<REPOSITORY> \
---namespace flux \
-weaveworks/flux
+--namespace fluxcd \
+fluxcd/flux
 ```
 
 At startup Flux generates a SSH key and logs the public key. Find the SSH public key with:
 
 ```bash
-kubectl -n flux logs deployment/flux | grep identity.pub | cut -d '"' -f2
+kubectl -n fluxcd logs deployment/flux | grep identity.pub | cut -d '"' -f2
 ```
 
 In order to sync your cluster state with Git you need to copy the public key and create a 
@@ -343,9 +344,9 @@ launch the `frontend` and `backend` apps.
 
 A CI/CD pipeline for the `frontend` release could look like this:
 
-* cut a release from the master branch of the podinfo code repo with the git tag `1.4.1`
-* CI builds the image and pushes the `podinfo:1.4.1` image to the container registry
-* Flux scans the registry and updates the Helm release `image.tag` to `1.4.1`
+* cut a release from the master branch of the podinfo code repo with the git tag `2.0.1`
+* CI builds the image and pushes the `podinfo:2.0.1` image to the container registry
+* Flux scans the registry and updates the Helm release `image.tag` to `2.0.1`
 * Flux commits and push the change to the cluster repo
 * Flux applies the updated Helm release on the cluster
 * Flux Helm Operator picks up the change and calls Tiller to upgrade the release
@@ -355,7 +356,7 @@ A CI/CD pipeline for the `frontend` release could look like this:
 * Based on the analysis result the canary deployment is promoted to production or rolled back
 * Flagger sends a Slack notification with the canary result
 
-If the canary fails, fix the bug, do another patch release eg `1.4.2` and the whole process will run again.
+If the canary fails, fix the bug, do another patch release eg `2.0.2` and the whole process will run again.
 
 A canary deployment can fail due to any of the following reasons:
 

docs/gitbook/tutorials/flagger-smi-istio.md

@@ -236,7 +236,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=quay.io/stefanprodan/podinfo:1.7.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

docs/gitbook/usage/ab-testing.md

@@ -131,7 +131,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/abtest \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=stefanprodan/podinfo:2.0.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:

docs/gitbook/usage/appmesh-progressive-delivery.md

@@ -37,8 +37,10 @@ Deploy the load testing service to generate traffic during the canary analysis:
 ```bash
 helm upgrade -i flagger-loadtester flagger/loadtester \
 --namespace=test \
---set meshName=global.appmesh-system \
---set "backends[0]=podinfo.test"
+--set meshName=global \
+--set "backends[0]=podinfo.test" \
+--set "backends[1]=podinfo-canary.test" \
+--set "backends[2]=podinfo-primary.test"
 ```
 
 Create a canary custom resource:
@@ -67,7 +69,7 @@ spec:
     # container port
     port: 9898
     # App Mesh reference
-    meshName: global.appmesh-system
+    meshName: global
     # App Mesh egress (optional) 
     backends:
       - backend.test
@@ -176,7 +178,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=stefanprodan/podinfo:2.0.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -239,7 +241,7 @@ Trigger a canary deployment:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.2
+podinfod=stefanprodan/podinfo:2.0.2
 ```
 
 Exec into the load tester pod with:

docs/gitbook/usage/blue-green.md

@@ -59,8 +59,8 @@ Create a deployment and a horizontal pod autoscaler:
 ```bash
 export REPO=https://raw.githubusercontent.com/weaveworks/flagger/master
 
-kubectl apply -f ${REPO}/artifacts/canary/deployment.yaml
-kubectl apply -f ${REPO}/artifacts/canary/hpa.yaml
+kubectl apply -f ${REPO}/artifacts/canaries/deployment.yaml
+kubectl apply -f ${REPO}/artifacts/canaries/hpa.yaml
 ```
 
 Deploy the load testing service to generate traffic during the analysis:
@@ -172,7 +172,7 @@ Trigger a deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=stefanprodan/podinfo:2.0.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -297,7 +297,7 @@ Trigger a deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.3
+podinfod=stefanprodan/podinfo:2.0.3
 ```
 
 Generate 404s:

docs/gitbook/usage/gloo-progressive-delivery.md

@@ -103,6 +103,7 @@ metadata:
   name: podinfo
   namespace: test
 spec:
+  provider: gloo
   # deployment reference
   targetRef:
     apiVersion: apps/v1
@@ -197,7 +198,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=stefanprodan/podinfo:2.0.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -251,7 +252,7 @@ Trigger another canary deployment:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.2
+podinfod=stefanprodan/podinfo:2.0.2
 ```
 
 Generate HTTP 500 errors:
@@ -334,7 +335,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.3
+podinfod=stefanprodan/podinfo:2.0.3
 ```
 
 Generate 404s:
@@ -362,5 +363,3 @@ Canary failed! Scaling down podinfo.test
 ```
 
 If you have Slack configured, Flagger will send a notification with the reason why the canary failed.
-
-

docs/gitbook/usage/linkerd-progressive-delivery.md

@@ -6,34 +6,23 @@ This guide shows you how to use Linkerd and Flagger to automate canary deploymen
 
 ### Prerequisites
 
-Flagger requires a Kubernetes cluster **v1.11** or newer and Linker with support for SMI Traffic Spit API.
+Flagger requires a Kubernetes cluster **v1.11** or newer and Linkerd **2.4** or newer.
 
 Install Flagger in the linkerd namespace:
 
 ```bash
-helm repo add flagger https://flagger.app
-
-helm upgrade -i flagger flagger/flagger \
---namespace linkerd \
---set metricsServer=http://linkerd-prometheus:9090 \
---set meshProvider=linkerd
+kubectl apply -k github.com/weaveworks/flagger//kustomize/linkerd
 ```
 
-Optionally you can enable Slack notifications:
+Note that you'll need kubectl 1.14 or newer to run the above command.
 
-```bash
-helm upgrade -i flagger flagger/flagger \
---reuse-values \
---namespace linkerd \
---set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \
---set slack.channel=general \
---set slack.user=flagger
-```
+To enable Slack or MS Teams notifications,
+see Flagger's [install docs](https://docs.flagger.app/install/flagger-install-on-kubernetes) for Kustomize or Helm options.
 
 ### Bootstrap
 
-Flagger takes a Kubernetes deployment and optionally a horizontal pod autoscaler (HPA), 
-then creates a series of objects (Kubernetes deployments, ClusterIP services and SMI traffic split). 
+Flagger takes a Kubernetes deployment and optionally a horizontal pod autoscaler (HPA),
+then creates a series of objects (Kubernetes deployments, ClusterIP services and SMI traffic split).
 These objects expose the application inside the mesh and drive the canary analysis and promotion.
 
 Create a test namespace and enable Linkerd proxy injection:
@@ -46,17 +35,13 @@ kubectl annotate namespace test linkerd.io/inject=enabled
 Install the load testing service to generate traffic during the canary analysis:
 
 ```bash
-helm upgrade -i flagger-loadtester flagger/loadtester \
---namespace=test
+kubectl apply -k github.com/weaveworks/flagger//kustomize/tester
 ```
 
 Create a deployment and a horizontal pod autoscaler:
 
 ```bash
-export REPO=https://raw.githubusercontent.com/weaveworks/flagger/master
-
-kubectl apply -f ${REPO}/artifacts/canary/deployment.yaml
-kubectl apply -f ${REPO}/artifacts/canary/hpa.yaml
+kubectl apply -k github.com/weaveworks/flagger//kustomize/podinfo
 ```
 
 Create a canary custom resource for the podinfo deployment:
@@ -129,19 +114,19 @@ Save the above resource as podinfo-canary.yaml and then apply it:
 kubectl apply -f ./podinfo-canary.yaml
 ```
 
-When the canary analysis starts, Flagger will call the pre-rollout webhooks before routing traffic to the canary. 
+When the canary analysis starts, Flagger will call the pre-rollout webhooks before routing traffic to the canary.
 The canary analysis will run for five minutes while validating the HTTP metrics and rollout hooks every half a minute.
 
 After a couple of seconds Flagger will create the canary objects:
 
 ```bash
-# applied 
+# applied
 deployment.apps/podinfo
 horizontalpodautoscaler.autoscaling/podinfo
 ingresses.extensions/podinfo
 canary.flagger.app/podinfo
 
-# generated 
+# generated
 deployment.apps/podinfo-primary
 horizontalpodautoscaler.autoscaling/podinfo-primary
 service/podinfo
@@ -150,12 +135,12 @@ service/podinfo-primary
 trafficsplits.split.smi-spec.io/podinfo
 ```
 
-After the boostrap, the podinfo deployment will be scaled to zero and the traffic to `podinfo.test` will be routed 
+After the boostrap, the podinfo deployment will be scaled to zero and the traffic to `podinfo.test` will be routed
 to the primary pods. During the canary analysis, the `podinfo-canary.test` address can be used to target directly the canary pods.
 
 ### Automated canary promotion
 
-Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance indicators 
+Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance indicators
 like HTTP requests success rate, requests average duration and pod health.
 Based on analysis of the KPIs a canary is promoted or aborted, and the analysis result is published to Slack.
 
@@ -165,7 +150,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=stefanprodan/podinfo:2.0.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -223,7 +208,7 @@ Trigger another canary deployment:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.2
+podinfod=stefanprodan/podinfo:2.0.2
 ```
 
 Exec into the load tester pod with:
@@ -244,7 +229,7 @@ Generate latency:
 watch -n 1 curl http://podinfo-canary.test:9898/delay/1
 ```
 
-When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, 
+When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary,
 the canary is scaled to zero and the rollout is marked as failed.
 
 ```text
@@ -271,7 +256,7 @@ Events:
 
 ### Custom metrics
 
-The canary analysis can be extended with Prometheus queries. 
+The canary analysis can be extended with Prometheus queries.
 
 Let's a define a check for not found errors. Edit the canary analysis and add the following metric:
 
@@ -290,8 +275,8 @@ Let's a define a check for not found errors. Edit the canary analysis and add th
                     direction="inbound"
                 }[1m]
             )
-        ) 
-        / 
+        )
+        /
         sum(
             rate(
                 response_total{
@@ -300,19 +285,19 @@ Let's a define a check for not found errors. Edit the canary analysis and add th
                     direction="inbound"
                 }[1m]
             )
-        ) 
+        )
         * 100
 ```
 
-The above configuration validates the canary version by checking if the HTTP 404 req/sec percentage is below 
-three percent of the total traffic. If the 404s rate reaches the 3% threshold, then the analysis is aborted and the 
+The above configuration validates the canary version by checking if the HTTP 404 req/sec percentage is below
+three percent of the total traffic. If the 404s rate reaches the 3% threshold, then the analysis is aborted and the
 canary is marked as failed.
 
 Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.3
+podinfod=stefanprodan/podinfo:2.0.3
 ```
 
 Generate 404s:
@@ -377,10 +362,10 @@ spec:
               servicePort: 9898
 ```
 
-When using an ingress controller, the Linkerd traffic split does not apply to incoming traffic since NGINX in running outside of 
+When using an ingress controller, the Linkerd traffic split does not apply to incoming traffic since NGINX in running outside of
 the mesh. In order to run a canary analysis for a frontend app, Flagger creates a shadow ingress and sets the NGINX specific annotations.
 
-### A/B Testing 
+### A/B Testing
 
 Besides weighted routing, Flagger can be configured to route traffic to the canary based on HTTP match conditions.
 In an A/B testing scenario, you'll be using HTTP headers or cookies to target a certain segment of your users.
@@ -450,16 +435,16 @@ spec:
           cmd: "hey -z 2m -q 10 -c 2 -H 'Cookie: canary=always' http://app.example.com"
 ```
 
-The above configuration will run an analysis for ten minutes targeting users that have a `canary` cookie set to `always` or 
-those that call the service using the `X-Canary: always` header. 
+The above configuration will run an analysis for ten minutes targeting users that have a `canary` cookie set to `always` or
+those that call the service using the `X-Canary: always` header.
 
-**Note** that the load test now targets the external address and uses the canary cookie. 
+**Note** that the load test now targets the external address and uses the canary cookie.
 
 Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.5.0
+podinfod=stefanprodan/podinfo:2.0.4
 ```
 
 Flagger detects that the deployment revision changed and starts the A/B testing:

docs/gitbook/usage/nginx-progressive-delivery.md

@@ -104,6 +104,7 @@ metadata:
   name: podinfo
   namespace: test
 spec:
+  provider: nginx
   # deployment reference
   targetRef:
     apiVersion: apps/v1
@@ -189,7 +190,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=stefanprodan/podinfo:2.0.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -243,7 +244,7 @@ Trigger another canary deployment:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.2
+podinfod=stefanprodan/podinfo:2.0.2
 ```
 
 Generate HTTP 500 errors:
@@ -313,7 +314,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.3
+podinfod=stefanprodan/podinfo:2.0.3
 ```
 
 Generate high response latency:
@@ -387,7 +388,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.5.0
+podinfod=stefanprodan/podinfo:2.0.4
 ```
 
 Flagger detects that the deployment revision changed and starts the A/B testing:

docs/gitbook/usage/progressive-delivery.md

@@ -79,8 +79,15 @@ spec:
       # milliseconds
       threshold: 500
       interval: 30s
-    # generate traffic during analysis
+    # testing (optional)
     webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 30s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary:9898/token | grep token"
       - name: load-test
         url: http://flagger-loadtester.test/
         timeout: 5s
@@ -94,6 +101,9 @@ Save the above resource as podinfo-canary.yaml and then apply it:
 kubectl apply -f ./podinfo-canary.yaml
 ```
 
+When the canary analysis starts, Flagger will call the pre-rollout webhooks before routing traffic to the canary.
+The canary analysis will run for five minutes while validating the HTTP metrics and rollout hooks every minute.
+
 After a couple of seconds Flagger will create the canary objects:
 
 ```bash
@@ -119,7 +129,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.4.1
+podinfod=stefanprodan/podinfo:2.0.1
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -169,14 +179,17 @@ prod        backend   Failed        0        2019-01-14T17:05:07Z
 
 During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses the rollout.
 
-Create a tester pod and exec into it:
+Trigger another canary deployment:
 
 ```bash
-kubectl -n test run tester \
---image=quay.io/stefanprodan/podinfo:1.2.1 \
--- ./podinfo --port=9898
+kubectl -n test set image deployment/podinfo \
+podinfod=stefanprodan/podinfo:2.0.2
+```
 
-kubectl -n test exec -it tester-xx-xx sh
+Exec into the load tester pod with:
+
+```bash
+kubectl -n test exec -it flagger-loadtester-xx-xx sh
 ```
 
 Generate HTTP 500 errors:
@@ -216,4 +229,3 @@ Events:
   Warning  Synced  1m    flagger  Rolling back podinfo.test failed checks threshold reached 10
   Warning  Synced  1m    flagger  Canary failed! Scaling down podinfo.test
 ```
-

go.mod

@@ -30,6 +30,8 @@ require (
 	github.com/mattn/go-isatty v0.0.7 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
+	github.com/mitchellh/hashstructure v1.0.0
+	github.com/pkg/errors v0.8.1
 	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
 	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 // indirect
 	github.com/prometheus/common v0.3.0 // indirect
@@ -54,6 +56,7 @@ require (
 )
 
 replace (
+	git.apache.org/thrift.git => github.com/apache/thrift v0.12.0
 	github.com/google/uuid => github.com/google/uuid v1.0.0
 	golang.org/x/crypto => golang.org/x/crypto v0.0.0-20181025213731-e84da0312774
 	golang.org/x/net => golang.org/x/net v0.0.0-20190206173232-65e2d4e15006

go.sum

@@ -31,6 +31,7 @@ github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30/go.mod h1:4AJxU
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/avast/retry-go v2.2.0+incompatible h1:m+w7mVLWa/oKqX2xYqiEKQQkeGH8DDEXB/XnjS54Wyw=
@@ -38,6 +39,7 @@ github.com/avast/retry-go v2.2.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevB
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
@@ -80,6 +82,7 @@ github.com/evanphx/json-patch v4.1.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi
 github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
+github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fgrosse/zaptest v1.1.0 h1:sK9hP0/xBoNX5qfFo3KWFluDXfc809APomI1QXuYELA=
 github.com/fgrosse/zaptest v1.1.0/go.mod h1:vMnRSul6kW7kIUXZgnZZcDwyTn8k49ODfAULL8nmL5w=
@@ -192,6 +195,7 @@ github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa
 github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
 github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=
 github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A=
+github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
@@ -202,6 +206,7 @@ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M=
@@ -265,6 +270,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
@@ -315,6 +321,7 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=

hack/update-codegen.sh

@@ -1,23 +1,38 @@
 #!/usr/bin/env bash
 
 set -o errexit
+set -o nounset
 set -o pipefail
 
-REPO_ROOT=$(git rev-parse --show-toplevel)
-CODEGEN_VERSION="@v0.0.0-20190620073620-d55040311883"
-CODEGEN_PKG=${CODEGEN_PKG:-$(echo `go env GOPATH`"/pkg/mod/k8s.io/code-generator${CODEGEN_VERSION}")}
+SCRIPT_ROOT=$(realpath $(dirname ${BASH_SOURCE})/..)
 
-if [[ "$CIRCLECI" ]]; then
-    ls $(echo `go env GOPATH`'/pkg/mod/k8s.io/');
-    mkdir -p ${REPO_ROOT}/bin;
-    cp -r ${CODEGEN_PKG} ${REPO_ROOT}/bin/;
-    CODEGEN_PKG=${REPO_ROOT}/bin/code-generator${CODEGEN_VERSION};
-    echo ">> $CODEGEN_PKG";
-fi
+# Grab code-generator version from go.sum.
+CODEGEN_VERSION=$(grep 'k8s.io/code-generator' go.sum | awk '{print $2}' | head -1)
+CODEGEN_PKG=$(echo `go env GOPATH`"/pkg/mod/k8s.io/code-generator@${CODEGEN_VERSION}")
 
+echo ">> Using ${CODEGEN_PKG}"
+
+# code-generator does work with go.mod but makes assumptions about
+# the project living in `$GOPATH/src`. To work around this and support
+# any location; create a temporary directory, use this as an output
+# base, and copy everything back once generated.
+TEMP_DIR=$(mktemp -d)
+cleanup() {
+    echo ">> Removing ${TEMP_DIR}"
+    rm -rf ${TEMP_DIR}
+}
+trap "cleanup" EXIT SIGINT
+
+echo ">> Temporary output directory ${TEMP_DIR}"
+
+# Ensure we can execute.
 chmod +x ${CODEGEN_PKG}/generate-groups.sh
 
 ${CODEGEN_PKG}/generate-groups.sh all \
     github.com/weaveworks/flagger/pkg/client github.com/weaveworks/flagger/pkg/apis \
     "appmesh:v1beta1 istio:v1alpha3 flagger:v1alpha3 smi:v1alpha1" \
-    --go-header-file ${REPO_ROOT}/hack/boilerplate.go.txt
+    --output-base "${TEMP_DIR}" \
+    --go-header-file ${SCRIPT_ROOT}/hack/boilerplate.go.txt
+
+# Copy everything back.
+cp -r "${TEMP_DIR}/github.com/weaveworks/flagger/." "${SCRIPT_ROOT}/"

kustomize/README.md

@@ -1,37 +1,50 @@
 # Flagger Kustomize installer
 
-As an alternative to Helm, Flagger can be installed with Kustomize.
+As an alternative to Helm, Flagger can be installed with [Kustomize](https://kustomize.io/).
 
 ## Service mesh specific installers
 
 Install Flagger for Istio:
 
 ```bash
-kubectl apply -k github.com/weaveworks/flagger/kustomize/istio
+kubectl apply -k github.com/weaveworks/flagger//kustomize/istio
 ```
 
-This deploys Flagger in the `istio-system` namespace and sets the metrics server URL to `http://prometheus.istio-system:9090`.
+This deploys Flagger in the `istio-system` namespace and sets the metrics server URL to Istio's Prometheus instance.
+
+Note that you'll need kubectl 1.14 to run the above the command or you can download the
+[kustomize binary](https://github.com/kubernetes-sigs/kustomize/releases) and run:
+
+```bash
+kustomize build github.com/weaveworks/flagger//kustomize/istio | kubectl apply -f -
+```
 
 Install Flagger for Linkerd:
 
 ```bash
-kubectl apply -k github.com/weaveworks/flagger/kustomize/linkerd
+kubectl apply -k github.com/weaveworks/flagger//kustomize/linkerd
 ```
 
-This deploys Flagger in the `linkerd` namespace and sets the metrics server URL to `http://linkerd-prometheus.linkerd:9090`.
+This deploys Flagger in the `linkerd` namespace and sets the metrics server URL to Linkerd's Prometheus instance.
+
+If you want to install a specific Flagger release, add the version number to the URL:
+
+```bash
+kubectl apply -k github.com/weaveworks/flagger//kustomize/linkerd?ref=0.18.0
+```
 
 ## Generic installer
 
 Install Flagger and Prometheus:
 
 ```bash
-kubectl apply -k github.com/weaveworks/flagger/kustomize/kubernetes
+kubectl apply -k github.com/weaveworks/flagger//kustomize/kubernetes
 ```
 
 This deploys Flagger and Prometheus in the `flagger-system` namespace,
 sets the metrics server URL to `http://flagger-prometheus.flagger-system:9090` and the mesh provider to `kubernetes`.
 
-To target a specific provider you need to specify it in the canary custom resource:
+To target a different provider you can specify it in the canary custom resource:
 
 ```yaml
 apiVersion: flagger.app/v1alpha3
@@ -40,12 +53,16 @@ metadata:
   name: app
   namespace: test
 spec:
-  # can be: kubernetes, istio, appmesh, linkerd, smi, nginx, gloo, supergloo
+  # can be: kubernetes, istio, linkerd, appmesh, nginx, gloo
   # use the kubernetes provider for Blue/Green style deployments
   provider: nginx
 ```
 
-## Customized installer
+You'll need Prometheus when using Flagger with AWS App Mesh, Gloo or NGINX ingress controller.
+The Prometheus instance has a two hours data retention and is configured to scrape all pods in your cluster that
+have the `prometheus.io/scrape: "true"` annotation.
+
+## Configure Slack notifications
 
 Create a kustomization file using flagger as base:
 
@@ -81,9 +98,48 @@ spec:
 EOF
 ```
 
-Install Flagger with Slack:
+Install Flagger for Istio with Slack notifications:
 
 ```bash
 kubectl apply -k .
 ```
 
+## Configure MS Teams notifications
+
+Create a kustomization file using flagger as base:
+
+```bash
+cat > kustomization.yaml <<EOF
+namespace: linkerd
+bases:
+  - github.com/weaveworks/flagger/kustomize/base/flagger
+patchesStrategicMerge:
+  - patch.yaml
+EOF
+```
+
+Create a patch and set the MS Teams webhook URL:
+
+```bash
+cat > patch.yaml <<EOF
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flagger
+spec:
+  template:
+    spec:
+      containers:
+        - name: flagger
+          args:
+            - -mesh-provider=linkerd
+            - -metrics-server=http://linkerd-prometheus:9090
+            - -msteams-url=https://outlook.office.com/webhook/YOUR/TEAMS/WEBHOOK
+EOF
+```
+
+Install Flagger for Linkerd with MS Teams notifications:
+
+```bash
+kubectl apply -k .
+```

kustomize/base/flagger/crd.yaml

@@ -33,6 +33,22 @@ spec:
     - name: Weight
       type: string
       JSONPath: .status.canaryWeight
+    - name: FailedChecks
+      type: string
+      JSONPath: .status.failedChecks
+      priority: 1
+    - name: Interval
+      type: string
+      JSONPath: .spec.canaryAnalysis.interval
+      priority: 1
+    - name: StepWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.stepWeight
+      priority: 1
+    - name: MaxWeight
+      type: string
+      JSONPath: .spec.canaryAnalysis.maxWeight
+      priority: 1
     - name: LastTransitionTime
       type: string
       JSONPath: .status.lastTransitionTime
@@ -46,12 +62,15 @@ spec:
             - canaryAnalysis
           properties:
             provider:
+              description: Traffic managent provider
               type: string
             progressDeadlineSeconds:
+              description: Deployment progress deadline
               type: number
             targetRef:
+              description: Deployment selector
               type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -60,10 +79,11 @@ spec:
                 name:
                   type: string
             autoscalerRef:
+              description: HPA selector
               anyOf:
                 - type: string
                 - type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -72,10 +92,11 @@ spec:
                 name:
                   type: string
             ingressRef:
+              description: NGINX ingress selector
               anyOf:
                 - type: string
                 - type: object
-              required: ['apiVersion', 'kind', 'name']
+              required: ["apiVersion", "kind", "name"]
               properties:
                 apiVersion:
                   type: string
@@ -85,88 +106,197 @@ spec:
                   type: string
             service:
               type: object
-              required: ['port']
+              required: ["port"]
               properties:
                 port:
+                  description: Container port number
                   type: number
                 portName:
+                  description: Container port name
                   type: string
                 portDiscovery:
+                  description: Enable port dicovery
                   type: boolean
                 meshName:
+                  description: AppMesh mesh name
                   type: string
+                backends:
+                  description: AppMesh backend array
+                  anyOf:
+                    - type: string
+                    - type: array
                 timeout:
+                  description: Istio HTTP or gRPC request timeout
                   type: string
+                trafficPolicy:
+                  description: Istio traffic policy
+                  anyOf:
+                    - type: string
+                    - type: object
+                match:
+                  description: Istio URL match conditions
+                  anyOf:
+                    - type: string
+                    - type: array
+                rewrite:
+                  description: Istio URL rewrite
+                  anyOf:
+                    - type: string
+                    - type: object
+                headers:
+                  description: Istio headers operations
+                  anyOf:
+                    - type: string
+                    - type: object
+                corsPolicy:
+                  description: Istio CORS policy
+                  anyOf:
+                    - type: string
+                    - type: object
+                gateways:
+                  description: Istio gateways list
+                  anyOf:
+                    - type: string
+                    - type: array
+                hosts:
+                  description: Istio hosts list
+                  anyOf:
+                    - type: string
+                    - type: array
             skipAnalysis:
               type: boolean
             canaryAnalysis:
               properties:
                 interval:
+                  description: Canary schedule interval
                   type: string
                   pattern: "^[0-9]+(m|s)"
                 iterations:
+                  description: Number of checks to run for A/B Testing and Blue/Green
                   type: number
                 threshold:
+                  description: Max number of failed checks before rollback
                   type: number
                 maxWeight:
+                  description: Max traffic percentage routed to canary
                   type: number
                 stepWeight:
+                  description: Canary incremental traffic percentage step
                   type: number
+                match:
+                  description: A/B testing match conditions
+                  anyOf:
+                    - type: string
+                    - type: array
                 metrics:
+                  description: Prometheus query list for this canary
                   type: array
                   properties:
                     items:
                       type: object
-                      required: ['name', 'threshold']
+                      required: ["name", "threshold"]
                       properties:
                         name:
+                          description: Name of the Prometheus metric
                           type: string
                         interval:
+                          description: Interval of the promql query
                           type: string
                           pattern: "^[0-9]+(m|s)"
                         threshold:
+                          description: Max scalar value accepted for this metric
                           type: number
                         query:
+                          description: Prometheus query
                           type: string
                 webhooks:
+                  description: Webhook list for this canary
                   type: array
                   properties:
                     items:
                       type: object
-                      required: ['name', 'url', 'timeout']
+                      required: ["name", "url", "timeout"]
                       properties:
                         name:
+                          description: Name of the webhook
                           type: string
                         type:
+                          description: Type of the webhook pre, post or during rollout
                           type: string
                           enum:
                             - ""
+                            - confirm-rollout
                             - pre-rollout
                             - rollout
                             - post-rollout
                         url:
+                          description: URL address of this webhook
                           type: string
                           format: url
                         timeout:
+                          description: Request timeout for this webhook
                           type: string
                           pattern: "^[0-9]+(m|s)"
+                        metadata:
+                          description: Metadata (key-value pairs) for this webhook
+                          anyOf:
+                            - type: string
+                            - type: object
         status:
           properties:
             phase:
+              description: Analysis phase of this canary
               type: string
               enum:
                 - ""
+                - Initializing
                 - Initialized
+                - Waiting
                 - Progressing
+                - Finalising
                 - Succeeded
                 - Failed
             canaryWeight:
+              description: Traffic weight percentage routed to canary
               type: number
             failedChecks:
+              description: Failed check count of the current canary analysis
               type: number
             iterations:
+              description: Iteration count of the current canary analysis
               type: number
             lastAppliedSpec:
+              description: LastAppliedSpec of this canary
               type: string
             lastTransitionTime:
+              description: LastTransitionTime of this canary
+              format: date-time
               type: string
+            conditions:
+              description: Status conditions of this canary
+              type: array
+              properties:
+                items:
+                  type: object
+                  required: ["type", "status", "reason"]
+                  properties:
+                    lastTransitionTime:
+                      description: LastTransitionTime of this condition
+                      format: date-time
+                      type: string
+                    lastUpdateTime:
+                      description: LastUpdateTime of this condition
+                      format: date-time
+                      type: string
+                    message:
+                      description: Message associated with this condition
+                      type: string
+                    reason:
+                      description: Reason for the current status of this condition
+                      type: string
+                    status:
+                      description: Status of this condition
+                      type: string
+                    type:
+                      description: Type of this condition
+                      type: string

kustomize/base/flagger/deployment.yaml

@@ -2,7 +2,6 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: flagger
-  namespace: flagger-system
 spec:
   replicas: 1
   strategy:

kustomize/base/flagger/kustomization.yaml

@@ -8,4 +8,4 @@ resources:
   - deployment.yaml
 images:
   - name: weaveworks/flagger
-    newTag: 0.17.0
+    newTag: 0.18.4

kustomize/istio/patch.yaml

@@ -8,6 +8,7 @@ spec:
       containers:
         - name: flagger
           args:
+            - -log-level=info
             - -mesh-provider=istio
             - -metrics-server=http://prometheus:9090
             - -slack-user=flagger

kustomize/kubernetes/patch.yaml

@@ -8,6 +8,7 @@ spec:
       containers:
         - name: flagger
           args:
+            - -log-level=info
             - -mesh-provider=kubernetes
             - -metrics-server=http://flagger-prometheus:9090
             - -slack-user=flagger

kustomize/linkerd/patch.yaml

@@ -8,6 +8,7 @@ spec:
       containers:
         - name: flagger
           args:
+            - -log-level=info
             - -mesh-provider=linkerd
             - -metrics-server=http://linkerd-prometheus:9090
             - -slack-user=flagger

kustomize/podinfo/deployment.yaml

@@ -2,11 +2,12 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: podinfo
-  namespace: test
   labels:
     app: podinfo
 spec:
-  replicas: 1
+  minReadySeconds: 5
+  revisionHistoryLimit: 5
+  progressDeadlineSeconds: 60
   strategy:
     rollingUpdate:
       maxUnavailable: 0
@@ -23,7 +24,7 @@ spec:
     spec:
       containers:
       - name: podinfod
-        image: quay.io/stefanprodan/podinfo:1.4.0
+        image: stefanprodan/podinfo:2.0.0
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9898
@@ -37,7 +38,7 @@ spec:
         - --random-error=false
         env:
         - name: PODINFO_UI_COLOR
-          value: green
+          value: blue
         livenessProbe:
           exec:
             command:
@@ -45,10 +46,8 @@ spec:
             - check
             - http
             - localhost:9898/healthz
-          failureThreshold: 3
-          periodSeconds: 10
-          successThreshold: 1
-          timeoutSeconds: 2
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
         readinessProbe:
           exec:
             command:
@@ -56,14 +55,12 @@ spec:
             - check
             - http
             - localhost:9898/readyz
-          failureThreshold: 3
-          periodSeconds: 3
-          successThreshold: 1
-          timeoutSeconds: 2
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
         resources:
           limits:
-            cpu: 1000m
-            memory: 256Mi
+            cpu: 2000m
+            memory: 512Mi
           requests:
             cpu: 100m
-            memory: 16Mi
+            memory: 64Mi

kustomize/podinfo/hpa.yaml

@@ -2,13 +2,12 @@ apiVersion: autoscaling/v2beta1
 kind: HorizontalPodAutoscaler
 metadata:
   name: podinfo
-  namespace: test
 spec:
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
     name: podinfo
-  minReplicas: 1
+  minReplicas: 2
   maxReplicas: 4
   metrics:
   - type: Resource

kustomize/podinfo/kustomization.yaml

@@ -0,0 +1,5 @@
+namespace: test
+resources:
+  - hpa.yaml
+  - deployment.yaml
+

kustomize/tester/deployment.yaml

@@ -0,0 +1,59 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flagger-loadtester
+  labels:
+    app: flagger-loadtester
+spec:
+  selector:
+    matchLabels:
+      app: flagger-loadtester
+  template:
+    metadata:
+      labels:
+        app: flagger-loadtester
+      annotations:
+        prometheus.io/scrape: "true"
+    spec:
+      containers:
+        - name: loadtester
+          image: weaveworks/flagger-loadtester:0.8.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 8080
+          command:
+            - ./loadtester
+            - -port=8080
+            - -log-level=info
+            - -timeout=1h
+          livenessProbe:
+            exec:
+              command:
+                - wget
+                - --quiet
+                - --tries=1
+                - --timeout=4
+                - --spider
+                - http://localhost:8080/healthz
+            timeoutSeconds: 5
+          readinessProbe:
+            exec:
+              command:
+                - wget
+                - --quiet
+                - --tries=1
+                - --timeout=4
+                - --spider
+                - http://localhost:8080/healthz
+            timeoutSeconds: 5
+          resources:
+            limits:
+              memory: "512Mi"
+              cpu: "1000m"
+            requests:
+              memory: "32Mi"
+              cpu: "10m"
+          securityContext:
+            readOnlyRootFilesystem: true
+            runAsUser: 10001

kustomize/tester/kustomization.yaml

@@ -0,0 +1,7 @@
+namespace: test
+resources:
+  - service.yaml
+  - deployment.yaml
+images:
+  - name: weaveworks/flagger-loadtester
+    newTag: 0.6.1

kustomize/tester/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: flagger-loadtester
+  labels:
+    app: flagger-loadtester
+spec:
+  type: ClusterIP
+  selector:
+    app: flagger-loadtester
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: http

pkg/apis/appmesh/v1beta1/types.go

@@ -102,13 +102,18 @@ type VirtualServiceSpec struct {
 	Routes []Route `json:"routes,omitempty"`
 }
 
+// VirtualRouter is the spec for a VirtualRouter resource
 type VirtualRouter struct {
-	Name string `json:"name"`
+	Name      string     `json:"name"`
+	Listeners []Listener `json:"listeners,omitempty"`
 }
 
 type Route struct {
-	Name string    `json:"name"`
-	Http HttpRoute `json:"http"`
+	Name string `json:"name"`
+	// +optional
+	Http *HttpRoute `json:"http,omitempty"`
+	// +optional
+	Tcp *TcpRoute `json:"tcp,omitempty"`
 }
 
 type HttpRoute struct {
@@ -124,6 +129,14 @@ type HttpRouteAction struct {
 	WeightedTargets []WeightedTarget `json:"weightedTargets"`
 }
 
+type TcpRoute struct {
+	Action TcpRouteAction `json:"action"`
+}
+
+type TcpRouteAction struct {
+	WeightedTargets []WeightedTarget `json:"weightedTargets"`
+}
+
 type WeightedTarget struct {
 	VirtualNodeName string `json:"virtualNodeName"`
 	Weight          int64  `json:"weight"`
@@ -203,6 +216,8 @@ type VirtualNodeSpec struct {
 	ServiceDiscovery *ServiceDiscovery `json:"serviceDiscovery,omitempty"`
 	// +optional
 	Backends []Backend `json:"backends,omitempty"`
+	// +optional
+	Logging *Logging `json:"logging,omitempty"`
 }
 
 type Listener struct {
@@ -237,6 +252,21 @@ type VirtualServiceBackend struct {
 	VirtualServiceName string `json:"virtualServiceName"`
 }
 
+// Logging refers to https://docs.aws.amazon.com/app-mesh/latest/APIReference/API_Logging.html
+type Logging struct {
+	AccessLog *AccessLog `json:"accessLog"`
+}
+
+// AccessLog refers to https://docs.aws.amazon.com/app-mesh/latest/APIReference/API_AccessLog.html
+type AccessLog struct {
+	File *FileAccessLog `json:"file"`
+}
+
+// FileAccessLog refers to https://docs.aws.amazon.com/app-mesh/latest/APIReference/API_FileAccessLog.html
+type FileAccessLog struct {
+	Path string `json:"path"`
+}
+
 // VirtualNodeStatus is the status for a VirtualNode resource
 type VirtualNodeStatus struct {
 	MeshArn *string `json:"meshArn,omitempty"`

pkg/apis/appmesh/v1beta1/zz_generated.deepcopy.go

@@ -24,6 +24,27 @@ import (
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AccessLog) DeepCopyInto(out *AccessLog) {
+	*out = *in
+	if in.File != nil {
+		in, out := &in.File, &out.File
+		*out = new(FileAccessLog)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AccessLog.
+func (in *AccessLog) DeepCopy() *AccessLog {
+	if in == nil {
+		return nil
+	}
+	out := new(AccessLog)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Backend) DeepCopyInto(out *Backend) {
 	*out = *in
@@ -73,6 +94,22 @@ func (in *DnsServiceDiscovery) DeepCopy() *DnsServiceDiscovery {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FileAccessLog) DeepCopyInto(out *FileAccessLog) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileAccessLog.
+func (in *FileAccessLog) DeepCopy() *FileAccessLog {
+	if in == nil {
+		return nil
+	}
+	out := new(FileAccessLog)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *HttpRoute) DeepCopyInto(out *HttpRoute) {
 	*out = *in
@@ -145,6 +182,27 @@ func (in *Listener) DeepCopy() *Listener {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Logging) DeepCopyInto(out *Logging) {
+	*out = *in
+	if in.AccessLog != nil {
+		in, out := &in.AccessLog, &out.AccessLog
+		*out = new(AccessLog)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Logging.
+func (in *Logging) DeepCopy() *Logging {
+	if in == nil {
+		return nil
+	}
+	out := new(Logging)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Mesh) DeepCopyInto(out *Mesh) {
 	*out = *in
@@ -304,7 +362,16 @@ func (in *PortMapping) DeepCopy() *PortMapping {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Route) DeepCopyInto(out *Route) {
 	*out = *in
-	in.Http.DeepCopyInto(&out.Http)
+	if in.Http != nil {
+		in, out := &in.Http, &out.Http
+		*out = new(HttpRoute)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Tcp != nil {
+		in, out := &in.Tcp, &out.Tcp
+		*out = new(TcpRoute)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -344,6 +411,44 @@ func (in *ServiceDiscovery) DeepCopy() *ServiceDiscovery {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TcpRoute) DeepCopyInto(out *TcpRoute) {
+	*out = *in
+	in.Action.DeepCopyInto(&out.Action)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TcpRoute.
+func (in *TcpRoute) DeepCopy() *TcpRoute {
+	if in == nil {
+		return nil
+	}
+	out := new(TcpRoute)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TcpRouteAction) DeepCopyInto(out *TcpRouteAction) {
+	*out = *in
+	if in.WeightedTargets != nil {
+		in, out := &in.WeightedTargets, &out.WeightedTargets
+		*out = make([]WeightedTarget, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TcpRouteAction.
+func (in *TcpRouteAction) DeepCopy() *TcpRouteAction {
+	if in == nil {
+		return nil
+	}
+	out := new(TcpRouteAction)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *VirtualNode) DeepCopyInto(out *VirtualNode) {
 	*out = *in
@@ -453,6 +558,11 @@ func (in *VirtualNodeSpec) DeepCopyInto(out *VirtualNodeSpec) {
 		*out = make([]Backend, len(*in))
 		copy(*out, *in)
 	}
+	if in.Logging != nil {
+		in, out := &in.Logging, &out.Logging
+		*out = new(Logging)
+		(*in).DeepCopyInto(*out)
+	}
 	return
 }
 
@@ -514,6 +624,11 @@ func (in *VirtualNodeStatus) DeepCopy() *VirtualNodeStatus {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *VirtualRouter) DeepCopyInto(out *VirtualRouter) {
 	*out = *in
+	if in.Listeners != nil {
+		in, out := &in.Listeners, &out.Listeners
+		*out = make([]Listener, len(*in))
+		copy(*out, *in)
+	}
 	return
 }
 
@@ -640,7 +755,7 @@ func (in *VirtualServiceSpec) DeepCopyInto(out *VirtualServiceSpec) {
 	if in.VirtualRouter != nil {
 		in, out := &in.VirtualRouter, &out.VirtualRouter
 		*out = new(VirtualRouter)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 	if in.Routes != nil {
 		in, out := &in.Routes, &out.Routes

pkg/apis/flagger/v1alpha3/status.go

@@ -0,0 +1,76 @@
+package v1alpha3
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// CanaryConditionType is the type of a CanaryCondition
+type CanaryConditionType string
+
+const (
+	// PromotedType refers to the result of the last canary analysis
+	PromotedType CanaryConditionType = "Promoted"
+)
+
+// CanaryCondition is a status condition for a Canary
+type CanaryCondition struct {
+	// Type of this condition
+	Type CanaryConditionType `json:"type"`
+
+	// Status of this condition
+	Status corev1.ConditionStatus `json:"status"`
+
+	// LastUpdateTime of this condition
+	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
+
+	// LastTransitionTime of this condition
+	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
+
+	// Reason for the current status of this condition
+	Reason string `json:"reason,omitempty"`
+
+	// Message associated with this condition
+	Message string `json:"message,omitempty"`
+}
+
+// CanaryPhase is a label for the condition of a canary at the current time
+type CanaryPhase string
+
+const (
+	// CanaryPhaseInitializing means the canary initializing is underway
+	CanaryPhaseInitializing CanaryPhase = "Initializing"
+	// CanaryPhaseInitialized means the primary deployment, hpa and ClusterIP services
+	// have been created along with the service mesh or ingress objects
+	CanaryPhaseInitialized CanaryPhase = "Initialized"
+	// CanaryPhaseWaiting means the canary rollout is paused (waiting for confirmation to proceed)
+	CanaryPhaseWaiting CanaryPhase = "Waiting"
+	// CanaryPhaseProgressing means the canary analysis is underway
+	CanaryPhaseProgressing CanaryPhase = "Progressing"
+	// CanaryPhaseProgressing means the canary analysis is finished and traffic has been routed back to primary
+	CanaryPhaseFinalising CanaryPhase = "Finalising"
+	// CanaryPhaseSucceeded means the canary analysis has been successful
+	// and the canary deployment has been promoted
+	CanaryPhaseSucceeded CanaryPhase = "Succeeded"
+	// CanaryPhaseFailed means the canary analysis failed
+	// and the canary deployment has been scaled to zero
+	CanaryPhaseFailed CanaryPhase = "Failed"
+)
+
+// CanaryStatus is used for state persistence (read-only)
+type CanaryStatus struct {
+	Phase        CanaryPhase `json:"phase"`
+	FailedChecks int         `json:"failedChecks"`
+	CanaryWeight int         `json:"canaryWeight"`
+	Iterations   int         `json:"iterations"`
+	// +optional
+	TrackedConfigs *map[string]string `json:"trackedConfigs,omitempty"`
+	// +optional
+	LastAppliedSpec string `json:"lastAppliedSpec,omitempty"`
+	// +optional
+	LastPromotedSpec string `json:"lastPromotedSpec,omitempty"`
+	// +optional
+	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
+	// +optional
+	Conditions []CanaryCondition `json:"conditions,omitempty"`
+}

pkg/apis/flagger/v1alpha3/types.go

@@ -85,37 +85,6 @@ type CanaryList struct {
 	Items []Canary `json:"items"`
 }
 
-// CanaryPhase is a label for the condition of a canary at the current time
-type CanaryPhase string
-
-const (
-	// CanaryInitialized means the primary deployment, hpa and ClusterIP services
-	// have been created along with the Istio virtual service
-	CanaryInitialized CanaryPhase = "Initialized"
-	// CanaryProgressing means the canary analysis is underway
-	CanaryProgressing CanaryPhase = "Progressing"
-	// CanarySucceeded means the canary analysis has been successful
-	// and the canary deployment has been promoted
-	CanarySucceeded CanaryPhase = "Succeeded"
-	// CanaryFailed means the canary analysis failed
-	// and the canary deployment has been scaled to zero
-	CanaryFailed CanaryPhase = "Failed"
-)
-
-// CanaryStatus is used for state persistence (read-only)
-type CanaryStatus struct {
-	Phase        CanaryPhase `json:"phase"`
-	FailedChecks int         `json:"failedChecks"`
-	CanaryWeight int         `json:"canaryWeight"`
-	Iterations   int         `json:"iterations"`
-	// +optional
-	TrackedConfigs *map[string]string `json:"trackedConfigs,omitempty"`
-	// +optional
-	LastAppliedSpec string `json:"lastAppliedSpec,omitempty"`
-	// +optional
-	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
-}
-
 // CanaryService is used to create ClusterIP services
 // and Istio Virtual Service
 type CanaryService struct {
@@ -168,6 +137,8 @@ const (
 	PreRolloutHook HookType = "pre-rollout"
 	// PreRolloutHook execute webhook after the canary analysis
 	PostRolloutHook HookType = "post-rollout"
+	// ConfirmRolloutHook halt canary analysis until webhook returns HTTP 200
+	ConfirmRolloutHook HookType = "confirm-rollout"
 )
 
 // CanaryWebhook holds the reference to external checks used for canary analysis

pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go

@@ -89,6 +89,24 @@ func (in *CanaryAnalysis) DeepCopy() *CanaryAnalysis {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CanaryCondition) DeepCopyInto(out *CanaryCondition) {
+	*out = *in
+	in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime)
+	in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CanaryCondition.
+func (in *CanaryCondition) DeepCopy() *CanaryCondition {
+	if in == nil {
+		return nil
+	}
+	out := new(CanaryCondition)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *CanaryList) DeepCopyInto(out *CanaryList) {
 	*out = *in
@@ -250,6 +268,13 @@ func (in *CanaryStatus) DeepCopyInto(out *CanaryStatus) {
 		}
 	}
 	in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime)
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]CanaryCondition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	return
 }
 

pkg/canary/deployer.go

@@ -2,20 +2,18 @@ package canary
 
 import (
 	"crypto/rand"
-	"encoding/base64"
-	"encoding/json"
 	"fmt"
+	"io"
+
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/mitchellh/hashstructure"
 	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
 	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
 	"go.uber.org/zap"
-	"io"
 	appsv1 "k8s.io/api/apps/v1"
 	hpav1 "k8s.io/api/autoscaling/v2beta1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes"
@@ -39,7 +37,7 @@ func (c *Deployer) Initialize(cd *flaggerv1.Canary, skipLivenessChecks bool) (la
 		return "", ports, fmt.Errorf("creating deployment %s.%s failed: %v", primaryName, cd.Namespace, err)
 	}
 
-	if cd.Status.Phase == "" {
+	if cd.Status.Phase == "" || cd.Status.Phase == flaggerv1.CanaryPhaseInitializing {
 		if !skipLivenessChecks {
 			_, readyErr := c.IsPrimaryReady(cd)
 			if readyErr != nil {
@@ -115,6 +113,7 @@ func (c *Deployer) Promote(cd *flaggerv1.Canary) error {
 
 	primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryName, label)
 
+	// apply update
 	_, err = c.KubeClient.AppsV1().Deployments(cd.Namespace).Update(primaryCopy)
 	if err != nil {
 		return fmt.Errorf("updating deployment %s.%s template spec failed: %v",
@@ -146,19 +145,17 @@ func (c *Deployer) HasDeploymentChanged(cd *flaggerv1.Canary) (bool, error) {
 		return true, nil
 	}
 
-	newSpec := &canary.Spec.Template.Spec
-	oldSpecJson, err := base64.StdEncoding.DecodeString(cd.Status.LastAppliedSpec)
+	newHash, err := hashstructure.Hash(canary.Spec.Template, nil)
 	if err != nil {
-		return false, fmt.Errorf("%s.%s decode error %v", cd.Name, cd.Namespace, err)
-	}
-	oldSpec := &corev1.PodSpec{}
-	err = json.Unmarshal(oldSpecJson, oldSpec)
-	if err != nil {
-		return false, fmt.Errorf("%s.%s unmarshal error %v", cd.Name, cd.Namespace, err)
+		return false, fmt.Errorf("hash error %v", err)
 	}
 
-	if diff := cmp.Diff(*newSpec, *oldSpec, cmpopts.IgnoreUnexported(resource.Quantity{})); diff != "" {
-		//fmt.Println(diff)
+	// do not trigger a canary deployment on manual rollback
+	if cd.Status.LastPromotedSpec == fmt.Sprintf("%d", newHash) {
+		return false, nil
+	}
+
+	if cd.Status.LastAppliedSpec != fmt.Sprintf("%d", newHash) {
 		return true, nil
 	}
 
@@ -337,7 +334,8 @@ func (c *Deployer) reconcilePrimaryHpa(cd *flaggerv1.Canary, init bool) error {
 	// update HPA
 	if !init && primaryHpa != nil {
 		diff := cmp.Diff(hpaSpec.Metrics, primaryHpa.Spec.Metrics)
-		if diff != "" || hpaSpec.MinReplicas != primaryHpa.Spec.MinReplicas || hpaSpec.MaxReplicas != primaryHpa.Spec.MaxReplicas {
+		if diff != "" || int32Default(hpaSpec.MinReplicas) != int32Default(primaryHpa.Spec.MinReplicas) || hpaSpec.MaxReplicas != primaryHpa.Spec.MaxReplicas {
+			fmt.Println(diff, hpaSpec.MinReplicas, primaryHpa.Spec.MinReplicas, hpaSpec.MaxReplicas, primaryHpa.Spec.MaxReplicas)
 			hpaClone := primaryHpa.DeepCopy()
 			hpaClone.Spec.MaxReplicas = hpaSpec.MaxReplicas
 			hpaClone.Spec.MinReplicas = hpaSpec.MinReplicas
@@ -434,3 +432,11 @@ func makePrimaryLabels(labels map[string]string, primaryName string, label strin
 func int32p(i int32) *int32 {
 	return &i
 }
+
+func int32Default(i *int32) int32 {
+	if i == nil {
+		return 1
+	}
+
+	return *i
+}

pkg/canary/deployer_test.go

@@ -229,7 +229,7 @@ func TestCanaryDeployer_SetState(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	err = mocks.deployer.SetStatusPhase(mocks.canary, v1alpha3.CanaryProgressing)
+	err = mocks.deployer.SetStatusPhase(mocks.canary, v1alpha3.CanaryPhaseProgressing)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -239,8 +239,8 @@ func TestCanaryDeployer_SetState(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	if res.Status.Phase != v1alpha3.CanaryProgressing {
-		t.Errorf("Got %v wanted %v", res.Status.Phase, v1alpha3.CanaryProgressing)
+	if res.Status.Phase != v1alpha3.CanaryPhaseProgressing {
+		t.Errorf("Got %v wanted %v", res.Status.Phase, v1alpha3.CanaryPhaseProgressing)
 	}
 }
 
@@ -252,7 +252,7 @@ func TestCanaryDeployer_SyncStatus(t *testing.T) {
 	}
 
 	status := v1alpha3.CanaryStatus{
-		Phase:        v1alpha3.CanaryProgressing,
+		Phase:        v1alpha3.CanaryPhaseProgressing,
 		FailedChecks: 2,
 	}
 	err = mocks.deployer.SyncStatus(mocks.canary, status)

pkg/canary/status.go

@@ -1,11 +1,13 @@
 package canary
 
 import (
-	"encoding/base64"
-	"encoding/json"
 	"fmt"
+	"k8s.io/client-go/util/retry"
 
+	"github.com/mitchellh/hashstructure"
+	ex "github.com/pkg/errors"
 	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -17,88 +19,227 @@ func (c *Deployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.CanaryStatu
 		if errors.IsNotFound(err) {
 			return fmt.Errorf("deployment %s.%s not found", cd.Spec.TargetRef.Name, cd.Namespace)
 		}
-		return fmt.Errorf("deployment %s.%s query error %v", cd.Spec.TargetRef.Name, cd.Namespace, err)
-	}
-
-	specJson, err := json.Marshal(dep.Spec.Template.Spec)
-	if err != nil {
-		return fmt.Errorf("deployment %s.%s marshal error %v", cd.Spec.TargetRef.Name, cd.Namespace, err)
+		return ex.Wrap(err, "SyncStatus deployment query error")
 	}
 
 	configs, err := c.ConfigTracker.GetConfigRefs(cd)
 	if err != nil {
-		return fmt.Errorf("configs query error %v", err)
+		return ex.Wrap(err, "SyncStatus configs query error")
 	}
 
-	cdCopy := cd.DeepCopy()
-	cdCopy.Status.Phase = status.Phase
-	cdCopy.Status.CanaryWeight = status.CanaryWeight
-	cdCopy.Status.FailedChecks = status.FailedChecks
-	cdCopy.Status.Iterations = status.Iterations
-	cdCopy.Status.LastAppliedSpec = base64.StdEncoding.EncodeToString(specJson)
-	cdCopy.Status.LastTransitionTime = metav1.Now()
-	cdCopy.Status.TrackedConfigs = configs
-
-	cd, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+	hash, err := hashstructure.Hash(dep.Spec.Template, nil)
 	if err != nil {
-		return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err)
+		return ex.Wrap(err, "SyncStatus hash error")
+	}
+
+	firstTry := true
+	err = retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+		var selErr error
+		if !firstTry {
+			cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{})
+			if selErr != nil {
+				return selErr
+			}
+		}
+		cdCopy := cd.DeepCopy()
+		cdCopy.Status.Phase = status.Phase
+		cdCopy.Status.CanaryWeight = status.CanaryWeight
+		cdCopy.Status.FailedChecks = status.FailedChecks
+		cdCopy.Status.Iterations = status.Iterations
+		cdCopy.Status.LastAppliedSpec = fmt.Sprintf("%d", hash)
+		cdCopy.Status.LastTransitionTime = metav1.Now()
+		cdCopy.Status.TrackedConfigs = configs
+
+		if ok, conditions := c.MakeStatusConditions(cd.Status, status.Phase); ok {
+			cdCopy.Status.Conditions = conditions
+		}
+
+		_, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		firstTry = false
+		return
+	})
+	if err != nil {
+		return ex.Wrap(err, "SyncStatus")
 	}
 	return nil
 }
 
 // SetStatusFailedChecks updates the canary failed checks counter
 func (c *Deployer) SetStatusFailedChecks(cd *flaggerv1.Canary, val int) error {
-	cdCopy := cd.DeepCopy()
-	cdCopy.Status.FailedChecks = val
-	cdCopy.Status.LastTransitionTime = metav1.Now()
+	firstTry := true
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+		var selErr error
+		if !firstTry {
+			cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{})
+			if selErr != nil {
+				return selErr
+			}
+		}
+		cdCopy := cd.DeepCopy()
+		cdCopy.Status.FailedChecks = val
+		cdCopy.Status.LastTransitionTime = metav1.Now()
 
-	cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		_, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		firstTry = false
+		return
+	})
 	if err != nil {
-		return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err)
+		return ex.Wrap(err, "SetStatusFailedChecks")
 	}
 	return nil
 }
 
 // SetStatusWeight updates the canary status weight value
 func (c *Deployer) SetStatusWeight(cd *flaggerv1.Canary, val int) error {
-	cdCopy := cd.DeepCopy()
-	cdCopy.Status.CanaryWeight = val
-	cdCopy.Status.LastTransitionTime = metav1.Now()
+	firstTry := true
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+		var selErr error
+		if !firstTry {
+			cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{})
+			if selErr != nil {
+				return selErr
+			}
+		}
+		cdCopy := cd.DeepCopy()
+		cdCopy.Status.CanaryWeight = val
+		cdCopy.Status.LastTransitionTime = metav1.Now()
 
-	cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		_, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		firstTry = false
+		return
+	})
 	if err != nil {
-		return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err)
+		return ex.Wrap(err, "SetStatusWeight")
 	}
 	return nil
 }
 
 // SetStatusIterations updates the canary status iterations value
 func (c *Deployer) SetStatusIterations(cd *flaggerv1.Canary, val int) error {
-	cdCopy := cd.DeepCopy()
-	cdCopy.Status.Iterations = val
-	cdCopy.Status.LastTransitionTime = metav1.Now()
+	firstTry := true
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+		var selErr error
+		if !firstTry {
+			cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{})
+			if selErr != nil {
+				return selErr
+			}
+		}
+
+		cdCopy := cd.DeepCopy()
+		cdCopy.Status.Iterations = val
+		cdCopy.Status.LastTransitionTime = metav1.Now()
+
+		_, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		firstTry = false
+		return
+	})
 
-	cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
 	if err != nil {
-		return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err)
+		return ex.Wrap(err, "SetStatusIterations")
 	}
 	return nil
 }
 
 // SetStatusPhase updates the canary status phase
 func (c *Deployer) SetStatusPhase(cd *flaggerv1.Canary, phase flaggerv1.CanaryPhase) error {
-	cdCopy := cd.DeepCopy()
-	cdCopy.Status.Phase = phase
-	cdCopy.Status.LastTransitionTime = metav1.Now()
+	firstTry := true
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+		var selErr error
+		if !firstTry {
+			cd, selErr = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).Get(cd.GetName(), metav1.GetOptions{})
+			if selErr != nil {
+				return selErr
+			}
+		}
+		cdCopy := cd.DeepCopy()
+		cdCopy.Status.Phase = phase
+		cdCopy.Status.LastTransitionTime = metav1.Now()
 
-	if phase != flaggerv1.CanaryProgressing {
-		cdCopy.Status.CanaryWeight = 0
-		cdCopy.Status.Iterations = 0
-	}
+		if phase != flaggerv1.CanaryPhaseProgressing && phase != flaggerv1.CanaryPhaseWaiting {
+			cdCopy.Status.CanaryWeight = 0
+			cdCopy.Status.Iterations = 0
+		}
 
-	cd, err := c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		// on promotion set primary spec hash
+		if phase == flaggerv1.CanaryPhaseInitialized || phase == flaggerv1.CanaryPhaseSucceeded {
+			cdCopy.Status.LastPromotedSpec = cd.Status.LastAppliedSpec
+		}
+
+		if ok, conditions := c.MakeStatusConditions(cdCopy.Status, phase); ok {
+			cdCopy.Status.Conditions = conditions
+		}
+
+		_, err = c.FlaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+		firstTry = false
+		return
+	})
 	if err != nil {
-		return fmt.Errorf("canary %s.%s status update error %v", cdCopy.Name, cdCopy.Namespace, err)
+		return ex.Wrap(err, "SetStatusPhase")
 	}
 	return nil
 }
+
+// GetStatusCondition returns a condition based on type
+func (c *Deployer) getStatusCondition(status flaggerv1.CanaryStatus, conditionType flaggerv1.CanaryConditionType) *flaggerv1.CanaryCondition {
+	for i := range status.Conditions {
+		c := status.Conditions[i]
+		if c.Type == conditionType {
+			return &c
+		}
+	}
+	return nil
+}
+
+// MakeStatusCondition updates the canary status conditions based on canary phase
+func (c *Deployer) MakeStatusConditions(canaryStatus flaggerv1.CanaryStatus,
+	phase flaggerv1.CanaryPhase) (bool, []flaggerv1.CanaryCondition) {
+	currentCondition := c.getStatusCondition(canaryStatus, flaggerv1.PromotedType)
+
+	message := "New deployment detected, starting initialization."
+	status := corev1.ConditionUnknown
+	switch phase {
+	case flaggerv1.CanaryPhaseInitializing:
+		status = corev1.ConditionUnknown
+		message = "New deployment detected, starting initialization."
+	case flaggerv1.CanaryPhaseInitialized:
+		status = corev1.ConditionTrue
+		message = "Deployment initialization completed."
+	case flaggerv1.CanaryPhaseWaiting:
+		status = corev1.ConditionUnknown
+		message = "Waiting for approval."
+	case flaggerv1.CanaryPhaseProgressing:
+		status = corev1.ConditionUnknown
+		message = "New revision detected, starting canary analysis."
+	case flaggerv1.CanaryPhaseFinalising:
+		status = corev1.ConditionUnknown
+		message = "Canary analysis completed, routing all traffic to primary."
+	case flaggerv1.CanaryPhaseSucceeded:
+		status = corev1.ConditionTrue
+		message = "Canary analysis completed successfully, promotion finished."
+	case flaggerv1.CanaryPhaseFailed:
+		status = corev1.ConditionFalse
+		message = "Canary analysis failed, deployment scaled to zero."
+	}
+
+	newCondition := &flaggerv1.CanaryCondition{
+		Type:               flaggerv1.PromotedType,
+		Status:             status,
+		LastUpdateTime:     metav1.Now(),
+		LastTransitionTime: metav1.Now(),
+		Message:            message,
+		Reason:             string(phase),
+	}
+
+	if currentCondition != nil &&
+		currentCondition.Status == newCondition.Status &&
+		currentCondition.Reason == newCondition.Reason {
+		return false, nil
+	}
+
+	if currentCondition != nil && currentCondition.Status == newCondition.Status {
+		newCondition.LastTransitionTime = currentCondition.LastTransitionTime
+	}
+
+	return true, []flaggerv1.CanaryCondition{*newCondition}
+}

pkg/controller/controller.go

@@ -5,20 +5,21 @@ import (
 	"sync"
 	"time"
 
-	"github.com/weaveworks/flagger/pkg/canary"
-	"github.com/weaveworks/flagger/pkg/metrics"
-	"github.com/weaveworks/flagger/pkg/router"
-
-	"github.com/google/go-cmp/cmp"
 	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3"
+	"github.com/weaveworks/flagger/pkg/canary"
 	clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
 	flaggerscheme "github.com/weaveworks/flagger/pkg/client/clientset/versioned/scheme"
 	flaggerinformers "github.com/weaveworks/flagger/pkg/client/informers/externalversions/flagger/v1alpha3"
 	flaggerlisters "github.com/weaveworks/flagger/pkg/client/listers/flagger/v1alpha3"
+	"github.com/weaveworks/flagger/pkg/metrics"
 	"github.com/weaveworks/flagger/pkg/notifier"
+	"github.com/weaveworks/flagger/pkg/router"
+
+	"github.com/google/go-cmp/cmp"
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
@@ -211,10 +212,25 @@ func (c *Controller) syncHandler(key string) error {
 	}
 	cd, err := c.flaggerLister.Canaries(namespace).Get(name)
 	if errors.IsNotFound(err) {
-		utilruntime.HandleError(fmt.Errorf("'%s' in work queue no longer exists", key))
+		utilruntime.HandleError(fmt.Errorf("%s in work queue no longer exists", key))
 		return nil
 	}
 
+	// set status condition for new canaries
+	if cd.Status.Conditions == nil {
+		if ok, conditions := c.deployer.MakeStatusConditions(cd.Status, flaggerv1.CanaryPhaseInitializing); ok {
+			cdCopy := cd.DeepCopy()
+			cdCopy.Status.Conditions = conditions
+			cdCopy.Status.LastTransitionTime = metav1.Now()
+			cdCopy.Status.Phase = flaggerv1.CanaryPhaseInitializing
+			_, err := c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
+			if err != nil {
+				c.logger.Errorf("%s status condition update error: %v", key, err)
+				return fmt.Errorf("%s status condition update error: %v", key, err)
+			}
+		}
+	}
+
 	c.canaries.Store(fmt.Sprintf("%s.%s", cd.Name, cd.Namespace), cd)
 	c.logger.Infof("Synced %s", key)
 

pkg/controller/controller_test.go

@@ -82,7 +82,7 @@ func SetupMocks(abtest bool) Mocks {
 	flaggerInformer := flaggerInformerFactory.Flagger().V1alpha3().Canaries()
 
 	// init router
-	rf := router.NewFactory(nil, kubeClient, flaggerClient, logger, flaggerClient)
+	rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", logger, flaggerClient)
 
 	// init observer
 	observerFactory, _ := metrics.NewFactory("fake", "istio", 5*time.Second)

pkg/controller/scheduler.go

@@ -134,6 +134,11 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		return
 	}
 
+	// check gates
+	if isApproved := c.runConfirmRolloutHooks(cd); !isApproved {
+		return
+	}
+
 	// set max weight default value to 100%
 	maxWeight := 100
 	if cd.Spec.CanaryAnalysis.MaxWeight > 0 {
@@ -178,7 +183,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 
 		// reset status
 		status := flaggerv1.CanaryStatus{
-			Phase:        flaggerv1.CanaryProgressing,
+			Phase:        flaggerv1.CanaryPhaseProgressing,
 			CanaryWeight: 0,
 			FailedChecks: 0,
 			Iterations:   0,
@@ -209,8 +214,28 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		return
 	}
 
+	// scale canary to zero if analysis has succeeded
+	if cd.Status.Phase == flaggerv1.CanaryPhaseFinalising {
+		if err := c.deployer.Scale(cd, 0); err != nil {
+			c.recordEventWarningf(cd, "%v", err)
+			return
+		}
+
+		// set status to succeeded
+		if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseSucceeded); err != nil {
+			c.recordEventWarningf(cd, "%v", err)
+			return
+		}
+		c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseSucceeded)
+		c.runPostRolloutHooks(cd, flaggerv1.CanaryPhaseSucceeded)
+		c.recordEventInfof(cd, "Promotion completed! Scaling down %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)
+		c.sendNotification(cd, "Canary analysis completed successfully, promotion finished.",
+			false, false)
+		return
+	}
+
 	// check if the number of failed checks reached the threshold
-	if cd.Status.Phase == flaggerv1.CanaryProgressing &&
+	if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing &&
 		(!retriable || cd.Status.FailedChecks >= cd.Spec.CanaryAnalysis.Threshold) {
 
 		if cd.Status.FailedChecks >= cd.Spec.CanaryAnalysis.Threshold {
@@ -246,13 +271,13 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		}
 
 		// mark canary as failed
-		if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryFailed, CanaryWeight: 0}); err != nil {
+		if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseFailed, CanaryWeight: 0}); err != nil {
 			c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Errorf("%v", err)
 			return
 		}
 
-		c.recorder.SetStatus(cd, flaggerv1.CanaryFailed)
-		c.runPostRolloutHooks(cd, flaggerv1.CanaryFailed)
+		c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseFailed)
+		c.runPostRolloutHooks(cd, flaggerv1.CanaryPhaseFailed)
 		return
 	}
 
@@ -314,31 +339,23 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 			return
 		}
 
-		// shutdown canary
+		// route all traffic to primary
 		if cd.Spec.CanaryAnalysis.Iterations < cd.Status.Iterations {
-			// route all traffic to the primary
-			if err := meshRouter.SetRoutes(cd, 100, 0); err != nil {
-				c.recordEventWarningf(cd, "%v", err)
-				return
-			}
-			c.recorder.SetWeight(cd, 100, 0)
-			c.recordEventInfof(cd, "Promotion completed! Scaling down %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)
-
-			// canary scale to zero
-			if err := c.deployer.Scale(cd, 0); err != nil {
+			primaryWeight = 100
+			canaryWeight = 0
+			if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
 				c.recordEventWarningf(cd, "%v", err)
 				return
 			}
+			c.recorder.SetWeight(cd, primaryWeight, canaryWeight)
 
 			// update status phase
-			if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanarySucceeded); err != nil {
+			if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseFinalising); err != nil {
 				c.recordEventWarningf(cd, "%v", err)
 				return
 			}
-			c.recorder.SetStatus(cd, flaggerv1.CanarySucceeded)
-			c.runPostRolloutHooks(cd, flaggerv1.CanarySucceeded)
-			c.sendNotification(cd, "Canary analysis completed successfully, promotion finished.",
-				false, false)
+
+			c.recordEventInfof(cd, "Routing all traffic to primary")
 			return
 		}
 
@@ -380,32 +397,23 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 			}
 		}
 	} else {
-		// route all traffic back to primary
+		// route all traffic to primary
 		primaryWeight = 100
 		canaryWeight = 0
 		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
 			c.recordEventWarningf(cd, "%v", err)
 			return
 		}
-
 		c.recorder.SetWeight(cd, primaryWeight, canaryWeight)
-		c.recordEventInfof(cd, "Promotion completed! Scaling down %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)
-
-		// shutdown canary
-		if err := c.deployer.Scale(cd, 0); err != nil {
-			c.recordEventWarningf(cd, "%v", err)
-			return
-		}
 
 		// update status phase
-		if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanarySucceeded); err != nil {
+		if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseFinalising); err != nil {
 			c.recordEventWarningf(cd, "%v", err)
 			return
 		}
-		c.recorder.SetStatus(cd, flaggerv1.CanarySucceeded)
-		c.runPostRolloutHooks(cd, flaggerv1.CanarySucceeded)
-		c.sendNotification(cd, "Canary analysis completed successfully, promotion finished.",
-			false, false)
+
+		c.recordEventInfof(cd, "Routing all traffic to primary")
+		return
 	}
 }
 
@@ -438,13 +446,13 @@ func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, meshRouter router.
 	}
 
 	// update status phase
-	if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanarySucceeded); err != nil {
+	if err := c.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseSucceeded); err != nil {
 		c.recordEventWarningf(cd, "%v", err)
 		return false
 	}
 
 	// notify
-	c.recorder.SetStatus(cd, flaggerv1.CanarySucceeded)
+	c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseSucceeded)
 	c.recordEventInfof(cd, "Promotion completed! Canary analysis was skipped for %s.%s",
 		cd.Spec.TargetRef.Name, cd.Namespace)
 	c.sendNotification(cd, "Canary analysis was skipped, promotion finished.",
@@ -454,7 +462,11 @@ func (c *Controller) shouldSkipAnalysis(cd *flaggerv1.Canary, meshRouter router.
 }
 
 func (c *Controller) shouldAdvance(cd *flaggerv1.Canary) (bool, error) {
-	if cd.Status.LastAppliedSpec == "" || cd.Status.Phase == flaggerv1.CanaryProgressing {
+	if cd.Status.LastAppliedSpec == "" ||
+		cd.Status.Phase == flaggerv1.CanaryPhaseInitializing ||
+		cd.Status.Phase == flaggerv1.CanaryPhaseProgressing ||
+		cd.Status.Phase == flaggerv1.CanaryPhaseWaiting ||
+		cd.Status.Phase == flaggerv1.CanaryPhaseFinalising {
 		return true, nil
 	}
 
@@ -477,16 +489,17 @@ func (c *Controller) shouldAdvance(cd *flaggerv1.Canary) (bool, error) {
 
 func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, shouldAdvance bool) bool {
 	c.recorder.SetStatus(cd, cd.Status.Phase)
-	if cd.Status.Phase == flaggerv1.CanaryProgressing {
+	if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing ||
+		cd.Status.Phase == flaggerv1.CanaryPhaseFinalising {
 		return true
 	}
 
-	if cd.Status.Phase == "" {
-		if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryInitialized}); err != nil {
+	if cd.Status.Phase == "" || cd.Status.Phase == flaggerv1.CanaryPhaseInitializing {
+		if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseInitialized}); err != nil {
 			c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Errorf("%v", err)
 			return false
 		}
-		c.recorder.SetStatus(cd, flaggerv1.CanaryInitialized)
+		c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseInitialized)
 		c.recordEventInfof(cd, "Initialization done! %s.%s", cd.Name, cd.Namespace)
 		c.sendNotification(cd, "New deployment detected, initialization completed.",
 			true, false)
@@ -501,18 +514,18 @@ func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, shouldAdvance bool)
 			c.recordEventErrorf(cd, "%v", err)
 			return false
 		}
-		if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryProgressing}); err != nil {
+		if err := c.deployer.SyncStatus(cd, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseProgressing}); err != nil {
 			c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Errorf("%v", err)
 			return false
 		}
-		c.recorder.SetStatus(cd, flaggerv1.CanaryProgressing)
+		c.recorder.SetStatus(cd, flaggerv1.CanaryPhaseProgressing)
 		return false
 	}
 	return false
 }
 
 func (c *Controller) hasCanaryRevisionChanged(cd *flaggerv1.Canary) bool {
-	if cd.Status.Phase == flaggerv1.CanaryProgressing {
+	if cd.Status.Phase == flaggerv1.CanaryPhaseProgressing {
 		if diff, _ := c.deployer.HasDeploymentChanged(cd); diff {
 			return true
 		}
@@ -523,10 +536,39 @@ func (c *Controller) hasCanaryRevisionChanged(cd *flaggerv1.Canary) bool {
 	return false
 }
 
+func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary) bool {
+	for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
+		if webhook.Type == flaggerv1.ConfirmRolloutHook {
+			err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
+			if err != nil {
+				if canary.Status.Phase != flaggerv1.CanaryPhaseWaiting {
+					if err := c.deployer.SetStatusPhase(canary, flaggerv1.CanaryPhaseWaiting); err != nil {
+						c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err)
+					}
+					c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for approval %s",
+						canary.Name, canary.Namespace, webhook.Name)
+					c.sendNotification(canary, "Canary is waiting for approval.", false, false)
+				}
+				return false
+			} else {
+				if canary.Status.Phase == flaggerv1.CanaryPhaseWaiting {
+					if err := c.deployer.SetStatusPhase(canary, flaggerv1.CanaryPhaseProgressing); err != nil {
+						c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)).Errorf("%v", err)
+						return false
+					}
+					c.recordEventInfof(canary, "Confirm-rollout check %s passed", webhook.Name)
+					return false
+				}
+			}
+		}
+	}
+	return true
+}
+
 func (c *Controller) runPreRolloutHooks(canary *flaggerv1.Canary) bool {
 	for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
 		if webhook.Type == flaggerv1.PreRolloutHook {
-			err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryProgressing, webhook)
+			err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
 			if err != nil {
 				c.recordEventWarningf(canary, "Halt %s.%s advancement pre-rollout check %s failed %v",
 					canary.Name, canary.Namespace, webhook.Name, err)
@@ -558,7 +600,7 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
 	// run external checks
 	for _, webhook := range r.Spec.CanaryAnalysis.Webhooks {
 		if webhook.Type == "" || webhook.Type == flaggerv1.RolloutHook {
-			err := CallWebhook(r.Name, r.Namespace, flaggerv1.CanaryProgressing, webhook)
+			err := CallWebhook(r.Name, r.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
 			if err != nil {
 				c.recordEventWarningf(r, "Halt %s.%s advancement external check %s failed %v",
 					r.Name, r.Namespace, webhook.Name, err)
@@ -572,8 +614,8 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
 	if r.Spec.Provider != "" {
 		metricsProvider = r.Spec.Provider
 
-		// set the metrics provider to Linkerd Prometheus when using NGINX as Linkerd Ingress
-		if r.Spec.Provider == "nginx" && strings.Contains(c.meshProvider, "linkerd") {
+		// set the metrics server to Linkerd Prometheus when Linkerd is the default mesh provider
+		if strings.Contains(c.meshProvider, "linkerd") {
 			metricsProvider = "linkerd"
 		}
 	}

pkg/controller/scheduler_test.go

@@ -47,7 +47,7 @@ func TestScheduler_Rollback(t *testing.T) {
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
 	// update failed checks to max
-	err := mocks.deployer.SyncStatus(mocks.canary, v1alpha3.CanaryStatus{Phase: v1alpha3.CanaryProgressing, FailedChecks: 11})
+	err := mocks.deployer.SyncStatus(mocks.canary, v1alpha3.CanaryStatus{Phase: v1alpha3.CanaryPhaseProgressing, FailedChecks: 11})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -60,8 +60,8 @@ func TestScheduler_Rollback(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	if c.Status.Phase != v1alpha3.CanaryFailed {
-		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryFailed)
+	if c.Status.Phase != v1alpha3.CanaryPhaseFailed {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseFailed)
 	}
 }
 
@@ -101,8 +101,8 @@ func TestScheduler_SkipAnalysis(t *testing.T) {
 		t.Errorf("Got skip analysis %v wanted %v", c.Spec.SkipAnalysis, true)
 	}
 
-	if c.Status.Phase != v1alpha3.CanarySucceeded {
-		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanarySucceeded)
+	if c.Status.Phase != v1alpha3.CanaryPhaseSucceeded {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseSucceeded)
 	}
 }
 
@@ -250,13 +250,22 @@ func TestScheduler_Promotion(t *testing.T) {
 		t.Errorf("Got primary secret %s wanted %s", secretPrimary.Data["apiKey"], secret2.Data["apiKey"])
 	}
 
+	// check finalising status
 	c, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	if c.Status.Phase != v1alpha3.CanarySucceeded {
-		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanarySucceeded)
+	// scale canary to zero
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	c, err = mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if c.Status.Phase != v1alpha3.CanaryPhaseSucceeded {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseSucceeded)
 	}
 }
 
@@ -302,9 +311,22 @@ func TestScheduler_ABTesting(t *testing.T) {
 		t.Fatal(err.Error())
 	}
 
-	// promote
+	// advance
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
+	// finalising
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	// check finalising status
+	c, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if c.Status.Phase != v1alpha3.CanaryPhaseFinalising {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseFinalising)
+	}
+
 	// check if the container image tag was updated
 	primaryDep, err := mocks.kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
 	if err != nil {
@@ -321,13 +343,13 @@ func TestScheduler_ABTesting(t *testing.T) {
 	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
 	// check rollout status
-	c, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	c, err = mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	if c.Status.Phase != v1alpha3.CanarySucceeded {
-		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanarySucceeded)
+	if c.Status.Phase != v1alpha3.CanaryPhaseSucceeded {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryPhaseSucceeded)
 	}
 }
 

pkg/controller/webhook_test.go

@@ -19,7 +19,7 @@ func TestCallWebhook(t *testing.T) {
 		Metadata: &map[string]string{"key1": "val1"},
 	}
 
-	err := CallWebhook("podinfo", "default", flaggerv1.CanaryProgressing, hook)
+	err := CallWebhook("podinfo", "default", flaggerv1.CanaryPhaseProgressing, hook)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -35,7 +35,7 @@ func TestCallWebhook_StatusCode(t *testing.T) {
 		URL:  ts.URL,
 	}
 
-	err := CallWebhook("podinfo", "default", flaggerv1.CanaryProgressing, hook)
+	err := CallWebhook("podinfo", "default", flaggerv1.CanaryPhaseProgressing, hook)
 	if err == nil {
 		t.Errorf("Got no error wanted %v", http.StatusInternalServerError)
 	}

pkg/loadtester/bash.go

@@ -24,7 +24,7 @@ func (task *BashTask) Run(ctx context.Context) (bool, error) {
 
 	if err != nil {
 		task.logger.With("canary", task.canary).Errorf("command failed %s %v %s", task.command, err, out)
-		return false, fmt.Errorf(" %v %v", err, out)
+		return false, fmt.Errorf(" %v %s", err, out)
 	} else {
 		if task.logCmdOutput {
 			fmt.Printf("%s\n", out)

pkg/loadtester/gate.go

@@ -0,0 +1,31 @@
+package loadtester
+
+import "sync"
+
+type GateStorage struct {
+	backend string
+	data    *sync.Map
+}
+
+func NewGateStorage(backend string) *GateStorage {
+	return &GateStorage{
+		backend: backend,
+		data:    new(sync.Map),
+	}
+}
+
+func (gs *GateStorage) open(key string) {
+	gs.data.Store(key, true)
+}
+
+func (gs *GateStorage) close(key string) {
+	gs.data.Store(key, false)
+}
+
+func (gs *GateStorage) isOpen(key string) (locked bool) {
+	val, ok := gs.data.LoadOrStore(key, false)
+	if ok {
+		return val.(bool)
+	}
+	return
+}

pkg/loadtester/helm.go

@@ -20,14 +20,14 @@ func (task *HelmTask) Hash() string {
 }
 
 func (task *HelmTask) Run(ctx context.Context) (bool, error) {
-	helmCmd := fmt.Sprintf("helm %s", task.command)
+	helmCmd := fmt.Sprintf("%s %s", TaskTypeHelm, task.command)
 	task.logger.With("canary", task.canary).Infof("running command %v", helmCmd)
 
-	cmd := exec.CommandContext(ctx, "helm", strings.Fields(task.command)...)
+	cmd := exec.CommandContext(ctx, TaskTypeHelm, strings.Fields(task.command)...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		task.logger.With("canary", task.canary).Errorf("command failed %s %v %s", task.command, err, out)
-		return false, fmt.Errorf(" %v %v", err, out)
+		return false, fmt.Errorf(" %v %s", err, out)
 	} else {
 		if task.logCmdOutput {
 			fmt.Printf("%s\n", out)

pkg/loadtester/helmv3.go

@@ -0,0 +1,42 @@
+package loadtester
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+const TaskTypeHelmv3 = "helmv3"
+
+type HelmTaskv3 struct {
+	TaskBase
+	command      string
+	logCmdOutput bool
+}
+
+func (task *HelmTaskv3) Hash() string {
+	return hash(task.canary + task.command)
+}
+
+func (task *HelmTaskv3) Run(ctx context.Context) (bool, error) {
+	helmCmd := fmt.Sprintf("%s %s", TaskTypeHelmv3, task.command)
+	task.logger.With("canary", task.canary).Infof("running command %v", helmCmd)
+
+	cmd := exec.CommandContext(ctx, TaskTypeHelmv3, strings.Fields(task.command)...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		task.logger.With("canary", task.canary).Errorf("command failed %s %v %s", task.command, err, out)
+		return false, fmt.Errorf(" %v %s", err, out)
+	} else {
+		if task.logCmdOutput {
+			fmt.Printf("%s\n", out)
+		}
+		task.logger.With("canary", task.canary).Infof("command finished %v", helmCmd)
+	}
+	return true, nil
+}
+
+func (task *HelmTaskv3) String() string {
+	return task.command
+}

pkg/loadtester/server.go

@@ -14,13 +14,101 @@ import (
 )
 
 // ListenAndServe starts a web server and waits for SIGTERM
-func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogger, taskRunner *TaskRunner, stopCh <-chan struct{}) {
+func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogger, taskRunner *TaskRunner, gate *GateStorage, stopCh <-chan struct{}) {
 	mux := http.DefaultServeMux
 	mux.Handle("/metrics", promhttp.Handler())
 	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 		w.Write([]byte("OK"))
 	})
+	mux.HandleFunc("/gate/approve", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("OK"))
+	})
+	mux.HandleFunc("/gate/halt", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusForbidden)
+		w.Write([]byte("Forbidden"))
+	})
+	mux.HandleFunc("/gate/check", func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			logger.Error("reading the request body failed", zap.Error(err))
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		defer r.Body.Close()
+
+		canary := &flaggerv1.CanaryWebhookPayload{}
+		err = json.Unmarshal(body, canary)
+		if err != nil {
+			logger.Error("decoding the request body failed", zap.Error(err))
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		canaryName := fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)
+		approved := gate.isOpen(canaryName)
+		if approved {
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("Approved"))
+		} else {
+			w.WriteHeader(http.StatusForbidden)
+			w.Write([]byte("Forbidden"))
+		}
+
+		logger.Infof("%s gate check: approved %v", canaryName, approved)
+	})
+
+	mux.HandleFunc("/gate/open", func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			logger.Error("reading the request body failed", zap.Error(err))
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		defer r.Body.Close()
+
+		canary := &flaggerv1.CanaryWebhookPayload{}
+		err = json.Unmarshal(body, canary)
+		if err != nil {
+			logger.Error("decoding the request body failed", zap.Error(err))
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		canaryName := fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)
+		gate.open(canaryName)
+
+		w.WriteHeader(http.StatusAccepted)
+
+		logger.Infof("%s gate opened", canaryName)
+	})
+
+	mux.HandleFunc("/gate/close", func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			logger.Error("reading the request body failed", zap.Error(err))
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		defer r.Body.Close()
+
+		canary := &flaggerv1.CanaryWebhookPayload{}
+		err = json.Unmarshal(body, canary)
+		if err != nil {
+			logger.Error("decoding the request body failed", zap.Error(err))
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		canaryName := fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)
+		gate.close(canaryName)
+
+		w.WriteHeader(http.StatusAccepted)
+
+		logger.Infof("%s gate closed", canaryName)
+	})
+
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		body, err := ioutil.ReadAll(r.Body)
 		if err != nil {
@@ -97,6 +185,31 @@ func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogge
 				return
 			}
 
+			// run helmv3 command (blocking task)
+			if typ == TaskTypeHelmv3 {
+				helm := HelmTaskv3{
+					command:      payload.Metadata["cmd"],
+					logCmdOutput: true,
+					TaskBase: TaskBase{
+						canary: fmt.Sprintf("%s.%s", payload.Name, payload.Namespace),
+						logger: logger,
+					},
+				}
+
+				ctx, cancel := context.WithTimeout(context.Background(), taskRunner.timeout)
+				defer cancel()
+
+				ok, err := helm.Run(ctx)
+				if !ok {
+					w.WriteHeader(http.StatusInternalServerError)
+					w.Write([]byte(err.Error()))
+					return
+				}
+
+				w.WriteHeader(http.StatusOK)
+				return
+			}
+
 			taskFactory, ok := GetTaskFactory(typ)
 			if !ok {
 				w.WriteHeader(http.StatusBadRequest)

pkg/metrics/factory.go

@@ -28,6 +28,10 @@ func (factory Factory) Observer(provider string) Interface {
 		return &HttpObserver{
 			client: factory.Client,
 		}
+	case provider == "kubernetes":
+		return &HttpObserver{
+			client: factory.Client,
+		}
 	case provider == "appmesh":
 		return &EnvoyObserver{
 			client: factory.Client,

pkg/metrics/recorder.go

@@ -87,9 +87,9 @@ func (cr *Recorder) SetTotal(namespace string, total int) {
 func (cr *Recorder) SetStatus(cd *flaggerv1.Canary, phase flaggerv1.CanaryPhase) {
 	status := 1
 	switch phase {
-	case flaggerv1.CanaryProgressing:
+	case flaggerv1.CanaryPhaseProgressing:
 		status = 0
-	case flaggerv1.CanaryFailed:
+	case flaggerv1.CanaryPhaseFailed:
 		status = 2
 	default:
 		status = 1