Release v0.5.0

Track changes in ConfigMaps and Secrets

.github/main.workflow

@@ -0,0 +1,17 @@
+workflow "Publish Helm charts" {
+  on = "push"
+  resolves = ["helm-push"]
+}
+
+action "helm-lint" {
+  uses = "stefanprodan/gh-actions/helm@master"
+  args = ["lint charts/*"]
+}
+
+action "helm-push" {
+  needs = ["helm-lint"]
+  uses = "stefanprodan/gh-actions/helm-gh-pages@master"
+  args = ["charts/*","https://flagger.app"]
+  secrets = ["GITHUB_TOKEN"]
+}
+

.travis.yml

@@ -23,9 +23,10 @@ after_success:
 - if [ -z "$DOCKER_USER" ]; then
     echo "PR build, skipping image push";
   else
-    docker tag stefanprodan/flagger:latest quay.io/stefanprodan/flagger:${TRAVIS_COMMIT};
+    BRANCH_COMMIT=${TRAVIS_BRANCH}-$(echo ${TRAVIS_COMMIT} | head -c7);
+    docker tag stefanprodan/flagger:latest quay.io/stefanprodan/flagger:${BRANCH_COMMIT};
     echo $DOCKER_PASS | docker login -u=$DOCKER_USER --password-stdin quay.io;
-    docker push quay.io/stefanprodan/flagger:${TRAVIS_COMMIT};
+    docker push quay.io/stefanprodan/flagger:${BRANCH_COMMIT};
   fi
 - if [ -z "$TRAVIS_TAG" ]; then
     echo "Not a release, skipping image push";

Dockerfile.loadtester

@@ -0,0 +1,44 @@
+FROM golang:1.11 AS hey-builder
+
+RUN mkdir -p /go/src/github.com/rakyll/hey/
+
+WORKDIR /go/src/github.com/rakyll/hey
+
+ADD https://github.com/rakyll/hey/archive/v0.1.1.tar.gz .
+
+RUN tar xzf v0.1.1.tar.gz --strip 1
+
+RUN go get ./...
+
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
+  go install -ldflags '-w -extldflags "-static"' \
+  /go/src/github.com/rakyll/hey
+
+FROM golang:1.11 AS builder
+
+RUN mkdir -p /go/src/github.com/stefanprodan/flagger/
+
+WORKDIR /go/src/github.com/stefanprodan/flagger
+
+COPY . .
+
+RUN go test -race ./pkg/loadtester/
+
+RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o loadtester ./cmd/loadtester/*
+
+FROM alpine:3.8
+
+RUN addgroup -S app \
+    && adduser -S -g app app \
+    && apk --no-cache add ca-certificates curl
+
+WORKDIR /home/app
+
+COPY --from=hey-builder /go/bin/hey /usr/local/bin/hey
+COPY --from=builder /go/src/github.com/stefanprodan/flagger/loadtester .
+
+RUN chown -R app:app ./
+
+USER app
+
+ENTRYPOINT ["./loadtester"]

Makefile

@@ -3,6 +3,8 @@ VERSION?=$(shell grep 'VERSION' pkg/version/version.go | awk '{ print $$4 }' | t
 VERSION_MINOR:=$(shell grep 'VERSION' pkg/version/version.go | awk '{ print $$4 }' | tr -d '"' | rev | cut -d'.' -f2- | rev)
 PATCH:=$(shell grep 'VERSION' pkg/version/version.go | awk '{ print $$4 }' | tr -d '"' | awk -F. '{print $$NF}')
 SOURCE_DIRS = cmd pkg/apis pkg/controller pkg/server pkg/logging pkg/version
+LT_VERSION?=$(shell grep 'VERSION' cmd/loadtester/main.go | awk '{ print $$4 }' | tr -d '"' | head -n1)
+
 run:
 	go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info \
 	-metrics-server=https://prometheus.iowa.weavedx.com \
@@ -29,7 +31,7 @@ test: test-fmt test-codegen
 	go test ./...
 
 helm-package:
-	cd charts/ && helm package flagger/ && helm package grafana/
+	cd charts/ && helm package flagger/ && helm package grafana/ && helm package loadtester/
 	mv charts/*.tgz docs/
 	helm repo index docs --url https://stefanprodan.github.io/flagger --merge ./docs/index.yaml
 
@@ -44,6 +46,7 @@ version-set:
 	sed -i '' "s/flagger:$$current/flagger:$$next/g" artifacts/flagger/deployment.yaml && \
 	sed -i '' "s/tag: $$current/tag: $$next/g" charts/flagger/values.yaml && \
 	sed -i '' "s/appVersion: $$current/appVersion: $$next/g" charts/flagger/Chart.yaml && \
+	sed -i '' "s/version: $$current/version: $$next/g" charts/flagger/Chart.yaml && \
 	echo "Version $$next set in code, deployment and charts"
 
 version-up:
@@ -77,3 +80,7 @@ reset-test:
 	kubectl delete -f ./artifacts/namespaces
 	kubectl apply -f ./artifacts/namespaces
 	kubectl apply -f ./artifacts/canaries
+
+loadtester-push:
+	docker build -t quay.io/stefanprodan/flagger-loadtester:$(LT_VERSION) . -f Dockerfile.loadtester
+	docker push quay.io/stefanprodan/flagger-loadtester:$(LT_VERSION)

README.md

@@ -38,6 +38,9 @@ ClusterIP [services](https://kubernetes.io/docs/concepts/services-networking/ser
 Istio [virtual services](https://istio.io/docs/reference/config/istio.networking.v1alpha3/#VirtualService)) 
 to drive the canary analysis and promotion.
 
+Flagger keeps track of ConfigMaps and Secrets referenced by a Kubernetes Deployment and triggers a canary analysis if any of those objects change. 
+When promoting a workload in production, both code (container images) and configuration (config maps and secrets) are being synchronised.
+
 ![flagger-overview](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/diagrams/flagger-canary-overview.png)
 
 Gated canary promotion stages:
@@ -48,28 +51,28 @@ Gated canary promotion stages:
     * halt advancement if a rolling update is underway
     * halt advancement if pods are unhealthy
 * increase canary traffic weight percentage from 0% to 5% (step weight)
+* call webhooks and check results
 * check canary HTTP request success rate and latency
     * halt advancement if any metric is under the specified threshold
     * increment the failed checks counter
 * check if the number of failed checks reached the threshold
     * route all traffic to primary
     * scale to zero the canary deployment and mark it as failed
-    * wait for the canary deployment to be updated (revision bump) and start over
+    * wait for the canary deployment to be updated and start over
 * increase canary traffic weight by 5% (step weight) till it reaches 50% (max weight) 
     * halt advancement while canary request success rate is under the threshold
     * halt advancement while canary request duration P99 is over the threshold
     * halt advancement if the primary or canary deployment becomes unhealthy 
     * halt advancement while canary deployment is being scaled up/down by HPA
 * promote canary to primary
+    * copy ConfigMaps and Secrets from canary to primary
     * copy canary deployment spec template over primary
 * wait for primary rolling update to finish
     * halt advancement if pods are unhealthy
 * route all traffic to primary
 * scale to zero the canary deployment
 * mark rollout as finished
-* wait for the canary deployment to be updated (revision bump) and start over
-
-You can change the canary analysis _max weight_ and the _step weight_ percentage in the Flagger's custom resource.
+* wait for the canary deployment to be updated and start over
 
 For a deployment named _podinfo_, a canary promotion can be defined using Flagger's custom resource:
 
@@ -127,12 +130,11 @@ spec:
       interval: 30s
     # external checks (optional)
     webhooks:
-      - name: integration-tests
-        url: http://podinfo.test:9898/echo
-        timeout: 1m
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
         metadata:
-          test: "all"
-          token: "16688eb5e9f289f1991c"
+          cmd: "hey -z 1m -q 10 -c 2 http://podinfo.test:9898/"
 ```
 
 The canary analysis is using the following promql queries:
@@ -211,6 +213,13 @@ kubectl apply -f ${REPO}/artifacts/canaries/deployment.yaml
 kubectl apply -f ${REPO}/artifacts/canaries/hpa.yaml
 ```
 
+Deploy the load testing service to generate traffic during the canary analysis:
+
+```bash
+kubectl -n test apply -f ${REPO}/artifacts/loadtester/deployment.yaml
+kubectl -n test apply -f ${REPO}/artifacts/loadtester/service.yaml
+```
+
 Create a canary promotion custom resource (replace the Istio gateway and the internet domain with your own):
 
 ```bash
@@ -239,9 +248,12 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.2.1
+podinfod=quay.io/stefanprodan/podinfo:1.4.0
 ```
 
+**Note** that Flagger tracks changes in the deployment `PodSpec` but also in `ConfigMaps` and `Secrets` 
+that are referenced in the pod's volumes and containers environment variables.
+
 Flagger detects that the deployment revision changed and starts a new canary analysis:
 
 ```
@@ -330,6 +342,8 @@ Events:
   Warning  Synced  1m    flagger  Canary failed! Scaling down podinfo.test
 ```
 
+**Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis.
+
 ### Monitoring
 
 Flagger comes with a Grafana dashboard made for canary analysis.

artifacts/canaries/canary.yaml

@@ -51,9 +51,8 @@ spec:
       interval: 30s
     # external checks (optional)
     webhooks:
-      - name: integration-tests
-        url: https://httpbin.org/post
-        timeout: 1m
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
         metadata:
-          test: "all"
-          token: "16688eb5e9f289f1991c"
+          cmd: "hey -z 1m -q 10 -c 2 http://podinfo.test:9898/"

artifacts/configs/canary.yaml

@@ -0,0 +1,58 @@
+apiVersion: flagger.app/v1alpha3
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  # deployment reference
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  # the maximum time in seconds for the canary deployment
+  # to make progress before it is rollback (default 600s)
+  progressDeadlineSeconds: 60
+  # HPA reference (optional)
+  autoscalerRef:
+    apiVersion: autoscaling/v2beta1
+    kind: HorizontalPodAutoscaler
+    name: podinfo
+  service:
+    # container port
+    port: 9898
+    # Istio gateways (optional)
+    gateways:
+    - public-gateway.istio-system.svc.cluster.local
+    # Istio virtual service host names (optional)
+    hosts:
+    - app.iowa.weavedx.com
+  canaryAnalysis:
+    # schedule interval (default 60s)
+    interval: 10s
+    # max number of failed metric checks before rollback
+    threshold: 10
+    # max traffic percentage routed to canary
+    # percentage (0-100)
+    maxWeight: 50
+    # canary increment step
+    # percentage (0-100)
+    stepWeight: 5
+    # Istio Prometheus checks
+    metrics:
+    - name: istio_requests_total
+      # minimum req success rate (non 5xx responses)
+      # percentage (0-100)
+      threshold: 99
+      interval: 1m
+    - name: istio_request_duration_seconds_bucket
+      # maximum req duration P99
+      # milliseconds
+      threshold: 500
+      interval: 30s
+    # external checks (optional)
+    webhooks:
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          cmd: "hey -z 1m -q 10 -c 2 http://podinfo.test:9898/"

artifacts/configs/configs.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: podinfo-config-env
+  namespace: test
+data:
+  color: blue
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: podinfo-config-vol
+  namespace: test
+data:
+  output: console
+  textmode: "true"

artifacts/configs/deployment.yaml

@@ -0,0 +1,89 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: podinfo
+  namespace: test
+  labels:
+    app: podinfo
+spec:
+  minReadySeconds: 5
+  revisionHistoryLimit: 5
+  progressDeadlineSeconds: 60
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 0
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: podinfo
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+      labels:
+        app: podinfo
+    spec:
+      containers:
+      - name: podinfod
+        image: quay.io/stefanprodan/podinfo:1.3.0
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 9898
+          name: http
+          protocol: TCP
+        command:
+        - ./podinfo
+        - --port=9898
+        - --level=info
+        - --random-delay=false
+        - --random-error=false
+        env:
+        - name: PODINFO_UI_COLOR
+          valueFrom:
+            configMapKeyRef:
+              name: podinfo-config-env
+              key: color
+        - name: SECRET_USER
+          valueFrom:
+            secretKeyRef:
+              name: podinfo-secret-env
+              key: user
+        livenessProbe:
+          exec:
+            command:
+            - podcli
+            - check
+            - http
+            - localhost:9898/healthz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        readinessProbe:
+          exec:
+            command:
+            - podcli
+            - check
+            - http
+            - localhost:9898/readyz
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        resources:
+          limits:
+            cpu: 2000m
+            memory: 512Mi
+          requests:
+            cpu: 100m
+            memory: 64Mi
+        volumeMounts:
+          - name: configs
+            mountPath: /etc/podinfo/configs
+            readOnly: true
+          - name: secrets
+            mountPath: /etc/podinfo/secrets
+            readOnly: true
+      volumes:
+        - name: configs
+          configMap:
+            name: podinfo-config-vol
+        - name: secrets
+          secret:
+            secretName: podinfo-secret-vol

artifacts/configs/hpa.yaml

@@ -0,0 +1,19 @@
+apiVersion: autoscaling/v2beta1
+kind: HorizontalPodAutoscaler
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  minReplicas: 1
+  maxReplicas: 4
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      # scale up if usage is above
+      # 99% of the requested CPU (100m)
+      targetAverageUtilization: 99

artifacts/configs/secrets.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: podinfo-secret-env
+  namespace: test
+data:
+  password: cGFzc3dvcmQ=
+  user: YWRtaW4=
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: podinfo-secret-vol
+  namespace: test
+data:
+  key: cGFzc3dvcmQ=

artifacts/flagger/deployment.yaml

@@ -22,7 +22,7 @@ spec:
       serviceAccountName: flagger
       containers:
       - name: flagger
-        image: quay.io/stefanprodan/flagger:0.4.0
+        image: quay.io/stefanprodan/flagger:0.5.0
         imagePullPolicy: Always
         ports:
         - name: http

artifacts/loadtester/deployment.yaml

@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flagger-loadtester
+  labels:
+    app: flagger-loadtester
+spec:
+  selector:
+    matchLabels:
+      app: flagger-loadtester
+  template:
+    metadata:
+      labels:
+        app: flagger-loadtester
+      annotations:
+        prometheus.io/scrape: "true"
+    spec:
+      containers:
+        - name: loadtester
+          image: quay.io/stefanprodan/flagger-loadtester:0.1.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 8080
+          command:
+            - ./loadtester
+            - -port=8080
+            - -log-level=info
+            - -timeout=1h
+            - -log-cmd-output=true
+          livenessProbe:
+            exec:
+              command:
+                - wget
+                - --quiet
+                - --tries=1
+                - --timeout=4
+                - --spider
+                - http://localhost:8080/healthz
+            timeoutSeconds: 5
+          readinessProbe:
+            exec:
+              command:
+                - wget
+                - --quiet
+                - --tries=1
+                - --timeout=4
+                - --spider
+                - http://localhost:8080/healthz
+            timeoutSeconds: 5
+          resources:
+            limits:
+              memory: "512Mi"
+              cpu: "1000m"
+            requests:
+              memory: "32Mi"
+              cpu: "10m"
+          securityContext:
+            readOnlyRootFilesystem: true
+            runAsUser: 10001

artifacts/loadtester/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: flagger-loadtester
+  labels:
+    app: flagger-loadtester
+spec:
+  type: ClusterIP
+  selector:
+    app: flagger-loadtester
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: http

charts/flagger/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 name: flagger
-version: 0.4.0
-appVersion: 0.4.0
+version: 0.5.0
+appVersion: 0.5.0
 kubeVersion: ">=1.11.0-0"
 engine: gotpl
 description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio routing for traffic shifting and Prometheus metrics for canary analysis.

charts/flagger/values.yaml

@@ -2,7 +2,7 @@
 
 image:
   repository: quay.io/stefanprodan/flagger
-  tag: 0.4.0
+  tag: 0.5.0
   pullPolicy: IfNotPresent
 
 metricsServer: "http://prometheus.istio-system.svc.cluster.local:9090"

charts/grafana/README.md

@@ -6,7 +6,7 @@ Grafana dashboards for monitoring progressive deployments powered by Istio, Prom
 
 ## Prerequisites
 
-* Kubernetes >= 1.9
+* Kubernetes >= 1.11
 * Istio >= 1.0
 * Prometheus >= 2.6
 
@@ -75,5 +75,5 @@ helm install flagger/grafana --name flagger-grafana -f values.yaml
 ```
 
 > **Tip**: You can use the default [values.yaml](values.yaml)
-```
+
 

charts/loadtester/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/loadtester/Chart.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+name: loadtester
+version: 0.1.0
+appVersion: 0.1.0
+kubeVersion: ">=1.11.0-0"
+engine: gotpl
+description: Flagger's load testing services based on rakyll/hey that generates traffic during canary analysis when configured as a webhook.
+home: https://docs.flagger.app
+icon: https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/logo/flagger-icon.png
+sources:
+  - https://github.com/stefanprodan/flagger
+maintainers:
+  - name: stefanprodan
+    url: https://github.com/stefanprodan
+    email: stefanprodan@users.noreply.github.com
+keywords:
+  - canary
+  - istio
+  - gitops
+  - load testing

charts/loadtester/README.md

@@ -0,0 +1,78 @@
+# Flagger load testing service
+
+[Flagger's](https://github.com/stefanprodan/flagger) load testing service is based on 
+[rakyll/hey](https://github.com/rakyll/hey) 
+and can be used to generates traffic during canary analysis when configured as a webhook.
+
+## Prerequisites
+
+* Kubernetes >= 1.11
+* Istio >= 1.0
+
+## Installing the Chart
+
+Add Flagger Helm repository:
+
+```console
+helm repo add flagger https://flagger.app
+```
+
+To install the chart with the release name `flagger-loadtester`:
+
+```console
+helm upgrade -i flagger-loadtester flagger/loadtester
+```
+
+The command deploys Grafana on the Kubernetes cluster in the default namespace.
+
+> **Tip**: Note that the namespace where you deploy the load tester should have the Istio sidecar injection enabled
+
+The [configuration](#configuration) section lists the parameters that can be configured during installation.
+
+## Uninstalling the Chart
+
+To uninstall/delete the `flagger-loadtester` deployment:
+
+```console
+helm delete --purge flagger-loadtester
+```
+
+The command removes all the Kubernetes components associated with the chart and deletes the release.
+
+## Configuration
+
+The following tables lists the configurable parameters of the load tester chart and their default values.
+
+Parameter | Description | Default
+--- | --- | ---
+`image.repository` | Image repository | `quay.io/stefanprodan/flagger-loadtester`
+`image.pullPolicy` | Image pull policy | `IfNotPresent`
+`image.tag` | Image tag | `<VERSION>`
+`replicaCount` | desired number of pods | `1`
+`resources.requests.cpu` | CPU requests | `10m`
+`resources.requests.memory` | memory requests | `64Mi`
+`tolerations` | List of node taints to tolerate | `[]`
+`affinity` | node/pod affinities | `node`
+`nodeSelector` | node labels for pod assignment | `{}`
+`service.type` | type of service | `ClusterIP`
+`service.port` | ClusterIP port | `80`
+`cmd.logOutput` | Log the command output to stderr | `true`
+`cmd.timeout` | Command execution timeout | `1h`
+`logLevel` | Log level can be debug, info, warning, error or panic | `info`
+
+Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example,
+
+```console
+helm install flagger/loadtester --name flagger-loadtester \
+--set cmd.logOutput=false
+```
+
+Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example,
+
+```console
+helm install flagger/loadtester --name flagger-loadtester -f values.yaml
+```
+
+> **Tip**: You can use the default [values.yaml](values.yaml)
+
+

charts/loadtester/templates/NOTES.txt

@@ -0,0 +1 @@
+Flagger's load testing service is available at http://{{ include "loadtester.fullname" . }}.{{ .Release.Namespace }}/

charts/loadtester/templates/_helpers.tpl

@@ -0,0 +1,32 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "loadtester.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "loadtester.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "loadtester.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}

charts/loadtester/templates/deployment.yaml

@@ -0,0 +1,66 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "loadtester.fullname" . }}
+  labels:
+    app.kubernetes.io/name: {{ include "loadtester.name" . }}
+    helm.sh/chart: {{ include "loadtester.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ include "loadtester.name" . }}
+  template:
+    metadata:
+      labels:
+        app: {{ include "loadtester.name" . }}
+    spec:
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: 8080
+          command:
+            - ./loadtester
+            - -port=8080
+            - -log-level={{ .Values.logLevel }}
+            - -timeout={{ .Values.cmd.timeout }}
+            - -log-cmd-output={{ .Values.cmd.logOutput }}
+          livenessProbe:
+            exec:
+              command:
+                - wget
+                - --quiet
+                - --tries=1
+                - --timeout=4
+                - --spider
+                - http://localhost:8080/healthz
+            timeoutSeconds: 5
+          readinessProbe:
+            exec:
+              command:
+                - wget
+                - --quiet
+                - --tries=1
+                - --timeout=4
+                - --spider
+                - http://localhost:8080/healthz
+            timeoutSeconds: 5
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}

charts/loadtester/templates/service.yaml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "loadtester.fullname" . }}
+  labels:
+    app.kubernetes.io/name: {{ include "loadtester.name" . }}
+    helm.sh/chart: {{ include "loadtester.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: {{ include "loadtester.name" . }}

charts/loadtester/values.yaml

@@ -0,0 +1,29 @@
+replicaCount: 1
+
+image:
+  repository: quay.io/stefanprodan/flagger-loadtester
+  tag: 0.1.0
+  pullPolicy: IfNotPresent
+
+logLevel: info
+cmd:
+  logOutput: true
+  timeout: 1h
+
+nameOverride: ""
+fullnameOverride: ""
+
+service:
+  type: ClusterIP
+  port: 80
+
+resources:
+  requests:
+    cpu: 10m
+    memory: 64Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

cmd/loadtester/main.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"flag"
+	"github.com/knative/pkg/signals"
+	"github.com/stefanprodan/flagger/pkg/loadtester"
+	"github.com/stefanprodan/flagger/pkg/logging"
+	"log"
+	"time"
+)
+
+var VERSION = "0.1.0"
+var (
+	logLevel     string
+	port         string
+	timeout      time.Duration
+	logCmdOutput bool
+)
+
+func init() {
+	flag.StringVar(&logLevel, "log-level", "debug", "Log level can be: debug, info, warning, error.")
+	flag.StringVar(&port, "port", "9090", "Port to listen on.")
+	flag.DurationVar(&timeout, "timeout", time.Hour, "Command exec timeout.")
+	flag.BoolVar(&logCmdOutput, "log-cmd-output", true, "Log command output to stderr")
+}
+
+func main() {
+	flag.Parse()
+
+	logger, err := logging.NewLogger(logLevel)
+	if err != nil {
+		log.Fatalf("Error creating logger: %v", err)
+	}
+	defer logger.Sync()
+
+	stopCh := signals.SetupSignalHandler()
+
+	taskRunner := loadtester.NewTaskRunner(logger, timeout, logCmdOutput)
+
+	go taskRunner.Start(100*time.Millisecond, stopCh)
+
+	logger.Infof("Starting load tester v%s API on port %s", VERSION, port)
+	loadtester.ListenAndServe(port, time.Minute, logger, taskRunner, stopCh)
+}

docs/CNAME

@@ -1 +0,0 @@
-flagger.app

docs/README.md

@@ -1,11 +0,0 @@
-# Flagger
-
-Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio routing for traffic 
-shifting and Prometheus metrics for canary analysis.
-
-Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance 
-indicators like HTTP requests success rate, requests average duration and pods health. Based on the KPIs analysis 
-a canary is promoted or aborted and the analysis result is published to Slack.
-
-### For the install instructions and usage examples please see [docs.flagger.app](https://docs.flagger.app)
-

docs/_config.yaml

@@ -1,55 +0,0 @@
-title: Flagger - Istio Progressive Delivery Kubernetes Operator
-
-remote_theme: errordeveloper/simple-project-homepage
-repository: stefanprodan/flagger
-by_weaveworks: true
-
-url: "https://flagger.app"
-baseurl: "/"
-
-twitter:
-  username: "stefanprodan"
-author:
-  twitter: "stefanprodan"
-
-# Set default og:image
-defaults:
-- scope: {path: ""}
-  values: {image: "diagrams/flagger-overview.png"}
-
-# See: https://material.io/guidelines/style/color.html
-# Use color-name-value, like pink-200 or deep-purple-100
-brand_color: "amber-400"
-
-# How article URLs are structured.
-# See: https://jekyllrb.com/docs/permalinks/
-permalink: posts/:title/
-
-# "UA-NNNNNNNN-N"
-google_analytics: ""
-
-# Language. For example, if you write in Japanese, use "ja"
-lang: "en"
-
-# How many posts are visible on the home page without clicking "View More"
-num_posts_visible_initially: 5
-
-# Date format: See http://strftime.net/
-date_format: "%b %-d, %Y"
-
-plugins:
-- jekyll-feed
-- jekyll-readme-index
-- jekyll-seo-tag
-- jekyll-sitemap
-- jemoji
-# # required for local builds with starefossen/github-pages
-# - jekyll-github-metadata
-# - jekyll-mentions
-# - jekyll-redirect-from
-# - jekyll-remote-theme
-
-exclude:
-- CNAME
-- gitbook
-

docs/gitbook/how-it-works.md

@@ -1,6 +1,8 @@
 # How it works
 
-[Flagger](https://github.com/stefanprodan/flagger) takes a Kubernetes deployment and optionally a horizontal pod autoscaler \(HPA\) and creates a series of objects \(Kubernetes deployments, ClusterIP services and Istio virtual services\) to drive the canary analysis and promotion. 
+[Flagger](https://github.com/stefanprodan/flagger) takes a Kubernetes deployment and optionally 
+a horizontal pod autoscaler \(HPA\) and creates a series of objects 
+\(Kubernetes deployments, ClusterIP services and Istio virtual services\) to drive the canary analysis and promotion. 
 
 ![Flagger Canary Process](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/diagrams/flagger-canary-hpa.png)
 
@@ -95,37 +97,42 @@ the Istio Virtual Service. The container port from the target deployment should
 
 ![Flagger Canary Stages](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/diagrams/flagger-canary-steps.png)
 
+A canary deployment is triggered by changes in any of the following objects:
+
+* Deployment PodSpec (container image, command, ports, env, resources, etc)
+* ConfigMaps mounted as volumes or mapped to environment variables
+* Secrets mounted as volumes or mapped to environment variables
+
 Gated canary promotion stages:
 
 * scan for canary deployments
-* creates the primary deployment if needed
 * check Istio virtual service routes are mapped to primary and canary ClusterIP services
 * check primary and canary deployments status
-  * halt advancement if a rolling update is underway
-  * halt advancement if pods are unhealthy
-* increase canary traffic weight percentage from 0% to 5% \(step weight\)
+    * halt advancement if a rolling update is underway
+    * halt advancement if pods are unhealthy
+* increase canary traffic weight percentage from 0% to 5% (step weight)
+* call webhooks and check results
 * check canary HTTP request success rate and latency
-  * halt advancement if any metric is under the specified threshold
-  * increment the failed checks counter
+    * halt advancement if any metric is under the specified threshold
+    * increment the failed checks counter
 * check if the number of failed checks reached the threshold
-  * route all traffic to primary
-  * scale to zero the canary deployment and mark it as failed
-  * wait for the canary deployment to be updated \(revision bump\) and start over
-* increase canary traffic weight by 5% \(step weight\) till it reaches 50% \(max weight\)
-  * halt advancement while canary request success rate is under the threshold
-  * halt advancement while canary request duration P99 is over the threshold
-  * halt advancement if the primary or canary deployment becomes unhealthy
-  * halt advancement while canary deployment is being scaled up/down by HPA
+    * route all traffic to primary
+    * scale to zero the canary deployment and mark it as failed
+    * wait for the canary deployment to be updated and start over
+* increase canary traffic weight by 5% (step weight) till it reaches 50% (max weight) 
+    * halt advancement while canary request success rate is under the threshold
+    * halt advancement while canary request duration P99 is over the threshold
+    * halt advancement if the primary or canary deployment becomes unhealthy 
+    * halt advancement while canary deployment is being scaled up/down by HPA
 * promote canary to primary
-  * copy canary deployment spec template over primary
+    * copy ConfigMaps and Secrets from canary to primary
+    * copy canary deployment spec template over primary
 * wait for primary rolling update to finish
-  * halt advancement if pods are unhealthy
+    * halt advancement if pods are unhealthy
 * route all traffic to primary
 * scale to zero the canary deployment
-* mark the canary deployment as finished
-* wait for the canary deployment to be updated \(revision bump\) and start over
-
-You can change the canary analysis _max weight_ and the _step weight_ percentage in the Flagger's custom resource.
+* mark rollout as finished
+* wait for the canary deployment to be updated and start over
 
 ### Canary Analysis
 
@@ -281,4 +288,78 @@ Response status codes:
 
 On a non-2xx response Flagger will include the response body (if any) in the failed checks log and Kubernetes events.
 
+### Load Testing
 
+For workloads that are not receiving constant traffic Flagger can be configured with a webhook, 
+that when called, will start a load test for the target workload.
+If the target workload doesn't receive any traffic during the canary analysis, 
+Flagger metric checks will fail with "no values found for metric istio_requests_total".
+
+Flagger comes with a load testing service based on [rakyll/hey](https://github.com/rakyll/hey) 
+that generates traffic during analysis when configured as a webhook.
+
+![Flagger Load Testing Webhook](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/diagrams/flagger-load-testing.png)
+
+First you need to deploy the load test runner in a namespace with Istio sidecar injection enabled:
+
+```bash
+export REPO=https://raw.githubusercontent.com/stefanprodan/flagger/master
+
+kubectl -n test apply -f ${REPO}/artifacts/loadtester/deployment.yaml
+kubectl -n test apply -f ${REPO}/artifacts/loadtester/service.yaml
+```
+
+Or by using Helm:
+
+```bash
+helm repo add flagger https://flagger.app
+
+helm upgrade -i flagger-loadtester flagger/loadtester \
+--namepace=test \
+--set cmd.logOutput=true \
+--set cmd.timeout=1h
+```
+
+When deployed the load tester API will be available at `http://flagger-loadtester.test/`. 
+
+Now you can add webhooks to the canary analysis spec:
+
+```yaml
+webhooks:
+  - name: load-test-get
+    url: http://flagger-loadtester.test/
+    timeout: 5s
+    metadata:
+      cmd: "hey -z 1m -q 10 -c 2 http://podinfo.test:9898/"
+  - name: load-test-post
+    url: http://flagger-loadtester.test/
+    timeout: 5s
+    metadata:
+      cmd: "hey -z 1m -q 10 -c 2 -m POST -d '{test: 2}' http://podinfo.test:9898/echo"
+```
+
+When the canary analysis starts, Flagger will call the webhooks and the load tester will run the `hey` commands 
+in the background, if they are not already running. This will ensure that during the 
+analysis, the `podinfo.test` virtual service will receive a steady steam of GET and POST requests.
+
+If your workload is exposed outside the mesh with the Istio Gateway and TLS you can point `hey` to the 
+public URL and use HTTP2.
+
+```yaml
+webhooks:
+  - name: load-test-get
+    url: http://flagger-loadtester.test/
+    timeout: 5s
+    metadata:
+      cmd: "hey -z 1m -q 10 -c 2 -h2 https://podinfo.example.com/"
+```
+
+The load tester can run arbitrary commands as long as the binary is present in the container image.
+For example if you you want to replace `hey` with another CLI, you can create your own Docker image:
+
+```dockerfile
+FROM quay.io/stefanprodan/flagger-loadtester:<VER>
+
+RUN curl -Lo /usr/local/bin/my-cli https://github.com/user/repo/releases/download/ver/my-cli \
+    && chmod +x /usr/local/bin/my-cli
+```

docs/gitbook/usage/progressive-delivery.md

@@ -17,7 +17,14 @@ kubectl apply -f ${REPO}/artifacts/canaries/deployment.yaml
 kubectl apply -f ${REPO}/artifacts/canaries/hpa.yaml
 ```
 
-Create a canary custom resource \(replace example.com with your own domain\):
+Deploy the load testing service to generate traffic during the canary analysis:
+
+```bash
+kubectl -n test apply -f ${REPO}/artifacts/loadtester/deployment.yaml
+kubectl -n test apply -f ${REPO}/artifacts/loadtester/service.yaml
+```
+
+Create a canary custom resource (replace example.com with your own domain):
 
 ```yaml
 apiVersion: flagger.app/v1alpha3
@@ -70,6 +77,13 @@ spec:
       # milliseconds
       threshold: 500
       interval: 30s
+    # generate traffic during analysis
+    webhooks:
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          cmd: "hey -z 1m -q 10 -c 2 http://podinfo.test:9898/"
 ```
 
 Save the above resource as podinfo-canary.yaml and then apply it:
@@ -99,7 +113,7 @@ Trigger a canary deployment by updating the container image:
 
 ```bash
 kubectl -n test set image deployment/podinfo \
-podinfod=quay.io/stefanprodan/podinfo:1.2.1
+podinfod=quay.io/stefanprodan/podinfo:1.4.0
 ```
 
 Flagger detects that the deployment revision changed and starts a new rollout:
@@ -132,6 +146,8 @@ Events:
   Normal   Synced  5s    flagger  Promotion completed! Scaling down podinfo.test
 ```
 
+**Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis.
+
 You can monitor all canaries with:
 
 ```bash
@@ -167,7 +183,8 @@ Generate latency:
 watch curl http://podinfo-canary:9898/delay/1
 ```
 
-When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, the canary is scaled to zero and the rollout is marked as failed.
+When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, 
+the canary is scaled to zero and the rollout is marked as failed.
 
 ```text
 kubectl -n test describe canary/podinfo

docs/index.yaml

@@ -1,146 +0,0 @@
-apiVersion: v1
-entries:
-  flagger:
-  - apiVersion: v1
-    appVersion: 0.4.0
-    created: 2019-01-18T12:49:18.099861+02:00
-    description: Flagger is a Kubernetes operator that automates the promotion of
-      canary deployments using Istio routing for traffic shifting and Prometheus metrics
-      for canary analysis.
-    digest: fe06de1c68c6cc414440ef681cde67ae02c771de9b1e4d2d264c38a7a9c37b3d
-    engine: gotpl
-    home: https://docs.flagger.app
-    icon: https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/logo/flagger-icon.png
-    keywords:
-    - canary
-    - istio
-    - gitops
-    kubeVersion: '>=1.11.0-0'
-    maintainers:
-    - email: stefanprodan@users.noreply.github.com
-      name: stefanprodan
-      url: https://github.com/stefanprodan
-    name: flagger
-    sources:
-    - https://github.com/stefanprodan/flagger
-    urls:
-    - https://stefanprodan.github.io/flagger/flagger-0.4.0.tgz
-    version: 0.4.0
-  - apiVersion: v1
-    appVersion: 0.3.0
-    created: 2019-01-18T12:49:18.099501+02:00
-    description: Flagger is a Kubernetes operator that automates the promotion of
-      canary deployments using Istio routing for traffic shifting and Prometheus metrics
-      for canary analysis.
-    digest: 8baa478cc802f4e6b7593934483359b8f70ec34413ca3b8de3a692e347a9bda4
-    engine: gotpl
-    home: https://docs.flagger.app
-    icon: https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/logo/flagger-icon.png
-    keywords:
-    - canary
-    - istio
-    - gitops
-    kubeVersion: '>=1.9.0-0'
-    maintainers:
-    - email: stefanprodan@users.noreply.github.com
-      name: stefanprodan
-      url: https://github.com/stefanprodan
-    name: flagger
-    sources:
-    - https://github.com/stefanprodan/flagger
-    urls:
-    - https://stefanprodan.github.io/flagger/flagger-0.3.0.tgz
-    version: 0.3.0
-  - apiVersion: v1
-    appVersion: 0.2.0
-    created: 2019-01-18T12:49:18.099162+02:00
-    description: Flagger is a Kubernetes operator that automates the promotion of
-      canary deployments using Istio routing for traffic shifting and Prometheus metrics
-      for canary analysis.
-    digest: 800b5fd1a0b2854ee8412b3170c36ecda3d382f209e18b475ee1d5e3c7fa2f83
-    engine: gotpl
-    home: https://flagger.app
-    icon: https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/logo/flagger-icon.png
-    keywords:
-    - canary
-    - istio
-    - gitops
-    kubeVersion: '>=1.9.0-0'
-    maintainers:
-    - email: stefanprodan@users.noreply.github.com
-      name: stefanprodan
-      url: https://github.com/stefanprodan
-    name: flagger
-    sources:
-    - https://github.com/stefanprodan/flagger
-    urls:
-    - https://stefanprodan.github.io/flagger/flagger-0.2.0.tgz
-    version: 0.2.0
-  - apiVersion: v1
-    appVersion: 0.1.2
-    created: 2019-01-18T12:49:18.098811+02:00
-    description: Flagger is a Kubernetes operator that automates the promotion of
-      canary deployments using Istio routing for traffic shifting and Prometheus metrics
-      for canary analysis.
-    digest: 0029ef8dd20ebead3d84638eaa4b44d60b3e2bd953b4b7a1169963ce93a4e87c
-    engine: gotpl
-    home: https://flagger.app
-    icon: https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/logo/flagger-icon.png
-    keywords:
-    - canary
-    - istio
-    - gitops
-    kubeVersion: '>=1.9.0-0'
-    maintainers:
-    - email: stefanprodan@users.noreply.github.com
-      name: stefanprodan
-      url: https://github.com/stefanprodan
-    name: flagger
-    sources:
-    - https://github.com/stefanprodan/flagger
-    urls:
-    - https://stefanprodan.github.io/flagger/flagger-0.1.2.tgz
-    version: 0.1.2
-  - apiVersion: v1
-    appVersion: 0.1.1
-    created: 2019-01-18T12:49:18.098439+02:00
-    description: Flagger is a Kubernetes operator that automates the promotion of
-      canary deployments using Istio routing for traffic shifting and Prometheus metrics
-      for canary analysis.
-    digest: 2bb8f72fcf63a5ba5ecbaa2ab0d0446f438ec93fbf3a598cd7de45e64d8f9628
-    home: https://github.com/stefanprodan/flagger
-    name: flagger
-    urls:
-    - https://stefanprodan.github.io/flagger/flagger-0.1.1.tgz
-    version: 0.1.1
-  - apiVersion: v1
-    appVersion: 0.1.0
-    created: 2019-01-18T12:49:18.098153+02:00
-    description: Flagger is a Kubernetes operator that automates the promotion of
-      canary deployments using Istio routing for traffic shifting and Prometheus metrics
-      for canary analysis.
-    digest: 03e05634149e13ddfddae6757266d65c271878a026c21c7d1429c16712bf3845
-    home: https://github.com/stefanprodan/flagger
-    name: flagger
-    urls:
-    - https://stefanprodan.github.io/flagger/flagger-0.1.0.tgz
-    version: 0.1.0
-  grafana:
-  - apiVersion: v1
-    appVersion: 5.4.2
-    created: 2019-01-18T12:49:18.100331+02:00
-    description: Grafana dashboards for monitoring Flagger canary deployments
-    digest: 97257d1742aca506f8703922d67863c459c1b43177870bc6050d453d19a683c0
-    home: https://flagger.app
-    icon: https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/logo/flagger-icon.png
-    maintainers:
-    - email: stefanprodan@users.noreply.github.com
-      name: stefanprodan
-      url: https://github.com/stefanprodan
-    name: grafana
-    sources:
-    - https://github.com/stefanprodan/flagger
-    urls:
-    - https://stefanprodan.github.io/flagger/grafana-0.1.0.tgz
-    version: 0.1.0
-generated: 2019-01-18T12:49:18.097682+02:00

pkg/apis/flagger/v1alpha3/types.go

@@ -93,6 +93,8 @@ type CanaryStatus struct {
 	FailedChecks int         `json:"failedChecks"`
 	CanaryWeight int         `json:"canaryWeight"`
 	// +optional
+	TrackedConfigs *map[string]string `json:"trackedConfigs,omitempty"`
+	// +optional
 	LastAppliedSpec string `json:"lastAppliedSpec,omitempty"`
 	// +optional
 	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
@@ -134,9 +136,9 @@ type CanaryWebhook struct {
 
 // CanaryWebhookPayload holds the deployment info and metadata sent to webhooks
 type CanaryWebhookPayload struct {
-	Name      string             `json:"name"`
-	Namespace string             `json:"namespace"`
-	Metadata  *map[string]string `json:"metadata,omitempty"`
+	Name      string            `json:"name"`
+	Namespace string            `json:"namespace"`
+	Metadata  map[string]string `json:"metadata,omitempty"`
 }
 
 // GetProgressDeadlineSeconds returns the progress deadline (default 600s)

pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go

@@ -188,6 +188,17 @@ func (in *CanarySpec) DeepCopy() *CanarySpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *CanaryStatus) DeepCopyInto(out *CanaryStatus) {
 	*out = *in
+	if in.TrackedConfigs != nil {
+		in, out := &in.TrackedConfigs, &out.TrackedConfigs
+		*out = new(map[string]string)
+		if **in != nil {
+			in, out := *in, *out
+			*out = make(map[string]string, len(*in))
+			for key, val := range *in {
+				(*out)[key] = val
+			}
+		}
+	}
 	in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime)
 	return
 }
@@ -234,13 +245,9 @@ func (in *CanaryWebhookPayload) DeepCopyInto(out *CanaryWebhookPayload) {
 	*out = *in
 	if in.Metadata != nil {
 		in, out := &in.Metadata, &out.Metadata
-		*out = new(map[string]string)
-		if **in != nil {
-			in, out := *in, *out
-			*out = make(map[string]string, len(*in))
-			for key, val := range *in {
-				(*out)[key] = val
-			}
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
 		}
 	}
 	return

pkg/controller/controller.go

@@ -74,6 +74,11 @@ func NewController(
 		kubeClient:    kubeClient,
 		istioClient:   istioClient,
 		flaggerClient: flaggerClient,
+		configTracker: ConfigTracker{
+			logger:        logger,
+			kubeClient:    kubeClient,
+			flaggerClient: flaggerClient,
+		},
 	}
 
 	router := CanaryRouter{

pkg/controller/controller_test.go

@@ -0,0 +1,523 @@
+package controller
+
+import (
+	istioclientset "github.com/knative/pkg/client/clientset/versioned"
+	fakeIstio "github.com/knative/pkg/client/clientset/versioned/fake"
+	"github.com/stefanprodan/flagger/pkg/apis/flagger/v1alpha3"
+	clientset "github.com/stefanprodan/flagger/pkg/client/clientset/versioned"
+	fakeFlagger "github.com/stefanprodan/flagger/pkg/client/clientset/versioned/fake"
+	informers "github.com/stefanprodan/flagger/pkg/client/informers/externalversions"
+	"github.com/stefanprodan/flagger/pkg/logging"
+	"go.uber.org/zap"
+	appsv1 "k8s.io/api/apps/v1"
+	hpav1 "k8s.io/api/autoscaling/v1"
+	hpav2 "k8s.io/api/autoscaling/v2beta1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/fake"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/util/workqueue"
+	"sync"
+	"time"
+)
+
+var (
+	alwaysReady        = func() bool { return true }
+	noResyncPeriodFunc = func() time.Duration { return 0 }
+)
+
+type Mocks struct {
+	canary        *v1alpha3.Canary
+	kubeClient    kubernetes.Interface
+	istioClient   istioclientset.Interface
+	flaggerClient clientset.Interface
+	deployer      CanaryDeployer
+	router        CanaryRouter
+	observer      CanaryObserver
+	ctrl          *Controller
+	logger        *zap.SugaredLogger
+}
+
+func SetupMocks() Mocks {
+	// init canary
+	canary := newTestCanary()
+	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
+
+	// init kube clientset and register mock objects
+	kubeClient := fake.NewSimpleClientset(
+		newTestDeployment(),
+		newTestHPA(),
+		NewTestConfigMap(),
+		NewTestConfigMapEnv(),
+		NewTestConfigMapVol(),
+		NewTestSecret(),
+		NewTestSecretEnv(),
+		NewTestSecretVol(),
+	)
+
+	istioClient := fakeIstio.NewSimpleClientset()
+	logger, _ := logging.NewLogger("debug")
+
+	// init controller helpers
+	deployer := CanaryDeployer{
+		flaggerClient: flaggerClient,
+		kubeClient:    kubeClient,
+		logger:        logger,
+		configTracker: ConfigTracker{
+			logger:        logger,
+			kubeClient:    kubeClient,
+			flaggerClient: flaggerClient,
+		},
+	}
+	router := CanaryRouter{
+		flaggerClient: flaggerClient,
+		kubeClient:    kubeClient,
+		istioClient:   istioClient,
+		logger:        logger,
+	}
+	observer := CanaryObserver{
+		metricsServer: "fake",
+	}
+
+	// init controller
+	flaggerInformerFactory := informers.NewSharedInformerFactory(flaggerClient, noResyncPeriodFunc())
+	flaggerInformer := flaggerInformerFactory.Flagger().V1alpha3().Canaries()
+
+	ctrl := &Controller{
+		kubeClient:    kubeClient,
+		istioClient:   istioClient,
+		flaggerClient: flaggerClient,
+		flaggerLister: flaggerInformer.Lister(),
+		flaggerSynced: flaggerInformer.Informer().HasSynced,
+		workqueue:     workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), controllerAgentName),
+		eventRecorder: &record.FakeRecorder{},
+		logger:        logger,
+		canaries:      new(sync.Map),
+		flaggerWindow: time.Second,
+		deployer:      deployer,
+		router:        router,
+		observer:      observer,
+		recorder:      NewCanaryRecorder(false),
+	}
+	ctrl.flaggerSynced = alwaysReady
+
+	return Mocks{
+		canary:        canary,
+		observer:      observer,
+		router:        router,
+		deployer:      deployer,
+		logger:        logger,
+		flaggerClient: flaggerClient,
+		istioClient:   istioClient,
+		kubeClient:    kubeClient,
+		ctrl:          ctrl,
+	}
+}
+
+func NewTestConfigMap() *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "default",
+			Name:      "podinfo-config-env",
+		},
+		Data: map[string]string{
+			"color": "red",
+		},
+	}
+}
+
+func NewTestConfigMapV2() *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "default",
+			Name:      "podinfo-config-env",
+		},
+		Data: map[string]string{
+			"color":  "blue",
+			"output": "console",
+		},
+	}
+}
+
+func NewTestConfigMapEnv() *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "default",
+			Name:      "podinfo-config-all-env",
+		},
+		Data: map[string]string{
+			"color": "red",
+		},
+	}
+}
+
+func NewTestConfigMapVol() *corev1.ConfigMap {
+	return &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "default",
+			Name:      "podinfo-config-vol",
+		},
+		Data: map[string]string{
+			"color": "red",
+		},
+	}
+}
+
+func NewTestSecret() *corev1.Secret {
+	return &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "default",
+			Name:      "podinfo-secret-env",
+		},
+		Type: corev1.SecretTypeOpaque,
+		Data: map[string][]byte{
+			"apiKey": []byte("test"),
+		},
+	}
+}
+
+func NewTestSecretV2() *corev1.Secret {
+	return &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "default",
+			Name:      "podinfo-secret-env",
+		},
+		Type: corev1.SecretTypeOpaque,
+		Data: map[string][]byte{
+			"apiKey": []byte("test2"),
+		},
+	}
+}
+
+func NewTestSecretEnv() *corev1.Secret {
+	return &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "default",
+			Name:      "podinfo-secret-all-env",
+		},
+		Type: corev1.SecretTypeOpaque,
+		Data: map[string][]byte{
+			"apiKey": []byte("test"),
+		},
+	}
+}
+
+func NewTestSecretVol() *corev1.Secret {
+	return &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "default",
+			Name:      "podinfo-secret-vol",
+		},
+		Type: corev1.SecretTypeOpaque,
+		Data: map[string][]byte{
+			"apiKey": []byte("test"),
+		},
+	}
+}
+
+func newTestCanary() *v1alpha3.Canary {
+	cd := &v1alpha3.Canary{
+		TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "default",
+			Name:      "podinfo",
+		},
+		Spec: v1alpha3.CanarySpec{
+			TargetRef: hpav1.CrossVersionObjectReference{
+				Name:       "podinfo",
+				APIVersion: "apps/v1",
+				Kind:       "Deployment",
+			},
+			AutoscalerRef: &hpav1.CrossVersionObjectReference{
+				Name:       "podinfo",
+				APIVersion: "autoscaling/v2beta1",
+				Kind:       "HorizontalPodAutoscaler",
+			}, Service: v1alpha3.CanaryService{
+				Port: 9898,
+			}, CanaryAnalysis: v1alpha3.CanaryAnalysis{
+				Threshold:  10,
+				StepWeight: 10,
+				MaxWeight:  50,
+				Metrics: []v1alpha3.CanaryMetric{
+					{
+						Name:      "istio_requests_total",
+						Threshold: 99,
+						Interval:  "1m",
+					},
+					{
+						Name:      "istio_request_duration_seconds_bucket",
+						Threshold: 500,
+						Interval:  "1m",
+					},
+				},
+			},
+		},
+	}
+	return cd
+}
+
+func newTestDeployment() *appsv1.Deployment {
+	d := &appsv1.Deployment{
+		TypeMeta: metav1.TypeMeta{APIVersion: appsv1.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "default",
+			Name:      "podinfo",
+		},
+		Spec: appsv1.DeploymentSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"app": "podinfo",
+				},
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"app": "podinfo",
+					},
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:  "podinfo",
+							Image: "quay.io/stefanprodan/podinfo:1.2.0",
+							Command: []string{
+								"./podinfo",
+								"--port=9898",
+							},
+							Args:       nil,
+							WorkingDir: "",
+							Ports: []corev1.ContainerPort{
+								{
+									Name:          "http",
+									ContainerPort: 9898,
+									Protocol:      corev1.ProtocolTCP,
+								},
+							},
+							Env: []corev1.EnvVar{
+								{
+									Name: "PODINFO_UI_COLOR",
+									ValueFrom: &corev1.EnvVarSource{
+										ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
+											LocalObjectReference: corev1.LocalObjectReference{
+												Name: "podinfo-config-env",
+											},
+											Key: "color",
+										},
+									},
+								},
+								{
+									Name: "API_KEY",
+									ValueFrom: &corev1.EnvVarSource{
+										SecretKeyRef: &corev1.SecretKeySelector{
+											LocalObjectReference: corev1.LocalObjectReference{
+												Name: "podinfo-secret-env",
+											},
+											Key: "apiKey",
+										},
+									},
+								},
+							},
+							EnvFrom: []corev1.EnvFromSource{
+								{
+									ConfigMapRef: &corev1.ConfigMapEnvSource{
+										LocalObjectReference: corev1.LocalObjectReference{
+											Name: "podinfo-config-all-env",
+										},
+									},
+								},
+								{
+									SecretRef: &corev1.SecretEnvSource{
+										LocalObjectReference: corev1.LocalObjectReference{
+											Name: "podinfo-secret-all-env",
+										},
+									},
+								},
+							},
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      "config",
+									MountPath: "/etc/podinfo/config",
+									ReadOnly:  true,
+								},
+								{
+									Name:      "secret",
+									MountPath: "/etc/podinfo/secret",
+									ReadOnly:  true,
+								},
+							},
+						},
+					},
+					Volumes: []corev1.Volume{
+						{
+							Name: "config",
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: "podinfo-config-vol",
+									},
+								},
+							},
+						},
+						{
+							Name: "secret",
+							VolumeSource: corev1.VolumeSource{
+								Secret: &corev1.SecretVolumeSource{
+									SecretName: "podinfo-secret-vol",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	return d
+}
+
+func newTestDeploymentV2() *appsv1.Deployment {
+	d := &appsv1.Deployment{
+		TypeMeta: metav1.TypeMeta{APIVersion: appsv1.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "default",
+			Name:      "podinfo",
+		},
+		Spec: appsv1.DeploymentSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"app": "podinfo",
+				},
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"app": "podinfo",
+					},
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:  "podinfo",
+							Image: "quay.io/stefanprodan/podinfo:1.2.1",
+							Ports: []corev1.ContainerPort{
+								{
+									Name:          "http",
+									ContainerPort: 9898,
+									Protocol:      corev1.ProtocolTCP,
+								},
+							},
+							Command: []string{
+								"./podinfo",
+								"--port=9898",
+							},
+							Env: []corev1.EnvVar{
+								{
+									Name: "PODINFO_UI_COLOR",
+									ValueFrom: &corev1.EnvVarSource{
+										ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
+											LocalObjectReference: corev1.LocalObjectReference{
+												Name: "podinfo-config-env",
+											},
+											Key: "color",
+										},
+									},
+								},
+								{
+									Name: "API_KEY",
+									ValueFrom: &corev1.EnvVarSource{
+										SecretKeyRef: &corev1.SecretKeySelector{
+											LocalObjectReference: corev1.LocalObjectReference{
+												Name: "podinfo-secret-env",
+											},
+											Key: "apiKey",
+										},
+									},
+								},
+							},
+							EnvFrom: []corev1.EnvFromSource{
+								{
+									ConfigMapRef: &corev1.ConfigMapEnvSource{
+										LocalObjectReference: corev1.LocalObjectReference{
+											Name: "podinfo-config-all-env",
+										},
+									},
+								},
+							},
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      "config",
+									MountPath: "/etc/podinfo/config",
+									ReadOnly:  true,
+								},
+								{
+									Name:      "secret",
+									MountPath: "/etc/podinfo/secret",
+									ReadOnly:  true,
+								},
+							},
+						},
+					},
+					Volumes: []corev1.Volume{
+						{
+							Name: "config",
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: "podinfo-config-vol",
+									},
+								},
+							},
+						},
+						{
+							Name: "secret",
+							VolumeSource: corev1.VolumeSource{
+								Secret: &corev1.SecretVolumeSource{
+									SecretName: "podinfo-secret-vol",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	return d
+}
+
+func newTestHPA() *hpav2.HorizontalPodAutoscaler {
+	h := &hpav2.HorizontalPodAutoscaler{
+		TypeMeta: metav1.TypeMeta{APIVersion: hpav2.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "default",
+			Name:      "podinfo",
+		},
+		Spec: hpav2.HorizontalPodAutoscalerSpec{
+			ScaleTargetRef: hpav2.CrossVersionObjectReference{
+				Name:       "podinfo",
+				APIVersion: "apps/v1",
+				Kind:       "Deployment",
+			},
+			Metrics: []hpav2.MetricSpec{
+				{
+					Type: "Resource",
+					Resource: &hpav2.ResourceMetricSource{
+						Name:                     "cpu",
+						TargetAverageUtilization: int32p(99),
+					},
+				},
+			},
+		},
+	}
+
+	return h
+}

pkg/controller/deployer.go

@@ -1,9 +1,11 @@
 package controller
 
 import (
+	"crypto/rand"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"io"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
@@ -28,9 +30,10 @@ type CanaryDeployer struct {
 	istioClient   istioclientset.Interface
 	flaggerClient clientset.Interface
 	logger        *zap.SugaredLogger
+	configTracker ConfigTracker
 }
 
-// Promote copies the pod spec from canary to primary
+// Promote copies the pod spec, secrets and config maps from canary to primary
 func (c *CanaryDeployer) Promote(cd *flaggerv1.Canary) error {
 	targetName := cd.Spec.TargetRef.Name
 	primaryName := fmt.Sprintf("%s-primary", targetName)
@@ -51,12 +54,30 @@ func (c *CanaryDeployer) Promote(cd *flaggerv1.Canary) error {
 		return fmt.Errorf("deployment %s.%s query error %v", primaryName, cd.Namespace, err)
 	}
 
+	// promote secrets and config maps
+	configRefs, err := c.configTracker.GetTargetConfigs(cd)
+	if err != nil {
+		return err
+	}
+	if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs); err != nil {
+		return err
+	}
+
 	primaryCopy := primary.DeepCopy()
 	primaryCopy.Spec.ProgressDeadlineSeconds = canary.Spec.ProgressDeadlineSeconds
 	primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds
 	primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit
 	primaryCopy.Spec.Strategy = canary.Spec.Strategy
-	primaryCopy.Spec.Template.Spec = canary.Spec.Template.Spec
+
+	// update spec with primary secrets and config maps
+	primaryCopy.Spec.Template.Spec = c.configTracker.ApplyPrimaryConfigs(canary.Spec.Template.Spec, configRefs)
+
+	// update pod annotations to ensure a rolling update
+	annotations, err := c.makeAnnotations(canary.Spec.Template.Annotations)
+	if err != nil {
+		return err
+	}
+	primaryCopy.Spec.Template.Annotations = annotations
 
 	_, err = c.kubeClient.AppsV1().Deployments(cd.Namespace).Update(primaryCopy)
 	if err != nil {
@@ -157,7 +178,22 @@ func (c *CanaryDeployer) ShouldAdvance(cd *flaggerv1.Canary) (bool, error) {
 	if cd.Status.LastAppliedSpec == "" || cd.Status.Phase == flaggerv1.CanaryProgressing {
 		return true, nil
 	}
-	return c.IsNewSpec(cd)
+
+	newDep, err := c.IsNewSpec(cd)
+	if err != nil {
+		return false, err
+	}
+	if newDep {
+		return newDep, nil
+	}
+
+	newCfg, err := c.configTracker.HasConfigChanged(cd)
+	if err != nil {
+		return false, err
+	}
+
+	return newCfg, nil
+
 }
 
 // SetStatusFailedChecks updates the canary failed checks counter
@@ -218,12 +254,18 @@ func (c *CanaryDeployer) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.Canar
 		return fmt.Errorf("deployment %s.%s marshal error %v", cd.Spec.TargetRef.Name, cd.Namespace, err)
 	}
 
+	configs, err := c.configTracker.GetConfigRefs(cd)
+	if err != nil {
+		return fmt.Errorf("configs query error %v", err)
+	}
+
 	cdCopy := cd.DeepCopy()
 	cdCopy.Status.Phase = status.Phase
 	cdCopy.Status.CanaryWeight = status.CanaryWeight
 	cdCopy.Status.FailedChecks = status.FailedChecks
 	cdCopy.Status.LastAppliedSpec = base64.StdEncoding.EncodeToString(specJson)
 	cdCopy.Status.LastTransitionTime = metav1.Now()
+	cdCopy.Status.TrackedConfigs = configs
 
 	cd, err = c.flaggerClient.FlaggerV1alpha3().Canaries(cd.Namespace).UpdateStatus(cdCopy)
 	if err != nil {
@@ -290,11 +332,25 @@ func (c *CanaryDeployer) createPrimaryDeployment(cd *flaggerv1.Canary) error {
 
 	primaryDep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(primaryName, metav1.GetOptions{})
 	if errors.IsNotFound(err) {
+		// create primary secrets and config maps
+		configRefs, err := c.configTracker.GetTargetConfigs(cd)
+		if err != nil {
+			return err
+		}
+		if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs); err != nil {
+			return err
+		}
+		annotations, err := c.makeAnnotations(canaryDep.Spec.Template.Annotations)
+		if err != nil {
+			return err
+		}
+
+		// create primary deployment
 		primaryDep = &appsv1.Deployment{
 			ObjectMeta: metav1.ObjectMeta{
-				Name:        primaryName,
-				Annotations: canaryDep.Annotations,
-				Namespace:   cd.Namespace,
+				Name:      primaryName,
+				Labels:    canaryDep.Labels,
+				Namespace: cd.Namespace,
 				OwnerReferences: []metav1.OwnerReference{
 					*metav1.NewControllerRef(cd, schema.GroupVersionKind{
 						Group:   flaggerv1.SchemeGroupVersion.Group,
@@ -317,9 +373,10 @@ func (c *CanaryDeployer) createPrimaryDeployment(cd *flaggerv1.Canary) error {
 				Template: corev1.PodTemplateSpec{
 					ObjectMeta: metav1.ObjectMeta{
 						Labels:      map[string]string{"app": primaryName},
-						Annotations: canaryDep.Spec.Template.Annotations,
+						Annotations: annotations,
 					},
-					Spec: canaryDep.Spec.Template.Spec,
+					// update spec with the primary secrets and config maps
+					Spec: c.configTracker.ApplyPrimaryConfigs(canaryDep.Spec.Template.Spec, configRefs),
 				},
 			},
 		}
@@ -353,6 +410,7 @@ func (c *CanaryDeployer) createPrimaryHpa(cd *flaggerv1.Canary) error {
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      primaryHpaName,
 				Namespace: cd.Namespace,
+				Labels:    hpa.Labels,
 				OwnerReferences: []metav1.OwnerReference{
 					*metav1.NewControllerRef(cd, schema.GroupVersionKind{
 						Group:   flaggerv1.SchemeGroupVersion.Group,
@@ -432,3 +490,26 @@ func (c *CanaryDeployer) getDeploymentCondition(
 	}
 	return nil
 }
+
+// makeAnnotations appends an unique ID to annotations map
+func (c *CanaryDeployer) makeAnnotations(annotations map[string]string) (map[string]string, error) {
+	idKey := "flagger-id"
+	res := make(map[string]string)
+	uuid := make([]byte, 16)
+	n, err := io.ReadFull(rand.Reader, uuid)
+	if n != len(uuid) || err != nil {
+		return res, err
+	}
+	uuid[8] = uuid[8]&^0xc0 | 0x80
+	uuid[6] = uuid[6]&^0xf0 | 0x40
+	id := fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:])
+
+	for k, v := range annotations {
+		if k != idKey {
+			res[k] = v
+		}
+	}
+	res[idKey] = id
+
+	return res, nil
+}

pkg/controller/deployer_test.go

@@ -4,200 +4,24 @@ import (
 	"testing"
 
 	"github.com/stefanprodan/flagger/pkg/apis/flagger/v1alpha3"
-	fakeFlagger "github.com/stefanprodan/flagger/pkg/client/clientset/versioned/fake"
-	"github.com/stefanprodan/flagger/pkg/logging"
-	appsv1 "k8s.io/api/apps/v1"
-	hpav1 "k8s.io/api/autoscaling/v1"
-	hpav2 "k8s.io/api/autoscaling/v2beta1"
-	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes/fake"
 )
 
-func newTestCanary() *v1alpha3.Canary {
-	cd := &v1alpha3.Canary{
-		TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()},
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: "default",
-			Name:      "podinfo",
-		},
-		Spec: v1alpha3.CanarySpec{
-			TargetRef: hpav1.CrossVersionObjectReference{
-				Name:       "podinfo",
-				APIVersion: "apps/v1",
-				Kind:       "Deployment",
-			},
-			AutoscalerRef: &hpav1.CrossVersionObjectReference{
-				Name:       "podinfo",
-				APIVersion: "autoscaling/v2beta1",
-				Kind:       "HorizontalPodAutoscaler",
-			}, Service: v1alpha3.CanaryService{
-				Port: 9898,
-			}, CanaryAnalysis: v1alpha3.CanaryAnalysis{
-				Threshold:  10,
-				StepWeight: 10,
-				MaxWeight:  50,
-				Metrics: []v1alpha3.CanaryMetric{
-					{
-						Name:      "istio_requests_total",
-						Threshold: 99,
-						Interval:  "1m",
-					},
-					{
-						Name:      "istio_request_duration_seconds_bucket",
-						Threshold: 500,
-						Interval:  "1m",
-					},
-				},
-			},
-		},
-	}
-	return cd
-}
-
-func newTestDeployment() *appsv1.Deployment {
-	d := &appsv1.Deployment{
-		TypeMeta: metav1.TypeMeta{APIVersion: appsv1.SchemeGroupVersion.String()},
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: "default",
-			Name:      "podinfo",
-		},
-		Spec: appsv1.DeploymentSpec{
-			Selector: &metav1.LabelSelector{
-				MatchLabels: map[string]string{
-					"app": "podinfo",
-				},
-			},
-			Template: corev1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: map[string]string{
-						"app": "podinfo",
-					},
-				},
-				Spec: corev1.PodSpec{
-					Containers: []corev1.Container{
-						{
-							Name:  "podinfo",
-							Image: "quay.io/stefanprodan/podinfo:1.2.0",
-							Ports: []corev1.ContainerPort{
-								{
-									Name:          "http",
-									ContainerPort: 9898,
-									Protocol:      corev1.ProtocolTCP,
-								},
-							},
-							Command: []string{
-								"./podinfo",
-								"--port=9898",
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	return d
-}
-
-func newTestDeploymentUpdated() *appsv1.Deployment {
-	d := &appsv1.Deployment{
-		TypeMeta: metav1.TypeMeta{APIVersion: appsv1.SchemeGroupVersion.String()},
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: "default",
-			Name:      "podinfo",
-		},
-		Spec: appsv1.DeploymentSpec{
-			Selector: &metav1.LabelSelector{
-				MatchLabels: map[string]string{
-					"app": "podinfo",
-				},
-			},
-			Template: corev1.PodTemplateSpec{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: map[string]string{
-						"app": "podinfo",
-					},
-				},
-				Spec: corev1.PodSpec{
-					Containers: []corev1.Container{
-						{
-							Name:  "podinfo",
-							Image: "quay.io/stefanprodan/podinfo:1.2.1",
-							Ports: []corev1.ContainerPort{
-								{
-									Name:          "http",
-									ContainerPort: 9898,
-									Protocol:      corev1.ProtocolTCP,
-								},
-							},
-							Command: []string{
-								"./podinfo",
-								"--port=9898",
-							},
-						},
-					},
-				},
-			},
-		},
-	}
-
-	return d
-}
-
-func newTestHPA() *hpav2.HorizontalPodAutoscaler {
-	h := &hpav2.HorizontalPodAutoscaler{
-		TypeMeta: metav1.TypeMeta{APIVersion: hpav2.SchemeGroupVersion.String()},
-		ObjectMeta: metav1.ObjectMeta{
-			Namespace: "default",
-			Name:      "podinfo",
-		},
-		Spec: hpav2.HorizontalPodAutoscalerSpec{
-			ScaleTargetRef: hpav2.CrossVersionObjectReference{
-				Name:       "podinfo",
-				APIVersion: "apps/v1",
-				Kind:       "Deployment",
-			},
-			Metrics: []hpav2.MetricSpec{
-				{
-					Type: "Resource",
-					Resource: &hpav2.ResourceMetricSource{
-						Name:                     "cpu",
-						TargetAverageUtilization: int32p(99),
-					},
-				},
-			},
-		},
-	}
-
-	return h
-}
-
 func TestCanaryDeployer_Sync(t *testing.T) {
-	canary := newTestCanary()
+	mocks := SetupMocks()
+	err := mocks.deployer.Sync(mocks.canary)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	depPrimary, err := mocks.kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
 	dep := newTestDeployment()
-	hpa := newTestHPA()
-
-	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
-
-	kubeClient := fake.NewSimpleClientset(dep, hpa)
-
-	logger, _ := logging.NewLogger("debug")
-	deployer := &CanaryDeployer{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		logger:        logger,
-	}
-
-	err := deployer.Sync(canary)
-	if err != nil {
-		t.Fatal(err.Error())
-	}
-
-	depPrimary, err := kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
-	if err != nil {
-		t.Fatal(err.Error())
-	}
+	configMap := NewTestConfigMap()
+	secret := NewTestSecret()
 
 	primaryImage := depPrimary.Spec.Template.Spec.Containers[0].Image
 	sourceImage := dep.Spec.Template.Spec.Containers[0].Image
@@ -205,7 +29,7 @@ func TestCanaryDeployer_Sync(t *testing.T) {
 		t.Errorf("Got image %s wanted %s", primaryImage, sourceImage)
 	}
 
-	hpaPrimary, err := kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers("default").Get("podinfo-primary", metav1.GetOptions{})
+	hpaPrimary, err := mocks.kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers("default").Get("podinfo-primary", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -213,37 +37,76 @@ func TestCanaryDeployer_Sync(t *testing.T) {
 	if hpaPrimary.Spec.ScaleTargetRef.Name != depPrimary.Name {
 		t.Errorf("Got HPA target %s wanted %s", hpaPrimary.Spec.ScaleTargetRef.Name, depPrimary.Name)
 	}
+
+	configPrimary, err := mocks.kubeClient.CoreV1().ConfigMaps("default").Get("podinfo-config-env-primary", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if configPrimary.Data["color"] != configMap.Data["color"] {
+		t.Errorf("Got ConfigMap color %s wanted %s", configPrimary.Data["color"], configMap.Data["color"])
+	}
+
+	configPrimaryEnv, err := mocks.kubeClient.CoreV1().ConfigMaps("default").Get("podinfo-config-all-env-primary", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if configPrimaryEnv.Data["color"] != configMap.Data["color"] {
+		t.Errorf("Got ConfigMap %s wanted %s", configPrimaryEnv.Data["a"], configMap.Data["color"])
+	}
+
+	configPrimaryVol, err := mocks.kubeClient.CoreV1().ConfigMaps("default").Get("podinfo-config-vol-primary", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if configPrimaryVol.Data["color"] != configMap.Data["color"] {
+		t.Errorf("Got ConfigMap color %s wanted %s", configPrimary.Data["color"], configMap.Data["color"])
+	}
+
+	secretPrimary, err := mocks.kubeClient.CoreV1().Secrets("default").Get("podinfo-secret-env-primary", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if string(secretPrimary.Data["apiKey"]) != string(secret.Data["apiKey"]) {
+		t.Errorf("Got primary secret %s wanted %s", secretPrimary.Data["apiKey"], secret.Data["apiKey"])
+	}
+
+	secretPrimaryEnv, err := mocks.kubeClient.CoreV1().Secrets("default").Get("podinfo-secret-all-env-primary", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if string(secretPrimaryEnv.Data["apiKey"]) != string(secret.Data["apiKey"]) {
+		t.Errorf("Got primary secret %s wanted %s", secretPrimary.Data["apiKey"], secret.Data["apiKey"])
+	}
+
+	secretPrimaryVol, err := mocks.kubeClient.CoreV1().Secrets("default").Get("podinfo-secret-vol-primary", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if string(secretPrimaryVol.Data["apiKey"]) != string(secret.Data["apiKey"]) {
+		t.Errorf("Got primary secret %s wanted %s", secretPrimary.Data["apiKey"], secret.Data["apiKey"])
+	}
 }
 
 func TestCanaryDeployer_IsNewSpec(t *testing.T) {
-	canary := newTestCanary()
-	dep := newTestDeployment()
-	dep2 := newTestDeploymentUpdated()
-
-	hpa := newTestHPA()
-
-	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
-
-	kubeClient := fake.NewSimpleClientset(dep, hpa)
-
-	logger, _ := logging.NewLogger("debug")
-	deployer := &CanaryDeployer{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		logger:        logger,
-	}
-
-	err := deployer.Sync(canary)
+	mocks := SetupMocks()
+	err := mocks.deployer.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	_, err = kubeClient.AppsV1().Deployments("default").Update(dep2)
+	dep2 := newTestDeploymentV2()
+	_, err = mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	isNew, err := deployer.IsNewSpec(canary)
+	isNew, err := mocks.deployer.IsNewSpec(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -254,39 +117,30 @@ func TestCanaryDeployer_IsNewSpec(t *testing.T) {
 }
 
 func TestCanaryDeployer_Promote(t *testing.T) {
-	canary := newTestCanary()
-	dep := newTestDeployment()
-	dep2 := newTestDeploymentUpdated()
-
-	hpa := newTestHPA()
-
-	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
-
-	kubeClient := fake.NewSimpleClientset(dep, hpa)
-
-	logger, _ := logging.NewLogger("debug")
-	deployer := &CanaryDeployer{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		logger:        logger,
-	}
-
-	err := deployer.Sync(canary)
+	mocks := SetupMocks()
+	err := mocks.deployer.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	_, err = kubeClient.AppsV1().Deployments("default").Update(dep2)
+	dep2 := newTestDeploymentV2()
+	_, err = mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	err = deployer.Promote(canary)
+	config2 := NewTestConfigMapV2()
+	_, err = mocks.kubeClient.CoreV1().ConfigMaps("default").Update(config2)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	depPrimary, err := kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
+	err = mocks.deployer.Promote(mocks.canary)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	depPrimary, err := mocks.kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -296,67 +150,48 @@ func TestCanaryDeployer_Promote(t *testing.T) {
 	if primaryImage != sourceImage {
 		t.Errorf("Got image %s wanted %s", primaryImage, sourceImage)
 	}
+
+	configPrimary, err := mocks.kubeClient.CoreV1().ConfigMaps("default").Get("podinfo-config-env-primary", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if configPrimary.Data["color"] != config2.Data["color"] {
+		t.Errorf("Got primary ConfigMap color %s wanted %s", configPrimary.Data["color"], config2.Data["color"])
+	}
 }
 
 func TestCanaryDeployer_IsReady(t *testing.T) {
-	canary := newTestCanary()
-	dep := newTestDeployment()
-	hpa := newTestHPA()
-
-	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
-
-	kubeClient := fake.NewSimpleClientset(dep, hpa)
-
-	logger, _ := logging.NewLogger("debug")
-	deployer := &CanaryDeployer{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		logger:        logger,
-	}
-
-	err := deployer.Sync(canary)
+	mocks := SetupMocks()
+	err := mocks.deployer.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	_, err = deployer.IsPrimaryReady(canary)
+	_, err = mocks.deployer.IsPrimaryReady(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	_, err = deployer.IsCanaryReady(canary)
+	_, err = mocks.deployer.IsCanaryReady(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 }
 
 func TestCanaryDeployer_SetFailedChecks(t *testing.T) {
-	canary := newTestCanary()
-	dep := newTestDeployment()
-	hpa := newTestHPA()
-
-	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
-
-	kubeClient := fake.NewSimpleClientset(dep, hpa)
-
-	logger, _ := logging.NewLogger("debug")
-	deployer := &CanaryDeployer{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		logger:        logger,
-	}
-
-	err := deployer.Sync(canary)
+	mocks := SetupMocks()
+	err := mocks.deployer.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	err = deployer.SetStatusFailedChecks(canary, 1)
+	err = mocks.deployer.SetStatusFailedChecks(mocks.canary, 1)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	res, err := flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	res, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -367,32 +202,18 @@ func TestCanaryDeployer_SetFailedChecks(t *testing.T) {
 }
 
 func TestCanaryDeployer_SetState(t *testing.T) {
-	canary := newTestCanary()
-	dep := newTestDeployment()
-	hpa := newTestHPA()
-
-	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
-
-	kubeClient := fake.NewSimpleClientset(dep, hpa)
-
-	logger, _ := logging.NewLogger("debug")
-	deployer := &CanaryDeployer{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		logger:        logger,
-	}
-
-	err := deployer.Sync(canary)
+	mocks := SetupMocks()
+	err := mocks.deployer.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	err = deployer.SetStatusPhase(canary, v1alpha3.CanaryProgressing)
+	err = mocks.deployer.SetStatusPhase(mocks.canary, v1alpha3.CanaryProgressing)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	res, err := flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	res, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -403,22 +224,8 @@ func TestCanaryDeployer_SetState(t *testing.T) {
 }
 
 func TestCanaryDeployer_SyncStatus(t *testing.T) {
-	canary := newTestCanary()
-	dep := newTestDeployment()
-	hpa := newTestHPA()
-
-	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
-
-	kubeClient := fake.NewSimpleClientset(dep, hpa)
-
-	logger, _ := logging.NewLogger("debug")
-	deployer := &CanaryDeployer{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		logger:        logger,
-	}
-
-	err := deployer.Sync(canary)
+	mocks := SetupMocks()
+	err := mocks.deployer.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -427,12 +234,12 @@ func TestCanaryDeployer_SyncStatus(t *testing.T) {
 		Phase:        v1alpha3.CanaryProgressing,
 		FailedChecks: 2,
 	}
-	err = deployer.SyncStatus(canary, status)
+	err = mocks.deployer.SyncStatus(mocks.canary, status)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	res, err := flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	res, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -444,32 +251,27 @@ func TestCanaryDeployer_SyncStatus(t *testing.T) {
 	if res.Status.FailedChecks != status.FailedChecks {
 		t.Errorf("Got failed checks %v wanted %v", res.Status.FailedChecks, status.FailedChecks)
 	}
+
+	if res.Status.TrackedConfigs == nil {
+		t.Fatalf("Status tracking configs are empty")
+	}
+	configs := *res.Status.TrackedConfigs
+	secret := NewTestSecret()
+	if _, exists := configs["secret/"+secret.GetName()]; !exists {
+		t.Errorf("Secret %s not found in status", secret.GetName())
+	}
 }
 
 func TestCanaryDeployer_Scale(t *testing.T) {
-	canary := newTestCanary()
-	dep := newTestDeployment()
-	hpa := newTestHPA()
-
-	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
-
-	kubeClient := fake.NewSimpleClientset(dep, hpa)
-
-	logger, _ := logging.NewLogger("debug")
-	deployer := &CanaryDeployer{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		logger:        logger,
-	}
-
-	err := deployer.Sync(canary)
+	mocks := SetupMocks()
+	err := mocks.deployer.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	err = deployer.Scale(canary, 2)
+	err = mocks.deployer.Scale(mocks.canary, 2)
 
-	c, err := kubeClient.AppsV1().Deployments("default").Get("podinfo", metav1.GetOptions{})
+	c, err := mocks.kubeClient.AppsV1().Deployments("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}

pkg/controller/router_test.go

@@ -5,37 +5,17 @@ import (
 	"testing"
 
 	istiov1alpha3 "github.com/knative/pkg/apis/istio/v1alpha3"
-	fakeIstio "github.com/knative/pkg/client/clientset/versioned/fake"
-	fakeFlagger "github.com/stefanprodan/flagger/pkg/client/clientset/versioned/fake"
-	"github.com/stefanprodan/flagger/pkg/logging"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes/fake"
 )
 
 func TestCanaryRouter_Sync(t *testing.T) {
-	canary := newTestCanary()
-	dep := newTestDeployment()
-	hpa := newTestHPA()
-
-	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
-	kubeClient := fake.NewSimpleClientset(dep, hpa)
-	istioClient := fakeIstio.NewSimpleClientset()
-
-	logger, _ := logging.NewLogger("debug")
-
-	router := &CanaryRouter{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		istioClient:   istioClient,
-		logger:        logger,
-	}
-
-	err := router.Sync(canary)
+	mocks := SetupMocks()
+	err := mocks.router.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	canarySvc, err := kubeClient.CoreV1().Services("default").Get("podinfo", metav1.GetOptions{})
+	canarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -48,7 +28,7 @@ func TestCanaryRouter_Sync(t *testing.T) {
 		t.Errorf("Got svc port %v wanted %v", canarySvc.Spec.Ports[0].Port, 9898)
 	}
 
-	primarySvc, err := kubeClient.CoreV1().Services("default").Get("podinfo-primary", metav1.GetOptions{})
+	primarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get("podinfo-primary", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -61,7 +41,7 @@ func TestCanaryRouter_Sync(t *testing.T) {
 		t.Errorf("Got primary svc port %v wanted %v", primarySvc.Spec.Ports[0].Port, 9898)
 	}
 
-	vs, err := istioClient.NetworkingV1alpha3().VirtualServices("default").Get("podinfo", metav1.GetOptions{})
+	vs, err := mocks.istioClient.NetworkingV1alpha3().VirtualServices("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -76,29 +56,13 @@ func TestCanaryRouter_Sync(t *testing.T) {
 }
 
 func TestCanaryRouter_GetRoutes(t *testing.T) {
-	canary := newTestCanary()
-	dep := newTestDeployment()
-	hpa := newTestHPA()
-
-	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
-	kubeClient := fake.NewSimpleClientset(dep, hpa)
-	istioClient := fakeIstio.NewSimpleClientset()
-
-	logger, _ := logging.NewLogger("debug")
-
-	router := &CanaryRouter{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		istioClient:   istioClient,
-		logger:        logger,
-	}
-
-	err := router.Sync(canary)
+	mocks := SetupMocks()
+	err := mocks.router.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	p, c, err := router.GetRoutes(canary)
+	p, c, err := mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -113,29 +77,13 @@ func TestCanaryRouter_GetRoutes(t *testing.T) {
 }
 
 func TestCanaryRouter_SetRoutes(t *testing.T) {
-	canary := newTestCanary()
-	dep := newTestDeployment()
-	hpa := newTestHPA()
-
-	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
-	kubeClient := fake.NewSimpleClientset(dep, hpa)
-	istioClient := fakeIstio.NewSimpleClientset()
-
-	logger, _ := logging.NewLogger("debug")
-
-	router := &CanaryRouter{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		istioClient:   istioClient,
-		logger:        logger,
-	}
-
-	err := router.Sync(canary)
+	mocks := SetupMocks()
+	err := mocks.router.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	p, c, err := router.GetRoutes(canary)
+	p, c, err := mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -143,12 +91,12 @@ func TestCanaryRouter_SetRoutes(t *testing.T) {
 	p.Weight = 50
 	c.Weight = 50
 
-	err = router.SetRoutes(canary, p, c)
+	err = mocks.router.SetRoutes(mocks.canary, p, c)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	vs, err := istioClient.NetworkingV1alpha3().VirtualServices("default").Get("podinfo", metav1.GetOptions{})
+	vs, err := mocks.istioClient.NetworkingV1alpha3().VirtualServices("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -158,10 +106,10 @@ func TestCanaryRouter_SetRoutes(t *testing.T) {
 
 	for _, http := range vs.Spec.Http {
 		for _, route := range http.Route {
-			if route.Destination.Host == fmt.Sprintf("%s-primary", canary.Spec.TargetRef.Name) {
+			if route.Destination.Host == fmt.Sprintf("%s-primary", mocks.canary.Spec.TargetRef.Name) {
 				pRoute = route
 			}
-			if route.Destination.Host == canary.Spec.TargetRef.Name {
+			if route.Destination.Host == mocks.canary.Spec.TargetRef.Name {
 				cRoute = route
 			}
 		}

pkg/controller/scheduler.go

@@ -2,6 +2,7 @@ package controller
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	flaggerv1 "github.com/stefanprodan/flagger/pkg/apis/flagger/v1alpha3"
@@ -90,10 +91,13 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 		return
 	}
 
-	if ok, err := c.deployer.ShouldAdvance(cd); !ok {
-		if err != nil {
-			c.recordEventWarningf(cd, "%v", err)
-		}
+	shouldAdvance, err := c.deployer.ShouldAdvance(cd)
+	if err != nil {
+		c.recordEventWarningf(cd, "%v", err)
+		return
+	}
+
+	if !shouldAdvance {
 		return
 	}
 
@@ -122,7 +126,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 	c.recorder.SetWeight(cd, primaryRoute.Weight, canaryRoute.Weight)
 
 	// check if canary analysis should start (canary revision has changes) or continue
-	if ok := c.checkCanaryStatus(cd); !ok {
+	if ok := c.checkCanaryStatus(cd, shouldAdvance); !ok {
 		return
 	}
 
@@ -290,7 +294,7 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 	}
 }
 
-func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary) bool {
+func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary, shouldAdvance bool) bool {
 	c.recorder.SetStatus(cd)
 	if cd.Status.Phase == flaggerv1.CanaryProgressing {
 		return true
@@ -308,11 +312,11 @@ func (c *Controller) checkCanaryStatus(cd *flaggerv1.Canary) bool {
 		return false
 	}
 
-	if diff, err := c.deployer.IsNewSpec(cd); diff {
+	if shouldAdvance {
 		c.recordEventInfof(cd, "New revision detected! Scaling up %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)
 		c.sendNotification(cd, "New revision detected, starting canary analysis.",
 			true, false)
-		if err = c.deployer.Scale(cd, 1); err != nil {
+		if err := c.deployer.Scale(cd, 1); err != nil {
 			c.recordEventErrorf(cd, "%v", err)
 			return false
 		}
@@ -331,17 +335,35 @@ func (c *Controller) hasCanaryRevisionChanged(cd *flaggerv1.Canary) bool {
 		if diff, _ := c.deployer.IsNewSpec(cd); diff {
 			return true
 		}
+		if diff, _ := c.deployer.configTracker.HasConfigChanged(cd); diff {
+			return true
+		}
 	}
 	return false
 }
 
 func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
+	// run external checks
+	for _, webhook := range r.Spec.CanaryAnalysis.Webhooks {
+		err := CallWebhook(r.Name, r.Namespace, webhook)
+		if err != nil {
+			c.recordEventWarningf(r, "Halt %s.%s advancement external check %s failed %v",
+				r.Name, r.Namespace, webhook.Name, err)
+			return false
+		}
+	}
+
 	// run metrics checks
 	for _, metric := range r.Spec.CanaryAnalysis.Metrics {
 		if metric.Name == "istio_requests_total" {
 			val, err := c.observer.GetDeploymentCounter(r.Spec.TargetRef.Name, r.Namespace, metric.Name, metric.Interval)
 			if err != nil {
-				c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observer.metricsServer, err)
+				if strings.Contains(err.Error(), "no values found") {
+					c.recordEventWarningf(r, "Halt advancement no values found for metric %s probably %s.%s is not receiving traffic",
+						metric.Name, r.Spec.TargetRef.Name, r.Namespace)
+				} else {
+					c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observer.metricsServer, err)
+				}
 				return false
 			}
 			if float64(metric.Threshold) > val {
@@ -366,15 +388,5 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
 		}
 	}
 
-	// run external checks
-	for _, webhook := range r.Spec.CanaryAnalysis.Webhooks {
-		err := CallWebhook(r.Name, r.Namespace, webhook)
-		if err != nil {
-			c.recordEventWarningf(r, "Halt %s.%s advancement external check %s failed %v",
-				r.Name, r.Namespace, webhook.Name, err)
-			return false
-		}
-	}
-
 	return true
 }

pkg/controller/scheduler_test.go

@@ -1,137 +1,36 @@
 package controller
 
 import (
-	"go.uber.org/zap"
-	"k8s.io/client-go/kubernetes"
-	"sync"
-	"testing"
-	"time"
-
-	istioclientset "github.com/knative/pkg/client/clientset/versioned"
-	fakeIstio "github.com/knative/pkg/client/clientset/versioned/fake"
 	"github.com/stefanprodan/flagger/pkg/apis/flagger/v1alpha3"
-	clientset "github.com/stefanprodan/flagger/pkg/client/clientset/versioned"
-	fakeFlagger "github.com/stefanprodan/flagger/pkg/client/clientset/versioned/fake"
-	informers "github.com/stefanprodan/flagger/pkg/client/informers/externalversions"
-	"github.com/stefanprodan/flagger/pkg/logging"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes/fake"
-	"k8s.io/client-go/tools/record"
-	"k8s.io/client-go/util/workqueue"
+	"testing"
 )
 
-var (
-	alwaysReady        = func() bool { return true }
-	noResyncPeriodFunc = func() time.Duration { return 0 }
-)
-
-func newTestController(
-	kubeClient kubernetes.Interface,
-	istioClient istioclientset.Interface,
-	flaggerClient clientset.Interface,
-	logger *zap.SugaredLogger,
-	deployer CanaryDeployer,
-	router CanaryRouter,
-	observer CanaryObserver,
-) *Controller {
-	flaggerInformerFactory := informers.NewSharedInformerFactory(flaggerClient, noResyncPeriodFunc())
-	flaggerInformer := flaggerInformerFactory.Flagger().V1alpha3().Canaries()
-
-	ctrl := &Controller{
-		kubeClient:    kubeClient,
-		istioClient:   istioClient,
-		flaggerClient: flaggerClient,
-		flaggerLister: flaggerInformer.Lister(),
-		flaggerSynced: flaggerInformer.Informer().HasSynced,
-		workqueue:     workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), controllerAgentName),
-		eventRecorder: &record.FakeRecorder{},
-		logger:        logger,
-		canaries:      new(sync.Map),
-		flaggerWindow: time.Second,
-		deployer:      deployer,
-		router:        router,
-		observer:      observer,
-		recorder:      NewCanaryRecorder(false),
-	}
-	ctrl.flaggerSynced = alwaysReady
-
-	return ctrl
-}
-
 func TestScheduler_Init(t *testing.T) {
-	canary := newTestCanary()
-	dep := newTestDeployment()
-	hpa := newTestHPA()
+	mocks := SetupMocks()
+	mocks.ctrl.advanceCanary("podinfo", "default", false)
 
-	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
-	kubeClient := fake.NewSimpleClientset(dep, hpa)
-	istioClient := fakeIstio.NewSimpleClientset()
-
-	logger, _ := logging.NewLogger("debug")
-	deployer := CanaryDeployer{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		logger:        logger,
-	}
-	router := CanaryRouter{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		istioClient:   istioClient,
-		logger:        logger,
-	}
-	observer := CanaryObserver{
-		metricsServer: "fake",
-	}
-	ctrl := newTestController(kubeClient, istioClient, flaggerClient, logger, deployer, router, observer)
-
-	ctrl.advanceCanary("podinfo", "default", false)
-
-	_, err := kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
+	_, err := mocks.kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 }
 
 func TestScheduler_NewRevision(t *testing.T) {
-	canary := newTestCanary()
-	dep := newTestDeployment()
-	hpa := newTestHPA()
-
-	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
-	kubeClient := fake.NewSimpleClientset(dep, hpa)
-	istioClient := fakeIstio.NewSimpleClientset()
-
-	logger, _ := logging.NewLogger("debug")
-	deployer := CanaryDeployer{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		logger:        logger,
-	}
-	router := CanaryRouter{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		istioClient:   istioClient,
-		logger:        logger,
-	}
-	observer := CanaryObserver{
-		metricsServer: "fake",
-	}
-	ctrl := newTestController(kubeClient, istioClient, flaggerClient, logger, deployer, router, observer)
-
-	// init
-	ctrl.advanceCanary("podinfo", "default", false)
+	mocks := SetupMocks()
+	mocks.ctrl.advanceCanary("podinfo", "default", false)
 
 	// update
-	dep2 := newTestDeploymentUpdated()
-	_, err := kubeClient.AppsV1().Deployments("default").Update(dep2)
+	dep2 := newTestDeploymentV2()
+	_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
 	// detect changes
-	ctrl.advanceCanary("podinfo", "default", false)
+	mocks.ctrl.advanceCanary("podinfo", "default", false)
 
-	c, err := kubeClient.AppsV1().Deployments("default").Get("podinfo", metav1.GetOptions{})
+	c, err := mocks.kubeClient.AppsV1().Deployments("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -142,44 +41,20 @@ func TestScheduler_NewRevision(t *testing.T) {
 }
 
 func TestScheduler_Rollback(t *testing.T) {
-	canary := newTestCanary()
-	dep := newTestDeployment()
-	hpa := newTestHPA()
-
-	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
-	kubeClient := fake.NewSimpleClientset(dep, hpa)
-	istioClient := fakeIstio.NewSimpleClientset()
-
-	logger, _ := logging.NewLogger("debug")
-	deployer := CanaryDeployer{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		logger:        logger,
-	}
-	router := CanaryRouter{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		istioClient:   istioClient,
-		logger:        logger,
-	}
-	observer := CanaryObserver{
-		metricsServer: "fake",
-	}
-	ctrl := newTestController(kubeClient, istioClient, flaggerClient, logger, deployer, router, observer)
-
+	mocks := SetupMocks()
 	// init
-	ctrl.advanceCanary("podinfo", "default", true)
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
 	// update failed checks to max
-	err := deployer.SyncStatus(canary, v1alpha3.CanaryStatus{Phase: v1alpha3.CanaryProgressing, FailedChecks: 11})
+	err := mocks.deployer.SyncStatus(mocks.canary, v1alpha3.CanaryStatus{Phase: v1alpha3.CanaryProgressing, FailedChecks: 11})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
 	// detect changes
-	ctrl.advanceCanary("podinfo", "default", true)
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
-	c, err := flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	c, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -190,47 +65,23 @@ func TestScheduler_Rollback(t *testing.T) {
 }
 
 func TestScheduler_NewRevisionReset(t *testing.T) {
-	canary := newTestCanary()
-	dep := newTestDeployment()
-	hpa := newTestHPA()
-
-	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
-	kubeClient := fake.NewSimpleClientset(dep, hpa)
-	istioClient := fakeIstio.NewSimpleClientset()
-
-	logger, _ := logging.NewLogger("debug")
-	deployer := CanaryDeployer{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		logger:        logger,
-	}
-	router := CanaryRouter{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		istioClient:   istioClient,
-		logger:        logger,
-	}
-	observer := CanaryObserver{
-		metricsServer: "fake",
-	}
-	ctrl := newTestController(kubeClient, istioClient, flaggerClient, logger, deployer, router, observer)
-
+	mocks := SetupMocks()
 	// init
-	ctrl.advanceCanary("podinfo", "default", false)
+	mocks.ctrl.advanceCanary("podinfo", "default", false)
 
 	// first update
-	dep2 := newTestDeploymentUpdated()
-	_, err := kubeClient.AppsV1().Deployments("default").Update(dep2)
+	dep2 := newTestDeploymentV2()
+	_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
 	// detect changes
-	ctrl.advanceCanary("podinfo", "default", true)
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
 	// advance
-	ctrl.advanceCanary("podinfo", "default", true)
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
-	primaryRoute, canaryRoute, err := router.GetRoutes(canary)
+	primaryRoute, canaryRoute, err := mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -245,15 +96,15 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
 
 	// second update
 	dep2.Spec.Template.Spec.ServiceAccountName = "test"
-	_, err = kubeClient.AppsV1().Deployments("default").Update(dep2)
+	_, err = mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
 	// detect changes
-	ctrl.advanceCanary("podinfo", "default", true)
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
-	primaryRoute, canaryRoute, err = router.GetRoutes(canary)
+	primaryRoute, canaryRoute, err = mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -268,63 +119,54 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
 }
 
 func TestScheduler_Promotion(t *testing.T) {
-	canary := newTestCanary()
-	dep := newTestDeployment()
-	hpa := newTestHPA()
-
-	flaggerClient := fakeFlagger.NewSimpleClientset(canary)
-	kubeClient := fake.NewSimpleClientset(dep, hpa)
-	istioClient := fakeIstio.NewSimpleClientset()
-
-	logger, _ := logging.NewLogger("debug")
-	deployer := CanaryDeployer{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		logger:        logger,
-	}
-	router := CanaryRouter{
-		flaggerClient: flaggerClient,
-		kubeClient:    kubeClient,
-		istioClient:   istioClient,
-		logger:        logger,
-	}
-	observer := CanaryObserver{
-		metricsServer: "fake",
-	}
-	ctrl := newTestController(kubeClient, istioClient, flaggerClient, logger, deployer, router, observer)
-
+	mocks := SetupMocks()
 	// init
-	ctrl.advanceCanary("podinfo", "default", false)
+	mocks.ctrl.advanceCanary("podinfo", "default", false)
 
 	// update
-	dep2 := newTestDeploymentUpdated()
-	_, err := kubeClient.AppsV1().Deployments("default").Update(dep2)
+	dep2 := newTestDeploymentV2()
+	_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
-	// detect changes
-	ctrl.advanceCanary("podinfo", "default", true)
+	// detect pod spec changes
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
-	primaryRoute, canaryRoute, err := router.GetRoutes(canary)
+	config2 := NewTestConfigMapV2()
+	_, err = mocks.kubeClient.CoreV1().ConfigMaps("default").Update(config2)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	secret2 := NewTestSecretV2()
+	_, err = mocks.kubeClient.CoreV1().Secrets("default").Update(secret2)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	// detect configs changes
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	primaryRoute, canaryRoute, err := mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
 	primaryRoute.Weight = 60
 	canaryRoute.Weight = 40
-	err = ctrl.router.SetRoutes(canary, primaryRoute, canaryRoute)
+	err = mocks.ctrl.router.SetRoutes(mocks.canary, primaryRoute, canaryRoute)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
 	// advance
-	ctrl.advanceCanary("podinfo", "default", true)
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
 	// promote
-	ctrl.advanceCanary("podinfo", "default", true)
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
 
-	primaryRoute, canaryRoute, err = router.GetRoutes(canary)
+	primaryRoute, canaryRoute, err = mocks.router.GetRoutes(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -337,7 +179,7 @@ func TestScheduler_Promotion(t *testing.T) {
 		t.Errorf("Got canary route %v wanted %v", canaryRoute.Weight, 0)
 	}
 
-	primaryDep, err := kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
+	primaryDep, err := mocks.kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -348,7 +190,25 @@ func TestScheduler_Promotion(t *testing.T) {
 		t.Errorf("Got primary image %v wanted %v", primaryImage, canaryImage)
 	}
 
-	c, err := flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	configPrimary, err := mocks.kubeClient.CoreV1().ConfigMaps("default").Get("podinfo-config-env-primary", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if configPrimary.Data["color"] != config2.Data["color"] {
+		t.Errorf("Got primary ConfigMap color %s wanted %s", configPrimary.Data["color"], config2.Data["color"])
+	}
+
+	secretPrimary, err := mocks.kubeClient.CoreV1().Secrets("default").Get("podinfo-secret-env-primary", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if string(secretPrimary.Data["apiKey"]) != string(secret2.Data["apiKey"]) {
+		t.Errorf("Got primary secret %s wanted %s", secretPrimary.Data["apiKey"], secret2.Data["apiKey"])
+	}
+
+	c, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
 	if err != nil {
 		t.Fatal(err.Error())
 	}

pkg/controller/tracker.go

@@ -0,0 +1,374 @@
+package controller
+
+import (
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	flaggerv1 "github.com/stefanprodan/flagger/pkg/apis/flagger/v1alpha3"
+	clientset "github.com/stefanprodan/flagger/pkg/client/clientset/versioned"
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/kubernetes"
+)
+
+// ConfigTracker is managing the operations for Kubernetes ConfigMaps and Secrets
+type ConfigTracker struct {
+	kubeClient    kubernetes.Interface
+	flaggerClient clientset.Interface
+	logger        *zap.SugaredLogger
+}
+
+type ConfigRefType string
+
+const (
+	ConfigRefMap    ConfigRefType = "configmap"
+	ConfigRefSecret ConfigRefType = "secret"
+)
+
+// ConfigRef holds the reference to a tracked Kubernetes ConfigMap or Secret
+type ConfigRef struct {
+	Name     string
+	Type     ConfigRefType
+	Checksum string
+}
+
+// GetName returns the config ref type and name
+func (c *ConfigRef) GetName() string {
+	return fmt.Sprintf("%s/%s", c.Type, c.Name)
+}
+
+func checksum(data interface{}) string {
+	jsonBytes, _ := json.Marshal(data)
+	hashBytes := sha256.Sum256(jsonBytes)
+
+	return fmt.Sprintf("%x", hashBytes[:8])
+}
+
+// getRefFromConfigMap transforms a Kubernetes ConfigMap into a ConfigRef
+// and computes the checksum of the ConfigMap data
+func (ct *ConfigTracker) getRefFromConfigMap(name string, namespace string) (*ConfigRef, error) {
+	config, err := ct.kubeClient.CoreV1().ConfigMaps(namespace).Get(name, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return &ConfigRef{
+		Name:     config.Name,
+		Type:     ConfigRefMap,
+		Checksum: checksum(config.Data),
+	}, nil
+}
+
+// getRefFromConfigMap transforms a Kubernetes Secret into a ConfigRef
+// and computes the checksum of the Secret data
+func (ct *ConfigTracker) getRefFromSecret(name string, namespace string) (*ConfigRef, error) {
+	secret, err := ct.kubeClient.CoreV1().Secrets(namespace).Get(name, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	// ignore registry secrets (those should be set via service account)
+	if secret.Type != corev1.SecretTypeOpaque &&
+		secret.Type != corev1.SecretTypeBasicAuth &&
+		secret.Type != corev1.SecretTypeSSHAuth &&
+		secret.Type != corev1.SecretTypeTLS {
+		ct.logger.Debugf("ignoring secret %s.%s type not supported %v", name, namespace, secret.Type)
+		return nil, nil
+	}
+
+	return &ConfigRef{
+		Name:     secret.Name,
+		Type:     ConfigRefSecret,
+		Checksum: checksum(secret.Data),
+	}, nil
+}
+
+// GetTargetConfigs scans the target deployment for Kubernetes ConfigMaps and Secretes
+// and returns a list of config references
+func (ct *ConfigTracker) GetTargetConfigs(cd *flaggerv1.Canary) (map[string]ConfigRef, error) {
+	res := make(map[string]ConfigRef)
+	targetName := cd.Spec.TargetRef.Name
+	targetDep, err := ct.kubeClient.AppsV1().Deployments(cd.Namespace).Get(targetName, metav1.GetOptions{})
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return res, fmt.Errorf("deployment %s.%s not found", targetName, cd.Namespace)
+		}
+		return res, fmt.Errorf("deployment %s.%s query error %v", targetName, cd.Namespace, err)
+	}
+
+	// scan volumes
+	for _, volume := range targetDep.Spec.Template.Spec.Volumes {
+		if cmv := volume.ConfigMap; cmv != nil {
+			config, err := ct.getRefFromConfigMap(cmv.Name, cd.Namespace)
+			if err != nil {
+				ct.logger.Errorf("configMap %s.%s query error %v", cmv.Name, cd.Namespace, err)
+				continue
+			}
+			if config != nil {
+				res[config.GetName()] = *config
+			}
+		}
+
+		if sv := volume.Secret; sv != nil {
+			secret, err := ct.getRefFromSecret(sv.SecretName, cd.Namespace)
+			if err != nil {
+				ct.logger.Errorf("secret %s.%s query error %v", sv.SecretName, cd.Namespace, err)
+				continue
+			}
+			if secret != nil {
+				res[secret.GetName()] = *secret
+			}
+		}
+	}
+	// scan containers
+	for _, container := range targetDep.Spec.Template.Spec.Containers {
+		// scan env
+		for _, env := range container.Env {
+			if env.ValueFrom != nil {
+				switch {
+				case env.ValueFrom.ConfigMapKeyRef != nil:
+					name := env.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name
+					config, err := ct.getRefFromConfigMap(name, cd.Namespace)
+					if err != nil {
+						ct.logger.Errorf("configMap %s.%s query error %v", name, cd.Namespace, err)
+						continue
+					}
+					if config != nil {
+						res[config.GetName()] = *config
+					}
+				case env.ValueFrom.SecretKeyRef != nil:
+					name := env.ValueFrom.SecretKeyRef.LocalObjectReference.Name
+					secret, err := ct.getRefFromSecret(name, cd.Namespace)
+					if err != nil {
+						ct.logger.Errorf("secret %s.%s query error %v", name, cd.Namespace, err)
+						continue
+					}
+					if secret != nil {
+						res[secret.GetName()] = *secret
+					}
+				}
+			}
+		}
+		// scan envFrom
+		for _, envFrom := range container.EnvFrom {
+			switch {
+			case envFrom.ConfigMapRef != nil:
+				name := envFrom.ConfigMapRef.LocalObjectReference.Name
+				config, err := ct.getRefFromConfigMap(name, cd.Namespace)
+				if err != nil {
+					ct.logger.Errorf("configMap %s.%s query error %v", name, cd.Namespace, err)
+					continue
+				}
+				if config != nil {
+					res[config.GetName()] = *config
+				}
+			case envFrom.SecretRef != nil:
+				name := envFrom.SecretRef.LocalObjectReference.Name
+				secret, err := ct.getRefFromSecret(name, cd.Namespace)
+				if err != nil {
+					ct.logger.Errorf("secret %s.%s query error %v", name, cd.Namespace, err)
+					continue
+				}
+				if secret != nil {
+					res[secret.GetName()] = *secret
+				}
+			}
+		}
+	}
+
+	return res, nil
+}
+
+// GetConfigRefs returns a map of configs and their checksum
+func (ct *ConfigTracker) GetConfigRefs(cd *flaggerv1.Canary) (*map[string]string, error) {
+	res := make(map[string]string)
+	configs, err := ct.GetTargetConfigs(cd)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, cfg := range configs {
+		res[cfg.GetName()] = cfg.Checksum
+	}
+
+	return &res, nil
+}
+
+// HasConfigChanged checks for changes in ConfigMaps and Secretes by comparing
+// the checksum for each ConfigRef stored in Canary.Status.TrackedConfigs
+func (ct *ConfigTracker) HasConfigChanged(cd *flaggerv1.Canary) (bool, error) {
+	configs, err := ct.GetTargetConfigs(cd)
+	if err != nil {
+		return false, err
+	}
+
+	if len(configs) == 0 && cd.Status.TrackedConfigs == nil {
+		return false, nil
+	}
+
+	if len(configs) > 0 && cd.Status.TrackedConfigs == nil {
+		return true, nil
+	}
+
+	trackedConfigs := *cd.Status.TrackedConfigs
+
+	if len(configs) != len(trackedConfigs) {
+		return true, nil
+	}
+
+	for _, cfg := range configs {
+		if trackedConfigs[cfg.GetName()] != cfg.Checksum {
+			ct.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).
+				Infof("%s %s has changed", cfg.Type, cfg.Name)
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+// CreatePrimaryConfigs syncs the primary Kubernetes ConfigMaps and Secretes
+// with those found in the target deployment
+func (ct *ConfigTracker) CreatePrimaryConfigs(cd *flaggerv1.Canary, refs map[string]ConfigRef) error {
+	for _, ref := range refs {
+		switch ref.Type {
+		case ConfigRefMap:
+			config, err := ct.kubeClient.CoreV1().ConfigMaps(cd.Namespace).Get(ref.Name, metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+			primaryName := fmt.Sprintf("%s-primary", config.GetName())
+			primaryConfigMap := &corev1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      primaryName,
+					Namespace: cd.Namespace,
+					Labels:    config.Labels,
+					OwnerReferences: []metav1.OwnerReference{
+						*metav1.NewControllerRef(cd, schema.GroupVersionKind{
+							Group:   flaggerv1.SchemeGroupVersion.Group,
+							Version: flaggerv1.SchemeGroupVersion.Version,
+							Kind:    flaggerv1.CanaryKind,
+						}),
+					},
+				},
+				Data: config.Data,
+			}
+
+			// update or insert primary ConfigMap
+			_, err = ct.kubeClient.CoreV1().ConfigMaps(cd.Namespace).Update(primaryConfigMap)
+			if err != nil {
+				if errors.IsNotFound(err) {
+					_, err = ct.kubeClient.CoreV1().ConfigMaps(cd.Namespace).Create(primaryConfigMap)
+					if err != nil {
+						return err
+					}
+				} else {
+					return err
+				}
+			}
+
+			ct.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).
+				Infof("ConfigMap %s synced", primaryConfigMap.GetName())
+		case ConfigRefSecret:
+			secret, err := ct.kubeClient.CoreV1().Secrets(cd.Namespace).Get(ref.Name, metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+			primaryName := fmt.Sprintf("%s-primary", secret.GetName())
+			primarySecret := &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      primaryName,
+					Namespace: cd.Namespace,
+					Labels:    secret.Labels,
+					OwnerReferences: []metav1.OwnerReference{
+						*metav1.NewControllerRef(cd, schema.GroupVersionKind{
+							Group:   flaggerv1.SchemeGroupVersion.Group,
+							Version: flaggerv1.SchemeGroupVersion.Version,
+							Kind:    flaggerv1.CanaryKind,
+						}),
+					},
+				},
+				Type: secret.Type,
+				Data: secret.Data,
+			}
+
+			// update or insert primary Secret
+			_, err = ct.kubeClient.CoreV1().Secrets(cd.Namespace).Update(primarySecret)
+			if err != nil {
+				if errors.IsNotFound(err) {
+					_, err = ct.kubeClient.CoreV1().Secrets(cd.Namespace).Create(primarySecret)
+					if err != nil {
+						return err
+					}
+				} else {
+					return err
+				}
+			}
+
+			ct.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).
+				Infof("Secret %s synced", primarySecret.GetName())
+		}
+	}
+
+	return nil
+}
+
+// ApplyPrimaryConfigs appends the primary suffix to all ConfigMaps and Secretes found in the PodSpec
+func (ct *ConfigTracker) ApplyPrimaryConfigs(spec corev1.PodSpec, refs map[string]ConfigRef) corev1.PodSpec {
+	// update volumes
+	for i, volume := range spec.Volumes {
+		if cmv := volume.ConfigMap; cmv != nil {
+			name := fmt.Sprintf("%s/%s", ConfigRefMap, cmv.Name)
+			if _, exists := refs[name]; exists {
+				spec.Volumes[i].ConfigMap.Name += "-primary"
+			}
+		}
+
+		if sv := volume.Secret; sv != nil {
+			name := fmt.Sprintf("%s/%s", ConfigRefSecret, sv.SecretName)
+			if _, exists := refs[name]; exists {
+				spec.Volumes[i].Secret.SecretName += "-primary"
+			}
+		}
+	}
+	// update containers
+	for _, container := range spec.Containers {
+		// update env
+		for i, env := range container.Env {
+			if env.ValueFrom != nil {
+				switch {
+				case env.ValueFrom.ConfigMapKeyRef != nil:
+					name := fmt.Sprintf("%s/%s", ConfigRefMap, env.ValueFrom.ConfigMapKeyRef.Name)
+					if _, exists := refs[name]; exists {
+						container.Env[i].ValueFrom.ConfigMapKeyRef.Name += "-primary"
+					}
+				case env.ValueFrom.SecretKeyRef != nil:
+					name := fmt.Sprintf("%s/%s", ConfigRefSecret, env.ValueFrom.SecretKeyRef.Name)
+					if _, exists := refs[name]; exists {
+						container.Env[i].ValueFrom.SecretKeyRef.Name += "-primary"
+					}
+				}
+			}
+		}
+		// update envFrom
+		for i, envFrom := range container.EnvFrom {
+			switch {
+			case envFrom.ConfigMapRef != nil:
+				name := fmt.Sprintf("%s/%s", ConfigRefMap, envFrom.ConfigMapRef.Name)
+				if _, exists := refs[name]; exists {
+					container.EnvFrom[i].ConfigMapRef.Name += "-primary"
+				}
+			case envFrom.SecretRef != nil:
+				name := fmt.Sprintf("%s/%s", ConfigRefSecret, envFrom.SecretRef.Name)
+				if _, exists := refs[name]; exists {
+					container.EnvFrom[i].SecretRef.Name += "-primary"
+				}
+			}
+		}
+	}
+
+	return spec
+}

pkg/controller/webhook.go

@@ -19,7 +19,10 @@ func CallWebhook(name string, namepace string, w flaggerv1.CanaryWebhook) error
 	payload := flaggerv1.CanaryWebhookPayload{
 		Name:      name,
 		Namespace: namepace,
-		Metadata:  w.Metadata,
+	}
+
+	if w.Metadata != nil {
+		payload.Metadata = *w.Metadata
 	}
 
 	payloadBin, err := json.Marshal(payload)

pkg/loadtester/runner.go

@@ -0,0 +1,107 @@
+package loadtester
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"go.uber.org/zap"
+	"hash/fnv"
+	"os/exec"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+type TaskRunner struct {
+	logger       *zap.SugaredLogger
+	timeout      time.Duration
+	todoTasks    *sync.Map
+	runningTasks *sync.Map
+	totalExecs   uint64
+	logCmdOutput bool
+}
+
+type Task struct {
+	Canary  string
+	Command string
+}
+
+func (t Task) Hash() string {
+	fnvHash := fnv.New32()
+	fnvBytes := fnvHash.Sum([]byte(t.Canary + t.Command))
+	return hex.EncodeToString(fnvBytes[:])
+}
+
+func NewTaskRunner(logger *zap.SugaredLogger, timeout time.Duration, logCmdOutput bool) *TaskRunner {
+	return &TaskRunner{
+		logger:       logger,
+		todoTasks:    new(sync.Map),
+		runningTasks: new(sync.Map),
+		timeout:      timeout,
+		logCmdOutput: logCmdOutput,
+	}
+}
+
+func (tr *TaskRunner) Add(task Task) {
+	tr.todoTasks.Store(task.Hash(), task)
+}
+
+func (tr *TaskRunner) GetTotalExecs() uint64 {
+	return atomic.LoadUint64(&tr.totalExecs)
+}
+
+func (tr *TaskRunner) runAll() {
+	tr.todoTasks.Range(func(key interface{}, value interface{}) bool {
+		task := value.(Task)
+		go func(t Task) {
+			// remove task from the to do list
+			tr.todoTasks.Delete(t.Hash())
+
+			// check if task is already running, if not run the task's command
+			if _, exists := tr.runningTasks.Load(t.Hash()); !exists {
+				// save the task in the running list
+				tr.runningTasks.Store(t.Hash(), t)
+
+				// create timeout context
+				ctx, cancel := context.WithTimeout(context.Background(), tr.timeout)
+				defer cancel()
+
+				// increment the total exec counter
+				atomic.AddUint64(&tr.totalExecs, 1)
+
+				tr.logger.With("canary", t.Canary).Infof("command starting %s", t.Command)
+				cmd := exec.CommandContext(ctx, "sh", "-c", t.Command)
+
+				// execute task
+				out, err := cmd.CombinedOutput()
+				if err != nil {
+					tr.logger.With("canary", t.Canary).Errorf("command failed %s %v %s", t.Command, err, out)
+				} else {
+					if tr.logCmdOutput {
+						fmt.Printf("%s\n", out)
+					}
+					tr.logger.With("canary", t.Canary).Infof("command finished %s", t.Command)
+				}
+
+				// remove task from the running list
+				tr.runningTasks.Delete(t.Hash())
+			} else {
+				tr.logger.With("canary", t.Canary).Infof("command skipped %s is already running", t.Command)
+			}
+		}(task)
+		return true
+	})
+}
+
+func (tr *TaskRunner) Start(interval time.Duration, stopCh <-chan struct{}) {
+	tickChan := time.NewTicker(interval).C
+	for {
+		select {
+		case <-tickChan:
+			tr.runAll()
+		case <-stopCh:
+			tr.logger.Info("shutting down the task runner")
+			return
+		}
+	}
+}

pkg/loadtester/runner_test.go

@@ -0,0 +1,52 @@
+package loadtester
+
+import (
+	"github.com/stefanprodan/flagger/pkg/logging"
+	"testing"
+	"time"
+)
+
+func TestTaskRunner_Start(t *testing.T) {
+	stop := make(chan struct{})
+	logger, _ := logging.NewLogger("debug")
+	tr := NewTaskRunner(logger, time.Hour, false)
+
+	go tr.Start(10*time.Millisecond, stop)
+
+	task1 := Task{
+		Canary:  "podinfo.default",
+		Command: "sleep 0.6",
+	}
+	task2 := Task{
+		Canary:  "podinfo.default",
+		Command: "sleep 0.7",
+	}
+
+	tr.Add(task1)
+	tr.Add(task2)
+
+	time.Sleep(100 * time.Millisecond)
+
+	tr.Add(task1)
+	tr.Add(task2)
+
+	time.Sleep(100 * time.Millisecond)
+
+	tr.Add(task1)
+	tr.Add(task2)
+
+	if tr.GetTotalExecs() != 2 {
+		t.Errorf("Got total executed commands %v wanted %v", tr.GetTotalExecs(), 2)
+	}
+
+	time.Sleep(time.Second)
+
+	tr.Add(task1)
+	tr.Add(task2)
+
+	time.Sleep(time.Second)
+
+	if tr.GetTotalExecs() != 4 {
+		t.Errorf("Got total executed commands %v wanted %v", tr.GetTotalExecs(), 4)
+	}
+}

pkg/loadtester/server.go

@@ -0,0 +1,85 @@
+package loadtester
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	flaggerv1 "github.com/stefanprodan/flagger/pkg/apis/flagger/v1alpha3"
+	"go.uber.org/zap"
+)
+
+// ListenAndServe starts a web server and waits for SIGTERM
+func ListenAndServe(port string, timeout time.Duration, logger *zap.SugaredLogger, taskRunner *TaskRunner, stopCh <-chan struct{}) {
+	mux := http.DefaultServeMux
+	mux.Handle("/metrics", promhttp.Handler())
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("OK"))
+	})
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		body, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			logger.Error("reading the request body failed", zap.Error(err))
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		defer r.Body.Close()
+
+		payload := &flaggerv1.CanaryWebhookPayload{}
+		err = json.Unmarshal(body, payload)
+		if err != nil {
+			logger.Error("decoding the request body failed", zap.Error(err))
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		if len(payload.Metadata) > 0 {
+			if cmd, ok := payload.Metadata["cmd"]; ok {
+				taskRunner.Add(Task{
+					Canary:  fmt.Sprintf("%s.%s", payload.Name, payload.Namespace),
+					Command: cmd,
+				})
+			} else {
+				w.WriteHeader(http.StatusBadRequest)
+				w.Write([]byte("cmd not found in metadata"))
+				return
+			}
+		} else {
+			w.WriteHeader(http.StatusBadRequest)
+			w.Write([]byte("metadata not found in payload"))
+			return
+		}
+
+		w.WriteHeader(http.StatusAccepted)
+	})
+	srv := &http.Server{
+		Addr:         ":" + port,
+		Handler:      mux,
+		ReadTimeout:  5 * time.Second,
+		WriteTimeout: 1 * time.Minute,
+		IdleTimeout:  15 * time.Second,
+	}
+
+	// run server in background
+	go func() {
+		if err := srv.ListenAndServe(); err != http.ErrServerClosed {
+			logger.Fatalf("HTTP server crashed %v", err)
+		}
+	}()
+
+	// wait for SIGTERM or SIGINT
+	<-stopCh
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	if err := srv.Shutdown(ctx); err != nil {
+		logger.Errorf("HTTP server graceful shutdown failed %v", err)
+	} else {
+		logger.Info("HTTP server stopped")
+	}
+}

pkg/version/version.go

@@ -1,4 +1,4 @@
 package version
 
-var VERSION = "0.4.0"
+var VERSION = "0.5.0"
 var REVISION = "unknown"