diff --git a/.circleci/config.yml b/.circleci/config.yml index 4a44d24e..f9164539 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ version: 2.1 jobs: - e2e-testing: + e2e-istio-testing: machine: true steps: - checkout @@ -9,14 +9,46 @@ jobs: - run: test/e2e-build.sh - run: test/e2e-tests.sh + e2e-supergloo-testing: + machine: true + steps: + - checkout + - run: test/e2e-kind.sh + - run: test/e2e-supergloo.sh + - run: test/e2e-build.sh supergloo:test.supergloo-system + - run: test/e2e-tests.sh canary + + e2e-nginx-testing: + machine: true + steps: + - checkout + - run: test/e2e-kind.sh + - run: test/e2e-nginx.sh + - run: test/e2e-nginx-build.sh + - run: test/e2e-nginx-tests.sh + workflows: version: 2 build-and-test: jobs: - - e2e-testing: + - e2e-istio-testing: filters: branches: ignore: - /gh-pages.*/ - /docs-.*/ - /release-.*/ + - e2e-supergloo-testing: + filters: + branches: + ignore: + - /gh-pages.*/ + - /docs-.*/ + - /release-.*/ + - e2e-nginx-testing: + filters: + branches: + ignore: + - /gh-pages.*/ + - /docs-.*/ + - /release-.*/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dae0295..db761266 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,36 @@ All notable changes to this project are documented in this file. +## 0.13.1 (2019-04-09) + +Fixes for custom metrics checks and NGINX Prometheus queries + +#### Fixes + +- Fix promql queries for custom checks and NGINX [#174](https://github.com/weaveworks/flagger/pull/174) + + +## 0.13.0 (2019-04-08) + +Adds support for [NGINX](https://docs.flagger.app/usage/nginx-progressive-delivery) ingress controller + +#### Features + +- Add support for nginx ingress controller (weighted traffic and A/B testing) [#170](https://github.com/weaveworks/flagger/pull/170) +- Add Prometheus add-on to Flagger Helm chart for App Mesh and NGINX [79b3370](https://github.com/weaveworks/flagger/pull/170/commits/79b337089294a92961bc8446fd185b38c50a32df) + +#### Fixes + +- Fix duplicate hosts Istio error when using wildcards [#162](https://github.com/weaveworks/flagger/pull/162) + +## 0.12.0 (2019-04-29) + +Adds support for [SuperGloo](https://docs.flagger.app/install/flagger-install-with-supergloo) + +#### Features + +- Supergloo support for canary deployment (weighted traffic) [#151](https://github.com/weaveworks/flagger/pull/151) + ## 0.11.1 (2019-04-18) Move Flagger and the load tester container images to Docker Hub diff --git a/Makefile b/Makefile index 3f656da4..262f98b9 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,12 @@ run-appmesh: -slack-url=https://hooks.slack.com/services/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 \ -slack-channel="devops-alerts" +run-nginx: + go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=nginx -namespace=nginx \ + -metrics-server=http://prometheus-weave.istio.weavedx.com \ + -slack-url=https://hooks.slack.com/services/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 \ + -slack-channel="devops-alerts" + build: docker build -t weaveworks/flagger:$(TAG) . -f Dockerfile diff --git a/README.md b/README.md index c25e5866..60e382c8 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![release](https://img.shields.io/github/release/weaveworks/flagger/all.svg)](https://github.com/weaveworks/flagger/releases) Flagger is a Kubernetes operator that automates the promotion of canary deployments -using Istio or App Mesh routing for traffic shifting and Prometheus metrics for canary analysis. +using Istio, App Mesh or NGINX routing for traffic shifting and Prometheus metrics for canary analysis. The canary analysis can be extended with webhooks for running acceptance tests, load tests or any other custom validation. @@ -25,6 +25,7 @@ Flagger documentation can be found at [docs.flagger.app](https://docs.flagger.ap * [Flagger install on Kubernetes](https://docs.flagger.app/install/flagger-install-on-kubernetes) * [Flagger install on GKE Istio](https://docs.flagger.app/install/flagger-install-on-google-cloud) * [Flagger install on EKS App Mesh](https://docs.flagger.app/install/flagger-install-on-eks-appmesh) + * [Flagger install with SuperGloo](https://docs.flagger.app/install/flagger-install-with-supergloo) * How it works * [Canary custom resource](https://docs.flagger.app/how-it-works#canary-custom-resource) * [Routing](https://docs.flagger.app/how-it-works#istio-routing) @@ -38,6 +39,7 @@ Flagger documentation can be found at [docs.flagger.app](https://docs.flagger.ap * [Istio canary deployments](https://docs.flagger.app/usage/progressive-delivery) * [Istio A/B testing](https://docs.flagger.app/usage/ab-testing) * [App Mesh canary deployments](https://docs.flagger.app/usage/appmesh-progressive-delivery) + * [NGINX ingress controller canary deployments](https://docs.flagger.app/usage/nginx-progressive-delivery) * [Monitoring](https://docs.flagger.app/usage/monitoring) * [Alerting](https://docs.flagger.app/usage/alerting) * Tutorials @@ -152,20 +154,20 @@ For more details on how the canary analysis and promotion works please [read the ## Features -| Feature | Istio | App Mesh | -| -------------------------------------------- | ------------------ | ------------------ | -| Canary deployments (weighted traffic) | :heavy_check_mark: | :heavy_check_mark: | -| A/B testing (headers and cookies filters) | :heavy_check_mark: | :heavy_minus_sign: | -| Load testing | :heavy_check_mark: | :heavy_check_mark: | -| Webhooks (custom acceptance tests) | :heavy_check_mark: | :heavy_check_mark: | -| Request success rate check (Envoy metric) | :heavy_check_mark: | :heavy_check_mark: | -| Request duration check (Envoy metric) | :heavy_check_mark: | :heavy_minus_sign: | -| Custom promql checks | :heavy_check_mark: | :heavy_check_mark: | -| Ingress gateway (CORS, retries and timeouts) | :heavy_check_mark: | :heavy_minus_sign: | +| Feature | Istio | App Mesh | SuperGloo | NGINX Ingress | +| -------------------------------------------- | ------------------ | ------------------ |------------------ |------------------ | +| Canary deployments (weighted traffic) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| A/B testing (headers and cookies filters) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_minus_sign: | :heavy_check_mark: | +| Load testing | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Webhooks (custom acceptance tests) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Request success rate check (L7 metric) | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Request duration check (L7 metric) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_check_mark: | +| Custom promql checks | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | +| Ingress gateway (CORS, retries and timeouts) | :heavy_check_mark: | :heavy_minus_sign: | :heavy_check_mark: | :heavy_check_mark: | ## Roadmap -* Integrate with other service mesh technologies like Linkerd v2, Super Gloo or Consul Mesh +* Integrate with other service mesh technologies like Linkerd v2 * Add support for comparing the canary metrics to the primary ones and do the validation based on the derivation between the two ## Contributing diff --git a/artifacts/flagger/account.yaml b/artifacts/flagger/account.yaml index 9bbd8410..0fac89ad 100644 --- a/artifacts/flagger/account.yaml +++ b/artifacts/flagger/account.yaml @@ -31,6 +31,12 @@ rules: resources: - horizontalpodautoscalers verbs: ["*"] + - apiGroups: + - "extensions" + resources: + - ingresses + - ingresses/status + verbs: ["*"] - apiGroups: - flagger.app resources: diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index 134020e6..61eac408 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -69,6 +69,18 @@ spec: type: string name: type: string + ingressRef: + anyOf: + - type: string + - type: object + required: ['apiVersion', 'kind', 'name'] + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string service: type: object required: ['port'] diff --git a/artifacts/flagger/deployment.yaml b/artifacts/flagger/deployment.yaml index b0141f1d..833be438 100644 --- a/artifacts/flagger/deployment.yaml +++ b/artifacts/flagger/deployment.yaml @@ -22,7 +22,7 @@ spec: serviceAccountName: flagger containers: - name: flagger - image: weaveworks/flagger:0.11.1 + image: weaveworks/flagger:0.13.1 imagePullPolicy: IfNotPresent ports: - name: http @@ -31,6 +31,7 @@ spec: - ./flagger - -log-level=info - -control-loop-interval=10s + - -mesh-provider=$(MESH_PROVIDER) - -metrics-server=http://prometheus.istio-system.svc.cluster.local:9090 livenessProbe: exec: diff --git a/artifacts/nginx/canary.yaml b/artifacts/nginx/canary.yaml new file mode 100644 index 00000000..6186889b --- /dev/null +++ b/artifacts/nginx/canary.yaml @@ -0,0 +1,68 @@ +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + # deployment reference + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + # ingress reference + ingressRef: + apiVersion: extensions/v1beta1 + kind: Ingress + name: podinfo + # HPA reference (optional) + autoscalerRef: + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + name: podinfo + # the maximum time in seconds for the canary deployment + # to make progress before it is rollback (default 600s) + progressDeadlineSeconds: 60 + service: + # container port + port: 9898 + canaryAnalysis: + # schedule interval (default 60s) + interval: 10s + # max number of failed metric checks before rollback + threshold: 10 + # max traffic percentage routed to canary + # percentage (0-100) + maxWeight: 50 + # canary increment step + # percentage (0-100) + stepWeight: 5 + # NGINX Prometheus checks + metrics: + - name: request-success-rate + # minimum req success rate (non 5xx responses) + # percentage (0-100) + threshold: 99 + interval: 1m + - name: "latency" + threshold: 0.5 + interval: 1m + query: | + histogram_quantile(0.99, + sum( + rate( + http_request_duration_seconds_bucket{ + kubernetes_namespace="test", + kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" + }[1m] + ) + ) by (le) + ) + # external checks (optional) + webhooks: + - name: load-test + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 1m -q 10 -c 2 http://app.example.com/" + logCmdOutput: "true" diff --git a/artifacts/nginx/deployment.yaml b/artifacts/nginx/deployment.yaml new file mode 100644 index 00000000..814dd9c2 --- /dev/null +++ b/artifacts/nginx/deployment.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podinfo + namespace: test + labels: + app: podinfo +spec: + replicas: 1 + strategy: + rollingUpdate: + maxUnavailable: 0 + type: RollingUpdate + selector: + matchLabels: + app: podinfo + template: + metadata: + annotations: + prometheus.io/scrape: "true" + labels: + app: podinfo + spec: + containers: + - name: podinfod + image: quay.io/stefanprodan/podinfo:1.4.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9898 + name: http + protocol: TCP + command: + - ./podinfo + - --port=9898 + - --level=info + - --random-delay=false + - --random-error=false + env: + - name: PODINFO_UI_COLOR + value: green + livenessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/healthz + failureThreshold: 3 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 2 + readinessProbe: + exec: + command: + - podcli + - check + - http + - localhost:9898/readyz + failureThreshold: 3 + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 2 + resources: + limits: + cpu: 1000m + memory: 256Mi + requests: + cpu: 100m + memory: 16Mi diff --git a/artifacts/nginx/hpa.yaml b/artifacts/nginx/hpa.yaml new file mode 100644 index 00000000..fa2b5a6f --- /dev/null +++ b/artifacts/nginx/hpa.yaml @@ -0,0 +1,19 @@ +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: podinfo + namespace: test +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + minReplicas: 2 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + # scale up if usage is above + # 99% of the requested CPU (100m) + targetAverageUtilization: 99 diff --git a/artifacts/nginx/ingress.yaml b/artifacts/nginx/ingress.yaml new file mode 100644 index 00000000..c5a6fa62 --- /dev/null +++ b/artifacts/nginx/ingress.yaml @@ -0,0 +1,17 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: podinfo + namespace: test + labels: + app: podinfo + annotations: + kubernetes.io/ingress.class: "nginx" +spec: + rules: + - host: app.example.com + http: + paths: + - backend: + serviceName: podinfo + servicePort: 9898 diff --git a/charts/flagger/Chart.yaml b/charts/flagger/Chart.yaml index 7c58e2c3..c7b1ca67 100644 --- a/charts/flagger/Chart.yaml +++ b/charts/flagger/Chart.yaml @@ -1,10 +1,10 @@ apiVersion: v1 name: flagger -version: 0.11.1 -appVersion: 0.11.1 +version: 0.13.1 +appVersion: 0.13.1 kubeVersion: ">=1.11.0-0" engine: gotpl -description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio routing for traffic shifting and Prometheus metrics for canary analysis. +description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio, App Mesh or NGINX routing for traffic shifting and Prometheus metrics for canary analysis. home: https://docs.flagger.app icon: https://raw.githubusercontent.com/weaveworks/flagger/master/docs/logo/flagger-icon.png sources: diff --git a/charts/flagger/README.md b/charts/flagger/README.md index 1ecbb95d..d47034a6 100644 --- a/charts/flagger/README.md +++ b/charts/flagger/README.md @@ -45,7 +45,7 @@ The following tables lists the configurable parameters of the Flagger chart and Parameter | Description | Default --- | --- | --- -`image.repository` | image repository | `quay.io/stefanprodan/flagger` +`image.repository` | image repository | `weaveworks/flagger` `image.tag` | image tag | `` `image.pullPolicy` | image pull policy | `IfNotPresent` `metricsServer` | Prometheus URL | `http://prometheus.istio-system:9090` diff --git a/charts/flagger/templates/crd.yaml b/charts/flagger/templates/crd.yaml index aad996b8..f189a31a 100644 --- a/charts/flagger/templates/crd.yaml +++ b/charts/flagger/templates/crd.yaml @@ -70,6 +70,18 @@ spec: type: string name: type: string + ingressRef: + anyOf: + - type: string + - type: object + required: ['apiVersion', 'kind', 'name'] + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string service: type: object required: ['port'] diff --git a/charts/flagger/templates/deployment.yaml b/charts/flagger/templates/deployment.yaml index c23ce93b..0df390b7 100644 --- a/charts/flagger/templates/deployment.yaml +++ b/charts/flagger/templates/deployment.yaml @@ -38,7 +38,11 @@ spec: {{- if .Values.meshProvider }} - -mesh-provider={{ .Values.meshProvider }} {{- end }} + {{- if .Values.prometheus.install }} + - -metrics-server=http://{{ template "flagger.fullname" . }}-prometheus:9090 + {{- else }} - -metrics-server={{ .Values.metricsServer }} + {{- end }} {{- if .Values.namespace }} - -namespace={{ .Values.namespace }} {{- end }} diff --git a/charts/flagger/templates/prometheus.yaml b/charts/flagger/templates/prometheus.yaml new file mode 100644 index 00000000..f1fe583b --- /dev/null +++ b/charts/flagger/templates/prometheus.yaml @@ -0,0 +1,292 @@ +{{- if .Values.prometheus.install }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: {{ template "flagger.fullname" . }}-prometheus + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +rules: + - apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + - nodes/proxy + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: + - configmaps + verbs: ["get"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ template "flagger.fullname" . }}-prometheus + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "flagger.fullname" . }}-prometheus +subjects: + - kind: ServiceAccount + name: {{ template "flagger.serviceAccountName" . }}-prometheus + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "flagger.serviceAccountName" . }}-prometheus + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "flagger.fullname" . }}-prometheus + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +data: + prometheus.yml: |- + global: + scrape_interval: 5s + scrape_configs: + + # Scrape config for AppMesh Envoy sidecar + - job_name: 'appmesh-envoy' + metrics_path: /stats/prometheus + kubernetes_sd_configs: + - role: pod + + relabel_configs: + - source_labels: [__meta_kubernetes_pod_container_name] + action: keep + regex: '^envoy$' + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: ${1}:9901 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + + # Exclude high cardinality metrics + metric_relabel_configs: + - source_labels: [ cluster_name ] + regex: '(outbound|inbound|prometheus_stats).*' + action: drop + - source_labels: [ tcp_prefix ] + regex: '(outbound|inbound|prometheus_stats).*' + action: drop + - source_labels: [ listener_address ] + regex: '(.+)' + action: drop + - source_labels: [ http_conn_manager_listener_prefix ] + regex: '(.+)' + action: drop + - source_labels: [ http_conn_manager_prefix ] + regex: '(.+)' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_tls.*' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_tcp_downstream.*' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_http_(stats|admin).*' + action: drop + - source_labels: [ __name__ ] + regex: 'envoy_cluster_(lb|retry|bind|internal|max|original).*' + action: drop + + # Scrape config for API servers + - job_name: 'kubernetes-apiservers' + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - default + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: kubernetes;https + + # Scrape config for nodes + - job_name: 'kubernetes-nodes' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics + + # scrape config for cAdvisor + - job_name: 'kubernetes-cadvisor' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + + # scrape config for pods + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - source_labels: [ __address__ ] + regex: '.*9901.*' + action: drop + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_pod_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: kubernetes_namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: kubernetes_pod_name +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "flagger.fullname" . }}-prometheus + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ template "flagger.name" . }}-prometheus + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ template "flagger.name" . }}-prometheus + app.kubernetes.io/instance: {{ .Release.Name }} + annotations: + appmesh.k8s.aws/sidecarInjectorWebhook: disabled + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: {{ template "flagger.serviceAccountName" . }}-prometheus + containers: + - name: prometheus + image: "docker.io/prom/prometheus:v2.7.1" + imagePullPolicy: IfNotPresent + args: + - '--storage.tsdb.retention=6h' + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - containerPort: 9090 + name: http + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + resources: + requests: + cpu: 10m + memory: 128Mi + volumeMounts: + - name: config-volume + mountPath: /etc/prometheus + - name: data-volume + mountPath: /prometheus/data + + volumes: + - name: config-volume + configMap: + name: {{ template "flagger.fullname" . }}-prometheus + - name: data-volume + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ template "flagger.fullname" . }}-prometheus + namespace: {{ .Release.Namespace }} + labels: + helm.sh/chart: {{ template "flagger.chart" . }} + app.kubernetes.io/name: {{ template "flagger.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +spec: + selector: + app.kubernetes.io/name: {{ template "flagger.name" . }}-prometheus + app.kubernetes.io/instance: {{ .Release.Name }} + ports: + - name: http + protocol: TCP + port: 9090 +{{- end }} diff --git a/charts/flagger/templates/rbac.yaml b/charts/flagger/templates/rbac.yaml index 95a2f249..e03755c0 100644 --- a/charts/flagger/templates/rbac.yaml +++ b/charts/flagger/templates/rbac.yaml @@ -27,6 +27,12 @@ rules: resources: - horizontalpodautoscalers verbs: ["*"] + - apiGroups: + - "extensions" + resources: + - ingresses + - ingresses/status + verbs: ["*"] - apiGroups: - flagger.app resources: diff --git a/charts/flagger/values.yaml b/charts/flagger/values.yaml index d624d440..6b6dafdd 100644 --- a/charts/flagger/values.yaml +++ b/charts/flagger/values.yaml @@ -2,12 +2,12 @@ image: repository: weaveworks/flagger - tag: 0.11.1 + tag: 0.13.1 pullPolicy: IfNotPresent metricsServer: "http://prometheus:9090" -# accepted values are istio or appmesh (defaults to istio) +# accepted values are istio, appmesh, nginx or supergloo:mesh.namespace (defaults to istio) meshProvider: "" # single namespace restriction @@ -49,3 +49,7 @@ nodeSelector: {} tolerations: [] affinity: {} + +prometheus: + # to be used with AppMesh or nginx ingress + install: false diff --git a/charts/grafana/Chart.yaml b/charts/grafana/Chart.yaml index 3ed9f1c2..7e057602 100644 --- a/charts/grafana/Chart.yaml +++ b/charts/grafana/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: grafana -version: 1.1.0 +version: 1.2.0 appVersion: 5.4.3 description: Grafana dashboards for monitoring Flagger canary deployments icon: https://raw.githubusercontent.com/weaveworks/flagger/master/docs/logo/flagger-icon.png diff --git a/charts/grafana/dashboards/istio.json b/charts/grafana/dashboards/istio.json index 539ef535..84bc0a64 100644 --- a/charts/grafana/dashboards/istio.json +++ b/charts/grafana/dashboards/istio.json @@ -1614,9 +1614,9 @@ "multi": false, "name": "primary", "options": [], - "query": "query_result(sum(istio_requests_total{destination_workload_namespace=~\"$namespace\"}) by (destination_service_name))", + "query": "query_result(sum(istio_requests_total{destination_workload_namespace=~\"$namespace\"}) by (destination_workload))", "refresh": 1, - "regex": "/.*destination_service_name=\"([^\"]*).*/", + "regex": "/.*destination_workload=\"([^\"]*).*/", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", @@ -1636,9 +1636,9 @@ "multi": false, "name": "canary", "options": [], - "query": "query_result(sum(istio_requests_total{destination_workload_namespace=~\"$namespace\"}) by (destination_service_name))", + "query": "query_result(sum(istio_requests_total{destination_workload_namespace=~\"$namespace\"}) by (destination_workload))", "refresh": 1, - "regex": "/.*destination_service_name=\"([^\"]*).*/", + "regex": "/.*destination_workload=\"([^\"]*).*/", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", diff --git a/charts/loadtester/Chart.yaml b/charts/loadtester/Chart.yaml index d0127e99..36ea2f42 100644 --- a/charts/loadtester/Chart.yaml +++ b/charts/loadtester/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: loadtester -version: 0.3.0 +version: 0.4.0 appVersion: 0.3.0 kubeVersion: ">=1.11.0-0" engine: gotpl diff --git a/charts/loadtester/values.yaml b/charts/loadtester/values.yaml index 3e11175c..9399a8ad 100644 --- a/charts/loadtester/values.yaml +++ b/charts/loadtester/values.yaml @@ -1,7 +1,7 @@ replicaCount: 1 image: - repository: quay.io/weaveworks/flagger-loadtester + repository: weaveworks/flagger-loadtester tag: 0.3.0 pullPolicy: IfNotPresent diff --git a/cmd/flagger/main.go b/cmd/flagger/main.go index 9297577e..0c09b7dc 100644 --- a/cmd/flagger/main.go +++ b/cmd/flagger/main.go @@ -99,7 +99,7 @@ func main() { canaryInformer := flaggerInformerFactory.Flagger().V1alpha3().Canaries() - logger.Infof("Starting flagger version %s revision %s", version.VERSION, version.REVISION) + logger.Infof("Starting flagger version %s revision %s mesh provider %s", version.VERSION, version.REVISION, meshProvider) ver, err := kubeClient.Discovery().ServerVersion() if err != nil { diff --git a/docs/diagrams/flagger-gitops-istio.png b/docs/diagrams/flagger-gitops-istio.png new file mode 100644 index 00000000..1974198e Binary files /dev/null and b/docs/diagrams/flagger-gitops-istio.png differ diff --git a/docs/diagrams/flagger-nginx-overview.png b/docs/diagrams/flagger-nginx-overview.png new file mode 100644 index 00000000..f8dcaadc Binary files /dev/null and b/docs/diagrams/flagger-nginx-overview.png differ diff --git a/docs/gitbook/README.md b/docs/gitbook/README.md index 4f6c4ed7..5f027115 100644 --- a/docs/gitbook/README.md +++ b/docs/gitbook/README.md @@ -5,7 +5,7 @@ description: Flagger is a progressive delivery Kubernetes operator # Introduction [Flagger](https://github.com/weaveworks/flagger) is a **Kubernetes** operator that automates the promotion of canary -deployments using **Istio** or **App Mesh** routing for traffic shifting and **Prometheus** metrics for canary analysis. +deployments using **Istio**, **App Mesh** or **NGINX** routing for traffic shifting and **Prometheus** metrics for canary analysis. The canary analysis can be extended with webhooks for running system integration/acceptance tests, load tests, or any other custom validation. diff --git a/docs/gitbook/SUMMARY.md b/docs/gitbook/SUMMARY.md index 4ce8974e..59625ced 100644 --- a/docs/gitbook/SUMMARY.md +++ b/docs/gitbook/SUMMARY.md @@ -8,12 +8,14 @@ * [Flagger Install on Kubernetes](install/flagger-install-on-kubernetes.md) * [Flagger Install on GKE Istio](install/flagger-install-on-google-cloud.md) * [Flagger Install on EKS App Mesh](install/flagger-install-on-eks-appmesh.md) +* [Flagger Install with SuperGloo](install/flagger-install-with-supergloo.md) ## Usage * [Istio Canary Deployments](usage/progressive-delivery.md) * [Istio A/B Testing](usage/ab-testing.md) * [App Mesh Canary Deployments](usage/appmesh-progressive-delivery.md) +* [NGINX Canary Deployments](usage/nginx-progressive-delivery.md) * [Monitoring](usage/monitoring.md) * [Alerting](usage/alerting.md) diff --git a/docs/gitbook/how-it-works.md b/docs/gitbook/how-it-works.md index b81b8900..d5c9160f 100644 --- a/docs/gitbook/how-it-works.md +++ b/docs/gitbook/how-it-works.md @@ -689,7 +689,7 @@ webhooks: When the canary analysis starts, Flagger will call the webhooks and the load tester will run the `hey` commands in the background, if they are not already running. This will ensure that during the -analysis, the `podinfo.test` virtual service will receive a steady steam of GET and POST requests. +analysis, the `podinfo.test` virtual service will receive a steady stream of GET and POST requests. If your workload is exposed outside the mesh with the Istio Gateway and TLS you can point `hey` to the public URL and use HTTP2. diff --git a/docs/gitbook/install/flagger-install-on-eks-appmesh.md b/docs/gitbook/install/flagger-install-on-eks-appmesh.md index 5e64dba8..761ba436 100644 --- a/docs/gitbook/install/flagger-install-on-eks-appmesh.md +++ b/docs/gitbook/install/flagger-install-on-eks-appmesh.md @@ -125,19 +125,6 @@ Status: Type: MeshActive ``` -### Install Prometheus - -In order to collect the App Mesh metrics that Flagger needs to run the canary analysis, -you'll need to setup a Prometheus instance to scrape the Envoy sidecars. - -Deploy Prometheus in the `appmesh-system` namespace: - -```bash -REPO=https://raw.githubusercontent.com/weaveworks/flagger/master - -kubectl apply -f ${REPO}/artifacts/eks/appmesh-prometheus.yaml -``` - ### Install Flagger and Grafana Add Flagger Helm repository: @@ -146,16 +133,17 @@ Add Flagger Helm repository: helm repo add flagger https://flagger.app ``` -Deploy Flagger in the _**appmesh-system**_ namespace: +Deploy Flagger and Prometheus in the _**appmesh-system**_ namespace: ```bash helm upgrade -i flagger flagger/flagger \ --namespace=appmesh-system \ --set meshProvider=appmesh \ ---set metricsServer=http://prometheus.appmesh-system:9090 +--set prometheus.install=true ``` -You can install Flagger in any namespace as long as it can talk to the Istio Prometheus service on port 9090. +In order to collect the App Mesh metrics that Flagger needs to run the canary analysis, +you'll need to setup a Prometheus instance to scrape the Envoy sidecars. You can enable **Slack** notifications with: diff --git a/docs/gitbook/install/flagger-install-on-kubernetes.md b/docs/gitbook/install/flagger-install-on-kubernetes.md index 73120559..80013f75 100644 --- a/docs/gitbook/install/flagger-install-on-kubernetes.md +++ b/docs/gitbook/install/flagger-install-on-kubernetes.md @@ -52,7 +52,8 @@ If you don't have Tiller you can use the helm template command and apply the gen ```bash # generate -helm template flagger/flagger \ +helm fetch --untar --untardir . flagger/flagger && +helm template flagger \ --name flagger \ --namespace=istio-system \ --set metricsServer=http://prometheus.istio-system:9090 \ @@ -98,12 +99,10 @@ Or use helm template command and apply the generated yaml with kubectl: ```bash # generate -helm template flagger/grafana \ +helm fetch --untar --untardir . flagger/grafana && +helm template grafana \ --name flagger-grafana \ --namespace=istio-system \ ---set url=http://prometheus.istio-system:9090 \ ---set user=admin \ ---set password=change-me \ > $HOME/flagger-grafana.yaml # apply @@ -132,10 +131,14 @@ helm upgrade -i flagger-loadtester flagger/loadtester \ Deploy with kubectl: ```bash -export REPO=https://raw.githubusercontent.com/weaveworks/flagger/master +helm fetch --untar --untardir . flagger/loadtester && +helm template loadtester \ +--name flagger-loadtester \ +--namespace=test +> $HOME/flagger-loadtester.yaml -kubectl -n test apply -f ${REPO}/artifacts/loadtester/deployment.yaml -kubectl -n test apply -f ${REPO}/artifacts/loadtester/service.yaml +# apply +kubectl apply -f $HOME/flagger-loadtester.yaml ``` > **Note** that the load tester should be deployed in a namespace with Istio sidecar injection enabled. diff --git a/docs/gitbook/install/flagger-install-with-supergloo.md b/docs/gitbook/install/flagger-install-with-supergloo.md new file mode 100644 index 00000000..7ec10ed1 --- /dev/null +++ b/docs/gitbook/install/flagger-install-with-supergloo.md @@ -0,0 +1,184 @@ +# Flagger install on Kubernetes with SuperGloo + +This guide walks you through setting up Flagger on a Kubernetes cluster using [SuperGloo](https://github.com/solo-io/supergloo). + +SuperGloo by [Solo.io](https://solo.io) is an opinionated abstraction layer that simplifies the installation, management, and operation of your service mesh. +It supports running multiple ingresses with multiple meshes (Istio, App Mesh, Consul Connect and Linkerd 2) in the same cluster. + +### Prerequisites + +Flagger requires a Kubernetes cluster **v1.11** or newer with the following admission controllers enabled: + +* MutatingAdmissionWebhook +* ValidatingAdmissionWebhook + +### Install Istio with SuperGloo + +#### Install SuperGloo command line interface helper + +SuperGloo includes a command line helper (CLI) that makes operation of SuperGloo easier. +The CLI is not required for SuperGloo to function correctly. + +If you use [Homebrew](https://brew.sh) package manager run the following +commands to install the SuperGloo CLI. + +```bash +brew tap solo-io/tap +brew solo-io/tap/supergloo +``` + +Or you can download SuperGloo CLI and add it to your path: + +```bash +curl -sL https://run.solo.io/supergloo/install | sh +export PATH=$HOME/.supergloo/bin:$PATH +``` + +#### Install SuperGloo controller + +Deploy the SuperGloo controller in the `supergloo-system` namespace: + +```bash +supergloo init +``` + +This is equivalent to installing SuperGloo using its Helm chart + +```bash +helm repo add supergloo http://storage.googleapis.com/supergloo-helm +helm upgrade --install supergloo supergloo/supergloo --namespace supergloo-system +``` + +#### Install Istio using SuperGloo + +Create the `istio-system` namespace and install Istio with traffic management, telemetry and Prometheus enabled: + +```bash +ISTIO_VER="1.0.6" + +kubectl create namespace istio-system + +supergloo install istio --name istio \ +--namespace=supergloo-system \ +--auto-inject=true \ +--installation-namespace=istio-system \ +--mtls=false \ +--prometheus=true \ +--version=${ISTIO_VER} +``` + +This creates a Kubernetes Custom Resource (CRD) like the following. + +```yaml +apiVersion: supergloo.solo.io/v1 +kind: Install +metadata: + name: istio + namespace: supergloo-system +spec: + installationNamespace: istio-system + mesh: + installedMesh: + name: istio + namespace: supergloo-system + istioMesh: + enableAutoInject: true + enableMtls: false + installGrafana: false + installJaeger: false + installPrometheus: true + istioVersion: 1.0.6 +``` + +#### Allow Flagger to manipulate SuperGloo + +Create a cluster role binding so that Flagger can manipulate SuperGloo custom resources: + +```bash +kubectl create clusterrolebinding flagger-supergloo \ +--clusterrole=mesh-discovery \ +--serviceaccount=istio-system:flagger +``` + +Wait for the Istio control plane to become available: + +```bash +kubectl --namespace istio-system rollout status deployment/istio-sidecar-injector +kubectl --namespace istio-system rollout status deployment/prometheus +``` + +### Install Flagger + +Add Flagger Helm repository: + +```bash +helm repo add flagger https://flagger.app +``` + +Deploy Flagger in the _**istio-system**_ namespace and set the service mesh provider to SuperGloo: + +```bash +helm upgrade -i flagger flagger/flagger \ +--namespace=istio-system \ +--set metricsServer=http://prometheus.istio-system:9090 \ +--set meshProvider=supergloo:istio.supergloo-system +``` + +When using SuperGloo the mesh provider format is `supergloo:.`. + +Optionally you can enable **Slack** notifications: + +```bash +helm upgrade -i flagger flagger/flagger \ +--reuse-values \ +--namespace=istio-system \ +--set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \ +--set slack.channel=general \ +--set slack.user=flagger +``` + +### Install Grafana + +Flagger comes with a Grafana dashboard made for monitoring the canary analysis. + +Deploy Grafana in the _**istio-system**_ namespace: + +```bash +helm upgrade -i flagger-grafana flagger/grafana \ +--namespace=istio-system \ +--set url=http://prometheus.istio-system:9090 +``` + +You can access Grafana using port forwarding: + +```bash +kubectl -n istio-system port-forward svc/flagger-grafana 3000:80 +``` + +### Install Load Tester + +Flagger comes with an optional load testing service that generates traffic +during canary analysis when configured as a webhook. + +Deploy the load test runner with Helm: + +```bash +helm upgrade -i flagger-loadtester flagger/loadtester \ +--namespace=test \ +--set cmd.timeout=1h +``` + +Deploy with kubectl: + +```bash +helm fetch --untar --untardir . flagger/loadtester && +helm template loadtester \ +--name flagger-loadtester \ +--namespace=test +> $HOME/flagger-loadtester.yaml + +# apply +kubectl apply -f $HOME/flagger-loadtester.yaml +``` + +> **Note** that the load tester should be deployed in a namespace with Istio sidecar injection enabled. diff --git a/docs/gitbook/usage/nginx-progressive-delivery.md b/docs/gitbook/usage/nginx-progressive-delivery.md new file mode 100644 index 00000000..4d660bc4 --- /dev/null +++ b/docs/gitbook/usage/nginx-progressive-delivery.md @@ -0,0 +1,421 @@ +# NGNIX Ingress Controller Canary Deployments + +This guide shows you how to use the NGINX ingress controller and Flagger to automate canary deployments and A/B testing. + +![Flagger NGINX Ingress Controller](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-nginx-overview.png) + +### Prerequisites + +Flagger requires a Kubernetes cluster **v1.11** or newer and NGINX ingress **0.24** or newer. + +Install NGINX with Helm: + +```bash +helm upgrade -i nginx-ingress stable/nginx-ingress \ +--namespace ingress-nginx \ +--set controller.stats.enabled=true \ +--set controller.metrics.enabled=true \ +--set controller.podAnnotations."prometheus\.io/scrape"=true \ +--set controller.podAnnotations."prometheus\.io/port"=10254 +``` + +Install Flagger and the Prometheus add-on in the same namespace as NGINX: + +```bash +helm repo add flagger https://flagger.app + +helm upgrade -i flagger flagger/flagger \ +--namespace ingress-nginx \ +--set prometheus.install=true \ +--set meshProvider=nginx +``` + +Optionally you can enable Slack notifications: + +```bash +helm upgrade -i flagger flagger/flagger \ +--reuse-values \ +--namespace ingress-nginx \ +--set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \ +--set slack.channel=general \ +--set slack.user=flagger +``` + +### Bootstrap + +Flagger takes a Kubernetes deployment and optionally a horizontal pod autoscaler (HPA), +then creates a series of objects (Kubernetes deployments, ClusterIP services and canary ingress). +These objects expose the application outside the cluster and drive the canary analysis and promotion. + +Create a test namespace: + +```bash +kubectl create ns test +``` + +Create a deployment and a horizontal pod autoscaler: + +```bash +kubectl apply -f ${REPO}/artifacts/nginx/deployment.yaml +kubectl apply -f ${REPO}/artifacts/nginx/hpa.yaml +``` + +Deploy the load testing service to generate traffic during the canary analysis: + +```bash +helm upgrade -i flagger-loadtester flagger/loadtester \ +--namespace=test +``` + +Create an ingress definition (replace `app.example.com` with your own domain): + +```yaml +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: podinfo + namespace: test + labels: + app: podinfo + annotations: + kubernetes.io/ingress.class: "nginx" +spec: + rules: + - host: app.example.com + http: + paths: + - backend: + serviceName: podinfo + servicePort: 9898 +``` + +Save the above resource as podinfo-ingress.yaml and then apply it: + +```bash +kubectl apply -f ./podinfo-ingress.yaml +``` + +Create a canary custom resource (replace `app.example.com` with your own domain): + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + # deployment reference + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + # ingress reference + ingressRef: + apiVersion: extensions/v1beta1 + kind: Ingress + name: podinfo + # HPA reference (optional) + autoscalerRef: + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + name: podinfo + # the maximum time in seconds for the canary deployment + # to make progress before it is rollback (default 600s) + progressDeadlineSeconds: 60 + service: + # container port + port: 9898 + canaryAnalysis: + # schedule interval (default 60s) + interval: 10s + # max number of failed metric checks before rollback + threshold: 10 + # max traffic percentage routed to canary + # percentage (0-100) + maxWeight: 50 + # canary increment step + # percentage (0-100) + stepWeight: 5 + # NGINX Prometheus checks + metrics: + - name: request-success-rate + # minimum req success rate (non 5xx responses) + # percentage (0-100) + threshold: 99 + interval: 1m + # load testing (optional) + webhooks: + - name: load-test + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 1m -q 10 -c 2 http://app.example.com/" +``` + +Save the above resource as podinfo-canary.yaml and then apply it: + +```bash +kubectl apply -f ./podinfo-canary.yaml +``` + +After a couple of seconds Flagger will create the canary objects: + +```bash +# applied +deployment.apps/podinfo +horizontalpodautoscaler.autoscaling/podinfo +ingresses.extensions/podinfo +canary.flagger.app/podinfo + +# generated +deployment.apps/podinfo-primary +horizontalpodautoscaler.autoscaling/podinfo-primary +service/podinfo +service/podinfo-canary +service/podinfo-primary +ingresses.extensions/podinfo-canary +``` + +### Automated canary promotion + +Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance indicators +like HTTP requests success rate, requests average duration and pod health. +Based on analysis of the KPIs a canary is promoted or aborted, and the analysis result is published to Slack. + +![Flagger Canary Stages](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-steps.png) + +Trigger a canary deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.4.1 +``` + +Flagger detects that the deployment revision changed and starts a new rollout: + +```text +kubectl -n test describe canary/podinfo + +Status: + Canary Weight: 0 + Failed Checks: 0 + Phase: Succeeded +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Synced 3m flagger New revision detected podinfo.test + Normal Synced 3m flagger Scaling up podinfo.test + Warning Synced 3m flagger Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available + Normal Synced 3m flagger Advance podinfo.test canary weight 5 + Normal Synced 3m flagger Advance podinfo.test canary weight 10 + Normal Synced 3m flagger Advance podinfo.test canary weight 15 + Normal Synced 2m flagger Advance podinfo.test canary weight 20 + Normal Synced 2m flagger Advance podinfo.test canary weight 25 + Normal Synced 1m flagger Advance podinfo.test canary weight 30 + Normal Synced 1m flagger Advance podinfo.test canary weight 35 + Normal Synced 55s flagger Advance podinfo.test canary weight 40 + Normal Synced 45s flagger Advance podinfo.test canary weight 45 + Normal Synced 35s flagger Advance podinfo.test canary weight 50 + Normal Synced 25s flagger Copying podinfo.test template spec to podinfo-primary.test + Warning Synced 15s flagger Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available + Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test +``` + +**Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis. + +You can monitor all canaries with: + +```bash +watch kubectl get canaries --all-namespaces + +NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME +test podinfo Progressing 15 2019-05-06T14:05:07Z +prod frontend Succeeded 0 2019-05-05T16:15:07Z +prod backend Failed 0 2019-05-04T17:05:07Z +``` + +### Automated rollback + +During the canary analysis you can generate HTTP 500 errors to test if Flagger pauses and rolls back the faulted version. + +Trigger another canary deployment: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.4.2 +``` + +Generate HTTP 500 errors: + +```bash +watch curl http://app.example.com/status/500 +``` + +When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, +the canary is scaled to zero and the rollout is marked as failed. + +```text +kubectl -n test describe canary/podinfo + +Status: + Canary Weight: 0 + Failed Checks: 10 + Phase: Failed +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Synced 3m flagger Starting canary deployment for podinfo.test + Normal Synced 3m flagger Advance podinfo.test canary weight 5 + Normal Synced 3m flagger Advance podinfo.test canary weight 10 + Normal Synced 3m flagger Advance podinfo.test canary weight 15 + Normal Synced 3m flagger Halt podinfo.test advancement success rate 69.17% < 99% + Normal Synced 2m flagger Halt podinfo.test advancement success rate 61.39% < 99% + Normal Synced 2m flagger Halt podinfo.test advancement success rate 55.06% < 99% + Normal Synced 2m flagger Halt podinfo.test advancement success rate 47.00% < 99% + Normal Synced 2m flagger (combined from similar events): Halt podinfo.test advancement success rate 38.08% < 99% + Warning Synced 1m flagger Rolling back podinfo.test failed checks threshold reached 10 + Warning Synced 1m flagger Canary failed! Scaling down podinfo.test +``` + +### Custom metrics + +The canary analysis can be extended with Prometheus queries. + +The demo app is instrumented with Prometheus so you can create a custom check that will use the HTTP request duration +histogram to validate the canary. + +Edit the canary analysis and add the following metric: + +```yaml + canaryAnalysis: + metrics: + - name: "latency" + threshold: 0.5 + interval: 1m + query: | + histogram_quantile(0.99, + sum( + rate( + http_request_duration_seconds_bucket{ + kubernetes_namespace="test", + kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" + }[1m] + ) + ) by (le) + ) +``` + +The threshold is set to 500ms so if the average request duration in the last minute +goes over half a second then the analysis will fail and the canary will not be promoted. + +Trigger a canary deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.4.3 +``` + +Generate high response latency: + +```bash +watch curl http://app.exmaple.com/delay/2 +``` + +Watch Flagger logs: + +``` +kubectl -n nginx-ingress logs deployment/flagger -f | jq .msg + +Starting canary deployment for podinfo.test +Advance podinfo.test canary weight 5 +Advance podinfo.test canary weight 10 +Advance podinfo.test canary weight 15 +Halt podinfo.test advancement latency 1.20 > 0.5 +Halt podinfo.test advancement latency 1.45 > 0.5 +Halt podinfo.test advancement latency 1.60 > 0.5 +Halt podinfo.test advancement latency 1.69 > 0.5 +Halt podinfo.test advancement latency 1.70 > 0.5 +Rolling back podinfo.test failed checks threshold reached 5 +Canary failed! Scaling down podinfo.test +``` + +If you have Slack configured, Flagger will send a notification with the reason why the canary failed. + +### A/B Testing + +Besides weighted routing, Flagger can be configured to route traffic to the canary based on HTTP match conditions. +In an A/B testing scenario, you'll be using HTTP headers or cookies to target a certain segment of your users. +This is particularly useful for frontend applications that require session affinity. + +![Flagger A/B Testing Stages](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-abtest-steps.png) + +Edit the canary analysis, remove the max/step weight and add the match conditions and iterations: + +```yaml + canaryAnalysis: + interval: 1m + threshold: 10 + iterations: 10 + match: + # curl -H 'X-Canary: insider' http://app.example.com + - headers: + x-canary: + exact: "insider" + # curl -b 'canary=always' http://app.example.com + - headers: + cookie: + exact: "canary" + metrics: + - name: request-success-rate + threshold: 99 + interval: 1m + webhooks: + - name: load-test + url: http://localhost:8888/ + timeout: 5s + metadata: + type: cmd + cmd: "hey -z 1m -q 10 -c 2 -H 'Cookie: canary=always' http://app.example.com/" + logCmdOutput: "true" +``` + +The above configuration will run an analysis for ten minutes targeting users that have a `canary` cookie set to `always` or +those that call the service using the `X-Canary: insider` header. + +Trigger a canary deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.5.0 +``` + +Flagger detects that the deployment revision changed and starts the A/B testing: + +```text +kubectl -n test describe canary/podinfo + +Status: + Failed Checks: 0 + Phase: Succeeded +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Synced 3m flagger New revision detected podinfo.test + Normal Synced 3m flagger Scaling up podinfo.test + Warning Synced 3m flagger Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available + Normal Synced 3m flagger Advance podinfo.test canary iteration 1/10 + Normal Synced 3m flagger Advance podinfo.test canary iteration 2/10 + Normal Synced 3m flagger Advance podinfo.test canary iteration 3/10 + Normal Synced 2m flagger Advance podinfo.test canary iteration 4/10 + Normal Synced 2m flagger Advance podinfo.test canary iteration 5/10 + Normal Synced 1m flagger Advance podinfo.test canary iteration 6/10 + Normal Synced 1m flagger Advance podinfo.test canary iteration 7/10 + Normal Synced 55s flagger Advance podinfo.test canary iteration 8/10 + Normal Synced 45s flagger Advance podinfo.test canary iteration 9/10 + Normal Synced 35s flagger Advance podinfo.test canary iteration 10/10 + Normal Synced 25s flagger Copying podinfo.test template spec to podinfo-primary.test + Warning Synced 15s flagger Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available + Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test +``` + diff --git a/pkg/apis/flagger/v1alpha3/types.go b/pkg/apis/flagger/v1alpha3/types.go index 4b0b9d12..e9eb9a49 100755 --- a/pkg/apis/flagger/v1alpha3/types.go +++ b/pkg/apis/flagger/v1alpha3/types.go @@ -52,6 +52,10 @@ type CanarySpec struct { // +optional AutoscalerRef *hpav1.CrossVersionObjectReference `json:"autoscalerRef,omitempty"` + // reference to NGINX ingress resource + // +optional + IngressRef *hpav1.CrossVersionObjectReference `json:"ingressRef,omitempty"` + // virtual service spec Service CanaryService `json:"service"` diff --git a/pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go index 4bb9551c..dc710448 100644 --- a/pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/flagger/v1alpha3/zz_generated.deepcopy.go @@ -205,6 +205,11 @@ func (in *CanarySpec) DeepCopyInto(out *CanarySpec) { *out = new(v1.CrossVersionObjectReference) **out = **in } + if in.IngressRef != nil { + in, out := &in.IngressRef, &out.IngressRef + *out = new(v1.CrossVersionObjectReference) + **out = **in + } in.Service.DeepCopyInto(&out.Service) in.CanaryAnalysis.DeepCopyInto(&out.CanaryAnalysis) if in.ProgressDeadlineSeconds != nil { diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 5c16cb4e..0fc06978 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -632,6 +632,41 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool { } } + // NGINX checks + if c.meshProvider == "nginx" { + if metric.Name == "request-success-rate" { + val, err := c.observer.GetNginxSuccessRate(r.Spec.IngressRef.Name, r.Namespace, metric.Name, metric.Interval) + if err != nil { + if strings.Contains(err.Error(), "no values found") { + c.recordEventWarningf(r, "Halt advancement no values found for metric %s probably %s.%s is not receiving traffic", + metric.Name, r.Spec.TargetRef.Name, r.Namespace) + } else { + c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observer.GetMetricsServer(), err) + } + return false + } + if float64(metric.Threshold) > val { + c.recordEventWarningf(r, "Halt %s.%s advancement success rate %.2f%% < %v%%", + r.Name, r.Namespace, val, metric.Threshold) + return false + } + } + + if metric.Name == "request-duration" { + val, err := c.observer.GetNginxRequestDuration(r.Spec.IngressRef.Name, r.Namespace, metric.Name, metric.Interval) + if err != nil { + c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.observer.GetMetricsServer(), err) + return false + } + t := time.Duration(metric.Threshold) * time.Millisecond + if val > t { + c.recordEventWarningf(r, "Halt %s.%s advancement request duration %v > %v", + r.Name, r.Namespace, val, t) + return false + } + } + } + // custom checks if metric.Query != "" { val, err := c.observer.GetScalar(metric.Query) diff --git a/pkg/metrics/nginx.go b/pkg/metrics/nginx.go new file mode 100644 index 00000000..bdd1c8f3 --- /dev/null +++ b/pkg/metrics/nginx.go @@ -0,0 +1,122 @@ +package metrics + +import ( + "fmt" + "net/url" + "strconv" + "time" +) + +const nginxSuccessRateQuery = ` +sum(rate( +nginx_ingress_controller_requests{namespace="{{ .Namespace }}", +ingress="{{ .Name }}", +status!~"5.*"} +[{{ .Interval }}])) +/ +sum(rate( +nginx_ingress_controller_requests{namespace="{{ .Namespace }}", +ingress="{{ .Name }}"} +[{{ .Interval }}])) +* 100 +` + +// GetNginxSuccessRate returns the requests success rate (non 5xx) using nginx_ingress_controller_requests metric +func (c *Observer) GetNginxSuccessRate(name string, namespace string, metric string, interval string) (float64, error) { + if c.metricsServer == "fake" { + return 100, nil + } + + meta := struct { + Name string + Namespace string + Interval string + }{ + name, + namespace, + interval, + } + + query, err := render(meta, nginxSuccessRateQuery) + if err != nil { + return 0, err + } + + var rate *float64 + querySt := url.QueryEscape(query) + result, err := c.queryMetric(querySt) + if err != nil { + return 0, err + } + + for _, v := range result.Data.Result { + metricValue := v.Value[1] + switch metricValue.(type) { + case string: + f, err := strconv.ParseFloat(metricValue.(string), 64) + if err != nil { + return 0, err + } + rate = &f + } + } + if rate == nil { + return 0, fmt.Errorf("no values found for metric %s", metric) + } + return *rate, nil +} + +const nginxRequestDurationQuery = ` +sum(rate( +nginx_ingress_controller_ingress_upstream_latency_seconds_sum{namespace="{{ .Namespace }}", +ingress="{{ .Name }}"}[{{ .Interval }}])) +/ +sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{namespace="{{ .Namespace }}", +ingress="{{ .Name }}"}[{{ .Interval }}])) * 1000 +` + +// GetNginxRequestDuration returns the avg requests latency using nginx_ingress_controller_ingress_upstream_latency_seconds_sum metric +func (c *Observer) GetNginxRequestDuration(name string, namespace string, metric string, interval string) (time.Duration, error) { + if c.metricsServer == "fake" { + return 1, nil + } + + meta := struct { + Name string + Namespace string + Interval string + }{ + name, + namespace, + interval, + } + + query, err := render(meta, nginxRequestDurationQuery) + if err != nil { + return 0, err + } + + var rate *float64 + querySt := url.QueryEscape(query) + result, err := c.queryMetric(querySt) + if err != nil { + return 0, err + } + + for _, v := range result.Data.Result { + metricValue := v.Value[1] + switch metricValue.(type) { + case string: + f, err := strconv.ParseFloat(metricValue.(string), 64) + if err != nil { + return 0, err + } + rate = &f + } + } + if rate == nil { + return 0, fmt.Errorf("no values found for metric %s", metric) + } + ms := time.Duration(int64(*rate)) * time.Millisecond + return ms, nil +} diff --git a/pkg/metrics/nginx_test.go b/pkg/metrics/nginx_test.go new file mode 100644 index 00000000..00e9ef90 --- /dev/null +++ b/pkg/metrics/nginx_test.go @@ -0,0 +1,51 @@ +package metrics + +import ( + "testing" +) + +func Test_NginxSuccessRateQueryRender(t *testing.T) { + meta := struct { + Name string + Namespace string + Interval string + }{ + "podinfo", + "nginx", + "1m", + } + + query, err := render(meta, nginxSuccessRateQuery) + if err != nil { + t.Fatal(err) + } + + expected := `sum(rate(nginx_ingress_controller_requests{namespace="nginx",ingress="podinfo",status!~"5.*"}[1m])) / sum(rate(nginx_ingress_controller_requests{namespace="nginx",ingress="podinfo"}[1m])) * 100` + + if query != expected { + t.Errorf("\nGot %s \nWanted %s", query, expected) + } +} + +func Test_NginxRequestDurationQueryRender(t *testing.T) { + meta := struct { + Name string + Namespace string + Interval string + }{ + "podinfo", + "nginx", + "1m", + } + + query, err := render(meta, nginxRequestDurationQuery) + if err != nil { + t.Fatal(err) + } + + expected := `sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_sum{namespace="nginx",ingress="podinfo"}[1m])) /sum(rate(nginx_ingress_controller_ingress_upstream_latency_seconds_count{namespace="nginx",ingress="podinfo"}[1m])) * 1000` + + if query != expected { + t.Errorf("\nGot %s \nWanted %s", query, expected) + } +} diff --git a/pkg/metrics/observer.go b/pkg/metrics/observer.go index ddb87119..ef0a5cfd 100644 --- a/pkg/metrics/observer.go +++ b/pkg/metrics/observer.go @@ -99,7 +99,9 @@ func (c *Observer) GetScalar(query string) (float64, error) { query = strings.Replace(query, " ", "", -1) var value *float64 - result, err := c.queryMetric(query) + + querySt := url.QueryEscape(query) + result, err := c.queryMetric(querySt) if err != nil { return 0, err } diff --git a/pkg/router/factory.go b/pkg/router/factory.go index 5b78243c..579a0570 100644 --- a/pkg/router/factory.go +++ b/pkg/router/factory.go @@ -41,9 +41,14 @@ func (factory *Factory) KubernetesRouter(label string) *KubernetesRouter { } } -// MeshRouter returns a service mesh router (Istio or AppMesh) +// MeshRouter returns a service mesh router func (factory *Factory) MeshRouter(provider string) Interface { switch { + case provider == "nginx": + return &IngressRouter{ + logger: factory.logger, + kubeClient: factory.kubeClient, + } case provider == "appmesh": return &AppMeshRouter{ logger: factory.logger, diff --git a/pkg/router/ingress.go b/pkg/router/ingress.go new file mode 100644 index 00000000..ffc663b1 --- /dev/null +++ b/pkg/router/ingress.go @@ -0,0 +1,231 @@ +package router + +import ( + "fmt" + "github.com/google/go-cmp/cmp" + flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1alpha3" + "go.uber.org/zap" + "k8s.io/api/extensions/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" + "strconv" + "strings" +) + +type IngressRouter struct { + kubeClient kubernetes.Interface + logger *zap.SugaredLogger +} + +func (i *IngressRouter) Reconcile(canary *flaggerv1.Canary) error { + if canary.Spec.IngressRef == nil || canary.Spec.IngressRef.Name == "" { + return fmt.Errorf("ingress selector is empty") + } + + targetName := canary.Spec.TargetRef.Name + canaryName := fmt.Sprintf("%s-canary", targetName) + canaryIngressName := fmt.Sprintf("%s-canary", canary.Spec.IngressRef.Name) + + ingress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canary.Spec.IngressRef.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + ingressClone := ingress.DeepCopy() + + // change backend to -canary + backendExists := false + for k, v := range ingressClone.Spec.Rules { + for x, y := range v.HTTP.Paths { + if y.Backend.ServiceName == targetName { + ingressClone.Spec.Rules[k].HTTP.Paths[x].Backend.ServiceName = canaryName + backendExists = true + break + } + } + } + + if !backendExists { + return fmt.Errorf("backend %s not found in ingress %s", targetName, canary.Spec.IngressRef.Name) + } + + canaryIngress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canaryIngressName, metav1.GetOptions{}) + + if errors.IsNotFound(err) { + ing := &v1beta1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: canaryIngressName, + Namespace: canary.Namespace, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(canary, schema.GroupVersionKind{ + Group: flaggerv1.SchemeGroupVersion.Group, + Version: flaggerv1.SchemeGroupVersion.Version, + Kind: flaggerv1.CanaryKind, + }), + }, + Annotations: i.makeAnnotations(ingressClone.Annotations), + Labels: ingressClone.Labels, + }, + Spec: ingressClone.Spec, + } + + _, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Create(ing) + if err != nil { + return err + } + + i.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). + Infof("Ingress %s.%s created", ing.GetName(), canary.Namespace) + return nil + } + + if err != nil { + return fmt.Errorf("ingress %s query error %v", canaryIngressName, err) + } + + if diff := cmp.Diff(ingressClone.Spec, canaryIngress.Spec); diff != "" { + iClone := canaryIngress.DeepCopy() + iClone.Spec = ingressClone.Spec + + _, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Update(iClone) + if err != nil { + return fmt.Errorf("ingress %s update error %v", canaryIngressName, err) + } + + i.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). + Infof("Ingress %s updated", canaryIngressName) + } + + return nil +} + +func (i *IngressRouter) GetRoutes(canary *flaggerv1.Canary) ( + primaryWeight int, + canaryWeight int, + err error, +) { + canaryIngressName := fmt.Sprintf("%s-canary", canary.Spec.IngressRef.Name) + canaryIngress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canaryIngressName, metav1.GetOptions{}) + if err != nil { + return 0, 0, err + } + + // A/B testing + if len(canary.Spec.CanaryAnalysis.Match) > 0 { + for k := range canaryIngress.Annotations { + if k == "nginx.ingress.kubernetes.io/canary-by-cookie" || k == "nginx.ingress.kubernetes.io/canary-by-header" { + return 0, 100, nil + } + } + } + + // Canary + for k, v := range canaryIngress.Annotations { + if k == "nginx.ingress.kubernetes.io/canary-weight" { + val, err := strconv.Atoi(v) + if err != nil { + return 0, 0, err + } + + canaryWeight = val + break + } + } + + primaryWeight = 100 - canaryWeight + return +} + +func (i *IngressRouter) SetRoutes( + canary *flaggerv1.Canary, + primaryWeight int, + canaryWeight int, +) error { + canaryIngressName := fmt.Sprintf("%s-canary", canary.Spec.IngressRef.Name) + canaryIngress, err := i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Get(canaryIngressName, metav1.GetOptions{}) + if err != nil { + return err + } + + iClone := canaryIngress.DeepCopy() + + // A/B testing + if len(canary.Spec.CanaryAnalysis.Match) > 0 { + cookie := "" + header := "" + headerValue := "" + for _, m := range canary.Spec.CanaryAnalysis.Match { + for k, v := range m.Headers { + if k == "cookie" { + cookie = v.Exact + } else { + header = k + headerValue = v.Exact + } + } + } + + iClone.Annotations = i.makeHeaderAnnotations(iClone.Annotations, header, headerValue, cookie) + } else { + // canary + iClone.Annotations["nginx.ingress.kubernetes.io/canary-weight"] = fmt.Sprintf("%v", canaryWeight) + } + + // toggle canary + if canaryWeight > 0 { + iClone.Annotations["nginx.ingress.kubernetes.io/canary"] = "true" + } else { + iClone.Annotations = i.makeAnnotations(iClone.Annotations) + } + + _, err = i.kubeClient.ExtensionsV1beta1().Ingresses(canary.Namespace).Update(iClone) + if err != nil { + return fmt.Errorf("ingress %s update error %v", canaryIngressName, err) + } + + return nil +} + +func (i *IngressRouter) makeAnnotations(annotations map[string]string) map[string]string { + res := make(map[string]string) + for k, v := range annotations { + if !strings.Contains(k, "nginx.ingress.kubernetes.io/canary") && + !strings.Contains(k, "kubectl.kubernetes.io/last-applied-configuration") { + res[k] = v + } + } + + res["nginx.ingress.kubernetes.io/canary"] = "false" + res["nginx.ingress.kubernetes.io/canary-weight"] = "0" + + return res +} + +func (i *IngressRouter) makeHeaderAnnotations(annotations map[string]string, + header string, headerValue string, cookie string) map[string]string { + res := make(map[string]string) + for k, v := range annotations { + if !strings.Contains(v, "nginx.ingress.kubernetes.io/canary") { + res[k] = v + } + } + + res["nginx.ingress.kubernetes.io/canary"] = "true" + res["nginx.ingress.kubernetes.io/canary-weight"] = "0" + + if cookie != "" { + res["nginx.ingress.kubernetes.io/canary-by-cookie"] = cookie + } + + if header != "" { + res["nginx.ingress.kubernetes.io/canary-by-header"] = header + } + + if headerValue != "" { + res["nginx.ingress.kubernetes.io/canary-by-header-value"] = headerValue + } + + return res +} diff --git a/pkg/router/ingress_test.go b/pkg/router/ingress_test.go new file mode 100644 index 00000000..7d2679e6 --- /dev/null +++ b/pkg/router/ingress_test.go @@ -0,0 +1,112 @@ +package router + +import ( + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" +) + +func TestIngressRouter_Reconcile(t *testing.T) { + mocks := setupfakeClients() + router := &IngressRouter{ + logger: mocks.logger, + kubeClient: mocks.kubeClient, + } + + err := router.Reconcile(mocks.ingressCanary) + if err != nil { + t.Fatal(err.Error()) + } + + canaryAn := "nginx.ingress.kubernetes.io/canary" + canaryWeightAn := "nginx.ingress.kubernetes.io/canary-weight" + + canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name) + inCanary, err := router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + if _, ok := inCanary.Annotations[canaryAn]; !ok { + t.Errorf("Canary annotation missing") + } + + // test initialisation + if inCanary.Annotations[canaryAn] != "false" { + t.Errorf("Got canary annotation %v wanted false", inCanary.Annotations[canaryAn]) + } + + if inCanary.Annotations[canaryWeightAn] != "0" { + t.Errorf("Got canary weight annotation %v wanted 0", inCanary.Annotations[canaryWeightAn]) + } +} + +func TestIngressRouter_GetSetRoutes(t *testing.T) { + mocks := setupfakeClients() + router := &IngressRouter{ + logger: mocks.logger, + kubeClient: mocks.kubeClient, + } + + err := router.Reconcile(mocks.ingressCanary) + if err != nil { + t.Fatal(err.Error()) + } + + p, c, err := router.GetRoutes(mocks.ingressCanary) + if err != nil { + t.Fatal(err.Error()) + } + + p = 50 + c = 50 + + err = router.SetRoutes(mocks.ingressCanary, p, c) + if err != nil { + t.Fatal(err.Error()) + } + + canaryAn := "nginx.ingress.kubernetes.io/canary" + canaryWeightAn := "nginx.ingress.kubernetes.io/canary-weight" + + canaryName := fmt.Sprintf("%s-canary", mocks.ingressCanary.Spec.IngressRef.Name) + inCanary, err := router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + if _, ok := inCanary.Annotations[canaryAn]; !ok { + t.Errorf("Canary annotation missing") + } + + // test rollout + if inCanary.Annotations[canaryAn] != "true" { + t.Errorf("Got canary annotation %v wanted true", inCanary.Annotations[canaryAn]) + } + + if inCanary.Annotations[canaryWeightAn] != "50" { + t.Errorf("Got canary weight annotation %v wanted 50", inCanary.Annotations[canaryWeightAn]) + } + + p = 100 + c = 0 + + err = router.SetRoutes(mocks.ingressCanary, p, c) + if err != nil { + t.Fatal(err.Error()) + } + + inCanary, err = router.kubeClient.ExtensionsV1beta1().Ingresses("default").Get(canaryName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + // test promotion + if inCanary.Annotations[canaryAn] != "false" { + t.Errorf("Got canary annotation %v wanted false", inCanary.Annotations[canaryAn]) + } + + if inCanary.Annotations[canaryWeightAn] != "0" { + t.Errorf("Got canary weight annotation %v wanted 0", inCanary.Annotations[canaryWeightAn]) + } +} diff --git a/pkg/router/istio.go b/pkg/router/istio.go index 40370d10..5a1bd227 100644 --- a/pkg/router/istio.go +++ b/pkg/router/istio.go @@ -32,7 +32,7 @@ func (ir *IstioRouter) Reconcile(canary *flaggerv1.Canary) error { hosts := canary.Spec.Service.Hosts var hasServiceHost bool for _, h := range hosts { - if h == targetName { + if h == targetName || h == "*" { hasServiceHost = true break } diff --git a/pkg/router/router_test.go b/pkg/router/router_test.go index 32c5f770..701cae0c 100644 --- a/pkg/router/router_test.go +++ b/pkg/router/router_test.go @@ -11,7 +11,9 @@ import ( appsv1 "k8s.io/api/apps/v1" hpav1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" ) @@ -20,6 +22,7 @@ type fakeClients struct { canary *v1alpha3.Canary abtest *v1alpha3.Canary appmeshCanary *v1alpha3.Canary + ingressCanary *v1alpha3.Canary kubeClient kubernetes.Interface meshClient clientset.Interface flaggerClient clientset.Interface @@ -30,9 +33,10 @@ func setupfakeClients() fakeClients { canary := newMockCanary() abtest := newMockABTest() appmeshCanary := newMockCanaryAppMesh() - flaggerClient := fakeFlagger.NewSimpleClientset(canary, abtest, appmeshCanary) + ingressCanary := newMockCanaryIngress() + flaggerClient := fakeFlagger.NewSimpleClientset(canary, abtest, appmeshCanary, ingressCanary) - kubeClient := fake.NewSimpleClientset(newMockDeployment(), newMockABTestDeployment()) + kubeClient := fake.NewSimpleClientset(newMockDeployment(), newMockABTestDeployment(), newMockIngress()) meshClient := fakeFlagger.NewSimpleClientset() logger, _ := logger.NewLogger("debug") @@ -41,6 +45,7 @@ func setupfakeClients() fakeClients { canary: canary, abtest: abtest, appmeshCanary: appmeshCanary, + ingressCanary: ingressCanary, kubeClient: kubeClient, meshClient: meshClient, flaggerClient: flaggerClient, @@ -266,3 +271,73 @@ func newMockABTestDeployment() *appsv1.Deployment { return d } + +func newMockCanaryIngress() *v1alpha3.Canary { + cd := &v1alpha3.Canary{ + TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "nginx", + }, + Spec: v1alpha3.CanarySpec{ + TargetRef: hpav1.CrossVersionObjectReference{ + Name: "podinfo", + APIVersion: "apps/v1", + Kind: "Deployment", + }, + IngressRef: &hpav1.CrossVersionObjectReference{ + Name: "podinfo", + APIVersion: "extensions/v1beta1", + Kind: "Ingress", + }, + Service: v1alpha3.CanaryService{ + Port: 9898, + }, CanaryAnalysis: v1alpha3.CanaryAnalysis{ + Threshold: 10, + StepWeight: 10, + MaxWeight: 50, + Metrics: []v1alpha3.CanaryMetric{ + { + Name: "request-success-rate", + Threshold: 99, + Interval: "1m", + }, + }, + }, + }, + } + return cd +} + +func newMockIngress() *v1beta1.Ingress { + return &v1beta1.Ingress{ + TypeMeta: metav1.TypeMeta{APIVersion: v1beta1.SchemeGroupVersion.String()}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "podinfo", + Annotations: map[string]string{ + "kubernetes.io/ingress.class": "nginx", + }, + }, + Spec: v1beta1.IngressSpec{ + Rules: []v1beta1.IngressRule{ + { + Host: "app.example.com", + IngressRuleValue: v1beta1.IngressRuleValue{ + HTTP: &v1beta1.HTTPIngressRuleValue{ + Paths: []v1beta1.HTTPIngressPath{ + { + Path: "/", + Backend: v1beta1.IngressBackend{ + ServiceName: "podinfo", + ServicePort: intstr.FromInt(9898), + }, + }, + }, + }, + }, + }, + }, + }, + } +} diff --git a/pkg/router/supergloo.go b/pkg/router/supergloo.go index 259a7637..762e845c 100644 --- a/pkg/router/supergloo.go +++ b/pkg/router/supergloo.go @@ -52,14 +52,14 @@ func NewSuperglooRouter(ctx context.Context, provider string, flaggerClient clie // remove the supergloo: prefix provider = strings.TrimPrefix(provider, "supergloo:") - // split namespace.name : + // split name.namespace: parts := strings.Split(provider, ".") if len(parts) != 2 { return nil, fmt.Errorf("invalid format for supergloo provider") } targetMesh := solokitcore.ResourceRef{ - Namespace: parts[0], - Name: parts[1], + Namespace: parts[1], + Name: parts[0], } return NewSuperglooRouterWithClient(ctx, routingRuleClient, targetMesh, logger), nil } @@ -74,12 +74,22 @@ func (sr *SuperglooRouter) Reconcile(canary *flaggerv1.Canary) error { if err := sr.setRetries(canary); err != nil { return err } + if err := sr.setHeaders(canary); err != nil { + return err + } if err := sr.setCors(canary); err != nil { return err } - return sr.SetRoutes(canary, 100, 0) - + // do we have routes already? + if _, _, err := sr.GetRoutes(canary); err == nil { + // we have routes, no need to do anything else + return nil + } else if solokiterror.IsNotExist(err) { + return sr.SetRoutes(canary, 100, 0) + } else { + return err + } } func (sr *SuperglooRouter) setRetries(canary *flaggerv1.Canary) error { @@ -98,6 +108,52 @@ func (sr *SuperglooRouter) setRetries(canary *flaggerv1.Canary) error { return sr.writeRuleForCanary(canary, rule) } +func (sr *SuperglooRouter) setHeaders(canary *flaggerv1.Canary) error { + if canary.Spec.Service.Headers == nil { + return nil + } + headerManipulation, err := convertHeaders(canary.Spec.Service.Headers) + if err != nil { + return err + } + if headerManipulation == nil { + return nil + } + rule := sr.createRule(canary, "headers", &supergloov1.RoutingRuleSpec{ + RuleType: &supergloov1.RoutingRuleSpec_HeaderManipulation{ + HeaderManipulation: headerManipulation, + }, + }) + + return sr.writeRuleForCanary(canary, rule) +} + +func convertHeaders(headers *istiov1alpha3.Headers) (*supergloov1.HeaderManipulation, error) { + var headersMaipulation *supergloov1.HeaderManipulation + + if headers.Request != nil { + headersMaipulation = &supergloov1.HeaderManipulation{} + + headersMaipulation.RemoveRequestHeaders = headers.Request.Remove + headersMaipulation.AppendRequestHeaders = make(map[string]string) + for k, v := range headers.Request.Add { + headersMaipulation.AppendRequestHeaders[k] = v + } + } + if headers.Response != nil { + if headersMaipulation == nil { + headersMaipulation = &supergloov1.HeaderManipulation{} + } + + headersMaipulation.RemoveResponseHeaders = headers.Response.Remove + headersMaipulation.AppendResponseHeaders = make(map[string]string) + for k, v := range headers.Response.Add { + headersMaipulation.AppendResponseHeaders[k] = v + } + } + + return headersMaipulation, nil +} func convertRetries(retries *istiov1alpha3.HTTPRetry) (*supergloov1.RetryPolicy, error) { perTryTimeout, err := time.ParseDuration(retries.PerTryTimeout) diff --git a/pkg/router/supergloo_test.go b/pkg/router/supergloo_test.go index 2e31aa40..aa500b63 100644 --- a/pkg/router/supergloo_test.go +++ b/pkg/router/supergloo_test.go @@ -25,10 +25,9 @@ func TestSuperglooRouter_Sync(t *testing.T) { if err := routingRuleClient.Register(); err != nil { t.Fatal(err.Error()) } - // TODO(yuval-k): un hard code this targetMesh := solokitcore.ResourceRef{ Namespace: "supergloo-system", - Name: "yuval", + Name: "mesh", } router := NewSuperglooRouterWithClient(context.TODO(), routingRuleClient, targetMesh, mocks.logger) err = router.Reconcile(mocks.canary) @@ -61,10 +60,9 @@ func TestSuperglooRouter_SetRoutes(t *testing.T) { if err := routingRuleClient.Register(); err != nil { t.Fatal(err.Error()) } - // TODO(yuval-k): un hard code this targetMesh := solokitcore.ResourceRef{ Namespace: "supergloo-system", - Name: "yuval", + Name: "mesh", } router := NewSuperglooRouterWithClient(context.TODO(), routingRuleClient, targetMesh, mocks.logger) @@ -126,10 +124,9 @@ func TestSuperglooRouter_GetRoutes(t *testing.T) { if err := routingRuleClient.Register(); err != nil { t.Fatal(err.Error()) } - // TODO(yuval-k): un hard code this targetMesh := solokitcore.ResourceRef{ Namespace: "supergloo-system", - Name: "yuval", + Name: "mesh", } router := NewSuperglooRouterWithClient(context.TODO(), routingRuleClient, targetMesh, mocks.logger) err = router.Reconcile(mocks.canary) diff --git a/pkg/version/version.go b/pkg/version/version.go index 2ae05041..d9f5035d 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -1,4 +1,4 @@ package version -var VERSION = "0.11.1" +var VERSION = "0.13.1" var REVISION = "unknown" diff --git a/test/Dockerfile.kind b/test/Dockerfile.kind deleted file mode 100644 index 28f5e083..00000000 --- a/test/Dockerfile.kind +++ /dev/null @@ -1,4 +0,0 @@ -FROM golang:1.11 - -RUN go get -u sigs.k8s.io/kind - diff --git a/test/README.md b/test/README.md index 3f8d586b..43c04b0a 100644 --- a/test/README.md +++ b/test/README.md @@ -2,7 +2,7 @@ The e2e testing infrastructure is powered by CircleCI and [Kubernetes Kind](https://github.com/kubernetes-sigs/kind). -CircleCI e2e workflow: +### CircleCI e2e Istio workflow * install latest stable kubectl [e2e-kind.sh](e2e-kind.sh) * install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh) @@ -21,4 +21,20 @@ CircleCI e2e workflow: * test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-tests.sh](e2e-tests.sh) * test the A/B testing analysis and promotion using cookies filters and pre/post rollout webhooks [e2e-tests.sh](e2e-tests.sh) +### CircleCI e2e NGINX ingress workflow +* install latest stable kubectl [e2e-kind.sh](e2e-kind.sh) +* install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh) +* create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh) +* install latest stable Helm CLI [e2e-nginx.sh](e2e-istio.sh) +* deploy Tiller on the local cluster [e2e-nginx.sh](e2e-istio.sh) +* install NGINX ingress with Helm [e2e-nginx.sh](e2e-istio.sh) +* build Flagger container image [e2e-nginx-build.sh](e2e-build.sh) +* load Flagger image onto the local cluster [e2e-nginx-build.sh](e2e-build.sh) +* install Flagger and Prometheus in the ingress-nginx namespace [e2e-nginx-build.sh](e2e-build.sh) +* create a test namespace [e2e-nginx-tests.sh](e2e-tests.sh) +* deploy the load tester in the test namespace [e2e-nginx-tests.sh](e2e-tests.sh) +* deploy the demo workload (podinfo) and ingress in the test namespace [e2e-nginx-tests.sh](e2e-tests.sh) +* test the canary initialization [e2e-nginx-tests.sh](e2e-tests.sh) +* test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-nginx-tests.sh](e2e-tests.sh) +* test the A/B testing analysis and promotion using header filters and pre/post rollout webhooks [e2e-nginx-tests.sh](e2e-tests.sh) diff --git a/test/e2e-build.sh b/test/e2e-build.sh index 43dff8d0..d0e4bfa8 100755 --- a/test/e2e-build.sh +++ b/test/e2e-build.sh @@ -11,5 +11,10 @@ cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile echo '>>> Installing Flagger' kind load docker-image test/flagger:latest kubectl apply -f ${REPO_ROOT}/artifacts/flagger/ + +if [ -n "$1" ]; then + kubectl -n istio-system set env deployment/flagger "MESH_PROVIDER=$1" +fi + kubectl -n istio-system set image deployment/flagger flagger=test/flagger:latest kubectl -n istio-system rollout status deployment/flagger diff --git a/test/e2e-ingress.yaml b/test/e2e-ingress.yaml new file mode 100644 index 00000000..c5a6fa62 --- /dev/null +++ b/test/e2e-ingress.yaml @@ -0,0 +1,17 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: podinfo + namespace: test + labels: + app: podinfo + annotations: + kubernetes.io/ingress.class: "nginx" +spec: + rules: + - host: app.example.com + http: + paths: + - backend: + serviceName: podinfo + servicePort: 9898 diff --git a/test/e2e-nginx-build.sh b/test/e2e-nginx-build.sh new file mode 100755 index 00000000..6ae5449a --- /dev/null +++ b/test/e2e-nginx-build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo '>>> Building Flagger' +cd ${REPO_ROOT} && docker build -t test/flagger:latest . -f Dockerfile + +echo '>>> Installing Flagger' +kind load docker-image test/flagger:latest + +echo '>>> Installing Flagger' +helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ +--wait \ +--namespace ingress-nginx \ +--set prometheus.install=true \ +--set meshProvider=nginx + +kubectl -n ingress-nginx set image deployment/flagger flagger=test/flagger:latest + +kubectl -n ingress-nginx rollout status deployment/flagger +kubectl -n ingress-nginx rollout status deployment/flagger-prometheus diff --git a/test/e2e-nginx-tests.sh b/test/e2e-nginx-tests.sh new file mode 100755 index 00000000..dac550f7 --- /dev/null +++ b/test/e2e-nginx-tests.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Canary initialization, analysis and promotion +# Prerequisites: Kubernetes Kind, Helm and NGINX ingress controller + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo '>>> Creating test namespace' +kubectl create namespace test + +echo '>>> Installing load tester' +kubectl -n test apply -f ${REPO_ROOT}/artifacts/loadtester/ +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Initialising canary' +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml +kubectl apply -f ${REPO_ROOT}/test/e2e-ingress.yaml + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false + sleep 10 + kubectl -n ingress-nginx logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +if [ "$1" = "canary" ]; then + exit 0 +fi + +cat <>> Triggering A/B testing' +kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.2 + +echo '>>> Waiting for A/B testing promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '1.4.2' && ok=true || ok=false + sleep 10 + kubectl -n ingress-nginx logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe deployment/podinfo + kubectl -n test describe deployment/podinfo-primary + kubectl -n ingress-nginx logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ A/B testing promotion test passed' + +kubectl -n ingress-nginx logs deployment/flagger + +echo '✔ All tests passed' \ No newline at end of file diff --git a/test/e2e-nginx.sh b/test/e2e-nginx.sh new file mode 100755 index 00000000..3fa97bb7 --- /dev/null +++ b/test/e2e-nginx.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo ">>> Installing Helm" +curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash + +echo '>>> Installing Tiller' +kubectl --namespace kube-system create sa tiller +kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller +helm init --service-account tiller --upgrade --wait + +echo '>>> Installing NGINX Ingress' +helm upgrade -i nginx-ingress stable/nginx-ingress \ +--wait \ +--namespace ingress-nginx \ +--set controller.stats.enabled=true \ +--set controller.metrics.enabled=true \ +--set controller.podAnnotations."prometheus\.io/scrape"=true \ +--set controller.podAnnotations."prometheus\.io/port"=10254 \ +--set controller.service.type=NodePort + +kubectl -n ingress-nginx rollout status deployment/nginx-ingress-controller +kubectl -n ingress-nginx get all + + diff --git a/test/e2e-supergloo.sh b/test/e2e-supergloo.sh new file mode 100755 index 00000000..428b9232 --- /dev/null +++ b/test/e2e-supergloo.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -o errexit + +ISTIO_VER="1.0.6" +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo ">>> Downloading Supergloo CLI" +curl -SsL https://github.com/solo-io/supergloo/releases/download/v0.3.13/supergloo-cli-linux-amd64 > supergloo-cli +chmod +x supergloo-cli + +echo ">>> Installing Supergloo" +./supergloo-cli init +echo ">>> Installing Istio ${ISTIO_VER}" +kubectl create ns istio-system +./supergloo-cli install istio --name test --namespace supergloo-system --auto-inject=true --installation-namespace istio-system --mtls=false --prometheus=true --version ${ISTIO_VER} + +echo '>>> Waiting for Istio to be ready' +until kubectl -n supergloo-system get mesh test +do + sleep 2 +done + +# add rbac rules +kubectl create clusterrolebinding flagger-supergloo --clusterrole=mesh-discovery --serviceaccount=istio-system:flagger + +kubectl -n istio-system rollout status deployment/istio-pilot +kubectl -n istio-system rollout status deployment/istio-policy +kubectl -n istio-system rollout status deployment/istio-sidecar-injector +kubectl -n istio-system rollout status deployment/istio-telemetry +kubectl -n istio-system rollout status deployment/prometheus + +kubectl -n istio-system get all diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index 2ef1b7bc..043d2cc9 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -125,6 +125,10 @@ done echo '✔ Canary promotion test passed' +if [ "$1" = "canary" ]; then + exit 0 +fi + cat <