diff --git a/.circleci/config.yml b/.circleci/config.yml index 478988a2..0497333b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -122,6 +122,17 @@ jobs: - run: test/e2e-nginx.sh - run: test/e2e-nginx-tests.sh + e2e-linkerd-testing: + machine: true + steps: + - checkout + - attach_workspace: + at: /tmp/bin + - run: test/container-build.sh + - run: test/e2e-kind.sh + - run: test/e2e-linkerd.sh + - run: test/e2e-linkerd-tests.sh + workflows: version: 2 build-test-push: @@ -146,6 +157,9 @@ workflows: - e2e-nginx-testing: requires: - build-binary + - e2e-linkerd-testing: + requires: + - build-binary - push-container: requires: - build-binary @@ -154,6 +168,7 @@ workflows: - e2e-supergloo-testing - e2e-gloo-testing - e2e-nginx-testing + - e2e-linkerd-testing release: jobs: diff --git a/Makefile b/Makefile index 145a9896..89ea050e 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,10 @@ run-nop: GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=none -namespace=bg \ -metrics-server=https://prometheus.istio.weavedx.com +run-linkerd: + GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=smi:linkerd -namespace=demo \ + -metrics-server=https://linkerd-prometheus.istio.weavedx.com + build: GIT_COMMIT=$$(git rev-list -1 HEAD) && GO111MODULE=on CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -w -X github.com/weaveworks/flagger/pkg/version.REVISION=$${GIT_COMMIT}" -a -installsuffix cgo -o ./bin/flagger ./cmd/flagger/* docker build -t weaveworks/flagger:$(TAG) . -f Dockerfile diff --git a/docs/diagrams/flagger-linkerd-traffic-split.png b/docs/diagrams/flagger-linkerd-traffic-split.png new file mode 100644 index 00000000..8ab024f5 Binary files /dev/null and b/docs/diagrams/flagger-linkerd-traffic-split.png differ diff --git a/docs/diagrams/flagger-nginx-linkerd.png b/docs/diagrams/flagger-nginx-linkerd.png new file mode 100644 index 00000000..55e7bfa8 Binary files /dev/null and b/docs/diagrams/flagger-nginx-linkerd.png differ diff --git a/docs/gitbook/usage/blue-green.md b/docs/gitbook/usage/blue-green.md index 9eb12e11..61ab6152 100644 --- a/docs/gitbook/usage/blue-green.md +++ b/docs/gitbook/usage/blue-green.md @@ -184,6 +184,7 @@ Events: New revision detected podinfo.test Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available +Pre-rollout check acceptance-test passed Advance podinfo.test canary iteration 1/10 Advance podinfo.test canary iteration 2/10 Advance podinfo.test canary iteration 3/10 diff --git a/docs/gitbook/usage/linkerd-progressive-delivery.md b/docs/gitbook/usage/linkerd-progressive-delivery.md new file mode 100644 index 00000000..40da3627 --- /dev/null +++ b/docs/gitbook/usage/linkerd-progressive-delivery.md @@ -0,0 +1,486 @@ +# Linkerd Canary Deployments + +This guide shows you how to use Linkerd and Flagger to automate canary deployments. + +![Flagger Linkerd Traffic Split](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-linkerd-traffic-split.png) + +### Prerequisites + +Flagger requires a Kubernetes cluster **v1.11** or newer and Linker with support for SMI Traffic Spit API. + +Install Flagger in the linkerd namespace: + +```bash +helm repo add flagger https://flagger.app + +helm upgrade -i flagger flagger/flagger \ +--namespace linkerd \ +--set metricsServer=http://linkerd-prometheus:9090 \ +--set meshProvider=linkerd +``` + +Optionally you can enable Slack notifications: + +```bash +helm upgrade -i flagger flagger/flagger \ +--reuse-values \ +--namespace linkerd \ +--set slack.url=https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK \ +--set slack.channel=general \ +--set slack.user=flagger +``` + +### Bootstrap + +Flagger takes a Kubernetes deployment and optionally a horizontal pod autoscaler (HPA), +then creates a series of objects (Kubernetes deployments, ClusterIP services and SMI traffic split). +These objects expose the application inside the mesh and drive the canary analysis and promotion. + +Create a test namespace and enable Linkerd proxy injection: + +```bash +kubectl create ns test +kubectl annotate namespace test linkerd.io/inject=enabled +``` + +Install the load testing service to generate traffic during the canary analysis: + +```bash +helm upgrade -i flagger-loadtester flagger/loadtester \ +--namespace=test +``` + +Create a deployment and a horizontal pod autoscaler: + +```bash +export REPO=https://raw.githubusercontent.com/weaveworks/flagger/master + +kubectl apply -f ${REPO}/artifacts/canary/deployment.yaml +kubectl apply -f ${REPO}/artifacts/canary/hpa.yaml +``` + +Create a canary custom resource for the podinfo deployment: + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + # deployment reference + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + # HPA reference (optional) + autoscalerRef: + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + name: podinfo + # the maximum time in seconds for the canary deployment + # to make progress before it is rollback (default 600s) + progressDeadlineSeconds: 60 + service: + # container port + port: 9898 + canaryAnalysis: + # schedule interval (default 60s) + interval: 30s + # max number of failed metric checks before rollback + threshold: 5 + # max traffic percentage routed to canary + # percentage (0-100) + maxWeight: 50 + # canary increment step + # percentage (0-100) + stepWeight: 5 + # Linkerd Prometheus checks + metrics: + - name: request-success-rate + # minimum req success rate (non 5xx responses) + # percentage (0-100) + threshold: 99 + interval: 1m + - name: request-duration + # maximum req duration P99 + # milliseconds + threshold: 500 + interval: 30s + # testing (optional) + webhooks: + - name: acceptance-test + type: pre-rollout + url: http://flagger-loadtester.test/ + timeout: 30s + metadata: + type: bash + cmd: "curl -sd 'test' http://podinfo-canary:9898/token | grep token" + - name: load-test + type: rollout + url: http://flagger-loadtester.test/ + metadata: + cmd: "hey -z 2m -q 10 -c 2 http://podinfo:9898/" +``` + +Save the above resource as podinfo-canary.yaml and then apply it: + +```bash +kubectl apply -f ./podinfo-canary.yaml +``` + +When the canary analysis starts, Flagger will call the pre-rollout webhooks before routing traffic to the canary. +The canary analysis will run for five minutes while validating the HTTP metrics and rollout hooks every half a minute. + +After a couple of seconds Flagger will create the canary objects: + +```bash +# applied +deployment.apps/podinfo +horizontalpodautoscaler.autoscaling/podinfo +ingresses.extensions/podinfo +canary.flagger.app/podinfo + +# generated +deployment.apps/podinfo-primary +horizontalpodautoscaler.autoscaling/podinfo-primary +service/podinfo +service/podinfo-canary +service/podinfo-primary +trafficsplits.split.smi-spec.io/podinfo +``` + +After the boostrap, the podinfo deployment will be scaled to zero and the traffic to `podinfo.test` will be routed +to the primary pods. During the canary analysis, the `podinfo-canary.test` address can be used to target directly the canary pods. + +### Automated canary promotion + +Flagger implements a control loop that gradually shifts traffic to the canary while measuring key performance indicators +like HTTP requests success rate, requests average duration and pod health. +Based on analysis of the KPIs a canary is promoted or aborted, and the analysis result is published to Slack. + +![Flagger Canary Stages](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-canary-steps.png) + +Trigger a canary deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.4.1 +``` + +Flagger detects that the deployment revision changed and starts a new rollout: + +```text +kubectl -n test describe canary/podinfo + +Status: + Canary Weight: 0 + Failed Checks: 0 + Phase: Succeeded +Events: + New revision detected! Scaling up podinfo.test + Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available + Pre-rollout check acceptance-test passed + Advance podinfo.test canary weight 5 + Advance podinfo.test canary weight 10 + Advance podinfo.test canary weight 15 + Advance podinfo.test canary weight 20 + Advance podinfo.test canary weight 25 + Waiting for podinfo.test rollout to finish: 1 of 2 updated replicas are available + Advance podinfo.test canary weight 30 + Advance podinfo.test canary weight 35 + Advance podinfo.test canary weight 40 + Advance podinfo.test canary weight 45 + Advance podinfo.test canary weight 50 + Copying podinfo.test template spec to podinfo-primary.test + Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available + Promotion completed! Scaling down podinfo.test +``` + +**Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis. + +A canary deployment is triggered by changes in any of the following objects: +* Deployment PodSpec (container image, command, ports, env, resources, etc) +* ConfigMaps mounted as volumes or mapped to environment variables +* Secrets mounted as volumes or mapped to environment variables + +You can monitor all canaries with: + +```bash +watch kubectl get canaries --all-namespaces + +NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME +test podinfo Progressing 15 2019-06-30T14:05:07Z +prod frontend Succeeded 0 2019-06-30T16:15:07Z +prod backend Failed 0 2019-06-30T17:05:07Z +``` + +### Automated rollback + +During the canary analysis you can generate HTTP 500 errors and high latency to test if Flagger pauses and rolls back the faulted version. + +Trigger another canary deployment: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.4.2 +``` + +Exec into the load tester pod with: + +```bash +kubectl -n test exec -it flagger-loadtester-xx-xx sh +``` + +Generate HTTP 500 errors: + +```bash +watch -n 1 curl http://podinfo-canary.test:9898/status/500 +``` + +Generate latency: + +```bash +watch -n 1 curl http://podinfo-canary.test:9898/delay/1 +``` + +When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, +the canary is scaled to zero and the rollout is marked as failed. + +```text +kubectl -n test describe canary/podinfo + +Status: + Canary Weight: 0 + Failed Checks: 10 + Phase: Failed +Events: + Starting canary analysis for podinfo.test + Pre-rollout check acceptance-test passed + Advance podinfo.test canary weight 5 + Advance podinfo.test canary weight 10 + Advance podinfo.test canary weight 15 + Halt podinfo.test advancement success rate 69.17% < 99% + Halt podinfo.test advancement success rate 61.39% < 99% + Halt podinfo.test advancement success rate 55.06% < 99% + Halt podinfo.test advancement request duration 1.20s > 0.5s + Halt podinfo.test advancement request duration 1.45s > 0.5s + Rolling back podinfo.test failed checks threshold reached 5 + Canary failed! Scaling down podinfo.test +``` + +### Custom metrics + +The canary analysis can be extended with Prometheus queries. + +Let's a define a check for not found errors. Edit the canary analysis and add the following metric: + +```yaml + canaryAnalysis: + metrics: + - name: "404s percentage" + threshold: 3 + query: | + 100 - sum( + rate( + response_total{ + namespace="test", + deployment="podinfo", + status_code!="404", + direction="inbound" + }[1m] + ) + ) + / + sum( + rate( + response_total{ + namespace="test", + deployment="podinfo", + direction="inbound" + }[1m] + ) + ) + * 100 +``` + +The above configuration validates the canary version by checking if the HTTP 404 req/sec percentage is below +three percent of the total traffic. If the 404s rate reaches the 3% threshold, then the analysis is aborted and the +canary is marked as failed. + +Trigger a canary deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.4.3 +``` + +Generate 404s: + +```bash +watch -n 1 curl http://podinfo-canary:9898/status/404 +``` + +Watch Flagger logs: + +``` +kubectl -n linkerd logs deployment/flagger -f | jq .msg + +Starting canary deployment for podinfo.test +Pre-rollout check acceptance-test passed +Advance podinfo.test canary weight 5 +Halt podinfo.test advancement 404s percentage 6.20 > 3 +Halt podinfo.test advancement 404s percentage 6.45 > 3 +Halt podinfo.test advancement 404s percentage 7.22 > 3 +Halt podinfo.test advancement 404s percentage 6.50 > 3 +Halt podinfo.test advancement 404s percentage 6.34 > 3 +Rolling back podinfo.test failed checks threshold reached 5 +Canary failed! Scaling down podinfo.test +``` + +If you have Slack configured, Flagger will send a notification with the reason why the canary failed. + +### Linkerd Ingress + +There are two ingress controllers that are compatible with both Flagger and Linkerd: NGINX and Gloo. + +Install NGINX: + +```bash +helm upgrade -i nginx-ingress stable/nginx-ingress \ +--namespace ingress-nginx +``` + +Create an ingress definition for podinfo that rewrites the incoming header to the internal service name (required by Linkerd): + +```yaml +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: podinfo + namespace: test + labels: + app: podinfo + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/configuration-snippet: | + proxy_set_header l5d-dst-override $service_name.$namespace.svc.cluster.local:9898; + proxy_hide_header l5d-remote-ip; + proxy_hide_header l5d-server-id; +spec: + rules: + - host: app.example.com + http: + paths: + - backend: + serviceName: podinfo + servicePort: 9898 +``` + +When using an ingress controller, the Linkerd traffic split does not apply to incoming traffic since NGINX in running outside of +the mesh. In order to run a canary analysis for a frontend app, Flagger creates a shadow ingress and sets the NGINX specific annotations. + +### A/B Testing + +Besides weighted routing, Flagger can be configured to route traffic to the canary based on HTTP match conditions. +In an A/B testing scenario, you'll be using HTTP headers or cookies to target a certain segment of your users. +This is particularly useful for frontend applications that require session affinity. + +![Flagger Linkerd Ingress](https://raw.githubusercontent.com/weaveworks/flagger/master/docs/diagrams/flagger-nginx-linkerd.png) + +Edit podinfo canary analysis, set the provider to `nginx`, add the ingress reference, remove the max/step weight and add the match conditions and iterations: + +```yaml +apiVersion: flagger.app/v1alpha3 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + # ingress reference + provider: nginx + ingressRef: + apiVersion: extensions/v1beta1 + kind: Ingress + name: podinfo + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + autoscalerRef: + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + name: podinfo + service: + # container port + port: 9898 + canaryAnalysis: + interval: 1m + threshold: 10 + iterations: 10 + match: + # curl -H 'X-Canary: always' http://app.example.com + - headers: + x-canary: + exact: "always" + # curl -b 'canary=always' http://app.example.com + - headers: + cookie: + exact: "canary" + # Linkerd Prometheus checks + metrics: + - name: request-success-rate + threshold: 99 + interval: 1m + - name: request-duration + threshold: 500 + interval: 30s + webhooks: + - name: acceptance-test + type: pre-rollout + url: http://flagger-loadtester.test/ + timeout: 30s + metadata: + type: bash + cmd: "curl -sd 'test' http://podinfo-canary:9898/token | grep token" + - name: load-test + type: rollout + url: http://flagger-loadtester.test/ + metadata: + cmd: "hey -z 2m -q 10 -c 2 -H 'Cookie: canary=always' http://app.example.com" +``` + +The above configuration will run an analysis for ten minutes targeting users that have a `canary` cookie set to `always` or +those that call the service using the `X-Canary: always` header. + +**Note** that the load test now targets the external address and uses the canary cookie. + +Trigger a canary deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.5.0 +``` + +Flagger detects that the deployment revision changed and starts the A/B testing: + +```text +kubectl -n test describe canary/podinfo + +Events: + Starting canary deployment for podinfo.test + Pre-rollout check acceptance-test passed + Advance podinfo.test canary iteration 1/10 + Advance podinfo.test canary iteration 2/10 + Advance podinfo.test canary iteration 3/10 + Advance podinfo.test canary iteration 4/10 + Advance podinfo.test canary iteration 5/10 + Advance podinfo.test canary iteration 6/10 + Advance podinfo.test canary iteration 7/10 + Advance podinfo.test canary iteration 8/10 + Advance podinfo.test canary iteration 9/10 + Advance podinfo.test canary iteration 10/10 + Copying podinfo.test template spec to podinfo-primary.test + Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available + Promotion completed! Scaling down podinfo.test +``` diff --git a/pkg/apis/flagger/v1alpha3/types.go b/pkg/apis/flagger/v1alpha3/types.go index 3d5b0364..d127a23b 100755 --- a/pkg/apis/flagger/v1alpha3/types.go +++ b/pkg/apis/flagger/v1alpha3/types.go @@ -208,6 +208,10 @@ func (c *Canary) GetAnalysisInterval() time.Duration { return AnalysisInterval } + if interval < 10*time.Second { + return time.Second * 10 + } + return interval } diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 943a9809..5ed080ad 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -567,8 +567,19 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool { } } + // override the global provider if one is specified in the canary spec + metricsProvider := c.meshProvider + if r.Spec.Provider != "" { + metricsProvider = r.Spec.Provider + + // set the metrics provider to Linkerd Prometheus when using NGINX as Linkerd Ingress + if r.Spec.Provider == "nginx" && strings.Contains(c.meshProvider, "linkerd") { + metricsProvider = "linkerd" + } + } + // create observer based on the mesh provider - observer := c.observerFactory.Observer() + observer := c.observerFactory.Observer(metricsProvider) // run metrics checks for _, metric := range r.Spec.CanaryAnalysis.Metrics { diff --git a/pkg/metrics/factory.go b/pkg/metrics/factory.go index cf669729..8e77f2df 100644 --- a/pkg/metrics/factory.go +++ b/pkg/metrics/factory.go @@ -22,25 +22,29 @@ func NewFactory(metricsServer string, meshProvider string, timeout time.Duration }, nil } -func (factory Factory) Observer() Interface { +func (factory Factory) Observer(provider string) Interface { switch { - case factory.MeshProvider == "none": + case provider == "none": return &HttpObserver{ client: factory.Client, } - case factory.MeshProvider == "appmesh": + case provider == "appmesh": return &EnvoyObserver{ client: factory.Client, } - case factory.MeshProvider == "nginx": + case provider == "nginx": return &NginxObserver{ client: factory.Client, } - case strings.HasPrefix(factory.MeshProvider, "gloo"): + case strings.HasPrefix(provider, "gloo"): return &GlooObserver{ client: factory.Client, } - case factory.MeshProvider == "smi:linkerd": + case provider == "smi:linkerd": + return &LinkerdObserver{ + client: factory.Client, + } + case provider == "linkerd": return &LinkerdObserver{ client: factory.Client, } diff --git a/pkg/metrics/linkerd.go b/pkg/metrics/linkerd.go index 4a9ee294..f4d5707a 100644 --- a/pkg/metrics/linkerd.go +++ b/pkg/metrics/linkerd.go @@ -11,7 +11,7 @@ var linkerdQueries = map[string]string{ response_total{ namespace="{{ .Namespace }}", deployment=~"{{ .Name }}", - classification="failure", + classification!="failure", direction="inbound" }[{{ .Interval }}] ) diff --git a/pkg/metrics/linkerd_test.go b/pkg/metrics/linkerd_test.go index 6dbfef5c..82b62dd5 100644 --- a/pkg/metrics/linkerd_test.go +++ b/pkg/metrics/linkerd_test.go @@ -8,7 +8,7 @@ import ( ) func TestLinkerdObserver_GetRequestSuccessRate(t *testing.T) { - expected := `sum(rate(response_total{namespace="default",deployment=~"podinfo",classification="failure",direction="inbound"}[1m]))/sum(rate(response_total{namespace="default",deployment=~"podinfo",direction="inbound"}[1m]))*100` + expected := `sum(rate(response_total{namespace="default",deployment=~"podinfo",classification!="failure",direction="inbound"}[1m]))/sum(rate(response_total{namespace="default",deployment=~"podinfo",direction="inbound"}[1m]))*100` ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { promql := r.URL.Query()["query"][0] diff --git a/pkg/router/factory.go b/pkg/router/factory.go index a14fef68..b99f440f 100644 --- a/pkg/router/factory.go +++ b/pkg/router/factory.go @@ -70,6 +70,14 @@ func (factory *Factory) MeshRouter(provider string) Interface { smiClient: factory.meshClient, targetMesh: mesh, } + case provider == "linkerd": + return &SmiRouter{ + logger: factory.logger, + flaggerClient: factory.flaggerClient, + kubeClient: factory.kubeClient, + smiClient: factory.meshClient, + targetMesh: "linkerd", + } case strings.HasPrefix(provider, "supergloo"): supergloo, err := NewSuperglooRouter(context.TODO(), provider, factory.flaggerClient, factory.logger, factory.kubeConfig) if err != nil { diff --git a/test/e2e-linkerd-tests.sh b/test/e2e-linkerd-tests.sh new file mode 100755 index 00000000..f5e623cb --- /dev/null +++ b/test/e2e-linkerd-tests.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash + +# This script runs Linkerd e2e tests for Canary initialization, analysis and promotion + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +echo '>>> Creating test namespace' +kubectl create namespace test +kubectl annotate namespace test linkerd.io/inject=enabled + +echo '>>> Installing the load tester' +kubectl -n test apply -f ${REPO_ROOT}/artifacts/loadtester/ +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Initialising canary' +kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '1.4.1' && ok=true || ok=false + sleep 10 + kubectl -n linkerd logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +cat <>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=quay.io/stefanprodan/podinfo:1.4.2 + +echo '>>> Waiting for canary rollback' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Failed' && ok=true || ok=false + sleep 10 + kubectl -n linkerd logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n linkerd logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary rollback test passed' \ No newline at end of file diff --git a/test/e2e-linkerd.sh b/test/e2e-linkerd.sh new file mode 100755 index 00000000..4545c64c --- /dev/null +++ b/test/e2e-linkerd.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -o errexit + +LINKERD_VER="edge-19.6.4" +REPO_ROOT=$(git rev-parse --show-toplevel) +export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" + +curl -SsL https://github.com/linkerd/linkerd2/releases/download/${LINKERD_VER}/linkerd2-cli-${LINKERD_VER}-linux > ${REPO_ROOT}/bin/linkerd +chmod +x ${REPO_ROOT}/bin/linkerd + +echo ">>> Installing Linkerd ${LINKERD_VER}" +${REPO_ROOT}/bin/linkerd install | kubectl apply -f - +${REPO_ROOT}/bin/linkerd check + +kubectl -n linkerd rollout status deployment/linkerd-controller +kubectl -n linkerd rollout status deployment/linkerd-proxy-injector + +echo '>>> Load Flagger image in Kind' +kind load docker-image test/flagger:latest + +echo '>>> Installing Flagger' +helm upgrade -i flagger ${REPO_ROOT}/charts/flagger \ +--namespace linkerd \ +--set metricsServer=http://linkerd-prometheus:9090 \ +--set meshProvider=smi:linkerd + +kubectl -n linkerd set image deployment/flagger flagger=test/flagger:latest +kubectl -n linkerd rollout status deployment/flagger \ No newline at end of file