Merge remote-tracking branch 'upstream/master'

test/README.md

@@ -53,4 +53,19 @@ The e2e testing infrastructure is powered by CircleCI and [Kubernetes Kind](http
 * test the A/B testing analysis and promotion using header filters and pre/post rollout webhooks [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
 * cleanup test environment [e2e-nginx-cleanup.sh](e2e-nginx-cleanup.sh)
 * install NGINX Ingress and Flagger with custom ingress annotations prefix [e2e-nginx-custom-annotations.sh](e2e-nginx-custom-annotations.sh)
-* repeat the canary and A/B testing workflow [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
+* repeat the canary and A/B testing workflow [e2e-nginx-tests.sh](e2e-nginx-tests.sh)
+
+### CircleCI e2e Skipper ingress workflow
+
+* install latest stable kubectl [e2e-kind.sh](e2e-kind.sh)
+* install Kubernetes Kind [e2e-kind.sh](e2e-kind.sh)
+* create local Kubernetes cluster with kind [e2e-kind.sh](e2e-kind.sh)
+* install Skipper ingress with Kustomize [e2e-skipper.sh](e2e-skipper.sh)
+* load Flagger image onto the local cluster [e2e-skipper.sh](e2e-skipper.sh)
+* install Flagger and Prometheus in the flagger-system namespace [e2e-skipper.sh](e2e-skipper.sh)
+* create a test namespace [e2e-skipper-tests.sh](e2e-skipper-tests.sh)
+* deploy the load tester in the test namespace [e2e-skipper-tests.sh](e2e-skipper-tests.sh)
+* deploy the demo workload (podinfo) and ingress in the test namespace [e2e-skipper-tests.sh](e2e-skipper-tests.sh)
+* test the canary initialization [e2e-skipper-tests.sh](e2e-skipper-tests.sh)
+* test the canary analysis and promotion using weighted traffic and the load testing webhook [e2e-skipper-tests.sh]e2e-skipper-tests.sh)
+* cleanup test environment [e2e-skipper-cleanup.sh](e2e-skipper-cleanup.sh)

test/e2e-contour.sh

@@ -4,7 +4,7 @@ set -o errexit
 
 REPO_ROOT=$(git rev-parse --show-toplevel)
 
-CONTOUR_VER="v1.5.0"
+CONTOUR_VER="v1.7.0"
 
 echo '>>> Installing Contour'
 kubectl apply -f https://projectcontour.io/quickstart/${CONTOUR_VER}/contour.yaml

test/e2e-istio.sh

@@ -2,7 +2,7 @@
 
 set -o errexit
 
-ISTIO_VER="1.6.1"
+ISTIO_VER="1.6.7"
 REPO_ROOT=$(git rev-parse --show-toplevel)
 
 echo ">>> Downloading Istio ${ISTIO_VER}"

test/e2e-kind.sh

@@ -3,7 +3,8 @@
 set -o errexit
 
 REPO_ROOT=$(git rev-parse --show-toplevel)
-HELM_VERSION=v3.0.3
+HELM_VERSION=v3.3.0
+KUSTOMIZE_VERSION=3.8.1
 KIND_VERSION=v0.8.1
 KUBE_VERSION=v1.16.9
 
@@ -26,6 +27,14 @@ kind create cluster --wait 5m --image kindest/node:${KUBE_VERSION}
 
 kubectl get pods --all-namespaces
 
+cd /tmp
+
 echo ">>> Installing Helm"
 curl -sSL https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz | tar xz && sudo mv linux-amd64/helm /usr/local/bin/ && rm -rf linux-amd64
 helm repo add stable https://kubernetes-charts.storage.googleapis.com/
+
+echo ">>> Installing Kustomize"
+kustomize_url=https://github.com/kubernetes-sigs/kustomize/releases/download && \
+curl -sL ${kustomize_url}/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz | tar xz
+chmod +x kustomize
+sudo mv kustomize /usr/local/bin/kustomize

test/e2e-linkerd.sh

@@ -2,7 +2,7 @@
 
 set -o errexit
 
-LINKERD_VER="stable-2.8.0"
+LINKERD_VER="stable-2.8.1"
 REPO_ROOT=$(git rev-parse --show-toplevel)
 
 curl -SsL https://github.com/linkerd/linkerd2/releases/download/${LINKERD_VER}/linkerd2-cli-${LINKERD_VER}-linux > ${REPO_ROOT}/bin/linkerd

test/e2e-skipper-canary.yaml

@@ -0,0 +1,66 @@
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  provider: skipper
+  progressDeadlineSeconds: 120
+  revertOnDeletion: true
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  ingressRef:
+    apiVersion: networking.k8s.io/v1beta1
+    kind: Ingress
+    name: podinfo-ingress
+  service:
+    # service name (defaults to targetRef.name)
+    name: podinfo-service
+    # ClusterIP port number
+    port: 80
+    # container port name or number (optional)
+    targetPort: http
+    # port name can be http or grpc (default http)
+    # portName: http
+    # add all the other container ports
+    # to the ClusterIP services (default false)
+    # portDiscovery: false
+  analysis:
+    interval: 15s
+    threshold: 5
+    maxWeight: 100
+    stepWeight: 10
+    metrics:
+    - name: request-success-rate
+      interval: 15s
+      # minimum req success rate (non 5xx responses)
+      # percentage (0-100)
+      thresholdRange:
+        min: 99
+    - name: request-duration
+      interval: 15s
+      # maximum req duration P99
+      # milliseconds
+      thresholdRange:
+        max: 500
+    webhooks:
+      - name: gate
+        type: confirm-rollout
+        url: http://flagger-loadtester.test/gate/approve
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 10s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-service-canary/token | grep token"
+      - name: "load test"
+        type: rollout
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          type: cmd
+          cmd: "hey -z 10m -q 10 -c 2 -host app.example.com http://skipper-ingress.kube-system"
+          logCmdOutput: "true"

test/e2e-skipper-cleanup.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+echo '>>> Deleting Skipper Ingress'
+kustomize build ${REPO_ROOT}/test/skipper | kubectl delete --force --wait=false -f -
+
+echo '>>> Deleting Flagger'
+kubectl delete namespace flagger-system --ignore-not-found=true --force --wait=false
+
+echo '>>> Cleanup test namespace'
+kubectl delete namespace test --ignore-not-found=true --force --wait=false
+
+exit 0

test/e2e-skipper-tests.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+
+# This script runs e2e tests for Canary initialization, analysis and promotion
+# Prerequisites: Kubernetes Kind and Skipper ingress controller
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+echo '>>> Creating test namespace'
+kubectl create namespace test || true
+
+echo '>>> Initialising workload'
+kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml
+kubectl apply -f ${REPO_ROOT}/test/e2e-workload-ingress.yaml
+
+echo '>>> Installing load tester'
+kubectl apply -k ${REPO_ROOT}/kustomize/tester
+kubectl -n test rollout status deployment/flagger-loadtester
+
+echo '>>> Create canary CRD'
+kubectl apply -f ${REPO_ROOT}/test/e2e-skipper-canary.yaml
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+  kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+  sleep 5
+  count=$(($count + 1))
+  if [[ ${count} -eq ${retries} ]]; then
+    kubectl -n flagger-system logs deployment/flagger
+    echo "No more retries left"
+    exit 1
+  fi
+done
+
+echo '✔ Canary initialization test passed'
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
+
+echo '>>> Waiting for canary promotion'
+retries=50
+count=0
+ok=false
+failed=false
+until ${ok}; do
+  kubectl -n test get canary/podinfo | grep 'Failed' && failed=true || failed=false
+  if ${failed}; then
+    kubectl -n flagger-system logs deployment/flagger
+    echo "Canary failed!"
+    exit 1
+  fi
+  kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
+  sleep 10
+  kubectl -n flagger-system logs deployment/flagger --tail 1
+  count=$(($count + 1))
+  if [[ ${count} -eq ${retries} ]]; then
+    kubectl -n test describe deployment/podinfo
+    kubectl -n test describe deployment/podinfo-primary
+    kubectl -n flagger-system logs deployment/flagger
+    echo "No more retries left"
+    exit 1
+  fi
+done
+
+echo '>>> Waiting for canary finalization'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+  kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
+  sleep 5
+  count=$(($count + 1))
+  if [[ ${count} -eq ${retries} ]]; then
+    kubectl -n flagger-system logs deployment/flagger
+    echo "No more retries left"
+    exit 1
+  fi
+done
+
+echo '✔ Canary promotion test passed'

test/e2e-skipper.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+echo '>>> Loading Flagger image'
+kind load docker-image test/flagger:latest
+
+echo '>>> Installing Skipper Ingress, Flagger and Prometheus'
+# use kustomize to avoid compatibility issues:
+# https://github.com/kubernetes-sigs/kustomize/issues/2390
+# Skipper will throw an Prometheus warning which can be ignored:
+# https://github.com/weaveworks/flagger/issues/664
+kustomize build ${REPO_ROOT}/test/skipper | kubectl apply -f -
+
+kubectl rollout status deployment/skipper-ingress -n kube-system
+kubectl rollout status deployment/flagger-prometheus -n flagger-system
+
+kubectl -n flagger-system set image deployment/flagger flagger=test/flagger:latest
+
+kubectl -n flagger-system rollout status deployment/flagger

test/e2e-workload-ingress.yaml

@@ -0,0 +1,17 @@
+apiVersion: networking.k8s.io/v1beta1
+kind: Ingress
+metadata:
+  name: podinfo-ingress
+  namespace: test
+  labels:
+    app: podinfo
+  annotations:
+    kubernetes.io/ingress.class: skipper
+spec:
+  rules:
+    - host: app.example.com
+      http:
+        paths:
+          - backend:
+              serviceName: podinfo-service
+              servicePort: 80

test/local/e2e-skipper.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# This script is intended for local workstation development convenience.
+# It will run the e2e tests for Skipper and leave a working setup to play with
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+cd $REPO_ROOT
+
+make test
+make build
+docker tag weaveworks/flagger:latest test/flagger:latest
+make loadtester-build
+(kind get clusters && kubectl delete ns/test --force) || kind create cluster --wait 5m --image kindest/node:v1.16.9
+./test/e2e-skipper.sh
+# port forward prometheus UI to localhost:9090
+kubectl port-forward $(kubectl get pods -l=app=flagger-prometheus -o name -n flagger-system | head -n 1) 9090:9090 -n flagger-system &
+
+./test/e2e-skipper-tests.sh

test/skipper/kustomization.yaml

@@ -0,0 +1,67 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../kustomize/base/prometheus/
+  - ../../kustomize/base/flagger/
+  - https://raw.githubusercontent.com/zalando/skipper/v0.11.140/docs/kubernetes/deploy/deployment/rbac.yaml
+  - https://raw.githubusercontent.com/zalando/skipper/v0.11.140/docs/kubernetes/deploy/deployment/service.yaml
+  - https://raw.githubusercontent.com/zalando/skipper/v0.11.140/docs/kubernetes/deploy/deployment/deployment.yaml
+  - namespace.yaml
+patchesStrategicMerge:
+  - patch.yaml
+patches:
+  - target:
+      kind: Deployment
+      name: skipper-ingress
+    patch: |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: skipper-ingress
+      spec:
+        template:
+          metadata:
+            annotations:
+              prometheus.io/path: /metrics
+              prometheus.io/port: "9911"
+              prometheus.io/scrape: "true"
+          spec:
+            nodeSelector:
+              $patch: delete
+            affinity:
+              $patch: delete
+            containers:
+              - name: skipper-ingress
+                image: registry.opensource.zalan.do/pathfinder/skipper:v0.11.141
+                ports:
+                  - name: metrics-port
+                    containerPort: 9911
+                resources:
+                  $patch: delete
+                readinessProbe:
+                  initialDelaySeconds: 5
+                args:
+                  - skipper
+                  - -default-filters-prepend=enableAccessLog(4,5)
+                  - -address=:9999
+                  - -disable-metrics-compat
+                  - -enable-connection-metrics
+                  - -enable-profile
+                  - -enable-ratelimits
+                  - -experimental-upgrade
+                  - -histogram-metric-buckets=.01,1,10,100
+                  - -kubernetes-in-cluster
+                  - -kubernetes-path-mode=path-prefix
+                  - -kubernetes
+                  - -lb-healthcheck-interval=3s
+                  - -max-audit-body=0
+                  - -max-idle-connection-backend=0
+                  - -metrics-exp-decay-sample
+                  - -metrics-flavour=prometheus
+                  - -proxy-preserve-host
+                  - -route-backend-metrics
+                  - -route-backend-error-counters
+                  - -route-response-metrics
+                  - -serve-host-metrics
+                  - -serve-route-metrics
+                  - -whitelisted-healthcheck-cidr=0.0.0.0/0 # kind uses other IP addresse

test/skipper/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: flagger-system

test/skipper/patch.yaml

@@ -0,0 +1,13 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flagger
+spec:
+  template:
+    spec:
+      containers:
+        - name: flagger
+          args:
+            - -log-level=debug
+            - -mesh-provider=skipper
+            - -metrics-server=http://flagger-prometheus:9090