Merge remote-tracking branch 'upstream/master'

test/e2e-contour-tests.sh

@@ -85,6 +85,34 @@ spec:
           logCmdOutput: "true"
 EOF
 
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo-service
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo-service
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+    portDiscovery: true
+    headers:
+      request:
+        add:
+          x-envoy-upstream-rq-timeout-ms: "15000"
+          x-envoy-max-retries: "10"
+          x-envoy-retry-on: "gateway-error,connect-failure,refused-stream"
+  analysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 30
+    stepWeight: 10
+EOF
+
 echo '>>> Waiting for primary to be ready'
 retries=50
 count=0
@@ -104,6 +132,19 @@ kubectl -n test get httpproxy podinfo -oyaml | grep 'projectcontour.io/ingress.c
 
 echo '✔ Canary initialization test passed'
 
+passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo selector test failed'
+  exit 1
+fi
+passed=$(kubectl -n test get svc/podinfo-service-canary -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo-service selector test failed'
+  exit 1
+fi
+
+echo '✔ Canary service custom metadata test passed'
+
 echo '>>> Triggering canary deployment'
 kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
 

test/e2e-gloo-tests.sh

@@ -88,6 +88,34 @@ spec:
           logCmdOutput: "true"
 EOF
 
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo-service
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo-service
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+    portDiscovery: true
+    headers:
+      request:
+        add:
+          x-envoy-upstream-rq-timeout-ms: "15000"
+          x-envoy-max-retries: "10"
+          x-envoy-retry-on: "gateway-error,connect-failure,refused-stream"
+  analysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 30
+    stepWeight: 10
+EOF
+
 echo '>>> Waiting for primary to be ready'
 retries=50
 count=0
@@ -105,6 +133,19 @@ done
 
 echo '✔ Canary initialization test passed'
 
+passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo selector test failed'
+  exit 1
+fi
+passed=$(kubectl -n test get svc/podinfo-service-canary -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo-service selector test failed'
+  exit 1
+fi
+
+echo '✔ Canary service custom metadata test passed'
+
 echo '>>> Triggering canary deployment'
 kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
 

test/e2e-istio-dependencies.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# This script setups the scenarios for istio tests by creating a Kubernetes namespace, installing the load tester and a test workload (podinfo)
+# Prerequisites: Kubernetes Kind and Istio
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+echo '>>> Creating test namespace'
+kubectl create namespace test
+kubectl label namespace test istio-injection=enabled
+
+echo '>>> Installing the load tester'
+kubectl apply -k ${REPO_ROOT}/kustomize/tester
+kubectl -n test rollout status deployment/flagger-loadtester
+
+echo '>>> Deploy podinfo'
+kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml

test/e2e-istio-tests-delegate.sh

@@ -0,0 +1,150 @@
+#!/usr/bin/env bash
+
+# This script runs e2e tests for when the canary delegation is enabled
+# Prerequisites: Kubernetes Kind and Istio
+
+set -o errexit
+
+echo '>>> Set pilot env to enable virtual service delegate'
+kubectl -n istio-system set env deploy istiod PILOT_ENABLE_VIRTUAL_SERVICE_DELEGATE=true
+kubectl -n istio-system rollout status deploy istiod
+
+echo '>>> Initialising Gateway'
+cat <<EOF | kubectl apply -f -
+apiVersion: networking.istio.io/v1alpha3
+kind: Gateway
+metadata:
+  name: istio-ingressgateway
+  namespace: istio-system
+spec:
+  selector:
+    app: istio-ingressgateway
+    istio: ingressgateway
+  servers:
+  - port:
+      number: 80
+      name: http
+      protocol: HTTP
+    hosts:
+      - "*"
+EOF
+
+echo '>>> Initialising root virtual service'
+cat <<EOF | kubectl apply -f -
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: root-vs
+  namespace: test
+spec:
+  gateways:
+    - istio-system/istio-ingressgateway
+  hosts:
+    - "*"
+  http:
+    - match:
+        - uri:
+            prefix: "/podinfo"
+      rewrite:
+        uri: "/"
+      delegate:
+        name: podinfo
+        namespace: test
+EOF
+
+echo '>>> Initialising canary for delegate'
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 80
+    targetPort: 9898
+    portDiscovery: true
+    delegation: true
+  skipAnalysis: true
+  analysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 30
+    stepWeight: 10
+    webhooks:
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          type: cmd
+          cmd: "hey -z 10m -q 10 -c 2 http://istio-ingressgateway.istio-system/podinfo"
+          logCmdOutput: "true"
+EOF
+
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary initialization test passed'
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
+
+echo '>>> Waiting for canary promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
+    sleep 10
+    kubectl -n istio-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n test describe deployment/podinfo
+        kubectl -n test describe deployment/podinfo-primary
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '>>> Waiting for canary finalization'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '>>> Set pilot env to disable virtual service delegate'
+kubectl -n istio-system set env deploy istiod PILOT_ENABLE_VIRTUAL_SERVICE_DELEGATE=false
+kubectl -n istio-system rollout status deploy istiod
+
+echo '✔ Canary promotion test passed'
+
+if [[ "$1" = "canary" ]]; then
+  exit 0
+fi

test/e2e-istio-tests-skip-analysis.sh

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+
+# This script runs e2e tests for when the canary analysis is skipped
+# Prerequisites: Kubernetes Kind and Istio
+
+set -o errexit
+
+echo '>>> Initialising canary'
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+    portDiscovery: true
+  skipAnalysis: true
+  analysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 30
+    stepWeight: 10
+    webhooks:
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          type: cmd
+          cmd: "hey -z 10m -q 10 -c 2 http://podinfo.test:9898/"
+          logCmdOutput: "true"
+EOF
+
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary initialization test passed'
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
+
+echo '>>> Waiting for canary promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
+    sleep 10
+    kubectl -n istio-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n test describe deployment/podinfo
+        kubectl -n test describe deployment/podinfo-primary
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '>>> Waiting for canary finalization'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary promotion test passed'
+
+if [[ "$1" = "canary" ]]; then
+  exit 0
+fi
+
+echo '>>> Triggering canary deployment with a bad release (non existent docker image)'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/potato:1.0.0
+
+echo '>>> Waiting for canary to fail'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl get canary/podinfo -n test -o=jsonpath='{.status.phase}' | grep 'Failed' && ok=true || ok=false
+    sleep 10
+    kubectl -n istio-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n test describe deployment/podinfo
+        kubectl -n test describe deployment/podinfo-primary
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '>>> Confirm primary pod is still running and with correct version'
+retries=50
+count=0
+ok=false
+until ${okImage} && ${okRunning}; do
+    kubectl get deployment podinfo-primary -n test -o jsonpath='{.spec.replicas}' | grep 1 && okRunning=true || okRunning=false
+    kubectl -n test describe deployment/podinfo-primary | grep '3.1.3' && okImage=true || okImage=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n istio-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+kubectl -n istio-system logs deployment/flagger
+
+echo '✔ All tests passed'

test/e2e-istio-tests.sh

@@ -5,19 +5,6 @@
 
 set -o errexit
 
-REPO_ROOT=$(git rev-parse --show-toplevel)
-
-echo '>>> Creating test namespace'
-kubectl create namespace test
-kubectl label namespace test istio-injection=enabled
-
-echo '>>> Installing the load tester'
-kubectl apply -k ${REPO_ROOT}/kustomize/tester
-kubectl -n test rollout status deployment/flagger-loadtester
-
-echo '>>> Deploy podinfo'
-kubectl apply -f ${REPO_ROOT}/test/e2e-workload.yaml
-
 echo '>>> Create latency metric template'
 cat <<EOF | kubectl apply -f -
 apiVersion: flagger.app/v1beta1
@@ -44,7 +31,7 @@ spec:
     )
 EOF
 
-echo '>>> Initialising canary'
+echo '>>> Initialising canaries'
 cat <<EOF | kubectl apply -f -
 apiVersion: flagger.app/v1beta1
 kind: Canary
@@ -98,12 +85,41 @@ spec:
           logCmdOutput: "true"
 EOF
 
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo-service
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo-service
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+    portDiscovery: true
+    headers:
+      request:
+        add:
+          x-envoy-upstream-rq-timeout-ms: "15000"
+          x-envoy-max-retries: "10"
+          x-envoy-retry-on: "gateway-error,connect-failure,refused-stream"
+  analysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 30
+    stepWeight: 10
+EOF
+
 echo '>>> Waiting for primary to be ready'
 retries=50
 count=0
 ok=false
 until ${ok}; do
     kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+    kubectl -n test get canary/podinfo-service | grep 'Initialized' && ok=true || ok=false
     sleep 5
     count=$(($count + 1))
     if [[ ${count} -eq ${retries} ]]; then
@@ -115,8 +131,26 @@ done
 
 echo '✔ Canary initialization test passed'
 
-kubectl -n test get svc/podinfo -oyaml | grep annotations-test
-kubectl -n test get svc/podinfo -oyaml | grep labels-test
+passed=$(kubectl -n test get svc/podinfo -oyaml 2>&1 | { grep annotations-test || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo annotations test failed'
+  exit 1
+fi
+passed=$(kubectl -n test get svc/podinfo -oyaml 2>&1 | { grep labels-test || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo labels test failed'
+  exit 1
+fi
+passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo selector test failed'
+  exit 1
+fi
+passed=$(kubectl -n test get svc/podinfo-service-canary -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo-service selector test failed'
+  exit 1
+fi
 
 echo '✔ Canary service custom metadata test passed'
 

test/e2e-istio.sh

@@ -2,7 +2,7 @@
 
 set -o errexit
 
-ISTIO_VER="1.6.7"
+ISTIO_VER="1.7.3"
 REPO_ROOT=$(git rev-parse --show-toplevel)
 
 echo ">>> Downloading Istio ${ISTIO_VER}"
@@ -10,8 +10,11 @@ cd ${REPO_ROOT}/bin && \
 curl -L https://istio.io/downloadIstio | ISTIO_VERSION=${ISTIO_VER} sh -
 
 echo ">>> Installing Istio ${ISTIO_VER}"
-${REPO_ROOT}/bin/istio-${ISTIO_VER}/bin/istioctl manifest apply --set profile=default
+${REPO_ROOT}/bin/istio-${ISTIO_VER}/bin/istioctl manifest install --set profile=default \
+  --set values.pilot.resources.requests.cpu=100m \
+  --set values.pilot.resources.requests.memory=100Mi
 
+kubectl apply -f https://raw.githubusercontent.com/istio/istio/release-1.7/samples/addons/prometheus.yaml
 kubectl -n istio-system rollout status deployment/prometheus
 
 kubectl -n istio-system get all

test/e2e-linkerd-tests.sh

@@ -100,6 +100,34 @@ spec:
           logCmdOutput: "true"
 EOF
 
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo-service
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo-service
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+    portDiscovery: true
+    headers:
+      request:
+        add:
+          x-envoy-upstream-rq-timeout-ms: "15000"
+          x-envoy-max-retries: "10"
+          x-envoy-retry-on: "gateway-error,connect-failure,refused-stream"
+  analysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 30
+    stepWeight: 10
+EOF
+
 echo '>>> Waiting for primary to be ready'
 retries=50
 count=0
@@ -117,6 +145,19 @@ done
 
 echo '✔ Canary initialization test passed'
 
+passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo selector test failed'
+  exit 1
+fi
+passed=$(kubectl -n test get svc/podinfo-service-canary -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo-service selector test failed'
+  exit 1
+fi
+
+echo '✔ Canary service custom metadata test passed'
+
 echo '>>> Triggering canary deployment'
 kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
 

test/e2e-nginx-tests.sh

@@ -124,6 +124,34 @@ spec:
           cmd: "hey -z 2m -q 10 -c 2 -host app.example.com http://nginx-ingress-controller.ingress-nginx"
 EOF
 
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo-service
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo-service
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+    portDiscovery: true
+    headers:
+      request:
+        add:
+          x-envoy-upstream-rq-timeout-ms: "15000"
+          x-envoy-max-retries: "10"
+          x-envoy-retry-on: "gateway-error,connect-failure,refused-stream"
+  analysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 30
+    stepWeight: 10
+EOF
+
 echo '>>> Waiting for primary to be ready'
 retries=50
 count=0
@@ -141,6 +169,19 @@ done
 
 echo '✔ Canary initialization test passed'
 
+passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' | { grep podinfo-primary || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo selector test failed'
+  exit 1
+fi
+passed=$(kubectl -n test get svc/podinfo-service-canary -o jsonpath='{.spec.selector.app}' | { grep podinfo || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo-service selector test failed'
+  exit 1
+fi
+
+echo '✔ Canary service custom metadata test passed'
+
 echo '>>> Triggering canary deployment'
 kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
 

test/e2e-workload.yaml

@@ -66,3 +66,72 @@ spec:
           requests:
             cpu: 1m
             memory: 16Mi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: podinfo-service
+  namespace: test
+  labels:
+    app: podinfo
+spec:
+  minReadySeconds: 5
+  revisionHistoryLimit: 5
+  progressDeadlineSeconds: 60
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 0
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: podinfo
+  template:
+    metadata:
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9797"
+      labels:
+        app: podinfo
+    spec:
+      containers:
+        - name: podinfod
+          image: stefanprodan/podinfo:3.1.0
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 9898
+              protocol: TCP
+            - name: http-metrics
+              containerPort: 9797
+              protocol: TCP
+            - name: grpc
+              containerPort: 9999
+              protocol: TCP
+          command:
+            - ./podinfo
+            - --port=9898
+            - --port-metrics=9797
+            - --grpc-port=9999
+            - --grpc-service-name=podinfo
+            - --level=info
+            - --random-delay=false
+            - --random-error=false
+          livenessProbe:
+            httpGet:
+              port: 9898
+              path: /healthz
+            initialDelaySeconds: 5
+            timeoutSeconds: 5
+          readinessProbe:
+            httpGet:
+              port: 9898
+              path: /readyz
+            initialDelaySeconds: 5
+            timeoutSeconds: 5
+          resources:
+            limits:
+              cpu: 1000m
+              memory: 128Mi
+            requests:
+              cpu: 1m
+              memory: 16Mi