Merge pull request #1108 from aryan9600/aryan9600/gateway-api

Add Gateway API as a provider

test/gatewayapi/install.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+CONTOUR_VER="release-1.20"
+GATEWAY_API_VER="v1alpha2"
+REPO_ROOT=$(git rev-parse --show-toplevel)
+KUSTOMIZE_VERSION=4.5.2
+OS=$(uname -s)
+ARCH=$(arch)
+if [[ $ARCH == "x86_64" ]]; then
+    ARCH="amd64"
+fi
+
+mkdir -p ${REPO_ROOT}/bin
+
+echo ">>> Installing Gateway API CRDs"
+kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v0.4.1" \
+| kubectl apply -f -
+
+echo ">>> Installing Contour components, GatewayClass and Gateway"
+kubectl apply -f https://raw.githubusercontent.com/projectcontour/contour/${CONTOUR_VER}/examples/render/contour-gateway.yaml
+
+kubectl -n projectcontour rollout status deployment/contour
+kubectl -n projectcontour get all
+kubectl get gatewayclass -oyaml
+kubectl -n projectcontour get gateway -oyaml
+
+echo '>>> Installing Kustomize'
+cd ${REPO_ROOT}/bin && \
+    curl -sL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_${OS}_${ARCH}.tar.gz | \
+    tar xz
+
+echo '>>> Installing Flagger'
+${REPO_ROOT}/bin/kustomize build ${REPO_ROOT}/kustomize/gatewayapi | kubectl apply -f -
+kubectl -n flagger-system set image deployment/flagger flagger=test/flagger:latest
+
+kubectl -n flagger-system rollout status deployment/flagger

test/gatewayapi/run.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+DIR="$(cd "$(dirname "$0")" && pwd)"
+
+"$DIR"/install.sh
+
+"$REPO_ROOT"/test/workloads/init.sh
+"$DIR"/test-canary.sh
+"$DIR"/test-bg.sh
+"$DIR"/test-ab.sh

test/gatewayapi/test-ab.sh

@@ -0,0 +1,125 @@
+#!/usr/bin/env bash
+
+# This script runs e2e tests for A/B traffic shifting, Canary analysis and promotion
+# Prerequisites: Kubernetes Kind and Contour with GatewayAPI
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+source ${REPO_ROOT}/test/gatewayapi/test-utils.sh
+
+create_latency_metric_template
+create_error_rate_metric_template
+
+echo '>>> Deploy podinfo in ab-test namespace'
+kubectl create ns ab-test
+kubectl apply -f ${REPO_ROOT}/test/workloads/secret.yaml -n ab-test
+kubectl apply -f ${REPO_ROOT}/test/workloads/deployment.yaml -n ab-test
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: ab-test
+spec:
+  progressDeadlineSeconds: 60
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  service:
+    port: 9898
+    portName: http
+    hosts:
+     - localproject.contour.io
+    gatewayRefs:
+      - name: contour
+        namespace: projectcontour
+  analysis:
+    interval: 15s
+    threshold: 10
+    iterations: 10
+    match:
+    - headers:
+        x-canary:
+          exact: "insider"
+    metrics:
+    # - name: request-success-rate
+      # thresholdRange:
+        # min: 99
+      # interval: 1m
+    # - name: request-duration
+      # threshold: 500
+      # interval: 30s
+
+    - name: error-rate
+      templateRef:
+        name: error-rate
+        namespace: flagger-system
+      thresholdRange:
+        max: 1
+      interval: 1m
+    - name: latency
+      templateRef:
+        name: latency
+        namespace: flagger-system
+      thresholdRange:
+        max: 0.5
+      interval: 30s
+    webhooks:
+      - name: load-test
+        type: rollout
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          cmd: "hey -z 2m -q 10 -c 2 -host localproject.contour.io -H 'X-Canary: insider' http://envoy.projectcontour/"
+          logCmdOutput: "true"
+EOF
+
+check_primary "ab-test"
+
+display_httproute "ab-test"
+
+echo '>>> Triggering A/B testing'
+kubectl -n ab-test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
+
+echo '>>> Waiting for A/B testing promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n ab-test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
+    sleep 10
+    kubectl -n flagger-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n ab-test describe deployment/podinfo
+        kubectl -n ab-test describe deployment/podinfo-primary
+        kubectl -n flagger-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+display_httproute "ab-test"
+
+echo '>>> Waiting for A/B finalization'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n ab-test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n flagger-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ A/B testing promotion test passed'
+
+kubectl delete -n ab-test canary podinfo

test/gatewayapi/test-bg.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+
+# This script runs e2e tests for Blue/Green traffic shifting, Canary analysis and promotion
+# Prerequisites: Kubernetes Kind and Contour with GatewayAPI
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+source ${REPO_ROOT}/test/gatewayapi/test-utils.sh
+
+echo '>>> Deploy podinfo in bg-test namespace'
+kubectl create ns bg-test
+kubectl apply -f ${REPO_ROOT}/test/workloads/secret.yaml -n bg-test
+kubectl apply -f ${REPO_ROOT}/test/workloads/deployment.yaml -n bg-test
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: bg-test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+    portName: http
+    hosts:
+     - localproject.contour.io
+    gatewayRefs:
+      - name: contour
+        namespace: projectcontour
+  analysis:
+    interval: 10s
+    threshold: 5
+    iterations: 5
+    metrics:
+    - name: request-success-rate
+      thresholdRange:
+        min: 99
+      interval: 1m
+    - name: request-duration
+      threshold: 500
+      interval: 30s
+    webhooks:
+      - name: smoke-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 15s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'anon' http://podinfo-canary.bg-test:9898/token | grep token"
+      - name: load-test
+        type: rollout
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          cmd: "hey -z 2m -q 10 -c 2 http://podinfo-canary.bg-test:9898"
+          logCmdOutput: "true"
+EOF
+ 
+check_primary "bg-test"
+
+display_httproute "bg-test"
+
+echo '>>> Triggering B/G deployment'
+kubectl -n bg-test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
+
+echo '>>> Waiting for B/G promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n bg-test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
+    sleep 10
+    kubectl -n flagger-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n bg-test describe deployment/podinfo
+        kubectl -n bg-test describe deployment/podinfo-primary
+        kubectl -n flagger-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+display_httproute "bg-test"
+
+echo '>>> Waiting for B/G finalization'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n bg-test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n flagger-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ B/G promotion test passed'
+
+kubectl delete -n bg-test canary podinfo

test/gatewayapi/test-canary.sh

@@ -0,0 +1,180 @@
+#!/usr/bin/env bash
+
+# This script runs e2e tests for progressive traffic shifting, Canary analysis and promotion
+# Prerequisites: Kubernetes Kind and Contour with GatewayAPI
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+source ${REPO_ROOT}/test/gatewayapi/test-utils.sh
+
+create_latency_metric_template
+create_error_rate_metric_template
+
+echo '>>> Installing Canary'
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+    portName: http
+    hosts:
+     - localproject.contour.io
+    gatewayRefs:
+      - name: contour
+        namespace: projectcontour
+  analysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 50
+    stepWeight: 10
+    metrics:
+      - name: error-rate
+        templateRef:
+          name: error-rate
+          namespace: flagger-system
+        thresholdRange:
+          max: 1
+        interval: 1m
+      - name: latency
+        templateRef:
+          name: latency
+          namespace: flagger-system
+        thresholdRange:
+          max: 0.5
+        interval: 30s
+    webhooks:
+      - name: load-test
+        type: rollout
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          cmd: "hey -z 2m -q 10 -c 2 -host localproject.contour.io http://envoy.projectcontour/"
+          logCmdOutput: "true"
+EOF
+
+check_primary "test"
+
+display_httproute "test"
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
+
+echo '>>> Waiting for canary promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
+    sleep 10
+    kubectl -n flagger-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n flagger-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+display_httproute "test"
+
+echo '>>> Waiting for canary finalization'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n flagger-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary promotion test passed'
+
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 30
+  service:
+    port: 9898
+    targetPort: 9898
+    portName: http
+    hosts:
+     - localproject.contour.io
+    gatewayRefs:
+      - name: contour
+        namespace: projectcontour
+  analysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 50
+    stepWeight: 10
+    metrics:
+    - name: error-rate
+      templateRef:
+        name: error-rate
+        namespace: flagger-system
+      thresholdRange:
+        max: 1
+      interval: 30s
+    - name: latency
+      templateRef:
+        name: latency
+        namespace: flagger-system
+      thresholdRange:
+        max: 0.5
+      interval: 30s
+    webhooks:
+      - name: load-test
+        type: rollout
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          cmd: "hey -z 2m -q 10 -c 2 -host localproject.contour.io http://envoy.projectcontour/status/500"
+          logCmdOutput: "true"
+EOF
+
+echo '>>> Triggering canary deployment rollback test'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2
+
+echo '>>> Waiting for canary rollback'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Failed' && ok=true || ok=false
+    sleep 10
+    kubectl -n flagger-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n flagger-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary rollback test passed'
+
+kubectl delete -n test canary podinfo

test/gatewayapi/test-utils.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+check_primary() {
+    echo '>>> Waiting for primary to be ready'
+    namespace=$1
+    retries=50
+    count=0
+    ok=false
+    until ${ok}; do
+        kubectl get -n ${namespace} canary/podinfo | grep 'Initialized' && ok=true || ok=false
+        sleep 5
+        count=$(($count + 1))
+        if [[ ${count} -eq ${retries} ]]; then
+            kubectl -n flagger-system logs deployment/flagger
+            echo "No more retries left"
+            exit 1
+        fi
+    done
+
+    echo '✔ Canary initialization test passed'
+
+    passed=$(kubectl -n $namespace get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; })
+    if [ -z "$passed" ]; then
+      echo -e '\u2716 podinfo selector test failed'
+      exit 1
+    fi
+
+    echo '✔ Canary service custom metadata test passed'
+}
+
+display_httproute() {
+    namespace=$1
+    if ! kubectl -n ${namespace} get httproute podinfo -oyaml; then
+        echo "Could not find HTTPRoute podinfo in ${namespace} namespace"
+        exit 1
+    fi
+}
+
+create_latency_metric_template() {
+    if ! kubectl -n flagger-system get metrictemplates latency; then
+        echo '>>> Create latency metric template'
+        cat <<EOF | kubectl apply -f -
+        apiVersion: flagger.app/v1beta1
+        kind: MetricTemplate
+        metadata:
+          name: latency
+          namespace: flagger-system
+        spec:
+          provider:
+            type: prometheus
+            address: http://flagger-prometheus:9090
+          query: |
+            histogram_quantile(0.99,
+              sum(
+                rate(
+                  envoy_cluster_upstream_rq_time_bucket{
+                    envoy_cluster_name=~"{{ namespace }}_{{ target }}-canary_[0-9a-zA-Z-]+",
+                  }[{{ interval }}]
+                )
+              ) by (le)
+            )/1000
+EOF
+    fi
+}
+
+create_error_rate_metric_template() {
+    if ! kubectl -n flagger-system get metrictemplates error-rate; then
+        echo '>>> Create latency metric template'
+        cat <<EOF | kubectl apply -f -
+        apiVersion: flagger.app/v1beta1
+        kind: MetricTemplate
+        metadata:
+          name: error-rate
+          namespace: flagger-system
+        spec:
+          provider:
+            type: prometheus
+            address: http://flagger-prometheus:9090
+          query: |
+            100 - sum(
+              rate(
+                envoy_cluster_upstream_rq{
+                  envoy_cluster_name=~"{{ namespace }}_{{ target }}-canary_[0-9a-zA-Z-]+",
+                  envoy_response_code!~"5.*"
+                }[{{ interval }}]
+              )
+            )
+            /
+            sum(
+              rate(
+                envoy_cluster_upstream_rq{
+                  envoy_cluster_name=~"{{ namespace }}_{{ target }}-canary_[0-9a-zA-Z-]+",
+                }[{{ interval }}]
+              )
+            )
+            * 100
+EOF
+    fi
+}

test/workloads/daemonset.yaml

@@ -2,7 +2,6 @@ apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: podinfo-ds
-  namespace: test
   labels:
     app.kubernetes.io/name: podinfo-ds
 spec:

test/workloads/deployment.yaml

@@ -2,7 +2,6 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: podinfo
-  namespace: test
   labels:
     app.kubernetes.io/name: podinfo
     app.kubernetes.io/managed-by: test-label-prefix

test/workloads/init.sh

@@ -20,6 +20,6 @@ kubectl apply -k ${REPO_ROOT}/kustomize/tester
 kubectl -n test rollout status deployment/flagger-loadtester
 
 echo '>>> Deploy podinfo'
-kubectl apply -f ${REPO_ROOT}/test/workloads/secret.yaml
-kubectl apply -f ${REPO_ROOT}/test/workloads/deployment.yaml
-kubectl apply -f ${REPO_ROOT}/test/workloads/daemonset.yaml
+kubectl apply -f ${REPO_ROOT}/test/workloads/secret.yaml -n test
+kubectl apply -f ${REPO_ROOT}/test/workloads/deployment.yaml -n test
+kubectl apply -f ${REPO_ROOT}/test/workloads/daemonset.yaml -n test

test/workloads/secret.yaml

@@ -2,6 +2,5 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: podinfo-secret
-  namespace: test
 stringData:
   value: s3cr3t