Merge pull request #955 from johnsonshi/support-osm-in-flagger

Support OSM progressive traffic shifting in Flagger

.github/workflows/e2e.yaml

@@ -22,6 +22,7 @@ jobs:
           - traefik
           - gloo
           - skipper
+          - osm
           - kubernetes
     steps:
       - name: Checkout

artifacts/examples/osm-canary-steps.yaml

@@ -0,0 +1,42 @@
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  provider: osm
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 600
+  service:
+    port: 9898
+    targetPort: 9898
+  analysis:
+    interval: 15s
+    threshold: 10
+    stepWeights: [5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55]
+    metrics:
+    - name: request-success-rate
+      thresholdRange:
+        min: 99
+      interval: 1m
+    - name: request-duration
+      thresholdRange:
+        max: 500
+      interval: 30s
+    webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 15s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token"
+      - name: load-test
+        type: rollout
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898/"

artifacts/examples/osm-canary.yaml

@@ -0,0 +1,43 @@
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  provider: osm
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 600
+  service:
+    port: 9898
+    targetPort: 9898
+  analysis:
+    interval: 15s
+    threshold: 10
+    maxWeight: 50
+    stepWeight: 5
+    metrics:
+    - name: request-success-rate
+      thresholdRange:
+        min: 99
+      interval: 1m
+    - name: request-duration
+      thresholdRange:
+        max: 500
+      interval: 30s
+    webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 15s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token"
+      - name: load-test
+        type: rollout
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898/"

charts/flagger/Chart.yaml

@@ -22,5 +22,7 @@ keywords:
   - contour
   - nginx
   - traefik
+  - osm
+  - smi
   - gitops
   - canary

charts/flagger/values.yaml

@@ -19,7 +19,7 @@ podPriorityClassName: ""
 
 metricsServer: "http://prometheus:9090"
 
-# accepted values are kubernetes, istio, linkerd, appmesh, contour, nginx, gloo, skipper, traefik
+# accepted values are kubernetes, istio, linkerd, appmesh, contour, nginx, gloo, skipper, traefik, osm
 meshProvider: ""
 
 # single namespace restriction

charts/loadtester/Chart.yaml

@@ -19,5 +19,7 @@ keywords:
   - appmesh
   - linkerd
   - gloo
+  - osm
+  - smi
   - gitops
   - load testing

charts/loadtester/templates/deployment.yaml

@@ -19,6 +19,7 @@ spec:
         app.kubernetes.io/name: {{ include "loadtester.name" . }}
       annotations:
         appmesh.k8s.aws/ports: "444"
+        openservicemesh.io/inbound-port-exclusion-list: "80, 8080"
         {{- if .Values.podAnnotations }}
 {{ toYaml .Values.podAnnotations | indent 8 }}
         {{- end }}

charts/podinfo/templates/tests/jwt.yaml

@@ -12,6 +12,7 @@ metadata:
     sidecar.istio.io/inject: "false"
     linkerd.io/inject: disabled
     appmesh.k8s.aws/sidecarInjectorWebhook: disabled
+    openservicemesh.io/sidecar-injection: disabled
 spec:
   containers:
     - name: tools

cmd/flagger/main.go

@@ -106,7 +106,7 @@ func init() {
 	flag.BoolVar(&zapReplaceGlobals, "zap-replace-globals", false, "Whether to change the logging level of the global zap logger.")
 	flag.StringVar(&zapEncoding, "zap-encoding", "json", "Zap logger encoding.")
 	flag.StringVar(&namespace, "namespace", "", "Namespace that flagger would watch canary object.")
-	flag.StringVar(&meshProvider, "mesh-provider", "istio", "Service mesh provider, can be istio, linkerd, appmesh, contour, gloo, nginx, skipper or traefik.")
+	flag.StringVar(&meshProvider, "mesh-provider", "istio", "Service mesh provider, can be istio, linkerd, appmesh, contour, gloo, nginx, skipper, traefik or osm.")
 	flag.StringVar(&selectorLabels, "selector-labels", "app,name,app.kubernetes.io/name", "List of pod labels that Flagger uses to create pod selectors.")
 	flag.StringVar(&ingressAnnotationsPrefix, "ingress-annotations-prefix", "nginx.ingress.kubernetes.io", "Annotations prefix for NGINX ingresses.")
 	flag.StringVar(&ingressClass, "ingress-class", "", "Ingress class used for annotating HTTPProxy objects.")

kustomize/osm/kustomization.yaml

@@ -0,0 +1,5 @@
+namespace: osm-system
+bases:
+  - ../base/flagger/
+patchesStrategicMerge:
+  - patch.yaml

kustomize/osm/patch.yaml

@@ -0,0 +1,27 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flagger
+spec:
+  template:
+    spec:
+      containers:
+        - name: flagger
+          args:
+            - -log-level=info
+            - -include-label-prefix=app.kubernetes.io
+            - -mesh-provider=osm
+            - -metrics-server=http://osm-prometheus.osm-system.svc:7070
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: flagger
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: flagger
+subjects:
+  - kind: ServiceAccount
+    name: flagger
+    namespace: osm-system

kustomize/tester/deployment.yaml

@@ -15,6 +15,7 @@ spec:
       annotations:
         prometheus.io/scrape: "true"
         prometheus.io/port: "8080"
+        openservicemesh.io/inbound-port-exclusion-list: "80, 8080"
     spec:
       containers:
         - name: loadtester

pkg/apis/flagger/v1beta1/provider.go

@@ -11,4 +11,5 @@ const (
 	KubernetesProvider string = "kubernetes"
 	SkipperProvider    string = "skipper"
 	TraefikProvider    string = "traefik"
+	OsmProvider        string = "osm"
 )

pkg/metrics/observers/factory.go

@@ -80,6 +80,10 @@ func (factory Factory) Observer(provider string) Interface {
 		return &TraefikObserver{
 			client: factory.Client,
 		}
+	case provider == flaggerv1.OsmProvider:
+		return &OsmObserver{
+			client: factory.Client,
+		}
 	default:
 		return &IstioObserver{
 			client: factory.Client,

pkg/metrics/observers/osm.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package observers
+
+import (
+	"fmt"
+	"time"
+
+	flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
+	"github.com/fluxcd/flagger/pkg/metrics/providers"
+)
+
+var osmQueries = map[string]string{
+	"request-success-rate": `
+    sum(
+        rate(
+            osm_request_total{
+				destination_namespace="{{ namespace }}",
+				destination_kind="Deployment",
+				destination_name="{{ target }}",
+				response_code!~"5.*"
+            }[{{ interval }}]
+        )
+    )
+    /
+    sum(
+        rate(
+            osm_request_total{
+				destination_namespace="{{ namespace }}",
+				destination_kind="Deployment",
+				destination_name="{{ target }}"
+            }[{{ interval }}]
+        )
+    )
+	* 100`,
+	"request-duration": `
+	histogram_quantile(
+		0.99,
+		sum(
+		  rate(
+			osm_request_duration_ms_bucket{
+				destination_namespace="{{ namespace }}",
+				destination_kind="Deployment",
+				destination_name="{{ target }}"
+			}[{{ interval }}]
+		  )
+		) by (le)
+	)`,
+}
+
+type OsmObserver struct {
+	client providers.Interface
+}
+
+func (ob *OsmObserver) GetRequestSuccessRate(model flaggerv1.MetricTemplateModel) (float64, error) {
+	query, err := RenderQuery(osmQueries["request-success-rate"], model)
+	if err != nil {
+		return 0, fmt.Errorf("rendering query failed: %w", err)
+	}
+
+	value, err := ob.client.RunQuery(query)
+	if err != nil {
+		return 0, fmt.Errorf("running query failed: %w", err)
+	}
+
+	return value, nil
+}
+
+func (ob *OsmObserver) GetRequestDuration(model flaggerv1.MetricTemplateModel) (time.Duration, error) {
+	query, err := RenderQuery(osmQueries["request-duration"], model)
+	if err != nil {
+		return 0, fmt.Errorf("rendering query failed: %w", err)
+	}
+
+	value, err := ob.client.RunQuery(query)
+	if err != nil {
+		return 0, fmt.Errorf("running query failed: %w", err)
+	}
+
+	ms := time.Duration(int64(value)) * time.Millisecond
+	return ms, nil
+}

pkg/metrics/observers/osm_test.go

@@ -0,0 +1,100 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package observers
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
+	"github.com/fluxcd/flagger/pkg/metrics/providers"
+)
+
+func TestOsmObserver_GetRequestSuccessRate(t *testing.T) {
+	expected := ` sum( rate( osm_request_total{ destination_namespace="default", destination_kind="Deployment", destination_name="podinfo", response_code!~"5.*" }[1m] ) ) / sum( rate( osm_request_total{ destination_namespace="default", destination_kind="Deployment", destination_name="podinfo" }[1m] ) ) * 100`
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		promql := r.URL.Query()["query"][0]
+		assert.Equal(t, expected, promql)
+
+		json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}`
+		w.Write([]byte(json))
+	}))
+	defer ts.Close()
+
+	client, err := providers.NewPrometheusProvider(flaggerv1.MetricTemplateProvider{
+		Type:      "prometheus",
+		Address:   ts.URL,
+		SecretRef: nil,
+	}, nil)
+	require.NoError(t, err)
+
+	observer := &OsmObserver{
+		client: client,
+	}
+
+	val, err := observer.GetRequestSuccessRate(flaggerv1.MetricTemplateModel{
+		Name:      "podinfo",
+		Namespace: "default",
+		Target:    "podinfo",
+		Service:   "podinfo",
+		Interval:  "1m",
+	})
+	require.NoError(t, err)
+
+	assert.Equal(t, float64(100), val)
+}
+
+func TestOsmObserver_GetRequestDuration(t *testing.T) {
+	expected := ` histogram_quantile( 0.99, sum( rate( osm_request_duration_ms_bucket{ destination_namespace="default", destination_kind="Deployment", destination_name="podinfo" }[1m] ) ) by (le) )`
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		promql := r.URL.Query()["query"][0]
+		assert.Equal(t, expected, promql)
+
+		json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}`
+		w.Write([]byte(json))
+	}))
+	defer ts.Close()
+
+	client, err := providers.NewPrometheusProvider(flaggerv1.MetricTemplateProvider{
+		Type:      "prometheus",
+		Address:   ts.URL,
+		SecretRef: nil,
+	}, nil)
+	require.NoError(t, err)
+
+	observer := &OsmObserver{
+		client: client,
+	}
+
+	val, err := observer.GetRequestDuration(flaggerv1.MetricTemplateModel{
+		Name:      "podinfo",
+		Namespace: "default",
+		Target:    "podinfo",
+		Service:   "podinfo",
+		Interval:  "1m",
+	})
+	require.NoError(t, err)
+
+	assert.Equal(t, 100*time.Millisecond, val)
+}

pkg/router/factory.go

@@ -162,6 +162,14 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf
 			logger:        factory.logger,
 			traefikClient: factory.meshClient,
 		}
+	case provider == flaggerv1.OsmProvider:
+		return &Smiv1alpha2Router{
+			logger:        factory.logger,
+			flaggerClient: factory.flaggerClient,
+			kubeClient:    factory.kubeClient,
+			smiClient:     factory.meshClient,
+			targetMesh:    flaggerv1.OsmProvider,
+		}
 	case provider == flaggerv1.KubernetesProvider:
 		return &NopRouter{}
 	default:

test/osm/init-pre-test.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+${REPO_ROOT}/bin/linux-amd64/osm namespace add test
+${REPO_ROOT}/bin/linux-amd64/osm metrics enable --namespace test
+
+kubectl patch deployment flagger-loadtester -n test -p '{"spec": {"template": {"metadata": {"annotations": {"openservicemesh.io/inbound-port-exclusion-list": "80, 8080"}}}}}' --type=merge
+kubectl -n test rollout restart deploy/flagger-loadtester
+kubectl -n test rollout status deploy/flagger-loadtester

test/osm/install.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+OSM_VER="v0.9.1"
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+mkdir -p ${REPO_ROOT}/bin
+
+cd ${REPO_ROOT}/bin && curl -sL "https://github.com/openservicemesh/osm/releases/download/${OSM_VER}/osm-${OSM_VER}-linux-amd64.tar.gz" | tar xz
+chmod +x ${REPO_ROOT}/bin/linux-amd64/osm
+
+echo ">>> Installing Open Service Mesh ${OSM_VER}"
+${REPO_ROOT}/bin/linux-amd64/osm install \
+--set=OpenServiceMesh.deployPrometheus=true \
+--set=OpenServiceMesh.enablePermissiveTrafficPolicy=true \
+--set=OpenServiceMesh.osmController.resource.limits.cpu=300m \
+--set=OpenServiceMesh.osmController.resource.requests.cpu=300m \
+--set=OpenServiceMesh.prometheus.resources.limits.cpu=300m \
+--set=OpenServiceMesh.prometheus.resources.requests.cpu=300m \
+--set=OpenServiceMesh.injector.resource.limits.cpu=300m \
+--set=OpenServiceMesh.injector.resource.requests.cpu=300m
+
+${REPO_ROOT}/bin/linux-amd64/osm version
+
+echo '>>> Installing Flagger'
+kubectl apply -k ${REPO_ROOT}/kustomize/osm
+
+kubectl -n osm-system set image deployment/flagger flagger=test/flagger:latest
+kubectl -n osm-system rollout status deployment/flagger

test/osm/run.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+DIR="$(cd "$(dirname "$0")" && pwd)"
+
+"$DIR"/install.sh
+
+${REPO_ROOT}/test/workloads/init.sh
+"$DIR"/init-pre-test.sh
+"$DIR"/test-canary.sh
+
+${REPO_ROOT}/test/workloads/init.sh
+"$DIR"/init-pre-test.sh
+"$DIR"/test-steps.sh

test/osm/test-canary.sh

@@ -0,0 +1,222 @@
+#!/usr/bin/env bash
+
+# This script runs OSM e2e tests for Canary initialization, analysis and promotion
+
+set -o errexit
+
+echo '>>> Create latency metric template'
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: MetricTemplate
+metadata:
+  name: latency
+  namespace: osm-system
+spec:
+  provider:
+    type: prometheus
+    address: http://osm-prometheus.osm-system.svc:7070
+  query: |
+    histogram_quantile(
+        0.99,
+        sum(
+            rate(
+                osm_request_duration_ms_bucket{
+                    destination_namespace="{{ namespace }}",
+                    destination_kind="Deployment",
+                    destination_name=~"{{ target }}"
+                }[{{ interval }}]
+            )
+        ) by (le)
+    )
+EOF
+
+echo '>>> Initialising canary'
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  provider: osm
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+    targetPort: 9898
+  analysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 50
+    stepWeight: 10
+    metrics:
+    - name: request-success-rate
+      thresholdRange:
+        min: 99
+      interval: 1m
+    - name: request-duration
+      thresholdRange:
+        max: 500
+      interval: 30s
+    - name: latency
+      templateRef:
+        name: latency
+        namespace: osm-system
+      threshold: 300
+      interval: 1m
+    webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 15s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token"
+      - name: load-test
+        type: rollout
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          cmd: "hey -z 10m -q 10 -c 2 http://podinfo-canary.test:9898/"
+EOF
+
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n osm-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary initialization test passed'
+
+passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; })
+if [ -z "$passed" ]; then
+  echo -e '\u2716 podinfo selector test failed'
+  exit 1
+fi
+
+echo '✔ Canary service custom metadata test passed'
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
+
+echo '>>> Waiting for canary promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
+    sleep 10
+    kubectl -n osm-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        echo ">>> kubectl -n osm-system logs deployment/flagger"
+        kubectl -n osm-system logs deployment/flagger
+        echo ">>> kubectl -n test logs pod/$(kubectl get pods -n test | grep flagger-loadtester | awk '{print $1}') loadtester"
+        kubectl -n test logs pod/$(kubectl get pods -n test | grep flagger-loadtester | awk '{print $1}') loadtester
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '>>> Waiting for canary finalization'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n osm-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary promotion test passed'
+
+echo '>>> Initialising canary for canary rollback test'
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  provider: osm
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 60
+  service:
+    port: 9898
+    targetPort: 9898
+  analysis:
+    interval: 15s
+    threshold: 15
+    maxWeight: 50
+    stepWeight: 10
+    metrics:
+    - name: request-success-rate
+      thresholdRange:
+        min: 99
+      interval: 1m
+    - name: request-duration
+      thresholdRange:
+        max: 500
+      interval: 30s
+    - name: latency
+      templateRef:
+        name: latency
+        namespace: osm-system
+      threshold: 300
+      interval: 1m
+    webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 15s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token"
+      - name: load-test
+        type: rollout
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          cmd: "hey -z 10m -q 10 -c 2 http://podinfo-canary.test:9898/status/500"
+EOF
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2
+
+echo '>>> Waiting for canary rollback'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Failed' && ok=true || ok=false
+    sleep 10
+    kubectl -n osm-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n osm-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary rollback test passed'

test/osm/test-steps.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+
+# This script runs OSM e2e tests for Canary initialization, analysis and promotion
+
+set -o errexit
+
+echo '>>> Initialising canary'
+cat <<EOF | kubectl apply -f -
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  provider: osm
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  progressDeadlineSeconds: 600
+  service:
+    port: 9898
+    targetPort: 9898
+  analysis:
+    interval: 15s
+    threshold: 15
+    stepWeights: [10, 20, 30, 40, 50, 60]
+    metrics:
+    - name: request-success-rate
+      thresholdRange:
+        min: 99
+      interval: 1m
+    - name: request-duration
+      thresholdRange:
+        max: 500
+      interval: 30s
+    webhooks:
+      - name: acceptance-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 15s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token"
+      - name: load-test
+        type: rollout
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          cmd: "hey -z 10m -q 10 -c 2 http://podinfo-canary.test:9898/"
+EOF
+
+echo '>>> Waiting for primary to be ready'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n osm-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary initialization test passed'
+
+echo '>>> Triggering canary deployment'
+kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1
+
+echo '>>> Waiting for canary promotion'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false
+    sleep 10
+    kubectl -n osm-system logs deployment/flagger --tail 1
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n osm-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '>>> Waiting for canary finalization'
+retries=50
+count=0
+ok=false
+until ${ok}; do
+    kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        kubectl -n osm-system logs deployment/flagger
+        echo "No more retries left"
+        exit 1
+    fi
+done
+
+echo '✔ Canary promotion test passed'