From 11375b68904b3b6f76733662bf82d3e039c2f29c Mon Sep 17 00:00:00 2001 From: Johnson Shi Date: Wed, 11 Aug 2021 06:00:47 -0700 Subject: [PATCH] Support OSM progressive traffic shifting Signed-off-by: Johnson Shi --- .github/workflows/e2e.yaml | 1 + artifacts/examples/osm-canary-steps.yaml | 42 ++++ artifacts/examples/osm-canary.yaml | 43 ++++ charts/flagger/Chart.yaml | 2 + charts/flagger/values.yaml | 2 +- charts/loadtester/Chart.yaml | 2 + charts/loadtester/templates/deployment.yaml | 1 + charts/podinfo/templates/tests/jwt.yaml | 1 + cmd/flagger/main.go | 2 +- kustomize/osm/kustomization.yaml | 5 + kustomize/osm/patch.yaml | 27 +++ kustomize/tester/deployment.yaml | 1 + pkg/apis/flagger/v1beta1/provider.go | 1 + pkg/metrics/observers/factory.go | 4 + pkg/metrics/observers/osm.go | 96 +++++++++ pkg/metrics/observers/osm_test.go | 100 +++++++++ pkg/router/factory.go | 8 + test/osm/init-pre-test.sh | 12 ++ test/osm/install.sh | 30 +++ test/osm/run.sh | 16 ++ test/osm/test-canary.sh | 222 ++++++++++++++++++++ test/osm/test-steps.sh | 104 +++++++++ 22 files changed, 720 insertions(+), 2 deletions(-) create mode 100644 artifacts/examples/osm-canary-steps.yaml create mode 100644 artifacts/examples/osm-canary.yaml create mode 100644 kustomize/osm/kustomization.yaml create mode 100644 kustomize/osm/patch.yaml create mode 100644 pkg/metrics/observers/osm.go create mode 100644 pkg/metrics/observers/osm_test.go create mode 100755 test/osm/init-pre-test.sh create mode 100755 test/osm/install.sh create mode 100755 test/osm/run.sh create mode 100755 test/osm/test-canary.sh create mode 100755 test/osm/test-steps.sh diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 4b3450e3..da3560ea 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -22,6 +22,7 @@ jobs: - traefik - gloo - skipper + - osm - kubernetes steps: - name: Checkout diff --git a/artifacts/examples/osm-canary-steps.yaml b/artifacts/examples/osm-canary-steps.yaml new file mode 100644 index 00000000..91bccc55 --- /dev/null +++ b/artifacts/examples/osm-canary-steps.yaml @@ -0,0 +1,42 @@ +apiVersion: flagger.app/v1beta1 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + provider: osm + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + progressDeadlineSeconds: 600 + service: + port: 9898 + targetPort: 9898 + analysis: + interval: 15s + threshold: 10 + stepWeights: [5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55] + metrics: + - name: request-success-rate + thresholdRange: + min: 99 + interval: 1m + - name: request-duration + thresholdRange: + max: 500 + interval: 30s + webhooks: + - name: acceptance-test + type: pre-rollout + url: http://flagger-loadtester.test/ + timeout: 15s + metadata: + type: bash + cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token" + - name: load-test + type: rollout + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898/" diff --git a/artifacts/examples/osm-canary.yaml b/artifacts/examples/osm-canary.yaml new file mode 100644 index 00000000..78208d8e --- /dev/null +++ b/artifacts/examples/osm-canary.yaml @@ -0,0 +1,43 @@ +apiVersion: flagger.app/v1beta1 +kind: Canary +metadata: + name: podinfo + namespace: test +spec: + provider: osm + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + progressDeadlineSeconds: 600 + service: + port: 9898 + targetPort: 9898 + analysis: + interval: 15s + threshold: 10 + maxWeight: 50 + stepWeight: 5 + metrics: + - name: request-success-rate + thresholdRange: + min: 99 + interval: 1m + - name: request-duration + thresholdRange: + max: 500 + interval: 30s + webhooks: + - name: acceptance-test + type: pre-rollout + url: http://flagger-loadtester.test/ + timeout: 15s + metadata: + type: bash + cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token" + - name: load-test + type: rollout + url: http://flagger-loadtester.test/ + timeout: 5s + metadata: + cmd: "hey -z 1m -q 10 -c 2 http://podinfo-canary.test:9898/" diff --git a/charts/flagger/Chart.yaml b/charts/flagger/Chart.yaml index 9cf431e3..6e9324fb 100644 --- a/charts/flagger/Chart.yaml +++ b/charts/flagger/Chart.yaml @@ -22,5 +22,7 @@ keywords: - contour - nginx - traefik + - osm + - smi - gitops - canary diff --git a/charts/flagger/values.yaml b/charts/flagger/values.yaml index 7fdd802e..96e6ab52 100644 --- a/charts/flagger/values.yaml +++ b/charts/flagger/values.yaml @@ -19,7 +19,7 @@ podPriorityClassName: "" metricsServer: "http://prometheus:9090" -# accepted values are kubernetes, istio, linkerd, appmesh, contour, nginx, gloo, skipper, traefik +# accepted values are kubernetes, istio, linkerd, appmesh, contour, nginx, gloo, skipper, traefik, osm meshProvider: "" # single namespace restriction diff --git a/charts/loadtester/Chart.yaml b/charts/loadtester/Chart.yaml index e65d0b0a..c9b94039 100644 --- a/charts/loadtester/Chart.yaml +++ b/charts/loadtester/Chart.yaml @@ -19,5 +19,7 @@ keywords: - appmesh - linkerd - gloo + - osm + - smi - gitops - load testing diff --git a/charts/loadtester/templates/deployment.yaml b/charts/loadtester/templates/deployment.yaml index f0a1bf7d..9f785ed9 100644 --- a/charts/loadtester/templates/deployment.yaml +++ b/charts/loadtester/templates/deployment.yaml @@ -19,6 +19,7 @@ spec: app.kubernetes.io/name: {{ include "loadtester.name" . }} annotations: appmesh.k8s.aws/ports: "444" + openservicemesh.io/inbound-port-exclusion-list: "80, 8080" {{- if .Values.podAnnotations }} {{ toYaml .Values.podAnnotations | indent 8 }} {{- end }} diff --git a/charts/podinfo/templates/tests/jwt.yaml b/charts/podinfo/templates/tests/jwt.yaml index e9b08b4a..7a1e7bed 100644 --- a/charts/podinfo/templates/tests/jwt.yaml +++ b/charts/podinfo/templates/tests/jwt.yaml @@ -12,6 +12,7 @@ metadata: sidecar.istio.io/inject: "false" linkerd.io/inject: disabled appmesh.k8s.aws/sidecarInjectorWebhook: disabled + openservicemesh.io/sidecar-injection: disabled spec: containers: - name: tools diff --git a/cmd/flagger/main.go b/cmd/flagger/main.go index 1bab4ec2..e26121db 100644 --- a/cmd/flagger/main.go +++ b/cmd/flagger/main.go @@ -106,7 +106,7 @@ func init() { flag.BoolVar(&zapReplaceGlobals, "zap-replace-globals", false, "Whether to change the logging level of the global zap logger.") flag.StringVar(&zapEncoding, "zap-encoding", "json", "Zap logger encoding.") flag.StringVar(&namespace, "namespace", "", "Namespace that flagger would watch canary object.") - flag.StringVar(&meshProvider, "mesh-provider", "istio", "Service mesh provider, can be istio, linkerd, appmesh, contour, gloo, nginx, skipper or traefik.") + flag.StringVar(&meshProvider, "mesh-provider", "istio", "Service mesh provider, can be istio, linkerd, appmesh, contour, gloo, nginx, skipper, traefik or osm.") flag.StringVar(&selectorLabels, "selector-labels", "app,name,app.kubernetes.io/name", "List of pod labels that Flagger uses to create pod selectors.") flag.StringVar(&ingressAnnotationsPrefix, "ingress-annotations-prefix", "nginx.ingress.kubernetes.io", "Annotations prefix for NGINX ingresses.") flag.StringVar(&ingressClass, "ingress-class", "", "Ingress class used for annotating HTTPProxy objects.") diff --git a/kustomize/osm/kustomization.yaml b/kustomize/osm/kustomization.yaml new file mode 100644 index 00000000..b69c136b --- /dev/null +++ b/kustomize/osm/kustomization.yaml @@ -0,0 +1,5 @@ +namespace: osm-system +bases: + - ../base/flagger/ +patchesStrategicMerge: + - patch.yaml diff --git a/kustomize/osm/patch.yaml b/kustomize/osm/patch.yaml new file mode 100644 index 00000000..f2cd0759 --- /dev/null +++ b/kustomize/osm/patch.yaml @@ -0,0 +1,27 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flagger +spec: + template: + spec: + containers: + - name: flagger + args: + - -log-level=info + - -include-label-prefix=app.kubernetes.io + - -mesh-provider=osm + - -metrics-server=http://osm-prometheus.osm-system.svc:7070 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: flagger +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flagger +subjects: + - kind: ServiceAccount + name: flagger + namespace: osm-system diff --git a/kustomize/tester/deployment.yaml b/kustomize/tester/deployment.yaml index 18e276e0..5dfd1e06 100644 --- a/kustomize/tester/deployment.yaml +++ b/kustomize/tester/deployment.yaml @@ -15,6 +15,7 @@ spec: annotations: prometheus.io/scrape: "true" prometheus.io/port: "8080" + openservicemesh.io/inbound-port-exclusion-list: "80, 8080" spec: containers: - name: loadtester diff --git a/pkg/apis/flagger/v1beta1/provider.go b/pkg/apis/flagger/v1beta1/provider.go index e81018ed..fb48d13f 100644 --- a/pkg/apis/flagger/v1beta1/provider.go +++ b/pkg/apis/flagger/v1beta1/provider.go @@ -11,4 +11,5 @@ const ( KubernetesProvider string = "kubernetes" SkipperProvider string = "skipper" TraefikProvider string = "traefik" + OsmProvider string = "osm" ) diff --git a/pkg/metrics/observers/factory.go b/pkg/metrics/observers/factory.go index 901d5857..9b54dbe5 100644 --- a/pkg/metrics/observers/factory.go +++ b/pkg/metrics/observers/factory.go @@ -80,6 +80,10 @@ func (factory Factory) Observer(provider string) Interface { return &TraefikObserver{ client: factory.Client, } + case provider == flaggerv1.OsmProvider: + return &OsmObserver{ + client: factory.Client, + } default: return &IstioObserver{ client: factory.Client, diff --git a/pkg/metrics/observers/osm.go b/pkg/metrics/observers/osm.go new file mode 100644 index 00000000..0c597508 --- /dev/null +++ b/pkg/metrics/observers/osm.go @@ -0,0 +1,96 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observers + +import ( + "fmt" + "time" + + flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" + "github.com/fluxcd/flagger/pkg/metrics/providers" +) + +var osmQueries = map[string]string{ + "request-success-rate": ` + sum( + rate( + osm_request_total{ + destination_namespace="{{ namespace }}", + destination_kind="Deployment", + destination_name="{{ target }}", + response_code!~"5.*" + }[{{ interval }}] + ) + ) + / + sum( + rate( + osm_request_total{ + destination_namespace="{{ namespace }}", + destination_kind="Deployment", + destination_name="{{ target }}" + }[{{ interval }}] + ) + ) + * 100`, + "request-duration": ` + histogram_quantile( + 0.99, + sum( + rate( + osm_request_duration_ms_bucket{ + destination_namespace="{{ namespace }}", + destination_kind="Deployment", + destination_name="{{ target }}" + }[{{ interval }}] + ) + ) by (le) + )`, +} + +type OsmObserver struct { + client providers.Interface +} + +func (ob *OsmObserver) GetRequestSuccessRate(model flaggerv1.MetricTemplateModel) (float64, error) { + query, err := RenderQuery(osmQueries["request-success-rate"], model) + if err != nil { + return 0, fmt.Errorf("rendering query failed: %w", err) + } + + value, err := ob.client.RunQuery(query) + if err != nil { + return 0, fmt.Errorf("running query failed: %w", err) + } + + return value, nil +} + +func (ob *OsmObserver) GetRequestDuration(model flaggerv1.MetricTemplateModel) (time.Duration, error) { + query, err := RenderQuery(osmQueries["request-duration"], model) + if err != nil { + return 0, fmt.Errorf("rendering query failed: %w", err) + } + + value, err := ob.client.RunQuery(query) + if err != nil { + return 0, fmt.Errorf("running query failed: %w", err) + } + + ms := time.Duration(int64(value)) * time.Millisecond + return ms, nil +} diff --git a/pkg/metrics/observers/osm_test.go b/pkg/metrics/observers/osm_test.go new file mode 100644 index 00000000..605390c3 --- /dev/null +++ b/pkg/metrics/observers/osm_test.go @@ -0,0 +1,100 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observers + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" + "github.com/fluxcd/flagger/pkg/metrics/providers" +) + +func TestOsmObserver_GetRequestSuccessRate(t *testing.T) { + expected := ` sum( rate( osm_request_total{ destination_namespace="default", destination_kind="Deployment", destination_name="podinfo", response_code!~"5.*" }[1m] ) ) / sum( rate( osm_request_total{ destination_namespace="default", destination_kind="Deployment", destination_name="podinfo" }[1m] ) ) * 100` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + promql := r.URL.Query()["query"][0] + assert.Equal(t, expected, promql) + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := providers.NewPrometheusProvider(flaggerv1.MetricTemplateProvider{ + Type: "prometheus", + Address: ts.URL, + SecretRef: nil, + }, nil) + require.NoError(t, err) + + observer := &OsmObserver{ + client: client, + } + + val, err := observer.GetRequestSuccessRate(flaggerv1.MetricTemplateModel{ + Name: "podinfo", + Namespace: "default", + Target: "podinfo", + Service: "podinfo", + Interval: "1m", + }) + require.NoError(t, err) + + assert.Equal(t, float64(100), val) +} + +func TestOsmObserver_GetRequestDuration(t *testing.T) { + expected := ` histogram_quantile( 0.99, sum( rate( osm_request_duration_ms_bucket{ destination_namespace="default", destination_kind="Deployment", destination_name="podinfo" }[1m] ) ) by (le) )` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + promql := r.URL.Query()["query"][0] + assert.Equal(t, expected, promql) + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := providers.NewPrometheusProvider(flaggerv1.MetricTemplateProvider{ + Type: "prometheus", + Address: ts.URL, + SecretRef: nil, + }, nil) + require.NoError(t, err) + + observer := &OsmObserver{ + client: client, + } + + val, err := observer.GetRequestDuration(flaggerv1.MetricTemplateModel{ + Name: "podinfo", + Namespace: "default", + Target: "podinfo", + Service: "podinfo", + Interval: "1m", + }) + require.NoError(t, err) + + assert.Equal(t, 100*time.Millisecond, val) +} diff --git a/pkg/router/factory.go b/pkg/router/factory.go index e8965bf5..5b7adc25 100644 --- a/pkg/router/factory.go +++ b/pkg/router/factory.go @@ -162,6 +162,14 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf logger: factory.logger, traefikClient: factory.meshClient, } + case provider == flaggerv1.OsmProvider: + return &Smiv1alpha2Router{ + logger: factory.logger, + flaggerClient: factory.flaggerClient, + kubeClient: factory.kubeClient, + smiClient: factory.meshClient, + targetMesh: flaggerv1.OsmProvider, + } case provider == flaggerv1.KubernetesProvider: return &NopRouter{} default: diff --git a/test/osm/init-pre-test.sh b/test/osm/init-pre-test.sh new file mode 100755 index 00000000..3dade4b4 --- /dev/null +++ b/test/osm/init-pre-test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +${REPO_ROOT}/bin/linux-amd64/osm namespace add test +${REPO_ROOT}/bin/linux-amd64/osm metrics enable --namespace test + +kubectl patch deployment flagger-loadtester -n test -p '{"spec": {"template": {"metadata": {"annotations": {"openservicemesh.io/inbound-port-exclusion-list": "80, 8080"}}}}}' --type=merge +kubectl -n test rollout restart deploy/flagger-loadtester +kubectl -n test rollout status deploy/flagger-loadtester diff --git a/test/osm/install.sh b/test/osm/install.sh new file mode 100755 index 00000000..78a1df55 --- /dev/null +++ b/test/osm/install.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -o errexit + +OSM_VER="v0.9.1" +REPO_ROOT=$(git rev-parse --show-toplevel) + +mkdir -p ${REPO_ROOT}/bin + +cd ${REPO_ROOT}/bin && curl -sL "https://github.com/openservicemesh/osm/releases/download/${OSM_VER}/osm-${OSM_VER}-linux-amd64.tar.gz" | tar xz +chmod +x ${REPO_ROOT}/bin/linux-amd64/osm + +echo ">>> Installing Open Service Mesh ${OSM_VER}" +${REPO_ROOT}/bin/linux-amd64/osm install \ +--set=OpenServiceMesh.deployPrometheus=true \ +--set=OpenServiceMesh.enablePermissiveTrafficPolicy=true \ +--set=OpenServiceMesh.osmController.resource.limits.cpu=300m \ +--set=OpenServiceMesh.osmController.resource.requests.cpu=300m \ +--set=OpenServiceMesh.prometheus.resources.limits.cpu=300m \ +--set=OpenServiceMesh.prometheus.resources.requests.cpu=300m \ +--set=OpenServiceMesh.injector.resource.limits.cpu=300m \ +--set=OpenServiceMesh.injector.resource.requests.cpu=300m + +${REPO_ROOT}/bin/linux-amd64/osm version + +echo '>>> Installing Flagger' +kubectl apply -k ${REPO_ROOT}/kustomize/osm + +kubectl -n osm-system set image deployment/flagger flagger=test/flagger:latest +kubectl -n osm-system rollout status deployment/flagger diff --git a/test/osm/run.sh b/test/osm/run.sh new file mode 100755 index 00000000..321fb67b --- /dev/null +++ b/test/osm/run.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +DIR="$(cd "$(dirname "$0")" && pwd)" + +"$DIR"/install.sh + +${REPO_ROOT}/test/workloads/init.sh +"$DIR"/init-pre-test.sh +"$DIR"/test-canary.sh + +${REPO_ROOT}/test/workloads/init.sh +"$DIR"/init-pre-test.sh +"$DIR"/test-steps.sh diff --git a/test/osm/test-canary.sh b/test/osm/test-canary.sh new file mode 100755 index 00000000..078df187 --- /dev/null +++ b/test/osm/test-canary.sh @@ -0,0 +1,222 @@ +#!/usr/bin/env bash + +# This script runs OSM e2e tests for Canary initialization, analysis and promotion + +set -o errexit + +echo '>>> Create latency metric template' +cat <>> Initialising canary' +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n osm-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +passed=$(kubectl -n test get svc/podinfo -o jsonpath='{.spec.selector.app}' 2>&1 | { grep podinfo-primary || true; }) +if [ -z "$passed" ]; then + echo -e '\u2716 podinfo selector test failed' + exit 1 +fi + +echo '✔ Canary service custom metadata test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n osm-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + echo ">>> kubectl -n osm-system logs deployment/flagger" + kubectl -n osm-system logs deployment/flagger + echo ">>> kubectl -n test logs pod/$(kubectl get pods -n test | grep flagger-loadtester | awk '{print $1}') loadtester" + kubectl -n test logs pod/$(kubectl get pods -n test | grep flagger-loadtester | awk '{print $1}') loadtester + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n osm-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +echo '>>> Initialising canary for canary rollback test' +cat <>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.2 + +echo '>>> Waiting for canary rollback' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Failed' && ok=true || ok=false + sleep 10 + kubectl -n osm-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n osm-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary rollback test passed' diff --git a/test/osm/test-steps.sh b/test/osm/test-steps.sh new file mode 100755 index 00000000..4b183ea6 --- /dev/null +++ b/test/osm/test-steps.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash + +# This script runs OSM e2e tests for Canary initialization, analysis and promotion + +set -o errexit + +echo '>>> Initialising canary' +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n osm-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test set image deployment/podinfo podinfod=stefanprodan/podinfo:3.1.1 + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test describe deployment/podinfo-primary | grep '3.1.1' && ok=true || ok=false + sleep 10 + kubectl -n osm-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n osm-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n osm-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed'