From 4da6c1b6e4fe2d83e77b710659543c5afe277830 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 3 Oct 2019 11:43:47 +0300 Subject: [PATCH 1/3] Create canary virtual service during App Mesh reconciliation Allows the canary pods to be accessed from inside the mesh during the canary analysis for conformance and load testing --- pkg/router/appmesh.go | 17 ++++++++++++----- pkg/router/appmesh_test.go | 19 ++++++++++++++++++- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/pkg/router/appmesh.go b/pkg/router/appmesh.go index c8b36ae0..6d25f117 100644 --- a/pkg/router/appmesh.go +++ b/pkg/router/appmesh.go @@ -56,9 +56,16 @@ func (ar *AppMeshRouter) Reconcile(canary *flaggerv1.Canary) error { return err } - // sync virtual service e.g. app.namespace + // sync main virtual service // DNS app.namespace - err = ar.reconcileVirtualService(canary, targetHost) + err = ar.reconcileVirtualService(canary, targetHost, 0) + if err != nil { + return err + } + + // sync canary virtual service + // DNS app-canary.namespace + err = ar.reconcileVirtualService(canary, fmt.Sprintf("%s.%s", canaryName, canary.Namespace), 100) if err != nil { return err } @@ -148,7 +155,7 @@ func (ar *AppMeshRouter) reconcileVirtualNode(canary *flaggerv1.Canary, name str } // reconcileVirtualService creates or updates a virtual service -func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name string) error { +func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name string, canaryWeight int64) error { targetName := canary.Spec.TargetRef.Name canaryVirtualNode := fmt.Sprintf("%s-canary", targetName) primaryVirtualNode := fmt.Sprintf("%s-primary", targetName) @@ -185,11 +192,11 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name WeightedTargets: []AppmeshV1beta1.WeightedTarget{ { VirtualNodeName: canaryVirtualNode, - Weight: 0, + Weight: canaryWeight, }, { VirtualNodeName: primaryVirtualNode, - Weight: 100, + Weight: 100 - canaryWeight, }, }, }, diff --git a/pkg/router/appmesh_test.go b/pkg/router/appmesh_test.go index 0fbed858..3e261d95 100644 --- a/pkg/router/appmesh_test.go +++ b/pkg/router/appmesh_test.go @@ -37,6 +37,23 @@ func TestAppmeshRouter_Reconcile(t *testing.T) { t.Errorf("Got routes %v wanted %v", targetsCount, 2) } + // check canary virtual service + vsCanaryName := fmt.Sprintf("%s-canary.%s", mocks.appmeshCanary.Spec.TargetRef.Name, mocks.appmeshCanary.Namespace) + vsCanary, err := router.appmeshClient.AppmeshV1beta1().VirtualServices("default").Get(vsCanaryName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + // check if the canary virtual service routes all traffic to the canary virtual node + target := vsCanary.Spec.Routes[0].Http.Action.WeightedTargets[0] + canaryVirtualNodeName := fmt.Sprintf("%s-canary", mocks.appmeshCanary.Spec.TargetRef.Name) + if target.VirtualNodeName != canaryVirtualNodeName { + t.Errorf("Got VirtualNodeName %v wanted %v", target.VirtualNodeName, canaryVirtualNodeName) + } + if target.Weight != 100 { + t.Errorf("Got weight %v wanted %v", target.Weight, 100) + } + // check virtual node vnName := mocks.appmeshCanary.Spec.TargetRef.Name vn, err := router.appmeshClient.AppmeshV1beta1().VirtualNodes("default").Get(vnName, metav1.GetOptions{}) @@ -103,7 +120,7 @@ func TestAppmeshRouter_Reconcile(t *testing.T) { weight := vs.Spec.Routes[0].Http.Action.WeightedTargets[0].Weight if weight != 50 { - t.Errorf("Got weight %v wanted %v", weight, 502) + t.Errorf("Got weight %v wanted %v", weight, 50) } // test URI update From 74c1c2f1efd5441ef5528df3891329e41bb3ca66 Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 3 Oct 2019 11:52:56 +0300 Subject: [PATCH 2/3] Add App Mesh request duration metric check to docs Fix: #143 depend on App Mesh Envoy >1.11 --- artifacts/appmesh/canary.yaml | 7 +++- .../usage/appmesh-progressive-delivery.md | 40 ++++++++++++------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/artifacts/appmesh/canary.yaml b/artifacts/appmesh/canary.yaml index d16ea07d..c9c38cd4 100644 --- a/artifacts/appmesh/canary.yaml +++ b/artifacts/appmesh/canary.yaml @@ -41,7 +41,12 @@ spec: # percentage (0-100) threshold: 99 interval: 1m - # external checks (optional) + - name: request-duration + # maximum req duration P99 + # milliseconds + threshold: 500 + interval: 30s + # testing (optional) webhooks: - name: load-test url: http://flagger-loadtester.test/ diff --git a/docs/gitbook/usage/appmesh-progressive-delivery.md b/docs/gitbook/usage/appmesh-progressive-delivery.md index 92263b21..ab7511dd 100644 --- a/docs/gitbook/usage/appmesh-progressive-delivery.md +++ b/docs/gitbook/usage/appmesh-progressive-delivery.md @@ -92,6 +92,11 @@ spec: # percentage (0-100) threshold: 99 interval: 1m + - name: request-duration + # maximum req duration P99 + # milliseconds + threshold: 500 + interval: 30s # external checks (optional) webhooks: - name: load-test @@ -247,7 +252,7 @@ podinfod=stefanprodan/podinfo:2.0.2 Exec into the load tester pod with: ```bash -kubectl -n test exec -it flagger-loadtester-xx-xx sh +kubectl -n test exec -it flagger-loadtester-xx-xx bash ``` Generate HTTP 500 errors: @@ -256,6 +261,12 @@ Generate HTTP 500 errors: hey -z 1m -c 5 -q 5 http://podinfo.test:9898/status/500 ``` +Generate latency: + +```bash +watch -n 1 curl http://podinfo-canary.test:9898/delay/1 +``` + When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, the canary is scaled to zero and the rollout is marked as failed. @@ -264,22 +275,21 @@ kubectl -n test describe canary/podinfo Status: Canary Weight: 0 - Failed Checks: 10 + Failed Checks: 5 Phase: Failed Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal Synced 3m flagger Starting canary deployment for podinfo.test - Normal Synced 3m flagger Advance podinfo.test canary weight 5 - Normal Synced 3m flagger Advance podinfo.test canary weight 10 - Normal Synced 3m flagger Advance podinfo.test canary weight 15 - Normal Synced 3m flagger Halt podinfo.test advancement success rate 69.17% < 99% - Normal Synced 2m flagger Halt podinfo.test advancement success rate 61.39% < 99% - Normal Synced 2m flagger Halt podinfo.test advancement success rate 55.06% < 99% - Normal Synced 2m flagger Halt podinfo.test advancement success rate 47.00% < 99% - Normal Synced 2m flagger (combined from similar events): Halt podinfo.test advancement success rate 38.08% < 99% - Warning Synced 1m flagger Rolling back podinfo.test failed checks threshold reached 10 - Warning Synced 1m flagger Canary failed! Scaling down podinfo.test + Starting canary analysis for podinfo.test + Pre-rollout check acceptance-test passed + Advance podinfo.test canary weight 5 + Advance podinfo.test canary weight 10 + Advance podinfo.test canary weight 15 + Halt podinfo.test advancement success rate 69.17% < 99% + Halt podinfo.test advancement success rate 61.39% < 99% + Halt podinfo.test advancement success rate 55.06% < 99% + Halt podinfo.test advancement request duration 1.20s > 0.5s + Halt podinfo.test advancement request duration 1.45s > 0.5s + Rolling back podinfo.test failed checks threshold reached 5 + Canary failed! Scaling down podinfo.test ``` If you’ve enabled the Slack notifications, you’ll receive a message if the progress deadline is exceeded, From fb2961715d82e91f9d61d3cfae431cef510f466f Mon Sep 17 00:00:00 2001 From: stefanprodan Date: Thu, 3 Oct 2019 12:11:11 +0300 Subject: [PATCH 3/3] Add App Mesh acceptance tests example to docs --- artifacts/appmesh/canary.yaml | 7 ++ artifacts/appmesh/deployment.yaml | 2 +- .../usage/appmesh-progressive-delivery.md | 72 +++++++++++-------- 3 files changed, 52 insertions(+), 29 deletions(-) diff --git a/artifacts/appmesh/canary.yaml b/artifacts/appmesh/canary.yaml index c9c38cd4..0cea6d6a 100644 --- a/artifacts/appmesh/canary.yaml +++ b/artifacts/appmesh/canary.yaml @@ -48,6 +48,13 @@ spec: interval: 30s # testing (optional) webhooks: + - name: acceptance-test + type: pre-rollout + url: http://flagger-loadtester.test/ + timeout: 30s + metadata: + type: bash + cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token" - name: load-test url: http://flagger-loadtester.test/ timeout: 5s diff --git a/artifacts/appmesh/deployment.yaml b/artifacts/appmesh/deployment.yaml index 21ae7cc0..5d1ee25f 100644 --- a/artifacts/appmesh/deployment.yaml +++ b/artifacts/appmesh/deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:2.0.0 + image: stefanprodan/podinfo:3.1.0 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/docs/gitbook/usage/appmesh-progressive-delivery.md b/docs/gitbook/usage/appmesh-progressive-delivery.md index ab7511dd..66b076e2 100644 --- a/docs/gitbook/usage/appmesh-progressive-delivery.md +++ b/docs/gitbook/usage/appmesh-progressive-delivery.md @@ -39,8 +39,7 @@ helm upgrade -i flagger-loadtester flagger/loadtester \ --namespace=test \ --set meshName=global \ --set "backends[0]=podinfo.test" \ ---set "backends[1]=podinfo-canary.test" \ ---set "backends[2]=podinfo-primary.test" +--set "backends[1]=podinfo-canary.test" ``` Create a canary custom resource: @@ -97,8 +96,15 @@ spec: # milliseconds threshold: 500 interval: 30s - # external checks (optional) + # testing (optional) webhooks: + - name: acceptance-test + type: pre-rollout + url: http://flagger-loadtester.test/ + timeout: 30s + metadata: + type: bash + cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token" - name: load-test url: http://flagger-loadtester.test/ timeout: 5s @@ -132,8 +138,12 @@ virtualnode.appmesh.k8s.aws/podinfo virtualnode.appmesh.k8s.aws/podinfo-canary virtualnode.appmesh.k8s.aws/podinfo-primary virtualservice.appmesh.k8s.aws/podinfo.test +virtualservice.appmesh.k8s.aws/podinfo-canary.test ``` +After the boostrap, the podinfo deployment will be scaled to zero and the traffic to `podinfo.test` will be routed +to the primary pods. During the canary analysis, the `podinfo-canary.test` address can be used to target directly the canary pods. + The App Mesh specific settings are: ```yaml @@ -183,7 +193,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=stefanprodan/podinfo:2.0.1 +podinfod=stefanprodan/podinfo:3.1.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: @@ -196,28 +206,34 @@ Status: Failed Checks: 0 Phase: Succeeded Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal Synced 3m flagger New revision detected podinfo.test - Normal Synced 3m flagger Scaling up podinfo.test - Warning Synced 3m flagger Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available - Normal Synced 3m flagger Advance podinfo.test canary weight 5 - Normal Synced 3m flagger Advance podinfo.test canary weight 10 - Normal Synced 3m flagger Advance podinfo.test canary weight 15 - Normal Synced 2m flagger Advance podinfo.test canary weight 20 - Normal Synced 2m flagger Advance podinfo.test canary weight 25 - Normal Synced 1m flagger Advance podinfo.test canary weight 30 - Normal Synced 1m flagger Advance podinfo.test canary weight 35 - Normal Synced 55s flagger Advance podinfo.test canary weight 40 - Normal Synced 45s flagger Advance podinfo.test canary weight 45 - Normal Synced 35s flagger Advance podinfo.test canary weight 50 - Normal Synced 25s flagger Copying podinfo.test template spec to podinfo-primary.test - Warning Synced 15s flagger Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available - Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test + New revision detected! Scaling up podinfo.test + Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available + Pre-rollout check acceptance-test passed + Advance podinfo.test canary weight 5 + Advance podinfo.test canary weight 10 + Advance podinfo.test canary weight 15 + Advance podinfo.test canary weight 20 + Advance podinfo.test canary weight 25 + Advance podinfo.test canary weight 30 + Advance podinfo.test canary weight 35 + Advance podinfo.test canary weight 40 + Advance podinfo.test canary weight 45 + Advance podinfo.test canary weight 50 + Copying podinfo.test template spec to podinfo-primary.test + Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available + Routing all traffic to primary + Promotion completed! Scaling down podinfo.test ``` +When the canary analysis starts, Flagger will call the pre-rollout webhooks before routing traffic to the canary. + **Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis. +A canary deployment is triggered by changes in any of the following objects: +* Deployment PodSpec (container image, command, ports, env, resources, etc) +* ConfigMaps mounted as volumes or mapped to environment variables +* Secrets mounted as volumes or mapped to environment variables + During the analysis the canary’s progress can be monitored with Grafana. The App Mesh dashboard URL is http://localhost:3000/d/flagger-appmesh/appmesh-canary?refresh=10s&orgId=1&var-namespace=test&var-primary=podinfo-primary&var-canary=podinfo @@ -229,9 +245,9 @@ You can monitor all canaries with: watch kubectl get canaries --all-namespaces NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME -test podinfo Progressing 15 2019-03-16T14:05:07Z -prod frontend Succeeded 0 2019-03-15T16:15:07Z -prod backend Failed 0 2019-03-14T17:05:07Z +test podinfo Progressing 15 2019-10-02T14:05:07Z +prod frontend Succeeded 0 2019-10-02T16:15:07Z +prod backend Failed 0 2019-10-02T17:05:07Z ``` If you’ve enabled the Slack notifications, you should receive the following messages: @@ -246,19 +262,19 @@ Trigger a canary deployment: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=stefanprodan/podinfo:2.0.2 +podinfod=stefanprodan/podinfo:3.1.2 ``` Exec into the load tester pod with: ```bash -kubectl -n test exec -it flagger-loadtester-xx-xx bash +kubectl -n test exec -it deploy/flagger-loadtester bash ``` Generate HTTP 500 errors: ```bash -hey -z 1m -c 5 -q 5 http://podinfo.test:9898/status/500 +hey -z 1m -c 5 -q 5 http://podinfo-canary.test:9898/status/500 ``` Generate latency: