diff --git a/artifacts/appmesh/canary.yaml b/artifacts/appmesh/canary.yaml index d16ea07d..0cea6d6a 100644 --- a/artifacts/appmesh/canary.yaml +++ b/artifacts/appmesh/canary.yaml @@ -41,8 +41,20 @@ spec: # percentage (0-100) threshold: 99 interval: 1m - # external checks (optional) + - name: request-duration + # maximum req duration P99 + # milliseconds + threshold: 500 + interval: 30s + # testing (optional) webhooks: + - name: acceptance-test + type: pre-rollout + url: http://flagger-loadtester.test/ + timeout: 30s + metadata: + type: bash + cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token" - name: load-test url: http://flagger-loadtester.test/ timeout: 5s diff --git a/artifacts/appmesh/deployment.yaml b/artifacts/appmesh/deployment.yaml index 21ae7cc0..5d1ee25f 100644 --- a/artifacts/appmesh/deployment.yaml +++ b/artifacts/appmesh/deployment.yaml @@ -25,7 +25,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:2.0.0 + image: stefanprodan/podinfo:3.1.0 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 diff --git a/docs/gitbook/usage/appmesh-progressive-delivery.md b/docs/gitbook/usage/appmesh-progressive-delivery.md index 92263b21..66b076e2 100644 --- a/docs/gitbook/usage/appmesh-progressive-delivery.md +++ b/docs/gitbook/usage/appmesh-progressive-delivery.md @@ -39,8 +39,7 @@ helm upgrade -i flagger-loadtester flagger/loadtester \ --namespace=test \ --set meshName=global \ --set "backends[0]=podinfo.test" \ ---set "backends[1]=podinfo-canary.test" \ ---set "backends[2]=podinfo-primary.test" +--set "backends[1]=podinfo-canary.test" ``` Create a canary custom resource: @@ -92,8 +91,20 @@ spec: # percentage (0-100) threshold: 99 interval: 1m - # external checks (optional) + - name: request-duration + # maximum req duration P99 + # milliseconds + threshold: 500 + interval: 30s + # testing (optional) webhooks: + - name: acceptance-test + type: pre-rollout + url: http://flagger-loadtester.test/ + timeout: 30s + metadata: + type: bash + cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token" - name: load-test url: http://flagger-loadtester.test/ timeout: 5s @@ -127,8 +138,12 @@ virtualnode.appmesh.k8s.aws/podinfo virtualnode.appmesh.k8s.aws/podinfo-canary virtualnode.appmesh.k8s.aws/podinfo-primary virtualservice.appmesh.k8s.aws/podinfo.test +virtualservice.appmesh.k8s.aws/podinfo-canary.test ``` +After the boostrap, the podinfo deployment will be scaled to zero and the traffic to `podinfo.test` will be routed +to the primary pods. During the canary analysis, the `podinfo-canary.test` address can be used to target directly the canary pods. + The App Mesh specific settings are: ```yaml @@ -178,7 +193,7 @@ Trigger a canary deployment by updating the container image: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=stefanprodan/podinfo:2.0.1 +podinfod=stefanprodan/podinfo:3.1.1 ``` Flagger detects that the deployment revision changed and starts a new rollout: @@ -191,28 +206,34 @@ Status: Failed Checks: 0 Phase: Succeeded Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal Synced 3m flagger New revision detected podinfo.test - Normal Synced 3m flagger Scaling up podinfo.test - Warning Synced 3m flagger Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available - Normal Synced 3m flagger Advance podinfo.test canary weight 5 - Normal Synced 3m flagger Advance podinfo.test canary weight 10 - Normal Synced 3m flagger Advance podinfo.test canary weight 15 - Normal Synced 2m flagger Advance podinfo.test canary weight 20 - Normal Synced 2m flagger Advance podinfo.test canary weight 25 - Normal Synced 1m flagger Advance podinfo.test canary weight 30 - Normal Synced 1m flagger Advance podinfo.test canary weight 35 - Normal Synced 55s flagger Advance podinfo.test canary weight 40 - Normal Synced 45s flagger Advance podinfo.test canary weight 45 - Normal Synced 35s flagger Advance podinfo.test canary weight 50 - Normal Synced 25s flagger Copying podinfo.test template spec to podinfo-primary.test - Warning Synced 15s flagger Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available - Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test + New revision detected! Scaling up podinfo.test + Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available + Pre-rollout check acceptance-test passed + Advance podinfo.test canary weight 5 + Advance podinfo.test canary weight 10 + Advance podinfo.test canary weight 15 + Advance podinfo.test canary weight 20 + Advance podinfo.test canary weight 25 + Advance podinfo.test canary weight 30 + Advance podinfo.test canary weight 35 + Advance podinfo.test canary weight 40 + Advance podinfo.test canary weight 45 + Advance podinfo.test canary weight 50 + Copying podinfo.test template spec to podinfo-primary.test + Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available + Routing all traffic to primary + Promotion completed! Scaling down podinfo.test ``` +When the canary analysis starts, Flagger will call the pre-rollout webhooks before routing traffic to the canary. + **Note** that if you apply new changes to the deployment during the canary analysis, Flagger will restart the analysis. +A canary deployment is triggered by changes in any of the following objects: +* Deployment PodSpec (container image, command, ports, env, resources, etc) +* ConfigMaps mounted as volumes or mapped to environment variables +* Secrets mounted as volumes or mapped to environment variables + During the analysis the canary’s progress can be monitored with Grafana. The App Mesh dashboard URL is http://localhost:3000/d/flagger-appmesh/appmesh-canary?refresh=10s&orgId=1&var-namespace=test&var-primary=podinfo-primary&var-canary=podinfo @@ -224,9 +245,9 @@ You can monitor all canaries with: watch kubectl get canaries --all-namespaces NAMESPACE NAME STATUS WEIGHT LASTTRANSITIONTIME -test podinfo Progressing 15 2019-03-16T14:05:07Z -prod frontend Succeeded 0 2019-03-15T16:15:07Z -prod backend Failed 0 2019-03-14T17:05:07Z +test podinfo Progressing 15 2019-10-02T14:05:07Z +prod frontend Succeeded 0 2019-10-02T16:15:07Z +prod backend Failed 0 2019-10-02T17:05:07Z ``` If you’ve enabled the Slack notifications, you should receive the following messages: @@ -241,19 +262,25 @@ Trigger a canary deployment: ```bash kubectl -n test set image deployment/podinfo \ -podinfod=stefanprodan/podinfo:2.0.2 +podinfod=stefanprodan/podinfo:3.1.2 ``` Exec into the load tester pod with: ```bash -kubectl -n test exec -it flagger-loadtester-xx-xx sh +kubectl -n test exec -it deploy/flagger-loadtester bash ``` Generate HTTP 500 errors: ```bash -hey -z 1m -c 5 -q 5 http://podinfo.test:9898/status/500 +hey -z 1m -c 5 -q 5 http://podinfo-canary.test:9898/status/500 +``` + +Generate latency: + +```bash +watch -n 1 curl http://podinfo-canary.test:9898/delay/1 ``` When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary, @@ -264,22 +291,21 @@ kubectl -n test describe canary/podinfo Status: Canary Weight: 0 - Failed Checks: 10 + Failed Checks: 5 Phase: Failed Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal Synced 3m flagger Starting canary deployment for podinfo.test - Normal Synced 3m flagger Advance podinfo.test canary weight 5 - Normal Synced 3m flagger Advance podinfo.test canary weight 10 - Normal Synced 3m flagger Advance podinfo.test canary weight 15 - Normal Synced 3m flagger Halt podinfo.test advancement success rate 69.17% < 99% - Normal Synced 2m flagger Halt podinfo.test advancement success rate 61.39% < 99% - Normal Synced 2m flagger Halt podinfo.test advancement success rate 55.06% < 99% - Normal Synced 2m flagger Halt podinfo.test advancement success rate 47.00% < 99% - Normal Synced 2m flagger (combined from similar events): Halt podinfo.test advancement success rate 38.08% < 99% - Warning Synced 1m flagger Rolling back podinfo.test failed checks threshold reached 10 - Warning Synced 1m flagger Canary failed! Scaling down podinfo.test + Starting canary analysis for podinfo.test + Pre-rollout check acceptance-test passed + Advance podinfo.test canary weight 5 + Advance podinfo.test canary weight 10 + Advance podinfo.test canary weight 15 + Halt podinfo.test advancement success rate 69.17% < 99% + Halt podinfo.test advancement success rate 61.39% < 99% + Halt podinfo.test advancement success rate 55.06% < 99% + Halt podinfo.test advancement request duration 1.20s > 0.5s + Halt podinfo.test advancement request duration 1.45s > 0.5s + Rolling back podinfo.test failed checks threshold reached 5 + Canary failed! Scaling down podinfo.test ``` If you’ve enabled the Slack notifications, you’ll receive a message if the progress deadline is exceeded, diff --git a/pkg/router/appmesh.go b/pkg/router/appmesh.go index c8b36ae0..6d25f117 100644 --- a/pkg/router/appmesh.go +++ b/pkg/router/appmesh.go @@ -56,9 +56,16 @@ func (ar *AppMeshRouter) Reconcile(canary *flaggerv1.Canary) error { return err } - // sync virtual service e.g. app.namespace + // sync main virtual service // DNS app.namespace - err = ar.reconcileVirtualService(canary, targetHost) + err = ar.reconcileVirtualService(canary, targetHost, 0) + if err != nil { + return err + } + + // sync canary virtual service + // DNS app-canary.namespace + err = ar.reconcileVirtualService(canary, fmt.Sprintf("%s.%s", canaryName, canary.Namespace), 100) if err != nil { return err } @@ -148,7 +155,7 @@ func (ar *AppMeshRouter) reconcileVirtualNode(canary *flaggerv1.Canary, name str } // reconcileVirtualService creates or updates a virtual service -func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name string) error { +func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name string, canaryWeight int64) error { targetName := canary.Spec.TargetRef.Name canaryVirtualNode := fmt.Sprintf("%s-canary", targetName) primaryVirtualNode := fmt.Sprintf("%s-primary", targetName) @@ -185,11 +192,11 @@ func (ar *AppMeshRouter) reconcileVirtualService(canary *flaggerv1.Canary, name WeightedTargets: []AppmeshV1beta1.WeightedTarget{ { VirtualNodeName: canaryVirtualNode, - Weight: 0, + Weight: canaryWeight, }, { VirtualNodeName: primaryVirtualNode, - Weight: 100, + Weight: 100 - canaryWeight, }, }, }, diff --git a/pkg/router/appmesh_test.go b/pkg/router/appmesh_test.go index 0fbed858..3e261d95 100644 --- a/pkg/router/appmesh_test.go +++ b/pkg/router/appmesh_test.go @@ -37,6 +37,23 @@ func TestAppmeshRouter_Reconcile(t *testing.T) { t.Errorf("Got routes %v wanted %v", targetsCount, 2) } + // check canary virtual service + vsCanaryName := fmt.Sprintf("%s-canary.%s", mocks.appmeshCanary.Spec.TargetRef.Name, mocks.appmeshCanary.Namespace) + vsCanary, err := router.appmeshClient.AppmeshV1beta1().VirtualServices("default").Get(vsCanaryName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err.Error()) + } + + // check if the canary virtual service routes all traffic to the canary virtual node + target := vsCanary.Spec.Routes[0].Http.Action.WeightedTargets[0] + canaryVirtualNodeName := fmt.Sprintf("%s-canary", mocks.appmeshCanary.Spec.TargetRef.Name) + if target.VirtualNodeName != canaryVirtualNodeName { + t.Errorf("Got VirtualNodeName %v wanted %v", target.VirtualNodeName, canaryVirtualNodeName) + } + if target.Weight != 100 { + t.Errorf("Got weight %v wanted %v", target.Weight, 100) + } + // check virtual node vnName := mocks.appmeshCanary.Spec.TargetRef.Name vn, err := router.appmeshClient.AppmeshV1beta1().VirtualNodes("default").Get(vnName, metav1.GetOptions{}) @@ -103,7 +120,7 @@ func TestAppmeshRouter_Reconcile(t *testing.T) { weight := vs.Spec.Routes[0].Http.Action.WeightedTargets[0].Weight if weight != 50 { - t.Errorf("Got weight %v wanted %v", weight, 502) + t.Errorf("Got weight %v wanted %v", weight, 50) } // test URI update