diff --git a/README.md b/README.md index 28da53ad..a0982a74 100644 --- a/README.md +++ b/README.md @@ -45,25 +45,25 @@ Gated canary promotion stages: * scan for canary deployments * check Istio virtual service routes are mapped to primary and canary ClusterIP services * check primary and canary deployments status - * halt rollout if a rolling update is underway - * halt rollout if pods are unhealthy + * halt advancement if a rolling update is underway + * halt advancement if pods are unhealthy * increase canary traffic weight percentage from 0% to 5% (step weight) * check canary HTTP request success rate and latency - * halt rollout if any metric is under the specified threshold + * halt advancement if any metric is under the specified threshold * increment the failed checks counter * check if the number of failed checks reached the threshold * route all traffic to primary * scale to zero the canary deployment and mark it as failed * wait for the canary deployment to be updated (revision bump) and start over * increase canary traffic weight by 5% (step weight) till it reaches 50% (max weight) - * halt rollout while canary request success rate is under the threshold - * halt rollout while canary request duration P99 is over the threshold - * halt rollout if the primary or canary deployment becomes unhealthy - * halt rollout while canary deployment is being scaled up/down by HPA + * halt advancement while canary request success rate is under the threshold + * halt advancement while canary request duration P99 is over the threshold + * halt advancement if the primary or canary deployment becomes unhealthy + * halt advancement while canary deployment is being scaled up/down by HPA * promote canary to primary * copy canary deployment spec template over primary * wait for primary rolling update to finish - * halt rollout if pods are unhealthy + * halt advancement if pods are unhealthy * route all traffic to primary * scale to zero the canary deployment * mark rollout as finished @@ -80,19 +80,27 @@ metadata: name: podinfo namespace: test spec: + # deployment reference targetRef: apiVersion: apps/v1 kind: Deployment name: podinfo + # hpa reference (optional) + autoscalerRef: + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + name: podinfo service: + # container port port: 9898 + # Istio gateways (optional) gateways: - public-gateway.istio-system.svc.cluster.local + # Istio virtual service host names (optional) hosts: - - podinfo.example.com + - app.istio.weavedx.com canaryAnalysis: - # max number of failed metric checks - # before rolling back the canary + # max number of failed metric checks before rollback threshold: 5 # max traffic percentage routed to canary # percentage (0-100) @@ -166,50 +174,68 @@ export REPO=https://raw.githubusercontent.com/stefanprodan/flagger/master kubectl apply -f ${REPO}/artifacts/namespaces/test.yaml ``` -Create a deployment: +Create a deployment and a horizontal pod autoscaler: ```bash kubectl apply -f ${REPO}/artifacts/canaries/deployment.yaml +kubectl apply -f ${REPO}/artifacts/canaries/hpa.yaml ``` -Create a canary promotion custom resource (replace the Istio gateway and the internet domain with your own):: +Create a canary promotion custom resource (replace the Istio gateway and the internet domain with your own): ```bash kubectl apply -f ${REPO}/artifacts/canaries/canary.yaml ``` +After a couple of seconds Flagger will create the canary objects: + +```bash +canaries.flagger.app/podinfo +deployment.apps/podinfo +deployment.apps/podinfo-primary +horizontalpodautoscaler.autoscaling/podinfo +horizontalpodautoscaler.autoscaling/podinfo-primary +service/podinfo +service/podinfo-canary +service/podinfo-primary +virtualservices.networking.istio.io/podinfo +``` + ![flagger-canary-steps](https://raw.githubusercontent.com/stefanprodan/flagger/master/docs/diagrams/flagger-canary-steps.png) -Canary promotion output: +Trigger a canary deployment by updating the container image: + +```bash +kubectl -n test set image deployment/podinfo \ +podinfod=quay.io/stefanprodan/podinfo:1.2.1 +``` + +Flagger detects that the deployment revision changed and starts a new rollout: ``` kubectl -n test describe canary/podinfo Status: - Canary Revision: 16271121 - Failed Checks: 6 + Canary Revision: 19871136 + Failed Checks: 0 State: finished Events: Type Reason Age From Message ---- ------ ---- ---- ------- - Normal Synced 3m flagger Starting canary deployment for podinfo.test + Normal Synced 3m flagger New revision detected podinfo.test + Normal Synced 3m flagger Scaling up podinfo.test + Warning Synced 3m flagger Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available Normal Synced 3m flagger Advance podinfo.test canary weight 5 Normal Synced 3m flagger Advance podinfo.test canary weight 10 Normal Synced 3m flagger Advance podinfo.test canary weight 15 - Warning Synced 3m flagger Halt podinfo.test advancement request duration 2.525s > 500ms - Warning Synced 3m flagger Halt podinfo.test advancement request duration 1.567s > 500ms - Warning Synced 3m flagger Halt podinfo.test advancement request duration 823ms > 500ms Normal Synced 2m flagger Advance podinfo.test canary weight 20 Normal Synced 2m flagger Advance podinfo.test canary weight 25 Normal Synced 1m flagger Advance podinfo.test canary weight 30 - Warning Synced 1m flagger Halt podinfo.test advancement success rate 82.33% < 99% - Warning Synced 1m flagger Halt podinfo.test advancement success rate 87.22% < 99% - Warning Synced 1m flagger Halt podinfo.test advancement success rate 94.74% < 99% Normal Synced 1m flagger Advance podinfo.test canary weight 35 Normal Synced 55s flagger Advance podinfo.test canary weight 40 Normal Synced 45s flagger Advance podinfo.test canary weight 45 Normal Synced 35s flagger Advance podinfo.test canary weight 50 Normal Synced 25s flagger Copying podinfo.test template spec to podinfo-primary.test - Warning Synced 15s flagger Waiting for podinfo.test rollout to finish: 1 of 2 updated replicas are available + Warning Synced 15s flagger Waiting for podinfo-primary.test rollout to finish: 1 of 2 updated replicas are available Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test ``` @@ -260,43 +286,6 @@ Events: Warning Synced 1m flagger Canary failed! Scaling down podinfo.test ``` -Trigger a new canary deployment by updating the canary image: - -```bash -kubectl -n test set image deployment/podinfo \ -podinfod=quay.io/stefanprodan/podinfo:1.2.1 -``` - -Steer detects that the canary revision changed and starts a new rollout: - -``` -kubectl -n test describe canary/podinfo - -Status: - Canary Revision: 19871136 - Failed Checks: 0 - State: finished -Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal Synced 3m flagger New revision detected podinfo-canary.test old 17211012 new 17246876 - Normal Synced 3m flagger Scaling up podinfo.test - Warning Synced 3m flagger Waiting for podinfo.test rollout to finish: 0 of 1 updated replicas are available - Normal Synced 3m flagger Advance podinfo.test canary weight 5 - Normal Synced 3m flagger Advance podinfo.test canary weight 10 - Normal Synced 3m flagger Advance podinfo.test canary weight 15 - Normal Synced 2m flagger Advance podinfo.test canary weight 20 - Normal Synced 2m flagger Advance podinfo.test canary weight 25 - Normal Synced 1m flagger Advance podinfo.test canary weight 30 - Normal Synced 1m flagger Advance podinfo.test canary weight 35 - Normal Synced 55s flagger Advance podinfo.test canary weight 40 - Normal Synced 45s flagger Advance podinfo.test canary weight 45 - Normal Synced 35s flagger Advance podinfo.test canary weight 50 - Normal Synced 25s flagger Copying podinfo.test template spec to podinfo-primary.test - Warning Synced 15s flagger Waiting for podinfo.test rollout to finish: 1 of 2 updated replicas are available - Normal Synced 5s flagger Promotion completed! Scaling down podinfo.test -``` - ### Monitoring Flagger comes with a Grafana dashboard made for canary analysis. @@ -331,9 +320,10 @@ Advance podinfo.test canary weight 40 Halt podinfo.test advancement request duration 1.515s > 500ms Advance podinfo.test canary weight 45 Advance podinfo.test canary weight 50 -Copying podinfo-canary.test template spec to podinfo.test -Scaling down podinfo-canary.test -Promotion completed! podinfo-canary.test revision 81289 +Copying podinfo.test template spec to podinfo-primary.test +Halt podinfo-primary.test advancement waiting for rollout to finish: 1 old replicas are pending termination +Scaling down podinfo.test +Promotion completed! podinfo.test ``` ### Roadmap diff --git a/artifacts/canaries/canary.yaml b/artifacts/canaries/canary.yaml index 21b71b76..5f901c9e 100644 --- a/artifacts/canaries/canary.yaml +++ b/artifacts/canaries/canary.yaml @@ -4,19 +4,27 @@ metadata: name: podinfo namespace: test spec: + # deployment reference targetRef: apiVersion: apps/v1 kind: Deployment name: podinfo + # HPA reference (optional) + autoscalerRef: + apiVersion: autoscaling/v2beta1 + kind: HorizontalPodAutoscaler + name: podinfo service: + # container port port: 9898 + # Istio gateways (optional) gateways: - public-gateway.istio-system.svc.cluster.local + # Istio virtual service host names (optional) hosts: - app.istio.weavedx.com canaryAnalysis: - # max number of failed metric checks - # before rolling back the canary + # max number of failed metric checks before rollback threshold: 5 # max traffic percentage routed to canary # percentage (0-100) diff --git a/artifacts/canaries/deployment.yaml b/artifacts/canaries/deployment.yaml index 895dadb1..3a89acda 100644 --- a/artifacts/canaries/deployment.yaml +++ b/artifacts/canaries/deployment.yaml @@ -22,7 +22,7 @@ spec: spec: containers: - name: podinfod - image: quay.io/stefanprodan/podinfo:1.2.0 + image: quay.io/stefanprodan/podinfo:1.2.1 imagePullPolicy: IfNotPresent ports: - containerPort: 9898 @@ -66,5 +66,5 @@ spec: cpu: 2000m memory: 512Mi requests: - cpu: 10m + cpu: 100m memory: 64Mi diff --git a/artifacts/canaries/hpa.yaml b/artifacts/canaries/hpa.yaml new file mode 100644 index 00000000..fa2b5a6f --- /dev/null +++ b/artifacts/canaries/hpa.yaml @@ -0,0 +1,19 @@ +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: podinfo + namespace: test +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: podinfo + minReplicas: 2 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + # scale up if usage is above + # 99% of the requested CPU (100m) + targetAverageUtilization: 99 diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index b3115a56..705f337e 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -31,6 +31,14 @@ spec: type: string name: type: string + autoscalerRef: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string service: properties: port: diff --git a/artifacts/flagger/deployment.yaml b/artifacts/flagger/deployment.yaml index 2f694fce..9cb278cc 100644 --- a/artifacts/flagger/deployment.yaml +++ b/artifacts/flagger/deployment.yaml @@ -22,7 +22,7 @@ spec: serviceAccountName: flagger containers: - name: flagger - image: stefanprodan/flagger:0.0.1 + image: stefanprodan/flagger:0.1.0-alpha.1 imagePullPolicy: Always ports: - name: http diff --git a/charts/flagger/Chart.yaml b/charts/flagger/Chart.yaml index a742bd98..7651ffc1 100644 --- a/charts/flagger/Chart.yaml +++ b/charts/flagger/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: flagger -version: 0.0.1 -appVersion: 0.0.1 +version: 0.1.0 +appVersion: 0.1.0-alpha.1 description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio routing for traffic shifting and Prometheus metrics for canary analysis. home: https://github.com/stefanprodan/flagger diff --git a/charts/flagger/templates/crd.yaml b/charts/flagger/templates/crd.yaml index ed4594f6..50f02017 100644 --- a/charts/flagger/templates/crd.yaml +++ b/charts/flagger/templates/crd.yaml @@ -32,6 +32,14 @@ spec: type: string name: type: string + autoscalerRef: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string service: properties: port: diff --git a/charts/flagger/values.yaml b/charts/flagger/values.yaml index d296bd82..b358e1f9 100644 --- a/charts/flagger/values.yaml +++ b/charts/flagger/values.yaml @@ -2,7 +2,7 @@ image: repository: stefanprodan/flagger - tag: 0.0.1 + tag: 0.1.0-alpha.1 pullPolicy: IfNotPresent controlLoopInterval: "10s" diff --git a/docs/flagger-0.1.0.tgz b/docs/flagger-0.1.0.tgz new file mode 100644 index 00000000..cc2283d9 Binary files /dev/null and b/docs/flagger-0.1.0.tgz differ diff --git a/docs/grafana-5.2.4.tgz b/docs/grafana-5.2.4.tgz index 53e83c7a..218430a0 100644 Binary files a/docs/grafana-5.2.4.tgz and b/docs/grafana-5.2.4.tgz differ diff --git a/docs/index.yaml b/docs/index.yaml index 68befafa..11a6a092 100755 --- a/docs/index.yaml +++ b/docs/index.yaml @@ -1,9 +1,21 @@ apiVersion: v1 entries: flagger: + - apiVersion: v1 + appVersion: 0.1.0-alpha.1 + created: 2018-10-11T11:16:00.30124+03:00 + description: Flagger is a Kubernetes operator that automates the promotion of + canary deployments using Istio routing for traffic shifting and Prometheus metrics + for canary analysis. + digest: 3368bc9efbabe6043f399db46884f8a18319756b028ebe007ca69f9e81b72b3b + home: https://github.com/stefanprodan/flagger + name: flagger + urls: + - https://stefanprodan.github.io/flagger/flagger-0.1.0.tgz + version: 0.1.0 - apiVersion: v1 appVersion: 0.0.1 - created: 2018-10-11T02:02:20.205592+03:00 + created: 2018-10-11T11:16:00.300274+03:00 description: Flagger is a Kubernetes operator that automates the promotion of canary deployments using Istio routing for traffic shifting and Prometheus metrics for canary analysis. @@ -16,10 +28,10 @@ entries: grafana: - apiVersion: v1 appVersion: 5.2.0 - created: 2018-10-11T02:02:20.206046+03:00 + created: 2018-10-11T11:16:00.302086+03:00 description: A Helm chart for monitoring progressive deployments powered by Istio and Flagger - digest: 82e43b6777c39dfc6cf2dabf5d51148a48d64ed0efb3fa0cd09290602ccbfb87 + digest: 109afd61ac5aaa212f18bfad4a1ad38425458f9d0a492547c5522d3898836faf home: https://github.com/stefanprodan/flagger name: grafana urls: @@ -28,9 +40,9 @@ entries: podinfo-flagger: - apiVersion: v1 appVersion: 1.2.1 - created: 2018-10-11T02:02:20.206405+03:00 + created: 2018-10-11T11:16:00.302902+03:00 description: Podinfo Helm chart for Flagger progressive delivery - digest: 2cddd159d311a1e20d08f486481c1743c70f9cecefc3ae7554b357a8186919d9 + digest: 72d95f4f4ec1eea85ffc2a7cbb9b5f95bb56ea9d934ba24c5dde078d4f9c7605 engine: gotpl home: https://github.com/stefanprodan/k8s-podinfo maintainers: @@ -58,4 +70,4 @@ entries: urls: - https://stefanprodan.github.io/flagger/podinfo-flagger-1.2.1.tgz version: 1.2.1 -generated: 2018-10-11T02:02:20.204687+03:00 +generated: 2018-10-11T11:16:00.298073+03:00 diff --git a/docs/podinfo-flagger-2.0.0.tgz b/docs/podinfo-flagger-2.0.0.tgz index 5819cd43..149a7f51 100644 Binary files a/docs/podinfo-flagger-2.0.0.tgz and b/docs/podinfo-flagger-2.0.0.tgz differ diff --git a/pkg/apis/flagger/v1alpha1/types.go b/pkg/apis/flagger/v1alpha1/types.go index 563c362c..9303de8b 100755 --- a/pkg/apis/flagger/v1alpha1/types.go +++ b/pkg/apis/flagger/v1alpha1/types.go @@ -40,6 +40,9 @@ type CanarySpec struct { // reference to target resource TargetRef hpav1.CrossVersionObjectReference `json:"targetRef"` + // reference to autoscaling resource + AutoscalerRef hpav1.CrossVersionObjectReference `json:"autoscalerRef"` + // virtual service spec Service CanaryService `json:"service"` diff --git a/pkg/apis/flagger/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/flagger/v1alpha1/zz_generated.deepcopy.go index dcc462be..4f3b8a0e 100644 --- a/pkg/apis/flagger/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/flagger/v1alpha1/zz_generated.deepcopy.go @@ -152,6 +152,7 @@ func (in *CanaryService) DeepCopy() *CanaryService { func (in *CanarySpec) DeepCopyInto(out *CanarySpec) { *out = *in out.TargetRef = in.TargetRef + out.AutoscalerRef = in.AutoscalerRef in.Service.DeepCopyInto(&out.Service) in.CanaryAnalysis.DeepCopyInto(&out.CanaryAnalysis) return diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 7f6610e6..72273006 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -182,10 +182,12 @@ func (c *Controller) syncHandler(key string) error { c.rollouts.Store(fmt.Sprintf("%s.%s", cd.Name, cd.Namespace), cd) - err = c.bootstrapDeployment(cd) - if err != nil { - c.logger.Warnf("%s.%s bootstrap error %v", cd.Name, cd.Namespace, err) - return err + if cd.Spec.TargetRef.Kind == "Deployment" { + err = c.bootstrapDeployment(cd) + if err != nil { + c.logger.Warnf("%s.%s bootstrap error %v", cd.Name, cd.Namespace, err) + return err + } } c.logger.Infof("Synced %s", key) diff --git a/pkg/controller/deployer.go b/pkg/controller/deployer.go index cd261caf..a3976a9e 100644 --- a/pkg/controller/deployer.go +++ b/pkg/controller/deployer.go @@ -6,6 +6,7 @@ import ( istiov1alpha3 "github.com/knative/pkg/apis/istio/v1alpha3" flaggerv1 "github.com/stefanprodan/flagger/pkg/apis/flagger/v1alpha1" appsv1 "k8s.io/api/apps/v1" + hpav1 "k8s.io/api/autoscaling/v2beta1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -242,5 +243,50 @@ func (c *Controller) bootstrapDeployment(cd *flaggerv1.Canary) error { c.recordEventInfof(cd, "VirtualService %s.%s created", virtualService.GetName(), cd.Namespace) } + if cd.Spec.AutoscalerRef.Kind == "HorizontalPodAutoscaler" { + hpa, err := c.kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers(cd.Namespace).Get(cd.Spec.AutoscalerRef.Name, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return fmt.Errorf("HorizontalPodAutoscaler %s.%s not found, retrying in %v", + cd.Spec.AutoscalerRef.Name, cd.Namespace, c.rolloutWindow) + } else { + return err + } + } + primaryHpaName := fmt.Sprintf("%s-primary", cd.Spec.AutoscalerRef.Name) + primaryHpa, err := c.kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers(cd.Namespace).Get(primaryHpaName, metav1.GetOptions{}) + + if errors.IsNotFound(err) { + primaryHpa = &hpav1.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: primaryHpaName, + Namespace: cd.Namespace, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(cd, schema.GroupVersionKind{ + Group: flaggerv1.SchemeGroupVersion.Group, + Version: flaggerv1.SchemeGroupVersion.Version, + Kind: flaggerv1.CanaryKind, + }), + }, + }, + Spec: hpav1.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: hpav1.CrossVersionObjectReference{ + Name: primaryName, + Kind: hpa.Spec.ScaleTargetRef.Kind, + APIVersion: hpa.Spec.ScaleTargetRef.APIVersion, + }, + MinReplicas: hpa.Spec.MinReplicas, + MaxReplicas: hpa.Spec.MaxReplicas, + Metrics: hpa.Spec.Metrics, + }, + } + + _, err = c.kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers(cd.Namespace).Create(primaryHpa) + if err != nil { + return err + } + c.recordEventInfof(cd, "HorizontalPodAutoscaler %s.%s created", primaryHpa.GetName(), cd.Namespace) + } + } return nil } diff --git a/pkg/version/version.go b/pkg/version/version.go index 1282bacb..17d4c116 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -1,4 +1,4 @@ package version -var VERSION = "0.0.1" +var VERSION = "0.1.0-alpha.1" var REVISION = "unknown"