package controller import ( "fmt" "time" istiov1alpha3 "github.com/knative/pkg/apis/istio/v1alpha3" rolloutv1 "github.com/stefanprodan/steerer/pkg/apis/rollout/v1beta1" appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( revisionAnnotation = "apps.weave.works/canary-revision" statusAnnotation = "apps.weave.works/status" ) func (c *Controller) doRollouts() { c.rollouts.Range(func(key interface{}, value interface{}) bool { r := value.(*rolloutv1.Rollout) if r.Spec.TargetKind == "Deployment" { go c.advanceDeploymentRollout(r.Name, r.Namespace) } return true }) } func (c *Controller) advanceDeploymentRollout(name string, namespace string) { // gate stage: check if the rollout exists r, ok := c.getRollout(name, namespace) if !ok { return } // set max weight default value to 100% maxWeight := 100 if r.Spec.CanaryAnalysis.MaxWeight > 0 { maxWeight = r.Spec.CanaryAnalysis.MaxWeight } // gate stage: check if canary deployment exists and is healthy canary, ok := c.getDeployment(r, r.Spec.Canary.Name, r.Namespace) if !ok { return } // gate stage: check if primary deployment exists and is healthy primary, ok := c.getDeployment(r, r.Spec.Primary.Name, r.Namespace) if !ok { return } // gate stage: check if virtual service exists // and if it contains weighted destination routes to the primary and canary services vs, primaryRoute, canaryRoute, ok := c.getVirtualService(r) if !ok { return } // gate stage: check if rollout should start (canary revision has changes) or continue if ok := c.checkRolloutStatus(r, canary.ResourceVersion); !ok { return } // gate stage: check if the canary success rate is above the threshold // skip check if no traffic is routed to canary if canaryRoute.Weight == 0 { c.recordEventInfof(r, "Starting rollout for %s.%s", r.Name, r.Namespace) } else { if ok := c.checkDeploymentMetrics(r); !ok { return } } // routing stage: increase canary traffic percentage if canaryRoute.Weight < maxWeight { primaryRoute.Weight -= r.Spec.CanaryAnalysis.StepWeight if primaryRoute.Weight < 0 { primaryRoute.Weight = 0 } canaryRoute.Weight += r.Spec.CanaryAnalysis.StepWeight if primaryRoute.Weight > 100 { primaryRoute.Weight = 100 } if ok := c.updateVirtualServiceRoutes(r, vs, primaryRoute, canaryRoute); !ok { return } c.recordEventInfof(r, "Advance rollout %s.%s weight %v", r.Name, r.Namespace, canaryRoute.Weight) // promotion stage: override primary.template.spec with the canary spec if canaryRoute.Weight == maxWeight { c.recordEventInfof(r, "Copying %s.%s template spec to %s.%s", canary.GetName(), canary.Namespace, primary.GetName(), primary.Namespace) primary.Spec.Template.Spec = canary.Spec.Template.Spec _, err := c.kubeClient.AppsV1().Deployments(primary.Namespace).Update(primary) if err != nil { c.recordEventErrorf(r, "Deployment %s.%s promotion failed: %v", primary.GetName(), primary.Namespace, err) return } } } else { // final stage: route all traffic back to primary primaryRoute.Weight = 100 canaryRoute.Weight = 0 if ok := c.updateVirtualServiceRoutes(r, vs, primaryRoute, canaryRoute); !ok { return } // final stage: mark rollout as finished and scale canary to zero replicas c.updateRolloutStatus(r, "finished") c.recordEventInfof(r, "%s.%s promotion complete! Scaling down %s.%s", r.Name, r.Namespace, canary.GetName(), canary.Namespace) c.scaleToZeroCanary(r) } } func (c *Controller) getRollout(name string, namespace string) (*rolloutv1.Rollout, bool) { r, err := c.rolloutClient.AppsV1beta1().Rollouts(namespace).Get(name, v1.GetOptions{}) if err != nil { c.logger.Errorf("Rollout %s.%s not found", name, namespace) return nil, false } return r, true } func (c *Controller) checkRolloutStatus(r *rolloutv1.Rollout, canaryVersion string) bool { var err error if val, ok := r.Annotations[revisionAnnotation]; !ok { r.Annotations[revisionAnnotation] = canaryVersion r.Annotations[statusAnnotation] = "running" r.Status.State = "running" r.Status.CanaryRevision = canaryVersion r, err = c.rolloutClient.AppsV1beta1().Rollouts(r.Namespace).Update(r) if err != nil { c.recordEventErrorf(r, "Rollout %s.%s status update failed: %v", r.Name, r.Namespace, err) return false } return true } else { if r.Annotations[statusAnnotation] == "running" { return true } if val != canaryVersion { r.Annotations[revisionAnnotation] = canaryVersion r.Annotations[statusAnnotation] = "running" r.Status.State = "running" r.Status.CanaryRevision = canaryVersion r, err = c.rolloutClient.AppsV1beta1().Rollouts(r.Namespace).Update(r) if err != nil { c.recordEventErrorf(r, "Rollout %s.%s status update failed: %v", r.Name, r.Namespace, err) return false } return true } } return false } func (c *Controller) updateRolloutStatus(r *rolloutv1.Rollout, status string) bool { var err error r.Annotations[statusAnnotation] = status r.Status.State = status r, err = c.rolloutClient.AppsV1beta1().Rollouts(r.Namespace).Update(r) if err != nil { c.recordEventErrorf(r, "Rollout %s.%s status update failed: %v", r.Name, r.Namespace, err) return false } return true } func (c *Controller) getDeployment(r *rolloutv1.Rollout, name string, namespace string) (*appsv1.Deployment, bool) { dep, err := c.kubeClient.AppsV1().Deployments(namespace).Get(name, v1.GetOptions{}) if err != nil { c.recordEventErrorf(r, "Deployment %s.%s not found", name, namespace) return nil, false } if msg, healthy := getDeploymentStatus(dep); !healthy { c.recordEventWarningf(r, "Halt rollout %s.%s %s", dep.GetName(), dep.Namespace, msg) return nil, false } if dep.Spec.Replicas == nil || *dep.Spec.Replicas == 0 { return nil, false } return dep, true } func (c *Controller) checkDeploymentMetrics(r *rolloutv1.Rollout) bool { for _, metric := range r.Spec.CanaryAnalysis.Metrics { if metric.Name == "istio_requests_total" { val, err := c.getDeploymentCounter(r.Spec.Canary.Name, r.Namespace, metric.Name, metric.Interval) if err != nil { c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.metricsServer, err) return false } if float64(metric.Threshold) > val { c.recordEventWarningf(r, "Halt rollout %s.%s success rate %.2f%% < %v%%", r.Name, r.Namespace, val, metric.Threshold) return false } } if metric.Name == "istio_request_duration_seconds_bucket" { val, err := c.GetDeploymentHistogram(r.Spec.Canary.Name, r.Namespace, metric.Name, metric.Interval) if err != nil { c.recordEventErrorf(r, "Metrics server %s query failed: %v", c.metricsServer, err) return false } t := time.Duration(metric.Threshold) * time.Millisecond if val > t { c.recordEventWarningf(r, "Halt rollout %s.%s request duration %v > %v", r.Name, r.Namespace, val, t) return false } } } return true } func (c *Controller) scaleToZeroCanary(r *rolloutv1.Rollout) { canary, err := c.kubeClient.AppsV1().Deployments(r.Namespace).Get(r.Spec.Canary.Name, v1.GetOptions{}) if err != nil { c.recordEventErrorf(r, "Deployment %s.%s not found", r.Spec.Canary.Name, r.Namespace) return } //HPA https://github.com/kubernetes/kubernetes/pull/29212 canary.Spec.Replicas = int32p(0) _, err = c.kubeClient.AppsV1().Deployments(canary.Namespace).Update(canary) if err != nil { c.recordEventErrorf(r, "Scaling down %s.%s failed: %v", canary.GetName(), canary.Namespace, err) return } } func (c *Controller) getVirtualService(r *rolloutv1.Rollout) ( vs *istiov1alpha3.VirtualService, primary istiov1alpha3.DestinationWeight, canary istiov1alpha3.DestinationWeight, ok bool, ) { var err error vs, err = c.istioClient.NetworkingV1alpha3().VirtualServices(r.Namespace).Get(r.Spec.VirtualService.Name, v1.GetOptions{}) if err != nil { c.recordEventErrorf(r, "VirtualService %s.%s not found", r.Spec.VirtualService.Name, r.Namespace) return } for _, http := range vs.Spec.Http { for _, route := range http.Route { if route.Destination.Host == r.Spec.Primary.Host { primary = route } if route.Destination.Host == r.Spec.Canary.Host { canary = route } } } if primary.Weight == 0 && canary.Weight == 0 { c.recordEventErrorf(r, "VirtualService %s.%s does not contain routes for %s and %s", r.Spec.VirtualService.Name, r.Namespace, r.Spec.Primary.Host, r.Spec.Canary.Host) return } ok = true return } func (c *Controller) updateVirtualServiceRoutes( r *rolloutv1.Rollout, vs *istiov1alpha3.VirtualService, primary istiov1alpha3.DestinationWeight, canary istiov1alpha3.DestinationWeight, ) bool { vs.Spec.Http = []istiov1alpha3.HTTPRoute{ { Route: []istiov1alpha3.DestinationWeight{primary, canary}, }, } var err error vs, err = c.istioClient.NetworkingV1alpha3().VirtualServices(r.Namespace).Update(vs) if err != nil { c.recordEventErrorf(r, "VirtualService %s.%s update failed: %v", r.Spec.VirtualService.Name, r.Namespace, err) return false } return true } func getDeploymentStatus(deployment *appsv1.Deployment) (string, bool) { if deployment.Generation <= deployment.Status.ObservedGeneration { cond := getDeploymentCondition(deployment.Status, appsv1.DeploymentProgressing) if cond != nil && cond.Reason == "ProgressDeadlineExceeded" { return fmt.Sprintf("deployment %q exceeded its progress deadline", deployment.GetName()), false } else if deployment.Spec.Replicas != nil && deployment.Status.UpdatedReplicas < *deployment.Spec.Replicas { return fmt.Sprintf("waiting for rollout to finish: %d out of %d new replicas have been updated", deployment.Status.UpdatedReplicas, *deployment.Spec.Replicas), false } else if deployment.Status.Replicas > deployment.Status.UpdatedReplicas { return fmt.Sprintf("waiting for rollout to finish: %d old replicas are pending termination", deployment.Status.Replicas-deployment.Status.UpdatedReplicas), false } else if deployment.Status.AvailableReplicas < deployment.Status.UpdatedReplicas { return fmt.Sprintf("waiting for rollout to finish: %d of %d updated replicas are available", deployment.Status.AvailableReplicas, deployment.Status.UpdatedReplicas), false } } else { return "waiting for rollout to finish: observed deployment generation less then desired generation", false } return "ready", true } func getDeploymentCondition( status appsv1.DeploymentStatus, conditionType appsv1.DeploymentConditionType, ) *appsv1.DeploymentCondition { for i := range status.Conditions { c := status.Conditions[i] if c.Type == conditionType { return &c } } return nil } func int32p(i int32) *int32 { return &i }