mirror of
https://github.com/fluxcd/flagger.git
synced 2026-04-15 06:57:34 +00:00
Tanner Altares
c9a07cec87
add e2e tests istio clean up comment from review add e2e tests istio clean up comment from review clean up logging statement add e2e tests istio clean up comment from review clean up logging statement add log statement on e2e iteration add e2e tests istio clean up comment from review clean up logging statement add log statement on e2e iteration extend timeout for finalizing add e2e tests istio clean up comment from review clean up logging statement add log statement on e2e iteration extend timeout for finalizing add phase to kustomize crd add e2e tests istio clean up comment from review clean up logging statement add log statement on e2e iteration extend timeout for finalizing add phase to kustomize crd revert timeout on circleci vs and svc checks for istio e2e tests fix fmt errors and tests add get statement in e2e test add get statement in e2e test add namespace to e2e use only selector for service revert
436 lines
16 KiB
Go
436 lines
16 KiB
Go
package canary
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"github.com/google/go-cmp/cmp"
|
|
"go.uber.org/zap"
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
hpav1 "k8s.io/api/autoscaling/v2beta1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
"k8s.io/client-go/kubernetes"
|
|
|
|
flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
|
|
clientset "github.com/weaveworks/flagger/pkg/client/clientset/versioned"
|
|
)
|
|
|
|
// DeploymentController is managing the operations for Kubernetes Deployment kind
|
|
type DeploymentController struct {
|
|
kubeClient kubernetes.Interface
|
|
flaggerClient clientset.Interface
|
|
logger *zap.SugaredLogger
|
|
configTracker Tracker
|
|
labels []string
|
|
}
|
|
|
|
// Initialize creates the primary deployment, hpa,
|
|
// scales to zero the canary deployment and returns the pod selector label and container ports
|
|
func (c *DeploymentController) Initialize(cd *flaggerv1.Canary, skipLivenessChecks bool) (err error) {
|
|
primaryName := fmt.Sprintf("%s-primary", cd.Spec.TargetRef.Name)
|
|
if err := c.createPrimaryDeployment(cd); err != nil {
|
|
return fmt.Errorf("createPrimaryDeployment failed: %w", err)
|
|
}
|
|
|
|
if cd.Status.Phase == "" || cd.Status.Phase == flaggerv1.CanaryPhaseInitializing {
|
|
if !skipLivenessChecks && !cd.SkipAnalysis() {
|
|
if err := c.IsPrimaryReady(cd); err != nil {
|
|
return fmt.Errorf("IsPrimaryReady failed: %w", err)
|
|
}
|
|
}
|
|
|
|
c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).
|
|
Infof("Scaling down Deployment %s.%s", cd.Spec.TargetRef.Name, cd.Namespace)
|
|
if err := c.ScaleToZero(cd); err != nil {
|
|
return fmt.Errorf("scaling down canary daemon set %s.%s failed: %w", cd.Spec.TargetRef.Name, cd.Namespace, err)
|
|
}
|
|
}
|
|
|
|
if cd.Spec.AutoscalerRef != nil {
|
|
if cd.Spec.AutoscalerRef.Kind == "HorizontalPodAutoscaler" {
|
|
if err := c.reconcilePrimaryHpa(cd, true); err != nil {
|
|
return fmt.Errorf(
|
|
"initial reconcilePrimaryHpa for %s.%s failed: %w", primaryName, cd.Namespace, err)
|
|
}
|
|
} else {
|
|
return fmt.Errorf("cd.Spec.AutoscalerRef.Kind is invalid: %s", cd.Spec.AutoscalerRef.Kind)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Promote copies the pod spec, secrets and config maps from canary to primary
|
|
func (c *DeploymentController) Promote(cd *flaggerv1.Canary) error {
|
|
targetName := cd.Spec.TargetRef.Name
|
|
primaryName := fmt.Sprintf("%s-primary", targetName)
|
|
|
|
canary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(targetName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("deployment %s.%s get query error: %w", targetName, cd.Namespace, err)
|
|
}
|
|
|
|
label, err := c.getSelectorLabel(canary)
|
|
if err != nil {
|
|
return fmt.Errorf("getSelectorLabel failed: %w", err)
|
|
}
|
|
|
|
primary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(primaryName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("deployment %s.%s get query error: %w", primaryName, cd.Namespace, err)
|
|
}
|
|
|
|
// promote secrets and config maps
|
|
configRefs, err := c.configTracker.GetTargetConfigs(cd)
|
|
if err != nil {
|
|
return fmt.Errorf("GetTargetConfigs failed: %w", err)
|
|
}
|
|
if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs); err != nil {
|
|
return fmt.Errorf("CreatePrimaryConfigs failed: %w", err)
|
|
}
|
|
|
|
primaryCopy := primary.DeepCopy()
|
|
primaryCopy.Spec.ProgressDeadlineSeconds = canary.Spec.ProgressDeadlineSeconds
|
|
primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds
|
|
primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit
|
|
primaryCopy.Spec.Strategy = canary.Spec.Strategy
|
|
|
|
// update spec with primary secrets and config maps
|
|
primaryCopy.Spec.Template.Spec = c.configTracker.ApplyPrimaryConfigs(canary.Spec.Template.Spec, configRefs)
|
|
|
|
// update pod annotations to ensure a rolling update
|
|
annotations, err := makeAnnotations(canary.Spec.Template.Annotations)
|
|
if err != nil {
|
|
return fmt.Errorf("makeAnnotations failed: %w", err)
|
|
}
|
|
|
|
primaryCopy.Spec.Template.Annotations = annotations
|
|
primaryCopy.Spec.Template.Labels = makePrimaryLabels(canary.Spec.Template.Labels, primaryName, label)
|
|
|
|
// apply update
|
|
_, err = c.kubeClient.AppsV1().Deployments(cd.Namespace).Update(primaryCopy)
|
|
if err != nil {
|
|
return fmt.Errorf("updating deployment %s.%s template spec failed: %w",
|
|
primaryCopy.GetName(), primaryCopy.Namespace, err)
|
|
}
|
|
|
|
// update HPA
|
|
if cd.Spec.AutoscalerRef != nil {
|
|
if cd.Spec.AutoscalerRef.Kind == "HorizontalPodAutoscaler" {
|
|
if err := c.reconcilePrimaryHpa(cd, false); err != nil {
|
|
return fmt.Errorf(
|
|
"reconcilePrimaryHpa for %s.%s failed: %w", primaryName, cd.Namespace, err)
|
|
}
|
|
} else {
|
|
return fmt.Errorf("cd.Spec.AutoscalerRef.Kind is invalid: %s", cd.Spec.AutoscalerRef.Kind)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// HasTargetChanged returns true if the canary deployment pod spec has changed
|
|
func (c *DeploymentController) HasTargetChanged(cd *flaggerv1.Canary) (bool, error) {
|
|
targetName := cd.Spec.TargetRef.Name
|
|
canary, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(targetName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return false, fmt.Errorf("deployment %s.%s get query error: %w", targetName, cd.Namespace, err)
|
|
}
|
|
|
|
return hasSpecChanged(cd, canary.Spec.Template)
|
|
}
|
|
|
|
// Scale sets the canary deployment replicas
|
|
func (c *DeploymentController) ScaleToZero(cd *flaggerv1.Canary) error {
|
|
targetName := cd.Spec.TargetRef.Name
|
|
dep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(targetName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("deployment %s.%s get query error: %w", targetName, cd.Namespace, err)
|
|
}
|
|
|
|
depCopy := dep.DeepCopy()
|
|
depCopy.Spec.Replicas = int32p(0)
|
|
|
|
_, err = c.kubeClient.AppsV1().Deployments(dep.Namespace).Update(depCopy)
|
|
if err != nil {
|
|
return fmt.Errorf("deployment %s.%s update query error: %w", targetName, cd.Namespace, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (c *DeploymentController) ScaleFromZero(cd *flaggerv1.Canary) error {
|
|
targetName := cd.Spec.TargetRef.Name
|
|
dep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(targetName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("deployment %s.%s get query error: %w", targetName, cd.Namespace, err)
|
|
}
|
|
|
|
replicas := int32p(1)
|
|
if dep.Spec.Replicas != nil && *dep.Spec.Replicas > 0 {
|
|
replicas = dep.Spec.Replicas
|
|
}
|
|
depCopy := dep.DeepCopy()
|
|
depCopy.Spec.Replicas = replicas
|
|
|
|
_, err = c.kubeClient.AppsV1().Deployments(dep.Namespace).Update(depCopy)
|
|
if err != nil {
|
|
return fmt.Errorf("scaling up %s.%s to %v failed: %v", depCopy.GetName(), depCopy.Namespace, replicas, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Scale sets the canary deployment replicas
|
|
func (c *DeploymentController) Scale(cd *flaggerv1.Canary, replicas int32) error {
|
|
targetName := cd.Spec.TargetRef.Name
|
|
dep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(targetName, metav1.GetOptions{})
|
|
if err != nil {
|
|
if errors.IsNotFound(err) {
|
|
return fmt.Errorf("deployment %s.%s not found", targetName, cd.Namespace)
|
|
}
|
|
return fmt.Errorf("deployment %s.%s query error %v", targetName, cd.Namespace, err)
|
|
}
|
|
|
|
depCopy := dep.DeepCopy()
|
|
depCopy.Spec.Replicas = int32p(replicas)
|
|
|
|
_, err = c.kubeClient.AppsV1().Deployments(dep.Namespace).Update(depCopy)
|
|
if err != nil {
|
|
return fmt.Errorf("scaling %s.%s to %v failed: %v", depCopy.GetName(), depCopy.Namespace, replicas, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// GetMetadata returns the pod label selector and svc ports
|
|
func (c *DeploymentController) GetMetadata(cd *flaggerv1.Canary) (string, map[string]int32, error) {
|
|
targetName := cd.Spec.TargetRef.Name
|
|
|
|
canaryDep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(targetName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return "", nil, fmt.Errorf("deployment %s.%s get query error: %w", targetName, cd.Namespace, err)
|
|
}
|
|
|
|
label, err := c.getSelectorLabel(canaryDep)
|
|
if err != nil {
|
|
return "", nil, fmt.Errorf("getSelectorLabel failed: %w", err)
|
|
}
|
|
|
|
var ports map[string]int32
|
|
if cd.Spec.Service.PortDiscovery {
|
|
ports = getPorts(cd, canaryDep.Spec.Template.Spec.Containers)
|
|
}
|
|
|
|
return label, ports, nil
|
|
}
|
|
func (c *DeploymentController) createPrimaryDeployment(cd *flaggerv1.Canary) error {
|
|
targetName := cd.Spec.TargetRef.Name
|
|
primaryName := fmt.Sprintf("%s-primary", cd.Spec.TargetRef.Name)
|
|
|
|
canaryDep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(targetName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("deplyoment %s.%s get query error: %w", targetName, cd.Namespace, err)
|
|
}
|
|
|
|
label, err := c.getSelectorLabel(canaryDep)
|
|
if err != nil {
|
|
return fmt.Errorf("getSelectorLabel failed: %w", err)
|
|
}
|
|
|
|
primaryDep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(primaryName, metav1.GetOptions{})
|
|
if errors.IsNotFound(err) {
|
|
// create primary secrets and config maps
|
|
configRefs, err := c.configTracker.GetTargetConfigs(cd)
|
|
if err != nil {
|
|
return fmt.Errorf("GetTargetConfigs failed: %w", err)
|
|
}
|
|
if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs); err != nil {
|
|
return fmt.Errorf("CreatePrimaryConfigs failed: %w", err)
|
|
}
|
|
annotations, err := makeAnnotations(canaryDep.Spec.Template.Annotations)
|
|
if err != nil {
|
|
return fmt.Errorf("makeAnnotations failed: %w", err)
|
|
}
|
|
|
|
replicas := int32(1)
|
|
if canaryDep.Spec.Replicas != nil && *canaryDep.Spec.Replicas > 0 {
|
|
replicas = *canaryDep.Spec.Replicas
|
|
}
|
|
|
|
// create primary deployment
|
|
primaryDep = &appsv1.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: primaryName,
|
|
Namespace: cd.Namespace,
|
|
Labels: map[string]string{
|
|
label: primaryName,
|
|
},
|
|
OwnerReferences: []metav1.OwnerReference{
|
|
*metav1.NewControllerRef(cd, schema.GroupVersionKind{
|
|
Group: flaggerv1.SchemeGroupVersion.Group,
|
|
Version: flaggerv1.SchemeGroupVersion.Version,
|
|
Kind: flaggerv1.CanaryKind,
|
|
}),
|
|
},
|
|
},
|
|
Spec: appsv1.DeploymentSpec{
|
|
ProgressDeadlineSeconds: canaryDep.Spec.ProgressDeadlineSeconds,
|
|
MinReadySeconds: canaryDep.Spec.MinReadySeconds,
|
|
RevisionHistoryLimit: canaryDep.Spec.RevisionHistoryLimit,
|
|
Replicas: int32p(replicas),
|
|
Strategy: canaryDep.Spec.Strategy,
|
|
Selector: &metav1.LabelSelector{
|
|
MatchLabels: map[string]string{
|
|
label: primaryName,
|
|
},
|
|
},
|
|
Template: corev1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: makePrimaryLabels(canaryDep.Spec.Template.Labels, primaryName, label),
|
|
Annotations: annotations,
|
|
},
|
|
// update spec with the primary secrets and config maps
|
|
Spec: c.configTracker.ApplyPrimaryConfigs(canaryDep.Spec.Template.Spec, configRefs),
|
|
},
|
|
},
|
|
}
|
|
|
|
_, err = c.kubeClient.AppsV1().Deployments(cd.Namespace).Create(primaryDep)
|
|
if err != nil {
|
|
return fmt.Errorf("creating deployment %s.%s failed: %w", primaryDep.Name, cd.Namespace, err)
|
|
}
|
|
|
|
c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).
|
|
Infof("Deployment %s.%s created", primaryDep.GetName(), cd.Namespace)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *DeploymentController) reconcilePrimaryHpa(cd *flaggerv1.Canary, init bool) error {
|
|
primaryName := fmt.Sprintf("%s-primary", cd.Spec.TargetRef.Name)
|
|
hpa, err := c.kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers(cd.Namespace).Get(cd.Spec.AutoscalerRef.Name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("HorizontalPodAutoscaler %s.%s get query error: %w",
|
|
cd.Spec.AutoscalerRef.Name, cd.Namespace, err)
|
|
}
|
|
|
|
hpaSpec := hpav1.HorizontalPodAutoscalerSpec{
|
|
ScaleTargetRef: hpav1.CrossVersionObjectReference{
|
|
Name: primaryName,
|
|
Kind: hpa.Spec.ScaleTargetRef.Kind,
|
|
APIVersion: hpa.Spec.ScaleTargetRef.APIVersion,
|
|
},
|
|
MinReplicas: hpa.Spec.MinReplicas,
|
|
MaxReplicas: hpa.Spec.MaxReplicas,
|
|
Metrics: hpa.Spec.Metrics,
|
|
}
|
|
|
|
primaryHpaName := fmt.Sprintf("%s-primary", cd.Spec.AutoscalerRef.Name)
|
|
primaryHpa, err := c.kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers(cd.Namespace).Get(primaryHpaName, metav1.GetOptions{})
|
|
|
|
// create HPA
|
|
if errors.IsNotFound(err) {
|
|
primaryHpa = &hpav1.HorizontalPodAutoscaler{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: primaryHpaName,
|
|
Namespace: cd.Namespace,
|
|
Labels: hpa.Labels,
|
|
OwnerReferences: []metav1.OwnerReference{
|
|
*metav1.NewControllerRef(cd, schema.GroupVersionKind{
|
|
Group: flaggerv1.SchemeGroupVersion.Group,
|
|
Version: flaggerv1.SchemeGroupVersion.Version,
|
|
Kind: flaggerv1.CanaryKind,
|
|
}),
|
|
},
|
|
},
|
|
Spec: hpaSpec,
|
|
}
|
|
|
|
_, err = c.kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers(cd.Namespace).Create(primaryHpa)
|
|
if err != nil {
|
|
return fmt.Errorf("creating HorizontalPodAutoscaler %s.%s failed: %w",
|
|
primaryHpa.Name, primaryHpa.Namespace, err)
|
|
}
|
|
c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).Infof(
|
|
"HorizontalPodAutoscaler %s.%s created", primaryHpa.GetName(), cd.Namespace)
|
|
return nil
|
|
} else if err != nil {
|
|
return fmt.Errorf("HorizontalPodAutoscaler %s.%s get query failed: %w",
|
|
primaryHpa.Name, primaryHpa.Namespace, err)
|
|
}
|
|
|
|
// update HPA
|
|
if !init && primaryHpa != nil {
|
|
diff := cmp.Diff(hpaSpec.Metrics, primaryHpa.Spec.Metrics)
|
|
if diff != "" || int32Default(hpaSpec.MinReplicas) != int32Default(primaryHpa.Spec.MinReplicas) || hpaSpec.MaxReplicas != primaryHpa.Spec.MaxReplicas {
|
|
fmt.Println(diff, hpaSpec.MinReplicas, primaryHpa.Spec.MinReplicas, hpaSpec.MaxReplicas, primaryHpa.Spec.MaxReplicas)
|
|
hpaClone := primaryHpa.DeepCopy()
|
|
hpaClone.Spec.MaxReplicas = hpaSpec.MaxReplicas
|
|
hpaClone.Spec.MinReplicas = hpaSpec.MinReplicas
|
|
hpaClone.Spec.Metrics = hpaSpec.Metrics
|
|
|
|
_, err := c.kubeClient.AutoscalingV2beta1().HorizontalPodAutoscalers(cd.Namespace).Update(hpaClone)
|
|
if err != nil {
|
|
return fmt.Errorf("updating HorizontalPodAutoscaler %s.%s failed: %w",
|
|
hpaClone.Name, hpaClone.Namespace, err)
|
|
}
|
|
c.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)).
|
|
Infof("HorizontalPodAutoscaler %s.%s updated", primaryHpa.GetName(), cd.Namespace)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// getSelectorLabel returns the selector match label
|
|
func (c *DeploymentController) getSelectorLabel(deployment *appsv1.Deployment) (string, error) {
|
|
for _, l := range c.labels {
|
|
if _, ok := deployment.Spec.Selector.MatchLabels[l]; ok {
|
|
return l, nil
|
|
}
|
|
}
|
|
|
|
return "", fmt.Errorf(
|
|
"deployment %s.%s spec.selector.matchLabels must contain one of %v",
|
|
deployment.Name, deployment.Namespace, c.labels,
|
|
)
|
|
}
|
|
|
|
func (c *DeploymentController) HaveDependenciesChanged(cd *flaggerv1.Canary) (bool, error) {
|
|
return c.configTracker.HasConfigChanged(cd)
|
|
}
|
|
|
|
// Finalize will set the replica count from the primary to the reference instance. This method is used
|
|
// during a delete to attempt to revert the deployment back to the original state. Error is returned if unable
|
|
// update the reference deployment replicas to the primary replicas
|
|
func (c *DeploymentController) Finalize(cd *flaggerv1.Canary) error {
|
|
|
|
primaryName := fmt.Sprintf("%s-primary", cd.Spec.TargetRef.Name)
|
|
|
|
//Get ref deployment
|
|
refDep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(cd.Spec.TargetRef.Name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
//2. Get primary if possible, if not scale from zero
|
|
primaryDep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(primaryName, metav1.GetOptions{})
|
|
if err != nil {
|
|
if errors.IsNotFound(err) {
|
|
if err := c.ScaleFromZero(cd); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
|
|
//3. If both ref and primary present update the replicas of the ref to match the primary
|
|
if refDep.Spec.Replicas != primaryDep.Spec.Replicas {
|
|
//3. Set the replicas value on the original reference deployment
|
|
if err := c.Scale(cd, int32Default(primaryDep.Spec.Replicas)); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|