introduction of finalizer

pkg/apis/flagger/v1beta1/status.go

@@ -57,6 +57,12 @@ const (
 	// CanaryPhaseFailed means the canary analysis failed
 	// and the canary deployment has been scaled to zero
 	CanaryPhaseFailed CanaryPhase = "Failed"
+	// CanaryPhaseTerminating means the canary has been marked
+	// for deletion and in the finalizing state
+	CanaryPhaseTerminating CanaryPhase = "Terminating"
+	// CanaryPhaseTerminated means the canary has been finalized
+	// and successfully deleted
+	CanaryPhaseTerminated CanaryPhase = "Terminated"
 )
 
 // CanaryStatus is used for state persistence (read-only)

pkg/canary/controller.go

@@ -19,4 +19,5 @@ type Controller interface {
 	HaveDependenciesChanged(canary *flaggerv1.Canary) (bool, error)
 	ScaleToZero(canary *flaggerv1.Canary) error
 	ScaleFromZero(canary *flaggerv1.Canary) error
+	Finalize(canary *flaggerv1.Canary) error
 }

pkg/canary/daemonset_controller.go

@@ -296,3 +296,7 @@ func (c *DaemonSetController) getSelectorLabel(daemonSet *appsv1.DaemonSet) (str
 func (c *DaemonSetController) HaveDependenciesChanged(cd *flaggerv1.Canary) (bool, error) {
 	return c.configTracker.HasConfigChanged(cd)
 }
+
+func (c *DaemonSetController) Finalize(cd *flaggerv1.Canary) error  {
+	return nil
+}

pkg/canary/deployment_controller.go

@@ -376,3 +376,40 @@ func (c *DeploymentController) getSelectorLabel(deployment *appsv1.Deployment) (
 func (c *DeploymentController) HaveDependenciesChanged(cd *flaggerv1.Canary) (bool, error) {
 	return c.configTracker.HasConfigChanged(cd)
 }
+
+
+// revertDeployment will set the replica count from the primary to the reference instance.  This method is used
+// during a delete to attempt to revert the deployment back to the original state.  Error is returned if unable
+// update the reference deployment replicas to the primary replicas
+func (c *DeploymentController) Finalize(cd *flaggerv1.Canary) error  {
+
+	//1. Get the Primary deployment if possible
+	primaryName := fmt.Sprintf("%s-primary", cd.Spec.TargetRef.Name)
+	//2. Get the replicas value
+	primaryDep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(primaryName, metav1.GetOptions{})
+	if err != nil {
+		/*if errors.IsNotFound(err) {
+			c.logger.Warnf("deployment %s.%s not found while finalizing", primaryName, cd.Namespace)
+			return nil
+		}*/
+		return err
+	}
+	refDep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(cd.Spec.TargetRef.Name, metav1.GetOptions{})
+	if err != nil {
+		/*if errors.IsNotFound(err) {
+			c.logger.Warnf("deployment %s.%s not found while finalizing", cd.Spec.TargetRef.Name, cd.Namespace)
+			return nil
+		}*/
+		return err
+	}
+
+	if refDep.Spec.Replicas != primaryDep.Spec.Replicas {
+		//3. Set the replicas value on the original reference deployment
+		desiredReplicas := primaryDep.Spec.Replicas
+		if err := c.Scale(cd, int32Default(desiredReplicas)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/canary/service_controller.go

@@ -221,3 +221,7 @@ func (c *ServiceController) IsPrimaryReady(_ *flaggerv1.Canary) error {
 func (c *ServiceController) IsCanaryReady(_ *flaggerv1.Canary) (bool, error) {
 	return true, nil
 }
+
+func (c *ServiceController) Finalize(cd *flaggerv1.Canary) error  {
+	return nil
+}

pkg/controller/finalizer.go

@@ -0,0 +1,221 @@
+package controller
+
+import (
+	"fmt"
+	ex "github.com/pkg/errors"
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
+	"github.com/weaveworks/flagger/pkg/canary"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/retry"
+)
+
+const finalizer = "finalizer.flagger.com"
+
+func (c *Controller) finalize(old interface{}) error {
+	var r *flaggerv1.Canary
+	var ok bool
+
+	//Ensure interface is a canary
+	if r, ok = old.(*flaggerv1.Canary); !ok {
+		c.logger.Warnf("Received unexpected object: %v", old)
+		return nil
+	}
+
+	//Retrieve a controller
+	canaryController := c.canaryFactory.Controller(r.Spec.TargetRef.Kind)
+
+	//Set the status to terminating if not already in that state
+	if r.Status.Phase != flaggerv1.CanaryPhaseTerminating {
+		if err := canaryController.SetStatusPhase(r, flaggerv1.CanaryPhaseTerminating); err != nil {
+			c.logger.Infof("Failed to update status to finalizing %s", err)
+			return err
+		}
+		//record event
+		c.recordEventInfof(r, "Terminating canary %s.%s", r.Name, r.Namespace)
+	}
+
+	err := c.revertTargetRef(canaryController, r)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			//No reason to wait not found
+			c.logger.Warnf("%s.%s failed due to %s not found", r.Name, r.Namespace, r.Kind)
+			return nil
+		}
+		c.logger.Errorf("%s.%s failed due to %s", r.Name, r.Namespace, err)
+		return err
+	} else {
+		//Ensure that targetRef has met a ready state
+		c.logger.Infof("Checking is canary is ready %s.%s", r.Name, r.Namespace)
+		ready, err := canaryController.IsCanaryReady(r)
+		if err != nil && ready {
+			return fmt.Errorf("%s.%s has not reached ready state during finalizing", r.Name, r.Namespace)
+		}
+
+	}
+
+	c.logger.Infof("%s.%s moving forward with router finalizing", r.Name, r.Namespace)
+	//TODO if I can't revert continue on?
+	labelSelector, ports, err := canaryController.GetMetadata(r)
+	if err != nil {
+		c.logger.Errorf("%s.%s failed to get metadata for router finalizing", r.Name, r.Namespace)
+		return err
+	}
+	//Revert the router
+	if err := c.revertRouter(r, labelSelector, ports); err != nil {
+		if errors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+
+	c.logger.Infof("%s.%s moving forward with mesh finalizing", r.Name, r.Namespace)
+	//TODO if I can't revert the mesh continue on?
+	//Revert the Mesh
+	if err := c.revertMesh(r); err != nil {
+		if errors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+
+	c.logger.Infof("Finalization complete for %s.%s", r.Name, r.Namespace)
+
+	return nil
+
+
+}
+
+func (c *Controller) revertTargetRef(ctrl canary.Controller, r *flaggerv1.Canary) error {
+	if err := ctrl.Finalize(r); err != nil {
+		return err
+	}
+	/*if err != nil {
+		if errors.IsNotFound(err) {
+			return false, err
+		}
+		return true, fmt.Errorf("%s.%s failed to revert deployment to original replicas", r.Name, r.Namespace)
+	}*/
+	c.logger.Infof("%s.%s kind %s reverted", r.Name, r.Namespace, r.Spec.TargetRef.Kind)
+	return nil
+}
+
+//revertRouter
+func (c *Controller) revertRouter(r *flaggerv1.Canary, labelSelector string, ports map[string]int32) error {
+	router := c.routerFactory.KubernetesRouter(r.Spec.TargetRef.Kind, labelSelector, map[string]string{}, ports)
+	if err := router.Finalize(r); err != nil {
+		c.logger.Errorf("%s.%s router failed with error %s", r.Name, r.Namespace, err)
+		return err
+	}
+	/*if err != nil {
+		return fmt.Errorf("%s.%s failed to revert service to original state", r.Name, r.Namespace)
+	}*/
+	c.logger.Infof("Service %s.%s reverted", r.Name, r.Namespace)
+	return nil
+}
+
+//revertMesh reverts defined mesh provider based upon the implementation's respective Finalize method.
+//If the Finalize method encounters and error that is returned, else revert is considered successful.
+func (c *Controller) revertMesh(r *flaggerv1.Canary) error {
+	provider := c.meshProvider
+	if r.Spec.Provider != "" {
+		provider = r.Spec.Provider
+	}
+
+	//Establish provider
+	meshRouter := c.routerFactory.MeshRouter(provider)
+
+	//Finalize mesh
+	err := meshRouter.Finalize(r)
+	if err != nil {
+		c.logger.Errorf("%s.%s mesh failed with error %s", r.Name, r.Namespace, err)
+		return err
+	}
+
+	c.logger.Infof("%s.%s mesh provider %s reverted", r.Name, r.Namespace, provider)
+	return nil
+}
+
+//hasFinalizer evaluates the finalizers of a given canary for for existence of a provide finalizer string.
+//It returns a boolean, true if the finalizer is found false otherwise.
+func hasFinalizer(canary *flaggerv1.Canary, finalizerString string) bool {
+	currentFinalizers := canary.ObjectMeta.Finalizers
+
+	for _, f := range currentFinalizers {
+		if f == finalizerString {
+			return true
+		}
+	}
+	return false
+}
+
+//addFinalizer adds a provided finalizer to the specified canary resource.
+//If failures occur the error will be returned otherwise the action is deemed successful
+//and error will be nil.
+func (c *Controller) addFinalizer(canary *flaggerv1.Canary, finalizerString string) error {
+	firstTry := true
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+
+		var selErr error
+		if !firstTry {
+			canary, selErr = c.flaggerClient.FlaggerV1beta1().Canaries(canary.Namespace).Get(canary.GetName(),metav1.GetOptions{})
+			if selErr != nil {
+				return selErr
+			}
+		}
+
+		copy := canary.DeepCopy()
+		copy.ObjectMeta.Finalizers = append(copy.ObjectMeta.Finalizers, finalizerString)
+
+		 _, err = c.flaggerClient.FlaggerV1beta1().Canaries(canary.Namespace).Update(copy)
+
+		firstTry = false
+		return
+	})
+
+	if err != nil {
+		return ex.Wrap(err, "Remove finalizer failed")
+	}
+	return nil
+}
+
+//removeFinalizer removes a provided finalizer to the specified canary resource.
+//If failures occur the error will be returned otherwise the action is deemed successful
+//and error will be nil.
+func (c *Controller) removeFinalizer(canary *flaggerv1.Canary, finalizerString string)  error {
+	firstTry := true
+	err := retry.RetryOnConflict(retry.DefaultBackoff, func() (err error) {
+
+		var selErr error
+		if !firstTry {
+			canary, selErr = c.flaggerClient.FlaggerV1beta1().Canaries(canary.Namespace).Get(canary.GetName(),metav1.GetOptions{})
+			if selErr != nil {
+				return selErr
+			}
+		}
+		copy := canary.DeepCopy()
+
+		newSlice := make([]string, 0)
+		for _, item := range copy.ObjectMeta.Finalizers {
+			if item == finalizerString {
+				continue
+			}
+			newSlice = append(newSlice, item)
+		}
+		if len(newSlice) == 0 {
+			newSlice = nil
+		}
+		copy.ObjectMeta.Finalizers = newSlice
+
+		_, err = c.flaggerClient.FlaggerV1beta1().Canaries(canary.Namespace).Update(copy)
+
+		firstTry = false
+		return
+	})
+
+	if err != nil {
+		return ex.Wrap(err, "Remove finalizer failed")
+	}
+	return nil
+
+}

pkg/controller/finalizer_test.go

@@ -0,0 +1,107 @@
+package controller
+
+import (
+	"fmt"
+	flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1"
+	fakeFlagger "github.com/weaveworks/flagger/pkg/client/clientset/versioned/fake"
+	"k8s.io/apimachinery/pkg/runtime"
+	k8sTesting "k8s.io/client-go/testing"
+
+	"testing"
+)
+
+//Test has finalizers
+func TestFinalizer_hasFinalizer(t *testing.T) {
+
+	withFinalizer := newTestCanary()
+	withFinalizer.Finalizers = append(withFinalizer.Finalizers, finalizer)
+
+	tables := []struct{
+		canary *flaggerv1.Canary
+		result bool
+	}{
+		{newTestCanary(), false},
+		{ withFinalizer, true},
+	}
+
+	for _, table := range tables {
+		isPresent := hasFinalizer(table.canary, finalizer)
+		if isPresent != table.result {
+			t.Errorf("Result of hasFinalizer returned [%t], but expected [%t]", isPresent, table.result)
+		}
+	}
+}
+
+func TestFinalizer_addFinalizer(t *testing.T) {
+
+	mockError := fmt.Errorf("failed to add finalizer to canary %s", "testCanary")
+	cs := fakeFlagger.NewSimpleClientset(newTestCanary())
+	//prepend so it is evaluated over the catch all *
+	cs.PrependReactor("update", "canaries", func(action k8sTesting.Action) (handled bool, ret runtime.Object, err error){
+		return true, nil, mockError
+	})
+	m := Mocks{
+		canary:newTestCanary(),
+		flaggerClient: cs,
+		ctrl:&Controller{flaggerClient:cs},
+	}
+
+
+	tables := []struct{
+		mock Mocks
+		canary *flaggerv1.Canary
+		error error
+	}{
+		{SetupMocks(nil), newTestCanary(), nil},
+		{m, m.canary, mockError},
+	}
+
+	for _, table := range tables {
+		response := table.mock.ctrl.addFinalizer(table.canary, finalizer)
+
+		if table.error != nil && response == nil {
+			t.Errorf("Expected an error from addFinalizer, but wasn't present")
+		} else if table.error == nil && response != nil {
+			t.Errorf("Expected no error from addFinalizer, but returned error %s", response)
+		}
+	}
+
+}
+
+func TestFinalizer_removeFinalizer(t *testing.T) {
+
+	withFinalizer := newTestCanary()
+	withFinalizer.Finalizers = append(withFinalizer.Finalizers, finalizer)
+
+	mockError := fmt.Errorf("failed to add finalizer to canary %s", "testCanary")
+	cs := fakeFlagger.NewSimpleClientset(newTestCanary())
+	//prepend so it is evaluated over the catch all *
+	cs.PrependReactor("update", "canaries", func(action k8sTesting.Action) (handled bool, ret runtime.Object, err error){
+		return true, nil, mockError
+	})
+	m := Mocks{
+		canary:withFinalizer,
+		flaggerClient: cs,
+		ctrl:&Controller{flaggerClient:cs},
+	}
+
+	tables := []struct{
+		mock Mocks
+		canary *flaggerv1.Canary
+		error error
+	}{
+		{SetupMocks(nil), withFinalizer, nil},
+		{m, m.canary, mockError},
+	}
+
+	for _, table := range tables {
+		response := table.mock.ctrl.removeFinalizer(table.canary, finalizer)
+
+		if table.error != nil && response == nil {
+			t.Errorf("Expected an error from addFinalizer, but wasn't present")
+		} else if table.error == nil && response != nil {
+			t.Errorf("Expected no error from addFinalizer, but returned error %s", response)
+		}
+
+	}
+}

pkg/router/kubernetes.go

@@ -10,4 +10,6 @@ type KubernetesRouter interface {
 	Initialize(canary *flaggerv1.Canary) error
 	// Reconcile creates or updates the main service
 	Reconcile(canary *flaggerv1.Canary) error
+	// Revert router
+	Finalize(canary *flaggerv1.Canary) error
 }

pkg/router/kubernetes_default.go

@@ -162,3 +162,41 @@ func (c *KubernetesDefaultRouter) reconcileService(canary *flaggerv1.Canary, nam
 
 	return nil
 }
+
+func (c *KubernetesDeploymentRouter) Finalize(canary *flaggerv1.Canary) error {
+	apexName, _, _ := canary.GetServiceNames()
+
+	svc, err := c.kubeClient.CoreV1().Services(canary.Namespace).Get(apexName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	if hasCanaryOwnerRef, isOwned := c.isOwnedByCanary(svc, canary.Name); !hasCanaryOwnerRef && !isOwned {
+		err = c.reconcileService(canary, apexName, canary.Spec.TargetRef.Name)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+//isOwnedByCanary evaluates if an object contains an OwnerReference declaration, that is of kind Canary and
+//has the same ref name as the Canary under evaluation.  It returns two bool the first returns true if
+//an OwnerReference is present and the second, returns if it is owned by the supplied name.
+func (c KubernetesDeploymentRouter) isOwnedByCanary(obj interface{}, name string) (bool, bool) {
+	var object metav1.Object
+	var ok bool
+	if object, ok = obj.(metav1.Object); ok {
+		if ownerRef := metav1.GetControllerOf(object); ownerRef != nil {
+			if ownerRef.Kind == flaggerv1.CanaryKind {
+				//And the name exists return true
+				if name == ownerRef.Name {
+					return true, true
+				}
+				return true, false
+			}
+		}
+	}
+	return false, false
+}

pkg/router/router.go

@@ -6,4 +6,5 @@ type Interface interface {
 	Reconcile(canary *flaggerv1.Canary) error
 	SetRoutes(canary *flaggerv1.Canary, primaryWeight int, canaryWeight int, mirrored bool) error
 	GetRoutes(canary *flaggerv1.Canary) (primaryWeight int, canaryWeight int, mirrored bool, err error)
+	Finalize(canary *flaggerv1.Canary) error
 }