From c9a07cec876ea75e2a2d4b0b5ab5dc4cca21d2a0 Mon Sep 17 00:00:00 2001 From: Tanner Altares Date: Mon, 16 Mar 2020 17:02:55 -0500 Subject: [PATCH] add e2e tests istio add e2e tests istio clean up comment from review add e2e tests istio clean up comment from review clean up logging statement add e2e tests istio clean up comment from review clean up logging statement add log statement on e2e iteration add e2e tests istio clean up comment from review clean up logging statement add log statement on e2e iteration extend timeout for finalizing add e2e tests istio clean up comment from review clean up logging statement add log statement on e2e iteration extend timeout for finalizing add phase to kustomize crd add e2e tests istio clean up comment from review clean up logging statement add log statement on e2e iteration extend timeout for finalizing add phase to kustomize crd revert timeout on circleci vs and svc checks for istio e2e tests fix fmt errors and tests add get statement in e2e test add get statement in e2e test add namespace to e2e use only selector for service revert --- kustomize/base/flagger/crd.yaml | 2 + pkg/apis/flagger/v1beta1/canary.go | 2 +- pkg/canary/deployment_controller.go | 2 +- pkg/controller/controller.go | 2 +- pkg/controller/finalizer.go | 5 +- pkg/controller/finalizer_test.go | 9 +- pkg/router/istio.go | 2 +- pkg/router/kubernetes_default.go | 2 +- test/e2e-istio-tests.sh | 231 ++++++++++++++++++++++++++++ 9 files changed, 249 insertions(+), 8 deletions(-) diff --git a/kustomize/base/flagger/crd.yaml b/kustomize/base/flagger/crd.yaml index 5d6ef4db..a39bb578 100644 --- a/kustomize/base/flagger/crd.yaml +++ b/kustomize/base/flagger/crd.yaml @@ -653,6 +653,8 @@ spec: - Finalising - Succeeded - Failed + - Terminating + - Terminated canaryWeight: description: Traffic weight percentage routed to canary type: number diff --git a/pkg/apis/flagger/v1beta1/canary.go b/pkg/apis/flagger/v1beta1/canary.go index 24a643df..25bafe0d 100644 --- a/pkg/apis/flagger/v1beta1/canary.go +++ b/pkg/apis/flagger/v1beta1/canary.go @@ -302,7 +302,7 @@ type CanaryWebhook struct { URL string `json:"url"` // Request timeout for this webhook - Timeout string `json:"timeout"` + Timeout string `json:"timeout,omitempty"` // Metadata (key-value pairs) for this webhook // +optional diff --git a/pkg/canary/deployment_controller.go b/pkg/canary/deployment_controller.go index 9eae0cc7..746c951c 100644 --- a/pkg/canary/deployment_controller.go +++ b/pkg/canary/deployment_controller.go @@ -398,7 +398,7 @@ func (c *DeploymentController) HaveDependenciesChanged(cd *flaggerv1.Canary) (bo return c.configTracker.HasConfigChanged(cd) } -// revertDeployment will set the replica count from the primary to the reference instance. This method is used +// Finalize will set the replica count from the primary to the reference instance. This method is used // during a delete to attempt to revert the deployment back to the original state. Error is returned if unable // update the reference deployment replicas to the primary replicas func (c *DeploymentController) Finalize(cd *flaggerv1.Canary) error { diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 37a2779a..9708909b 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -140,7 +140,7 @@ func NewController( ctrl.logger.Infof("%s.%s opting out, deleting finalizers", newCanary.Name, newCanary.Namespace) err := ctrl.removeFinalizer(&newCanary, finalizer) if err != nil { - ctrl.logger.Warnf("Failed to finalizers for %s.%s", oldCanary.Name, oldCanary.Namespace) + ctrl.logger.Warnf("Failed to remove finalizers for %s.%s", oldCanary.Name, oldCanary.Namespace) return } } diff --git a/pkg/controller/finalizer.go b/pkg/controller/finalizer.go index 49301ba8..d3f76bf4 100644 --- a/pkg/controller/finalizer.go +++ b/pkg/controller/finalizer.go @@ -43,7 +43,7 @@ func (c *Controller) finalize(old interface{}) error { c.logger.Warnf("%s.%s failed due to %s not found", r.Name, r.Namespace, r.Spec.TargetRef.Kind) return nil } - c.logger.Errorf("%s.%s failed due to %s", r.Name, r.Namespace, err) + c.logger.Debugf("%s.%s failed due to %s", r.Name, r.Namespace, err) return err } else { //Ensure that targetRef has met a ready state @@ -163,7 +163,8 @@ func (c *Controller) addFinalizer(canary *flaggerv1.Canary, finalizerString stri }) if err != nil { - return ex.Wrap(err, "Remove finalizer failed") + c.logger.Errorf("Failed to add finalizer %s", err) + return ex.Wrap(err, "Add finalizer failed") } return nil } diff --git a/pkg/controller/finalizer_test.go b/pkg/controller/finalizer_test.go index f5861934..02d18907 100644 --- a/pkg/controller/finalizer_test.go +++ b/pkg/controller/finalizer_test.go @@ -5,6 +5,7 @@ import ( flaggerv1 "github.com/weaveworks/flagger/pkg/apis/flagger/v1beta1" fakeFlagger "github.com/weaveworks/flagger/pkg/client/clientset/versioned/fake" + "github.com/weaveworks/flagger/pkg/logger" "k8s.io/apimachinery/pkg/runtime" k8sTesting "k8s.io/client-go/testing" @@ -41,10 +42,16 @@ func TestFinalizer_addFinalizer(t *testing.T) { cs.PrependReactor("update", "canaries", func(action k8sTesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, mockError }) + + logger, _ := logger.NewLogger("debug") m := fixture{ canary: newDeploymentTestCanary(), flaggerClient: cs, - ctrl: &Controller{flaggerClient: cs}, + ctrl: &Controller{ + flaggerClient: cs, + logger: logger, + }, + logger: logger, } tables := []struct { diff --git a/pkg/router/istio.go b/pkg/router/istio.go index 613d4513..54718319 100644 --- a/pkg/router/istio.go +++ b/pkg/router/istio.go @@ -214,7 +214,7 @@ func (ir *IstioRouter) reconcileVirtualService(canary *flaggerv1.Canary) error { //If annotation kubectl.kubernetes.io/last-applied-configuration is present no need to duplicate //serialization. If not present store the serialized object in annotation - //flagger.kubernetes.io/original-configuration + //flagger.kubernetes.app/original-configuration if _, ok := vtClone.Annotations[kubectlAnnotation]; !ok { b, err := json.Marshal(virtualService.Spec) if err != nil { diff --git a/pkg/router/kubernetes_default.go b/pkg/router/kubernetes_default.go index 4c23f0fb..80ecb58a 100644 --- a/pkg/router/kubernetes_default.go +++ b/pkg/router/kubernetes_default.go @@ -184,7 +184,7 @@ func (c *KubernetesDefaultRouter) Finalize(canary *flaggerv1.Canary) error { svc.Name, svc.Namespace, kubectlAnnotation) } clone := svc.DeepCopy() - clone.Spec = storedSvc.Spec + clone.Spec.Selector = storedSvc.Spec.Selector _, err = c.kubeClient.CoreV1().Services(canary.Namespace).Update(clone) if err != nil { diff --git a/test/e2e-istio-tests.sh b/test/e2e-istio-tests.sh index 31600601..53231439 100755 --- a/test/e2e-istio-tests.sh +++ b/test/e2e-istio-tests.sh @@ -328,6 +328,237 @@ done echo '✔ A/B testing promotion test passed' +cat <>> Waiting for finalizers to be present' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl get canary podinfo -n test -o jsonpath='{.metadata.finalizers}' | grep "finalizer.flagger.app" && ok=true || ok=false + sleep 10 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe canary/podinfo + echo "No more retries left" + exit 1 + fi +done + +kubectl delete canary podinfo -n test + +echo '>>> Waiting for primary to revert' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl get deployment podinfo -n test -o jsonpath='{.spec.replicas}' | grep 1 && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 10 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe canary/podinfo + echo "No more retries left" + exit 1 + fi +done +echo '✔ Delete testing passed' + + + +cat <>> Waiting for canary to initialize' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n istio-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +kubectl delete canary podinfo -n test + +echo '>>> Waiting for revert' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl get svc/podinfo vs/podinfo -n test -o jsonpath="{range .items[*]}{.metadata.name}{'\n'}{end}" | wc -l | grep 2 && ok=true || ok=false + sleep 10 + kubectl -n istio-system logs deployment/flagger --tail 10 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n test describe canary/podinfo + kubectl -n test describe svc/podinfo + kubectl -n test describe vs/podinfo + echo "No more retries left" + exit 1 + fi +done +echo '✔ Revert testing passed' + + kubectl -n istio-system logs deployment/flagger echo '✔ All tests passed'