Add A/B testing scheduling unit tests

2026-04-15 06:57:34 +00:00 · 2019-03-08 13:06:39 +02:00
parent 780ba82385
commit e80a3d3232
3 changed files with 140 additions and 15 deletions
@@ -2,6 +2,8 @@ package controller

 import (
 	"github.com/stefanprodan/flagger/pkg/apis/flagger/v1alpha3"
+	istiov1alpha1 "github.com/stefanprodan/flagger/pkg/apis/istio/common/v1alpha1"
+	istiov1alpha3 "github.com/stefanprodan/flagger/pkg/apis/istio/v1alpha3"
 	clientset "github.com/stefanprodan/flagger/pkg/client/clientset/versioned"
 	fakeFlagger "github.com/stefanprodan/flagger/pkg/client/clientset/versioned/fake"
 	informers "github.com/stefanprodan/flagger/pkg/client/informers/externalversions"
@@ -38,9 +40,12 @@ type Mocks struct {
 	router        router.Interface
 }

-func SetupMocks() Mocks {
+func SetupMocks(abtest bool) Mocks {
 	// init canary
 	canary := newTestCanary()
+	if abtest {
+		canary = newTestCanaryAB()
+	}
 	flaggerClient := fakeFlagger.NewSimpleClientset(canary)

 	// init kube clientset and register mock objects
@@ -261,6 +266,55 @@ func newTestCanary() *v1alpha3.Canary {
 	return cd
 }

+func newTestCanaryAB() *v1alpha3.Canary {
+	cd := &v1alpha3.Canary{
+		TypeMeta: metav1.TypeMeta{APIVersion: v1alpha3.SchemeGroupVersion.String()},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "default",
+			Name:      "podinfo",
+		},
+		Spec: v1alpha3.CanarySpec{
+			TargetRef: hpav1.CrossVersionObjectReference{
+				Name:       "podinfo",
+				APIVersion: "apps/v1",
+				Kind:       "Deployment",
+			},
+			AutoscalerRef: &hpav1.CrossVersionObjectReference{
+				Name:       "podinfo",
+				APIVersion: "autoscaling/v2beta1",
+				Kind:       "HorizontalPodAutoscaler",
+			}, Service: v1alpha3.CanaryService{
+				Port: 9898,
+			}, CanaryAnalysis: v1alpha3.CanaryAnalysis{
+				Threshold:  10,
+				Iterations: 10,
+				Match: []istiov1alpha3.HTTPMatchRequest{
+					{
+						Headers: map[string]istiov1alpha1.StringMatch{
+							"x-user-type": {
+								Exact: "test",
+							},
+						},
+					},
+				},
+				Metrics: []v1alpha3.CanaryMetric{
+					{
+						Name:      "istio_requests_total",
+						Threshold: 99,
+						Interval:  "1m",
+					},
+					{
+						Name:      "istio_request_duration_seconds_bucket",
+						Threshold: 500,
+						Interval:  "1m",
+					},
+				},
+			},
+		},
+	}
+	return cd
+}
+
 func newTestDeployment() *appsv1.Deployment {
 	d := &appsv1.Deployment{
 		TypeMeta: metav1.TypeMeta{APIVersion: appsv1.SchemeGroupVersion.String()},
@@ -8,7 +8,7 @@ import (
 )

 func TestCanaryDeployer_Sync(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := SetupMocks(false)
 	err := mocks.deployer.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -94,7 +94,7 @@ func TestCanaryDeployer_Sync(t *testing.T) {
 }

 func TestCanaryDeployer_IsNewSpec(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := SetupMocks(false)
 	err := mocks.deployer.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -117,7 +117,7 @@ func TestCanaryDeployer_IsNewSpec(t *testing.T) {
 }

 func TestCanaryDeployer_Promote(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := SetupMocks(false)
 	err := mocks.deployer.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -162,7 +162,7 @@ func TestCanaryDeployer_Promote(t *testing.T) {
 }

 func TestCanaryDeployer_IsReady(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := SetupMocks(false)
 	err := mocks.deployer.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -180,7 +180,7 @@ func TestCanaryDeployer_IsReady(t *testing.T) {
 }

 func TestCanaryDeployer_SetFailedChecks(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := SetupMocks(false)
 	err := mocks.deployer.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -202,7 +202,7 @@ func TestCanaryDeployer_SetFailedChecks(t *testing.T) {
 }

 func TestCanaryDeployer_SetState(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := SetupMocks(false)
 	err := mocks.deployer.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -224,7 +224,7 @@ func TestCanaryDeployer_SetState(t *testing.T) {
 }

 func TestCanaryDeployer_SyncStatus(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := SetupMocks(false)
 	err := mocks.deployer.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -263,7 +263,7 @@ func TestCanaryDeployer_SyncStatus(t *testing.T) {
 }

 func TestCanaryDeployer_Scale(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := SetupMocks(false)
 	err := mocks.deployer.Sync(mocks.canary)
 	if err != nil {
 		t.Fatal(err.Error())
@@ -7,7 +7,7 @@ import (
 )

 func TestScheduler_Init(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := SetupMocks(false)
 	mocks.ctrl.advanceCanary("podinfo", "default", false)

 	_, err := mocks.kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
@@ -17,7 +17,7 @@ func TestScheduler_Init(t *testing.T) {
 }

 func TestScheduler_NewRevision(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := SetupMocks(false)
 	mocks.ctrl.advanceCanary("podinfo", "default", false)

 	// update
@@ -41,7 +41,7 @@ func TestScheduler_NewRevision(t *testing.T) {
 }

 func TestScheduler_Rollback(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := SetupMocks(false)
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", true)

@@ -65,7 +65,7 @@ func TestScheduler_Rollback(t *testing.T) {
 }

 func TestScheduler_SkipAnalysis(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := SetupMocks(false)
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", false)

@@ -106,7 +106,7 @@ func TestScheduler_SkipAnalysis(t *testing.T) {
 }

 func TestScheduler_NewRevisionReset(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := SetupMocks(false)
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", false)

@@ -160,7 +160,7 @@ func TestScheduler_NewRevisionReset(t *testing.T) {
 }

 func TestScheduler_Promotion(t *testing.T) {
-	mocks := SetupMocks()
+	mocks := SetupMocks(false)
 	// init
 	mocks.ctrl.advanceCanary("podinfo", "default", false)

@@ -258,3 +258,74 @@ func TestScheduler_Promotion(t *testing.T) {
 		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanarySucceeded)
 	}
 }
+
+func TestScheduler_ABTesting(t *testing.T) {
+	mocks := SetupMocks(true)
+	// init
+	mocks.ctrl.advanceCanary("podinfo", "default", false)
+
+	// update
+	dep2 := newTestDeploymentV2()
+	_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(dep2)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	// detect pod spec changes
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	// advance
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	// check if traffic is routed to canary
+	primaryWeight, canaryWeight, err := mocks.router.GetRoutes(mocks.canary)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if primaryWeight != 0 {
+		t.Errorf("Got primary route %v wanted %v", primaryWeight, 0)
+	}
+
+	if canaryWeight != 100 {
+		t.Errorf("Got canary route %v wanted %v", canaryWeight, 100)
+	}
+
+	cd, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	// set max iterations
+	if err := mocks.deployer.SetStatusIterations(cd, 10); err != nil {
+		t.Fatal(err.Error())
+	}
+
+	// promote
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	// check if the container image tag was updated
+	primaryDep, err := mocks.kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	primaryImage := primaryDep.Spec.Template.Spec.Containers[0].Image
+	canaryImage := dep2.Spec.Template.Spec.Containers[0].Image
+	if primaryImage != canaryImage {
+		t.Errorf("Got primary image %v wanted %v", primaryImage, canaryImage)
+	}
+
+	// shutdown canary
+	mocks.ctrl.advanceCanary("podinfo", "default", true)
+
+	// check rollout status
+	c, err := mocks.flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	if c.Status.Phase != v1alpha3.CanarySucceeded {
+		t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanarySucceeded)
+	}
+}