github/flagger
1
0
Fork 0
mirror of https://github.com/fluxcd/flagger.git synced 2026-04-15 06:57:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add tests for analysis restart and canary promotion

Browse Source
This commit is contained in:
stefanprodan
2019-01-18 11:05:40 +02:00
parent aff8b117d4
commit 6d8a7343b7
4 changed files with 234 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/deployer.go
+3 -7
View File
@@ -82,15 +82,11 @@ func (c *CanaryDeployer) IsPrimaryReady(cd *flaggerv1.Canary) (bool, error) {
retriable, err := c.isDeploymentReady(primary, cd.GetProgressDeadlineSeconds())
if err != nil {
if retriable {
return retriable, fmt.Errorf("Halt %s.%s advancement %s", cd.Name, cd.Namespace, err.Error())
} else {
return retriable, err
}
return retriable, fmt.Errorf("Halt advancement %s.%s %s", primaryName, cd.Namespace, err.Error())
}
if primary.Spec.Replicas == int32p(0) {
return true, fmt.Errorf("halt %s.%s advancement primary deployment is scaled to zero",
return true, fmt.Errorf("Halt %s.%s advancement primary deployment is scaled to zero",
cd.Name, cd.Namespace)
}
return true, nil
@@ -112,7 +108,7 @@ func (c *CanaryDeployer) IsCanaryReady(cd *flaggerv1.Canary) (bool, error) {
retriable, err := c.isDeploymentReady(canary, cd.GetProgressDeadlineSeconds())
if err != nil {
if retriable {
return retriable, fmt.Errorf("Halt %s.%s advancement %s", cd.Name, cd.Namespace, err.Error())
return retriable, fmt.Errorf("Halt advancement %s.%s %s", targetName, cd.Namespace, err.Error())
} else {
return retriable, fmt.Errorf("deployment does not have minimum availability for more than %vs",
cd.GetProgressDeadlineSeconds())
pkg/controller/job.go
+4 -3
View File
@@ -6,7 +6,8 @@ import "time"
type CanaryJob struct {
Name string
Namespace string
function func(name string, namespace string)
SkipTests bool
function func(name string, namespace string, skipTests bool)
done chan bool
ticker *time.Ticker
}
@@ -15,11 +16,11 @@ type CanaryJob struct {
func (j CanaryJob) Start() {
go func() {
// run the infra bootstrap on job creation
j.function(j.Name, j.Namespace)
j.function(j.Name, j.Namespace, j.SkipTests)
for {
select {
case <-j.ticker.C:
j.function(j.Name, j.Namespace)
j.function(j.Name, j.Namespace, j.SkipTests)
case <-j.done:
return
}
pkg/controller/scheduler.go
+13 -8
View File
@@ -69,7 +69,7 @@ func (c *Controller) scheduleCanaries() {
}
}
func (c *Controller) advanceCanary(name string, namespace string) {
func (c *Controller) advanceCanary(name string, namespace string, skipLivenessChecks bool) {
begin := time.Now()
// check if the canary exists
cd, err := c.flaggerClient.FlaggerV1alpha3().Canaries(namespace).Get(name, v1.GetOptions{})
@@ -104,9 +104,11 @@ func (c *Controller) advanceCanary(name string, namespace string) {
}
// check primary deployment status
if _, err := c.deployer.IsPrimaryReady(cd); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
if !skipLivenessChecks {
if _, err := c.deployer.IsPrimaryReady(cd); err != nil {
c.recordEventWarningf(cd, "%v", err)
return
}
}
// check if virtual service exists
@@ -155,10 +157,13 @@ func (c *Controller) advanceCanary(name string, namespace string) {
}()
// check canary deployment status
retriable, err := c.deployer.IsCanaryReady(cd)
if err != nil && retriable {
c.recordEventWarningf(cd, "%v", err)
return
var retriable = true
if !skipLivenessChecks {
retriable, err = c.deployer.IsCanaryReady(cd)
if err != nil && retriable {
c.recordEventWarningf(cd, "%v", err)
return
}
}
// check if the number of failed checks reached the threshold
pkg/controller/scheduler_test.go
+214 -5
View File
@@ -67,7 +67,7 @@ func TestScheduler_Init(t *testing.T) {
}
ctrl.flaggerSynced = alwaysReady
ctrl.advanceCanary("podinfo", "default")
ctrl.advanceCanary("podinfo", "default", false)
_, err := kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
if err != nil {
@@ -122,7 +122,7 @@ func TestScheduler_NewRevision(t *testing.T) {
ctrl.flaggerSynced = alwaysReady
// init
ctrl.advanceCanary("podinfo", "default")
ctrl.advanceCanary("podinfo", "default", false)
// update
dep2 := newTestDeploymentUpdated()
@@ -132,7 +132,7 @@ func TestScheduler_NewRevision(t *testing.T) {
}
// detect changes
ctrl.advanceCanary("podinfo", "default")
ctrl.advanceCanary("podinfo", "default", false)
c, err := kubeClient.AppsV1().Deployments("default").Get("podinfo", metav1.GetOptions{})
if err != nil {
@@ -191,7 +191,7 @@ func TestScheduler_Rollback(t *testing.T) {
ctrl.flaggerSynced = alwaysReady
// init
ctrl.advanceCanary("podinfo", "default")
ctrl.advanceCanary("podinfo", "default", true)
// update failed checks to max
err := deployer.SyncStatus(canary, v1alpha3.CanaryStatus{Phase: v1alpha3.CanaryProgressing, FailedChecks: 11})
@@ -200,7 +200,7 @@ func TestScheduler_Rollback(t *testing.T) {
}
// detect changes
ctrl.advanceCanary("podinfo", "default")
ctrl.advanceCanary("podinfo", "default", true)
c, err := flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
if err != nil {
@@ -211,3 +211,212 @@ func TestScheduler_Rollback(t *testing.T) {
t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanaryFailed)
}
}
func TestScheduler_NewRevisionReset(t *testing.T) {
canary := newTestCanary()
dep := newTestDeployment()
hpa := newTestHPA()
flaggerClient := fakeFlagger.NewSimpleClientset(canary)
kubeClient := fake.NewSimpleClientset(dep, hpa)
istioClient := fakeIstio.NewSimpleClientset()
logger, _ := logging.NewLogger("debug")
deployer := CanaryDeployer{
flaggerClient: flaggerClient,
kubeClient: kubeClient,
logger: logger,
}
router := CanaryRouter{
flaggerClient: flaggerClient,
kubeClient: kubeClient,
istioClient: istioClient,
logger: logger,
}
observer := CanaryObserver{
metricsServer: "fake",
}
flaggerInformerFactory := informers.NewSharedInformerFactory(flaggerClient, noResyncPeriodFunc())
flaggerInformer := flaggerInformerFactory.Flagger().V1alpha3().Canaries()
ctrl := &Controller{
kubeClient: kubeClient,
istioClient: istioClient,
flaggerClient: flaggerClient,
flaggerLister: flaggerInformer.Lister(),
flaggerSynced: flaggerInformer.Informer().HasSynced,
workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), controllerAgentName),
eventRecorder: &record.FakeRecorder{},
logger: logger,
canaries: new(sync.Map),
flaggerWindow: time.Second,
deployer: deployer,
router: router,
observer: observer,
recorder: NewCanaryRecorder(false),
}
ctrl.flaggerSynced = alwaysReady
// init
ctrl.advanceCanary("podinfo", "default", false)
// first update
dep2 := newTestDeploymentUpdated()
_, err := kubeClient.AppsV1().Deployments("default").Update(dep2)
if err != nil {
t.Fatal(err.Error())
}
// detect changes
ctrl.advanceCanary("podinfo", "default", true)
// advance
ctrl.advanceCanary("podinfo", "default", true)
primaryRoute, canaryRoute, err := router.GetRoutes(canary)
if err != nil {
t.Fatal(err.Error())
}
if primaryRoute.Weight != 90 {
t.Errorf("Got primary route %v wanted %v", primaryRoute.Weight, 90)
}
if canaryRoute.Weight != 10 {
t.Errorf("Got canary route %v wanted %v", canaryRoute.Weight, 10)
}
// second update
dep2.Spec.Template.Spec.ServiceAccountName = "test"
_, err = kubeClient.AppsV1().Deployments("default").Update(dep2)
if err != nil {
t.Fatal(err.Error())
}
// detect changes
ctrl.advanceCanary("podinfo", "default", true)
primaryRoute, canaryRoute, err = router.GetRoutes(canary)
if err != nil {
t.Fatal(err.Error())
}
if primaryRoute.Weight != 100 {
t.Errorf("Got primary route %v wanted %v", primaryRoute.Weight, 100)
}
if canaryRoute.Weight != 0 {
t.Errorf("Got canary route %v wanted %v", canaryRoute.Weight, 0)
}
}
func TestScheduler_Promotion(t *testing.T) {
canary := newTestCanary()
dep := newTestDeployment()
hpa := newTestHPA()
flaggerClient := fakeFlagger.NewSimpleClientset(canary)
kubeClient := fake.NewSimpleClientset(dep, hpa)
istioClient := fakeIstio.NewSimpleClientset()
logger, _ := logging.NewLogger("debug")
deployer := CanaryDeployer{
flaggerClient: flaggerClient,
kubeClient: kubeClient,
logger: logger,
}
router := CanaryRouter{
flaggerClient: flaggerClient,
kubeClient: kubeClient,
istioClient: istioClient,
logger: logger,
}
observer := CanaryObserver{
metricsServer: "fake",
}
flaggerInformerFactory := informers.NewSharedInformerFactory(flaggerClient, noResyncPeriodFunc())
flaggerInformer := flaggerInformerFactory.Flagger().V1alpha3().Canaries()
ctrl := &Controller{
kubeClient: kubeClient,
istioClient: istioClient,
flaggerClient: flaggerClient,
flaggerLister: flaggerInformer.Lister(),
flaggerSynced: flaggerInformer.Informer().HasSynced,
workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), controllerAgentName),
eventRecorder: &record.FakeRecorder{},
logger: logger,
canaries: new(sync.Map),
flaggerWindow: time.Second,
deployer: deployer,
router: router,
observer: observer,
recorder: NewCanaryRecorder(false),
}
ctrl.flaggerSynced = alwaysReady
// init
ctrl.advanceCanary("podinfo", "default", false)
// update
dep2 := newTestDeploymentUpdated()
_, err := kubeClient.AppsV1().Deployments("default").Update(dep2)
if err != nil {
t.Fatal(err.Error())
}
// detect changes
ctrl.advanceCanary("podinfo", "default", true)
primaryRoute, canaryRoute, err := router.GetRoutes(canary)
if err != nil {
t.Fatal(err.Error())
}
primaryRoute.Weight = 60
canaryRoute.Weight = 40
err = ctrl.router.SetRoutes(canary, primaryRoute, canaryRoute)
if err != nil {
t.Fatal(err.Error())
}
// advance
ctrl.advanceCanary("podinfo", "default", true)
// promote
ctrl.advanceCanary("podinfo", "default", true)
primaryRoute, canaryRoute, err = router.GetRoutes(canary)
if err != nil {
t.Fatal(err.Error())
}
if primaryRoute.Weight != 100 {
t.Errorf("Got primary route %v wanted %v", primaryRoute.Weight, 100)
}
if canaryRoute.Weight != 0 {
t.Errorf("Got canary route %v wanted %v", canaryRoute.Weight, 0)
}
primaryDep, err := kubeClient.AppsV1().Deployments("default").Get("podinfo-primary", metav1.GetOptions{})
if err != nil {
t.Fatal(err.Error())
}
primaryImage := primaryDep.Spec.Template.Spec.Containers[0].Image
canaryImage := dep2.Spec.Template.Spec.Containers[0].Image
if primaryImage != canaryImage {
t.Errorf("Got primary image %v wanted %v", primaryImage, canaryImage)
}
c, err := flaggerClient.FlaggerV1alpha3().Canaries("default").Get("podinfo", metav1.GetOptions{})
if err != nil {
t.Fatal(err.Error())
}
if c.Status.Phase != v1alpha3.CanarySucceeded {
t.Errorf("Got canary state %v wanted %v", c.Status.Phase, v1alpha3.CanarySucceeded)
}
}