mirror of
https://github.com/fluxcd/flagger.git
synced 2026-09-05 02:47:17 +00:00
Pedram Pourmohammad
e461dd9b20
On a failed promotion the canary keeps serving, but the traffic may already have been shifted to the primary by runPromotionTrafficShift before it started failing. Route all traffic back to the canary and report the matching canary weight instead of zeroing it. Addresses review feedback on #1931. Signed-off-by: Pedram Pourmohammad <eragon.pedy@gmail.com>
776 lines
26 KiB
Go
776 lines
26 KiB
Go
/*
|
|
Copyright 2020 The Flux authors
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package controller
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
|
|
flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
|
|
"github.com/fluxcd/flagger/pkg/notifier"
|
|
)
|
|
|
|
func TestScheduler_DeploymentInit(t *testing.T) {
|
|
mocks := newDeploymentFixture(nil)
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
_, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
func TestScheduler_DeploymentNewRevision(t *testing.T) {
|
|
mocks := newDeploymentFixture(nil)
|
|
|
|
// initializing ...
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// make primary ready
|
|
mocks.makePrimaryReady(t)
|
|
|
|
// initialization done
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// check if ScaleToZero was performed
|
|
dp, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, int32(0), *dp.Spec.Replicas)
|
|
|
|
// update
|
|
dep2 := newDeploymentTestDeploymentV2()
|
|
_, err = mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// detect changes
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
c, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, int32(1), *c.Spec.Replicas)
|
|
}
|
|
|
|
func TestScheduler_DeploymentRollback(t *testing.T) {
|
|
mocks := newDeploymentFixture(nil)
|
|
// initializing
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// make primary ready
|
|
mocks.makePrimaryReady(t)
|
|
|
|
// initialized
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// update failed checks to max
|
|
err := mocks.deployer.SyncStatus(mocks.canary, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseProgressing, FailedChecks: 10})
|
|
require.NoError(t, err)
|
|
|
|
// set a metric check to fail
|
|
c, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
cd := c.DeepCopy()
|
|
cd.Spec.Analysis.Metrics = append(c.Spec.Analysis.Metrics, flaggerv1.CanaryMetric{
|
|
Name: "fail",
|
|
Interval: "1m",
|
|
ThresholdRange: &flaggerv1.CanaryThresholdRange{
|
|
Min: toFloatPtr(0),
|
|
Max: toFloatPtr(50),
|
|
},
|
|
Query: "fail",
|
|
})
|
|
_, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// run metric checks
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// finalise analysis
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// check status
|
|
c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
|
|
assert.Equal(t, flaggerv1.CanaryPhaseFailed, c.Status.Phase)
|
|
}
|
|
|
|
// when the primary fails to become ready during promotion, the healthy canary
|
|
// must be kept instead of rolled back to the broken primary (#1898)
|
|
func TestScheduler_DeploymentPromotionPrimaryNotReady(t *testing.T) {
|
|
mocks := newDeploymentFixture(nil)
|
|
|
|
// initializing
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
mocks.makePrimaryReady(t)
|
|
|
|
// initialized
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// update
|
|
dep2 := newDeploymentTestDeploymentV2()
|
|
_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// detect changes -> progressing, canary scaled up
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
mocks.makeCanaryReady(t)
|
|
require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing))
|
|
|
|
canaryDep, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
require.NotNil(t, canaryDep.Spec.Replicas)
|
|
canaryReplicas := *canaryDep.Spec.Replicas
|
|
require.Greater(t, canaryReplicas, int32(0))
|
|
|
|
// simulate: analysis succeeded, spec promoted to primary, now finishing promotion
|
|
cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
err = mocks.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhasePromoting)
|
|
require.NoError(t, err)
|
|
|
|
// known routing state at promotion time (split between primary and canary)
|
|
require.NoError(t, mocks.router.SetRoutes(mocks.canary, 50, 50, false))
|
|
|
|
// the promoted primary fails to roll out
|
|
mocks.makePrimaryNotReady(t)
|
|
|
|
// advance: Flagger observes the primary is stuck
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// the canary must NOT be scaled to zero - it is the only healthy copy serving traffic
|
|
canaryDep, err = mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
require.NotNil(t, canaryDep.Spec.Replicas)
|
|
assert.Equal(t, canaryReplicas, *canaryDep.Spec.Replicas, "canary must not be scaled to zero when promotion fails")
|
|
|
|
// traffic must be routed to the healthy canary, off the broken primary
|
|
primaryWeight, canaryWeight, _, err := mocks.router.GetRoutes(mocks.canary)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 0, primaryWeight, "no traffic must remain on the unhealthy primary")
|
|
assert.Equal(t, 100, canaryWeight, "all traffic must be routed to the healthy canary")
|
|
|
|
// the rollout is reported as failed so it stops advancing and alerts
|
|
c, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, flaggerv1.CanaryPhaseFailed, c.Status.Phase)
|
|
assert.Equal(t, 100, c.Status.CanaryWeight, "reported canary weight must match the traffic on the canary")
|
|
}
|
|
|
|
// when the primary fails after promotion traffic has already been shifted to it
|
|
// (Finalising phase), traffic must be routed back to the healthy canary (#1898)
|
|
func TestScheduler_DeploymentPromotionFailedAfterTrafficShift(t *testing.T) {
|
|
mocks := newDeploymentFixture(nil)
|
|
|
|
// initializing
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
mocks.makePrimaryReady(t)
|
|
|
|
// initialized
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// update
|
|
dep2 := newDeploymentTestDeploymentV2()
|
|
_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// detect changes -> progressing, canary scaled up
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
mocks.makeCanaryReady(t)
|
|
|
|
canaryDep, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
require.NotNil(t, canaryDep.Spec.Replicas)
|
|
canaryReplicas := *canaryDep.Spec.Replicas
|
|
require.Greater(t, canaryReplicas, int32(0))
|
|
|
|
// simulate: promotion already shifted all traffic to the primary (Finalising)
|
|
cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
err = mocks.deployer.SetStatusPhase(cd, flaggerv1.CanaryPhaseFinalising)
|
|
require.NoError(t, err)
|
|
require.NoError(t, mocks.router.SetRoutes(mocks.canary, 100, 0, false))
|
|
|
|
// the promoted primary then fails to stay ready
|
|
mocks.makePrimaryNotReady(t)
|
|
|
|
// advance: Flagger observes the primary is stuck
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// traffic must be routed back to the healthy canary, off the broken primary
|
|
primaryWeight, canaryWeight, _, err := mocks.router.GetRoutes(mocks.canary)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 0, primaryWeight, "no traffic must remain on the unhealthy primary")
|
|
assert.Equal(t, 100, canaryWeight, "all traffic must be routed to the healthy canary")
|
|
|
|
// the canary must not be scaled to zero
|
|
canaryDep, err = mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
require.NotNil(t, canaryDep.Spec.Replicas)
|
|
assert.Equal(t, canaryReplicas, *canaryDep.Spec.Replicas, "canary must not be scaled to zero when promotion fails")
|
|
|
|
// reported canary weight must match the routing (not zeroed)
|
|
c, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, flaggerv1.CanaryPhaseFailed, c.Status.Phase)
|
|
assert.Equal(t, 100, c.Status.CanaryWeight, "reported canary weight must match the traffic on the canary")
|
|
}
|
|
|
|
func TestScheduler_DeploymentSkipAnalysis(t *testing.T) {
|
|
mocks := newDeploymentFixture(nil)
|
|
// initializing
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// make primary ready
|
|
mocks.makePrimaryReady(t)
|
|
|
|
// initialized
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// enable skip
|
|
cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
cd.Spec.SkipAnalysis = true
|
|
_, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// update
|
|
dep2 := newDeploymentTestDeploymentV2()
|
|
_, err = mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// detect changes
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
mocks.makeCanaryReady(t)
|
|
|
|
// advance
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
c, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
assert.True(t, c.Spec.SkipAnalysis)
|
|
assert.Equal(t, flaggerv1.CanaryPhaseSucceeded, c.Status.Phase)
|
|
}
|
|
|
|
func TestScheduler_DeploymentAnalysisPhases(t *testing.T) {
|
|
cd := newDeploymentTestCanary()
|
|
cd.Spec.Analysis = &flaggerv1.CanaryAnalysis{
|
|
Interval: "1m",
|
|
StepWeight: 100,
|
|
StepWeightPromotion: 50,
|
|
}
|
|
mocks := newDeploymentFixture(cd)
|
|
|
|
// initializing
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// make primary ready
|
|
mocks.makePrimaryReady(t)
|
|
|
|
// initialized
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseInitialized))
|
|
|
|
// update
|
|
dep2 := newDeploymentTestDeploymentV2()
|
|
_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// detect changes
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing))
|
|
require.NoError(t, assertCanaryWeight(mocks.flaggerClient, "podinfo", 0))
|
|
mocks.makeCanaryReady(t)
|
|
|
|
// progressing
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing))
|
|
require.NoError(t, assertCanaryWeight(mocks.flaggerClient, "podinfo", 100))
|
|
|
|
// start promotion
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhasePromoting))
|
|
require.NoError(t, assertCanaryWeight(mocks.flaggerClient, "podinfo", 100))
|
|
|
|
// end promotion
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhasePromoting))
|
|
require.NoError(t, assertCanaryWeight(mocks.flaggerClient, "podinfo", 50))
|
|
|
|
// finalising
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseFinalising))
|
|
require.NoError(t, assertCanaryWeight(mocks.flaggerClient, "podinfo", 0))
|
|
|
|
// succeeded
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseSucceeded))
|
|
require.NoError(t, assertCanaryWeight(mocks.flaggerClient, "podinfo", 0))
|
|
}
|
|
|
|
func TestScheduler_DeploymentBlueGreenAnalysisPhases(t *testing.T) {
|
|
cd := newDeploymentTestCanary()
|
|
cd.Spec.Analysis = &flaggerv1.CanaryAnalysis{
|
|
Interval: "1m",
|
|
Iterations: 1,
|
|
}
|
|
mocks := newDeploymentFixture(cd)
|
|
|
|
// initializing
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// make primary ready
|
|
mocks.makePrimaryReady(t)
|
|
|
|
// initialized
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseInitialized))
|
|
|
|
// update
|
|
dep2 := newDeploymentTestDeploymentV2()
|
|
_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// detect changes (progressing)
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing))
|
|
mocks.makeCanaryReady(t)
|
|
|
|
// advance (progressing)
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing))
|
|
|
|
// route traffic to primary (progressing)
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing))
|
|
|
|
// promoting
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhasePromoting))
|
|
|
|
// finalising
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseFinalising))
|
|
|
|
// succeeded
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseSucceeded))
|
|
}
|
|
|
|
func TestScheduler_DeploymentNewRevisionReset(t *testing.T) {
|
|
mocks := newDeploymentFixture(nil)
|
|
// init
|
|
// initializing
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// make primary ready
|
|
mocks.makePrimaryReady(t)
|
|
|
|
// initialized
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// first update
|
|
dep2 := newDeploymentTestDeploymentV2()
|
|
_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// detect changes
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
mocks.makeCanaryReady(t)
|
|
|
|
// advance
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 90, primaryWeight)
|
|
assert.Equal(t, 10, canaryWeight)
|
|
assert.False(t, mirrored)
|
|
|
|
// second update
|
|
dep2.Spec.Template.Spec.ServiceAccountName = "test"
|
|
_, err = mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// detect changes
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
primaryWeight, canaryWeight, mirrored, err = mocks.router.GetRoutes(mocks.canary)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 100, primaryWeight)
|
|
assert.Equal(t, 0, canaryWeight)
|
|
assert.False(t, mirrored)
|
|
}
|
|
|
|
func TestScheduler_DeploymentPromotion(t *testing.T) {
|
|
mocks := newDeploymentFixture(nil)
|
|
|
|
// initializing
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// make primary ready
|
|
mocks.makePrimaryReady(t)
|
|
|
|
// initialized
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// check initialized status
|
|
c, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, flaggerv1.CanaryPhaseInitialized, c.Status.Phase)
|
|
|
|
// update
|
|
dep2 := newDeploymentTestDeploymentV2()
|
|
_, err = mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// detect pod spec changes
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
mocks.makeCanaryReady(t)
|
|
|
|
config2 := newDeploymentTestConfigMapV2()
|
|
_, err = mocks.kubeClient.CoreV1().ConfigMaps("default").Update(context.TODO(), config2, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
secret2 := newDeploymentTestSecretV2()
|
|
_, err = mocks.kubeClient.CoreV1().Secrets("default").Update(context.TODO(), secret2, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// detect configs changes
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
_, _, _, err = mocks.router.GetRoutes(mocks.canary)
|
|
require.NoError(t, err)
|
|
|
|
primaryWeight := 60
|
|
canaryWeight := 40
|
|
err = mocks.router.SetRoutes(mocks.canary, primaryWeight, canaryWeight, false)
|
|
require.NoError(t, err)
|
|
|
|
// advance
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// check progressing status
|
|
c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, flaggerv1.CanaryPhaseProgressing, c.Status.Phase)
|
|
|
|
// promote
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// check promoting status
|
|
c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, flaggerv1.CanaryPhasePromoting, c.Status.Phase)
|
|
|
|
// finalise
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 100, primaryWeight)
|
|
assert.Equal(t, 0, canaryWeight)
|
|
assert.False(t, mirrored)
|
|
|
|
primaryDep, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
|
|
primaryImage := primaryDep.Spec.Template.Spec.Containers[0].Image
|
|
canaryImage := dep2.Spec.Template.Spec.Containers[0].Image
|
|
assert.Equal(t, canaryImage, primaryImage)
|
|
|
|
configPrimary, err := mocks.kubeClient.CoreV1().ConfigMaps("default").Get(context.TODO(), "podinfo-config-env-primary", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, config2.Data["color"], configPrimary.Data["color"])
|
|
|
|
secretPrimary, err := mocks.kubeClient.CoreV1().Secrets("default").Get(context.TODO(), "podinfo-secret-env-primary", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, string(secret2.Data["apiKey"]), string(secretPrimary.Data["apiKey"]))
|
|
|
|
// check finalising status
|
|
c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, flaggerv1.CanaryPhaseFinalising, c.Status.Phase)
|
|
|
|
// scale canary to zero
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, flaggerv1.CanaryPhaseSucceeded, c.Status.Phase)
|
|
}
|
|
|
|
func TestScheduler_DeploymentMirroring(t *testing.T) {
|
|
mocks := newDeploymentFixture(newDeploymentTestCanaryMirror())
|
|
|
|
// initializing
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// make primary ready
|
|
mocks.makePrimaryReady(t)
|
|
|
|
// initialized
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// update
|
|
dep2 := newDeploymentTestDeploymentV2()
|
|
_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// detect pod spec changes
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
mocks.makeCanaryReady(t)
|
|
|
|
// advance
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// check if traffic is mirrored to canary
|
|
primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 100, primaryWeight)
|
|
assert.Equal(t, 0, canaryWeight)
|
|
assert.True(t, mirrored)
|
|
|
|
// advance
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// check if traffic is mirrored to canary
|
|
primaryWeight, canaryWeight, mirrored, err = mocks.router.GetRoutes(mocks.canary)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 90, primaryWeight)
|
|
assert.Equal(t, 10, canaryWeight)
|
|
assert.False(t, mirrored)
|
|
}
|
|
|
|
func TestScheduler_DeploymentABTesting(t *testing.T) {
|
|
mocks := newDeploymentFixture(newDeploymentTestCanaryAB())
|
|
// initializing
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// make primary ready
|
|
mocks.makePrimaryReady(t)
|
|
|
|
// initialized
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// update
|
|
dep2 := newDeploymentTestDeploymentV2()
|
|
_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// detect pod spec changes
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
mocks.makeCanaryReady(t)
|
|
|
|
// advance
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// check if traffic is routed to canary
|
|
primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, 0, primaryWeight)
|
|
assert.Equal(t, 100, canaryWeight)
|
|
assert.False(t, mirrored)
|
|
|
|
cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// set max iterations
|
|
err = mocks.deployer.SetStatusIterations(cd, 10)
|
|
require.NoError(t, err)
|
|
|
|
// advance
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// finalising
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// check finalising status
|
|
c, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, flaggerv1.CanaryPhaseFinalising, c.Status.Phase)
|
|
|
|
// check if the container image tag was updated
|
|
primaryDep, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
|
|
primaryImage := primaryDep.Spec.Template.Spec.Containers[0].Image
|
|
canaryImage := dep2.Spec.Template.Spec.Containers[0].Image
|
|
assert.Equal(t, canaryImage, primaryImage)
|
|
|
|
// shutdown canary
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// check rollout status
|
|
c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, flaggerv1.CanaryPhaseSucceeded, c.Status.Phase)
|
|
}
|
|
|
|
func TestScheduler_DeploymentPortDiscovery(t *testing.T) {
|
|
mocks := newDeploymentFixture(nil)
|
|
|
|
// enable port discovery
|
|
cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
cd.Spec.Service.PortDiscovery = true
|
|
_, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
canarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get(context.TODO(), "podinfo-canary", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
require.Len(t, canarySvc.Spec.Ports, 3)
|
|
|
|
matchPorts := func(lookup string) bool {
|
|
switch lookup {
|
|
case
|
|
"http 9898",
|
|
"http-metrics 8080",
|
|
"tcp-podinfo-2 8888":
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
for _, port := range canarySvc.Spec.Ports {
|
|
require.True(t, matchPorts(fmt.Sprintf("%s %v", port.Name, port.Port)))
|
|
}
|
|
}
|
|
|
|
func TestScheduler_DeploymentTargetPortNumber(t *testing.T) {
|
|
mocks := newDeploymentFixture(nil)
|
|
|
|
cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
cd.Spec.Service.Port = 80
|
|
cd.Spec.Service.TargetPort = intstr.FromInt(9898)
|
|
cd.Spec.Service.PortDiscovery = true
|
|
_, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
canarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get(context.TODO(), "podinfo-canary", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
require.Len(t, canarySvc.Spec.Ports, 3)
|
|
|
|
matchPorts := func(lookup string) bool {
|
|
switch lookup {
|
|
case
|
|
"http 80",
|
|
"http-metrics 8080",
|
|
"tcp-podinfo-2 8888":
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
for _, port := range canarySvc.Spec.Ports {
|
|
require.True(t, matchPorts(fmt.Sprintf("%s %v", port.Name, port.Port)))
|
|
}
|
|
}
|
|
|
|
func TestScheduler_DeploymentTargetPortName(t *testing.T) {
|
|
mocks := newDeploymentFixture(nil)
|
|
|
|
cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
cd.Spec.Service.Port = 8080
|
|
cd.Spec.Service.TargetPort = intstr.FromString("http")
|
|
cd.Spec.Service.PortDiscovery = true
|
|
_, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
canarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get(context.TODO(), "podinfo-canary", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
require.Len(t, canarySvc.Spec.Ports, 3)
|
|
|
|
matchPorts := func(lookup string) bool {
|
|
switch lookup {
|
|
case
|
|
"http 8080",
|
|
"http-metrics 8080",
|
|
"tcp-podinfo-2 8888":
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
for _, port := range canarySvc.Spec.Ports {
|
|
require.True(t, matchPorts(fmt.Sprintf("%s %v", port.Name, port.Port)))
|
|
}
|
|
}
|
|
|
|
func TestScheduler_DeploymentAlerts(t *testing.T) {
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
b, err := io.ReadAll(r.Body)
|
|
require.NoError(t, err)
|
|
var payload = notifier.SlackPayload{}
|
|
err = json.Unmarshal(b, &payload)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "podinfo.default", payload.Attachments[0].AuthorName)
|
|
}))
|
|
defer ts.Close()
|
|
|
|
canary := newDeploymentTestCanary()
|
|
canary.Spec.Analysis.Alerts = []flaggerv1.CanaryAlert{
|
|
{
|
|
Name: "slack-dev",
|
|
Severity: "info",
|
|
ProviderRef: flaggerv1.CrossNamespaceObjectReference{
|
|
Name: "slack",
|
|
Namespace: "default",
|
|
},
|
|
},
|
|
{
|
|
Name: "slack-prod",
|
|
Severity: "info",
|
|
ProviderRef: flaggerv1.CrossNamespaceObjectReference{
|
|
Name: "slack",
|
|
},
|
|
},
|
|
}
|
|
mocks := newDeploymentFixture(canary)
|
|
|
|
secret := newDeploymentTestAlertProviderSecret()
|
|
secret.Data = map[string][]byte{
|
|
"address": []byte(ts.URL),
|
|
}
|
|
_, err := mocks.kubeClient.CoreV1().Secrets("default").Update(context.TODO(), secret, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// init canary
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
|
|
// make primary ready
|
|
mocks.makePrimaryReady(t)
|
|
|
|
// initialization done - now send alert
|
|
mocks.ctrl.advanceCanary("podinfo", "default")
|
|
}
|