/* Copyright 2020 The Flux authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package controller import ( "context" "encoding/json" "fmt" "io" "net/http" "net/http/httptest" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" "github.com/fluxcd/flagger/pkg/notifier" ) func TestScheduler_DeploymentInit(t *testing.T) { mocks := newDeploymentFixture(nil) mocks.ctrl.advanceCanary("podinfo", "default") _, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{}) require.NoError(t, err) } func TestScheduler_DeploymentNewRevision(t *testing.T) { mocks := newDeploymentFixture(nil) // initializing ... mocks.ctrl.advanceCanary("podinfo", "default") // make primary ready mocks.makePrimaryReady(t) // initialization done mocks.ctrl.advanceCanary("podinfo", "default") // check if ScaleToZero was performed dp, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) assert.Equal(t, int32(0), *dp.Spec.Replicas) // update dep2 := newDeploymentTestDeploymentV2() _, err = mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{}) require.NoError(t, err) // detect changes mocks.ctrl.advanceCanary("podinfo", "default") c, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) assert.Equal(t, int32(1), *c.Spec.Replicas) } func TestScheduler_DeploymentRollback(t *testing.T) { mocks := newDeploymentFixture(nil) // initializing mocks.ctrl.advanceCanary("podinfo", "default") // make primary ready mocks.makePrimaryReady(t) // initialized mocks.ctrl.advanceCanary("podinfo", "default") // update failed checks to max err := mocks.deployer.SyncStatus(mocks.canary, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseProgressing, FailedChecks: 10}) require.NoError(t, err) // set a metric check to fail c, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) cd := c.DeepCopy() cd.Spec.Analysis.Metrics = append(c.Spec.Analysis.Metrics, flaggerv1.CanaryMetric{ Name: "fail", Interval: "1m", ThresholdRange: &flaggerv1.CanaryThresholdRange{ Min: toFloatPtr(0), Max: toFloatPtr(50), }, Query: "fail", }) _, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{}) require.NoError(t, err) // run metric checks mocks.ctrl.advanceCanary("podinfo", "default") // finalise analysis mocks.ctrl.advanceCanary("podinfo", "default") // check status c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) assert.Equal(t, flaggerv1.CanaryPhaseFailed, c.Status.Phase) } func TestScheduler_DeploymentSkipAnalysis(t *testing.T) { mocks := newDeploymentFixture(nil) // initializing mocks.ctrl.advanceCanary("podinfo", "default") // make primary ready mocks.makePrimaryReady(t) // initialized mocks.ctrl.advanceCanary("podinfo", "default") // enable skip cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) cd.Spec.SkipAnalysis = true _, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{}) require.NoError(t, err) // update dep2 := newDeploymentTestDeploymentV2() _, err = mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{}) require.NoError(t, err) // detect changes mocks.ctrl.advanceCanary("podinfo", "default") mocks.makeCanaryReady(t) // advance mocks.ctrl.advanceCanary("podinfo", "default") c, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) assert.True(t, c.Spec.SkipAnalysis) assert.Equal(t, flaggerv1.CanaryPhaseSucceeded, c.Status.Phase) } func TestScheduler_DeploymentAnalysisPhases(t *testing.T) { cd := newDeploymentTestCanary() cd.Spec.Analysis = &flaggerv1.CanaryAnalysis{ Interval: "1m", StepWeight: 100, StepWeightPromotion: 50, } mocks := newDeploymentFixture(cd) // initializing mocks.ctrl.advanceCanary("podinfo", "default") // make primary ready mocks.makePrimaryReady(t) // initialized mocks.ctrl.advanceCanary("podinfo", "default") require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseInitialized)) // update dep2 := newDeploymentTestDeploymentV2() _, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{}) require.NoError(t, err) // detect changes mocks.ctrl.advanceCanary("podinfo", "default") require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing)) mocks.makeCanaryReady(t) // progressing mocks.ctrl.advanceCanary("podinfo", "default") require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing)) // start promotion mocks.ctrl.advanceCanary("podinfo", "default") require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhasePromoting)) // end promotion mocks.ctrl.advanceCanary("podinfo", "default") require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhasePromoting)) // finalising mocks.ctrl.advanceCanary("podinfo", "default") require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseFinalising)) // succeeded mocks.ctrl.advanceCanary("podinfo", "default") require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseSucceeded)) } func TestScheduler_DeploymentBlueGreenAnalysisPhases(t *testing.T) { cd := newDeploymentTestCanary() cd.Spec.Analysis = &flaggerv1.CanaryAnalysis{ Interval: "1m", Iterations: 1, } mocks := newDeploymentFixture(cd) // initializing mocks.ctrl.advanceCanary("podinfo", "default") // make primary ready mocks.makePrimaryReady(t) // initialized mocks.ctrl.advanceCanary("podinfo", "default") require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseInitialized)) // update dep2 := newDeploymentTestDeploymentV2() _, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{}) require.NoError(t, err) // detect changes (progressing) mocks.ctrl.advanceCanary("podinfo", "default") require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing)) mocks.makeCanaryReady(t) // advance (progressing) mocks.ctrl.advanceCanary("podinfo", "default") require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing)) // route traffic to primary (progressing) mocks.ctrl.advanceCanary("podinfo", "default") require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing)) // promoting mocks.ctrl.advanceCanary("podinfo", "default") require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhasePromoting)) // finalising mocks.ctrl.advanceCanary("podinfo", "default") require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseFinalising)) // succeeded mocks.ctrl.advanceCanary("podinfo", "default") require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseSucceeded)) } func TestScheduler_DeploymentNewRevisionReset(t *testing.T) { mocks := newDeploymentFixture(nil) // init // initializing mocks.ctrl.advanceCanary("podinfo", "default") // make primary ready mocks.makePrimaryReady(t) // initialized mocks.ctrl.advanceCanary("podinfo", "default") // first update dep2 := newDeploymentTestDeploymentV2() _, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{}) require.NoError(t, err) // detect changes mocks.ctrl.advanceCanary("podinfo", "default") mocks.makeCanaryReady(t) // advance mocks.ctrl.advanceCanary("podinfo", "default") primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary) require.NoError(t, err) assert.Equal(t, 90, primaryWeight) assert.Equal(t, 10, canaryWeight) assert.False(t, mirrored) // second update dep2.Spec.Template.Spec.ServiceAccountName = "test" _, err = mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{}) require.NoError(t, err) // detect changes mocks.ctrl.advanceCanary("podinfo", "default") primaryWeight, canaryWeight, mirrored, err = mocks.router.GetRoutes(mocks.canary) require.NoError(t, err) assert.Equal(t, 100, primaryWeight) assert.Equal(t, 0, canaryWeight) assert.False(t, mirrored) } func TestScheduler_DeploymentPromotion(t *testing.T) { mocks := newDeploymentFixture(nil) // initializing mocks.ctrl.advanceCanary("podinfo", "default") // make primary ready mocks.makePrimaryReady(t) // initialized mocks.ctrl.advanceCanary("podinfo", "default") // check initialized status c, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) assert.Equal(t, flaggerv1.CanaryPhaseInitialized, c.Status.Phase) // update dep2 := newDeploymentTestDeploymentV2() _, err = mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{}) require.NoError(t, err) // detect pod spec changes mocks.ctrl.advanceCanary("podinfo", "default") mocks.makeCanaryReady(t) config2 := newDeploymentTestConfigMapV2() _, err = mocks.kubeClient.CoreV1().ConfigMaps("default").Update(context.TODO(), config2, metav1.UpdateOptions{}) require.NoError(t, err) secret2 := newDeploymentTestSecretV2() _, err = mocks.kubeClient.CoreV1().Secrets("default").Update(context.TODO(), secret2, metav1.UpdateOptions{}) require.NoError(t, err) // detect configs changes mocks.ctrl.advanceCanary("podinfo", "default") _, _, _, err = mocks.router.GetRoutes(mocks.canary) require.NoError(t, err) primaryWeight := 60 canaryWeight := 40 err = mocks.router.SetRoutes(mocks.canary, primaryWeight, canaryWeight, false) require.NoError(t, err) // advance mocks.ctrl.advanceCanary("podinfo", "default") // check progressing status c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) assert.Equal(t, flaggerv1.CanaryPhaseProgressing, c.Status.Phase) // promote mocks.ctrl.advanceCanary("podinfo", "default") // check promoting status c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) assert.Equal(t, flaggerv1.CanaryPhasePromoting, c.Status.Phase) // finalise mocks.ctrl.advanceCanary("podinfo", "default") primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary) require.NoError(t, err) assert.Equal(t, 100, primaryWeight) assert.Equal(t, 0, canaryWeight) assert.False(t, mirrored) primaryDep, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{}) require.NoError(t, err) primaryImage := primaryDep.Spec.Template.Spec.Containers[0].Image canaryImage := dep2.Spec.Template.Spec.Containers[0].Image assert.Equal(t, canaryImage, primaryImage) configPrimary, err := mocks.kubeClient.CoreV1().ConfigMaps("default").Get(context.TODO(), "podinfo-config-env-primary", metav1.GetOptions{}) require.NoError(t, err) assert.Equal(t, config2.Data["color"], configPrimary.Data["color"]) secretPrimary, err := mocks.kubeClient.CoreV1().Secrets("default").Get(context.TODO(), "podinfo-secret-env-primary", metav1.GetOptions{}) require.NoError(t, err) assert.Equal(t, string(secret2.Data["apiKey"]), string(secretPrimary.Data["apiKey"])) // check finalising status c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) assert.Equal(t, flaggerv1.CanaryPhaseFinalising, c.Status.Phase) // scale canary to zero mocks.ctrl.advanceCanary("podinfo", "default") c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) assert.Equal(t, flaggerv1.CanaryPhaseSucceeded, c.Status.Phase) } func TestScheduler_DeploymentMirroring(t *testing.T) { mocks := newDeploymentFixture(newDeploymentTestCanaryMirror()) // initializing mocks.ctrl.advanceCanary("podinfo", "default") // make primary ready mocks.makePrimaryReady(t) // initialized mocks.ctrl.advanceCanary("podinfo", "default") // update dep2 := newDeploymentTestDeploymentV2() _, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{}) require.NoError(t, err) // detect pod spec changes mocks.ctrl.advanceCanary("podinfo", "default") mocks.makeCanaryReady(t) // advance mocks.ctrl.advanceCanary("podinfo", "default") // check if traffic is mirrored to canary primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary) require.NoError(t, err) assert.Equal(t, 100, primaryWeight) assert.Equal(t, 0, canaryWeight) assert.True(t, mirrored) // advance mocks.ctrl.advanceCanary("podinfo", "default") // check if traffic is mirrored to canary primaryWeight, canaryWeight, mirrored, err = mocks.router.GetRoutes(mocks.canary) require.NoError(t, err) assert.Equal(t, 90, primaryWeight) assert.Equal(t, 10, canaryWeight) assert.False(t, mirrored) } func TestScheduler_DeploymentABTesting(t *testing.T) { mocks := newDeploymentFixture(newDeploymentTestCanaryAB()) // initializing mocks.ctrl.advanceCanary("podinfo", "default") // make primary ready mocks.makePrimaryReady(t) // initialized mocks.ctrl.advanceCanary("podinfo", "default") // update dep2 := newDeploymentTestDeploymentV2() _, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{}) require.NoError(t, err) // detect pod spec changes mocks.ctrl.advanceCanary("podinfo", "default") mocks.makeCanaryReady(t) // advance mocks.ctrl.advanceCanary("podinfo", "default") // check if traffic is routed to canary primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary) require.NoError(t, err) assert.Equal(t, 0, primaryWeight) assert.Equal(t, 100, canaryWeight) assert.False(t, mirrored) cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) // set max iterations err = mocks.deployer.SetStatusIterations(cd, 10) require.NoError(t, err) // advance mocks.ctrl.advanceCanary("podinfo", "default") // finalising mocks.ctrl.advanceCanary("podinfo", "default") // check finalising status c, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) assert.Equal(t, flaggerv1.CanaryPhaseFinalising, c.Status.Phase) // check if the container image tag was updated primaryDep, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{}) require.NoError(t, err) primaryImage := primaryDep.Spec.Template.Spec.Containers[0].Image canaryImage := dep2.Spec.Template.Spec.Containers[0].Image assert.Equal(t, canaryImage, primaryImage) // shutdown canary mocks.ctrl.advanceCanary("podinfo", "default") // check rollout status c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) assert.Equal(t, flaggerv1.CanaryPhaseSucceeded, c.Status.Phase) } func TestScheduler_DeploymentPortDiscovery(t *testing.T) { mocks := newDeploymentFixture(nil) // enable port discovery cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) cd.Spec.Service.PortDiscovery = true _, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{}) require.NoError(t, err) mocks.ctrl.advanceCanary("podinfo", "default") canarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get(context.TODO(), "podinfo-canary", metav1.GetOptions{}) require.NoError(t, err) require.Len(t, canarySvc.Spec.Ports, 3) matchPorts := func(lookup string) bool { switch lookup { case "http 9898", "http-metrics 8080", "tcp-podinfo-2 8888": return true } return false } for _, port := range canarySvc.Spec.Ports { require.True(t, matchPorts(fmt.Sprintf("%s %v", port.Name, port.Port))) } } func TestScheduler_DeploymentTargetPortNumber(t *testing.T) { mocks := newDeploymentFixture(nil) cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) cd.Spec.Service.Port = 80 cd.Spec.Service.TargetPort = intstr.FromInt(9898) cd.Spec.Service.PortDiscovery = true _, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{}) require.NoError(t, err) mocks.ctrl.advanceCanary("podinfo", "default") canarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get(context.TODO(), "podinfo-canary", metav1.GetOptions{}) require.NoError(t, err) require.Len(t, canarySvc.Spec.Ports, 3) matchPorts := func(lookup string) bool { switch lookup { case "http 80", "http-metrics 8080", "tcp-podinfo-2 8888": return true } return false } for _, port := range canarySvc.Spec.Ports { require.True(t, matchPorts(fmt.Sprintf("%s %v", port.Name, port.Port))) } } func TestScheduler_DeploymentTargetPortName(t *testing.T) { mocks := newDeploymentFixture(nil) cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) require.NoError(t, err) cd.Spec.Service.Port = 8080 cd.Spec.Service.TargetPort = intstr.FromString("http") cd.Spec.Service.PortDiscovery = true _, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{}) require.NoError(t, err) mocks.ctrl.advanceCanary("podinfo", "default") canarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get(context.TODO(), "podinfo-canary", metav1.GetOptions{}) require.NoError(t, err) require.Len(t, canarySvc.Spec.Ports, 3) matchPorts := func(lookup string) bool { switch lookup { case "http 8080", "http-metrics 8080", "tcp-podinfo-2 8888": return true } return false } for _, port := range canarySvc.Spec.Ports { require.True(t, matchPorts(fmt.Sprintf("%s %v", port.Name, port.Port))) } } func TestScheduler_DeploymentAlerts(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { b, err := io.ReadAll(r.Body) require.NoError(t, err) var payload = notifier.SlackPayload{} err = json.Unmarshal(b, &payload) require.NoError(t, err) require.Equal(t, "podinfo.default", payload.Attachments[0].AuthorName) })) defer ts.Close() canary := newDeploymentTestCanary() canary.Spec.Analysis.Alerts = []flaggerv1.CanaryAlert{ { Name: "slack-dev", Severity: "info", ProviderRef: flaggerv1.CrossNamespaceObjectReference{ Name: "slack", Namespace: "default", }, }, { Name: "slack-prod", Severity: "info", ProviderRef: flaggerv1.CrossNamespaceObjectReference{ Name: "slack", }, }, } mocks := newDeploymentFixture(canary) secret := newDeploymentTestAlertProviderSecret() secret.Data = map[string][]byte{ "address": []byte(ts.URL), } _, err := mocks.kubeClient.CoreV1().Secrets("default").Update(context.TODO(), secret, metav1.UpdateOptions{}) require.NoError(t, err) // init canary mocks.ctrl.advanceCanary("podinfo", "default") // make primary ready mocks.makePrimaryReady(t) // initialization done - now send alert mocks.ctrl.advanceCanary("podinfo", "default") }