flagger/pkg/controller/scheduler_deployment_test.go

/*
Copyright 2020 The Flux authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"

	flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
	"github.com/fluxcd/flagger/pkg/notifier"
)

func TestScheduler_DeploymentInit(t *testing.T) {
	mocks := newDeploymentFixture(nil)
	mocks.ctrl.advanceCanary("podinfo", "default")

	_, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
	require.NoError(t, err)
}

func TestScheduler_DeploymentNewRevision(t *testing.T) {
	mocks := newDeploymentFixture(nil)

	// initializing ...
	mocks.ctrl.advanceCanary("podinfo", "default")

	// make primary ready
	mocks.makePrimaryReady(t)

	// initialization done
	mocks.ctrl.advanceCanary("podinfo", "default")

	// update
	dep2 := newDeploymentTestDeploymentV2()
	_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
	require.NoError(t, err)

	// detect changes
	mocks.ctrl.advanceCanary("podinfo", "default")

	c, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
	require.NoError(t, err)
	assert.Equal(t, int32(1), *c.Spec.Replicas)
}

func TestScheduler_DeploymentRollback(t *testing.T) {
	mocks := newDeploymentFixture(nil)
	// initializing
	mocks.ctrl.advanceCanary("podinfo", "default")

	// make primary ready
	mocks.makePrimaryReady(t)

	// initialized
	mocks.ctrl.advanceCanary("podinfo", "default")

	// update failed checks to max
	err := mocks.deployer.SyncStatus(mocks.canary, flaggerv1.CanaryStatus{Phase: flaggerv1.CanaryPhaseProgressing, FailedChecks: 10})
	require.NoError(t, err)

	// set a metric check to fail
	c, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
	require.NoError(t, err)
	cd := c.DeepCopy()
	cd.Spec.Analysis.Metrics = append(c.Spec.Analysis.Metrics, flaggerv1.CanaryMetric{
		Name:     "fail",
		Interval: "1m",
		ThresholdRange: &flaggerv1.CanaryThresholdRange{
			Min: toFloatPtr(0),
			Max: toFloatPtr(50),
		},
		Query: "fail",
	})
	_, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{})
	require.NoError(t, err)

	// run metric checks
	mocks.ctrl.advanceCanary("podinfo", "default")

	// finalise analysis
	mocks.ctrl.advanceCanary("podinfo", "default")

	// check status
	c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
	require.NoError(t, err)

	assert.Equal(t, flaggerv1.CanaryPhaseFailed, c.Status.Phase)
}

func TestScheduler_DeploymentSkipAnalysis(t *testing.T) {
	mocks := newDeploymentFixture(nil)
	// initializing
	mocks.ctrl.advanceCanary("podinfo", "default")

	// make primary ready
	mocks.makePrimaryReady(t)

	// initialized
	mocks.ctrl.advanceCanary("podinfo", "default")

	// enable skip
	cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
	require.NoError(t, err)
	cd.Spec.SkipAnalysis = true
	_, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{})
	require.NoError(t, err)

	// update
	dep2 := newDeploymentTestDeploymentV2()
	_, err = mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
	require.NoError(t, err)

	// detect changes
	mocks.ctrl.advanceCanary("podinfo", "default")
	mocks.makeCanaryReady(t)

	// advance
	mocks.ctrl.advanceCanary("podinfo", "default")

	c, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
	require.NoError(t, err)
	assert.True(t, c.Spec.SkipAnalysis)
	assert.Equal(t, flaggerv1.CanaryPhaseSucceeded, c.Status.Phase)
}

func TestScheduler_DeploymentAnalysisPhases(t *testing.T) {
	cd := newDeploymentTestCanary()
	cd.Spec.Analysis = &flaggerv1.CanaryAnalysis{
		Interval:            "1m",
		StepWeight:          100,
		StepWeightPromotion: 50,
	}
	mocks := newDeploymentFixture(cd)

	// initializing
	mocks.ctrl.advanceCanary("podinfo", "default")

	// make primary ready
	mocks.makePrimaryReady(t)

	// initialized
	mocks.ctrl.advanceCanary("podinfo", "default")
	require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseInitialized))

	// update
	dep2 := newDeploymentTestDeploymentV2()
	_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
	require.NoError(t, err)

	// detect changes
	mocks.ctrl.advanceCanary("podinfo", "default")
	require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing))
	mocks.makeCanaryReady(t)

	// progressing
	mocks.ctrl.advanceCanary("podinfo", "default")
	require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing))

	// start promotion
	mocks.ctrl.advanceCanary("podinfo", "default")
	require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhasePromoting))

	// end promotion
	mocks.ctrl.advanceCanary("podinfo", "default")
	require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhasePromoting))

	// finalising
	mocks.ctrl.advanceCanary("podinfo", "default")
	require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseFinalising))

	// succeeded
	mocks.ctrl.advanceCanary("podinfo", "default")
	require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseSucceeded))
}

func TestScheduler_DeploymentBlueGreenAnalysisPhases(t *testing.T) {
	cd := newDeploymentTestCanary()
	cd.Spec.Analysis = &flaggerv1.CanaryAnalysis{
		Interval:   "1m",
		Iterations: 1,
	}
	mocks := newDeploymentFixture(cd)

	// initializing
	mocks.ctrl.advanceCanary("podinfo", "default")

	// make primary ready
	mocks.makePrimaryReady(t)

	// initialized
	mocks.ctrl.advanceCanary("podinfo", "default")
	require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseInitialized))

	// update
	dep2 := newDeploymentTestDeploymentV2()
	_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
	require.NoError(t, err)

	// detect changes (progressing)
	mocks.ctrl.advanceCanary("podinfo", "default")
	require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing))
	mocks.makeCanaryReady(t)

	// advance (progressing)
	mocks.ctrl.advanceCanary("podinfo", "default")
	require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing))

	// route traffic to primary (progressing)
	mocks.ctrl.advanceCanary("podinfo", "default")
	require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseProgressing))

	// promoting
	mocks.ctrl.advanceCanary("podinfo", "default")
	require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhasePromoting))

	// finalising
	mocks.ctrl.advanceCanary("podinfo", "default")
	require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseFinalising))

	// succeeded
	mocks.ctrl.advanceCanary("podinfo", "default")
	require.NoError(t, assertPhase(mocks.flaggerClient, "podinfo", flaggerv1.CanaryPhaseSucceeded))
}

func TestScheduler_DeploymentNewRevisionReset(t *testing.T) {
	mocks := newDeploymentFixture(nil)
	// init
	// initializing
	mocks.ctrl.advanceCanary("podinfo", "default")

	// make primary ready
	mocks.makePrimaryReady(t)

	// initialized
	mocks.ctrl.advanceCanary("podinfo", "default")

	// first update
	dep2 := newDeploymentTestDeploymentV2()
	_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
	require.NoError(t, err)

	// detect changes
	mocks.ctrl.advanceCanary("podinfo", "default")
	mocks.makeCanaryReady(t)

	// advance
	mocks.ctrl.advanceCanary("podinfo", "default")

	primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
	require.NoError(t, err)
	assert.Equal(t, 90, primaryWeight)
	assert.Equal(t, 10, canaryWeight)
	assert.False(t, mirrored)

	// second update
	dep2.Spec.Template.Spec.ServiceAccountName = "test"
	_, err = mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
	require.NoError(t, err)

	// detect changes
	mocks.ctrl.advanceCanary("podinfo", "default")

	primaryWeight, canaryWeight, mirrored, err = mocks.router.GetRoutes(mocks.canary)
	require.NoError(t, err)
	assert.Equal(t, 100, primaryWeight)
	assert.Equal(t, 0, canaryWeight)
	assert.False(t, mirrored)
}

func TestScheduler_DeploymentPromotion(t *testing.T) {
	mocks := newDeploymentFixture(nil)

	// initializing
	mocks.ctrl.advanceCanary("podinfo", "default")

	// make primary ready
	mocks.makePrimaryReady(t)

	// initialized
	mocks.ctrl.advanceCanary("podinfo", "default")

	// check initialized status
	c, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
	require.NoError(t, err)
	assert.Equal(t, flaggerv1.CanaryPhaseInitialized, c.Status.Phase)

	// update
	dep2 := newDeploymentTestDeploymentV2()
	_, err = mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
	require.NoError(t, err)

	// detect pod spec changes
	mocks.ctrl.advanceCanary("podinfo", "default")
	mocks.makeCanaryReady(t)

	config2 := newDeploymentTestConfigMapV2()
	_, err = mocks.kubeClient.CoreV1().ConfigMaps("default").Update(context.TODO(), config2, metav1.UpdateOptions{})
	require.NoError(t, err)

	secret2 := newDeploymentTestSecretV2()
	_, err = mocks.kubeClient.CoreV1().Secrets("default").Update(context.TODO(), secret2, metav1.UpdateOptions{})
	require.NoError(t, err)

	// detect configs changes
	mocks.ctrl.advanceCanary("podinfo", "default")

	_, _, _, err = mocks.router.GetRoutes(mocks.canary)
	require.NoError(t, err)

	primaryWeight := 60
	canaryWeight := 40
	err = mocks.router.SetRoutes(mocks.canary, primaryWeight, canaryWeight, false)
	require.NoError(t, err)

	// advance
	mocks.ctrl.advanceCanary("podinfo", "default")

	// check progressing status
	c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
	require.NoError(t, err)
	assert.Equal(t, flaggerv1.CanaryPhaseProgressing, c.Status.Phase)

	// promote
	mocks.ctrl.advanceCanary("podinfo", "default")

	// check promoting status
	c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
	require.NoError(t, err)
	assert.Equal(t, flaggerv1.CanaryPhasePromoting, c.Status.Phase)

	// finalise
	mocks.ctrl.advanceCanary("podinfo", "default")

	primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
	require.NoError(t, err)
	assert.Equal(t, 100, primaryWeight)
	assert.Equal(t, 0, canaryWeight)
	assert.False(t, mirrored)

	primaryDep, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
	require.NoError(t, err)

	primaryImage := primaryDep.Spec.Template.Spec.Containers[0].Image
	canaryImage := dep2.Spec.Template.Spec.Containers[0].Image
	assert.Equal(t, canaryImage, primaryImage)

	configPrimary, err := mocks.kubeClient.CoreV1().ConfigMaps("default").Get(context.TODO(), "podinfo-config-env-primary", metav1.GetOptions{})
	require.NoError(t, err)
	assert.Equal(t, config2.Data["color"], configPrimary.Data["color"])

	secretPrimary, err := mocks.kubeClient.CoreV1().Secrets("default").Get(context.TODO(), "podinfo-secret-env-primary", metav1.GetOptions{})
	require.NoError(t, err)
	assert.Equal(t, string(secret2.Data["apiKey"]), string(secretPrimary.Data["apiKey"]))

	// check finalising status
	c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
	require.NoError(t, err)
	assert.Equal(t, flaggerv1.CanaryPhaseFinalising, c.Status.Phase)

	// scale canary to zero
	mocks.ctrl.advanceCanary("podinfo", "default")

	c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
	require.NoError(t, err)
	assert.Equal(t, flaggerv1.CanaryPhaseSucceeded, c.Status.Phase)
}

func TestScheduler_DeploymentMirroring(t *testing.T) {
	mocks := newDeploymentFixture(newDeploymentTestCanaryMirror())

	// initializing
	mocks.ctrl.advanceCanary("podinfo", "default")

	// make primary ready
	mocks.makePrimaryReady(t)

	// initialized
	mocks.ctrl.advanceCanary("podinfo", "default")

	// update
	dep2 := newDeploymentTestDeploymentV2()
	_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
	require.NoError(t, err)

	// detect pod spec changes
	mocks.ctrl.advanceCanary("podinfo", "default")
	mocks.makeCanaryReady(t)

	// advance
	mocks.ctrl.advanceCanary("podinfo", "default")

	// check if traffic is mirrored to canary
	primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
	require.NoError(t, err)
	assert.Equal(t, 100, primaryWeight)
	assert.Equal(t, 0, canaryWeight)
	assert.True(t, mirrored)

	// advance
	mocks.ctrl.advanceCanary("podinfo", "default")

	// check if traffic is mirrored to canary
	primaryWeight, canaryWeight, mirrored, err = mocks.router.GetRoutes(mocks.canary)
	require.NoError(t, err)
	assert.Equal(t, 90, primaryWeight)
	assert.Equal(t, 10, canaryWeight)
	assert.False(t, mirrored)
}

func TestScheduler_DeploymentABTesting(t *testing.T) {
	mocks := newDeploymentFixture(newDeploymentTestCanaryAB())
	// initializing
	mocks.ctrl.advanceCanary("podinfo", "default")

	// make primary ready
	mocks.makePrimaryReady(t)

	// initialized
	mocks.ctrl.advanceCanary("podinfo", "default")

	// update
	dep2 := newDeploymentTestDeploymentV2()
	_, err := mocks.kubeClient.AppsV1().Deployments("default").Update(context.TODO(), dep2, metav1.UpdateOptions{})
	require.NoError(t, err)

	// detect pod spec changes
	mocks.ctrl.advanceCanary("podinfo", "default")
	mocks.makeCanaryReady(t)

	// advance
	mocks.ctrl.advanceCanary("podinfo", "default")

	// check if traffic is routed to canary
	primaryWeight, canaryWeight, mirrored, err := mocks.router.GetRoutes(mocks.canary)
	require.NoError(t, err)
	assert.Equal(t, 0, primaryWeight)
	assert.Equal(t, 100, canaryWeight)
	assert.False(t, mirrored)

	cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
	require.NoError(t, err)

	// set max iterations
	err = mocks.deployer.SetStatusIterations(cd, 10)
	require.NoError(t, err)

	// advance
	mocks.ctrl.advanceCanary("podinfo", "default")

	// finalising
	mocks.ctrl.advanceCanary("podinfo", "default")

	// check finalising status
	c, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
	require.NoError(t, err)
	assert.Equal(t, flaggerv1.CanaryPhaseFinalising, c.Status.Phase)

	// check if the container image tag was updated
	primaryDep, err := mocks.kubeClient.AppsV1().Deployments("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
	require.NoError(t, err)

	primaryImage := primaryDep.Spec.Template.Spec.Containers[0].Image
	canaryImage := dep2.Spec.Template.Spec.Containers[0].Image
	assert.Equal(t, canaryImage, primaryImage)

	// shutdown canary
	mocks.ctrl.advanceCanary("podinfo", "default")

	// check rollout status
	c, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
	require.NoError(t, err)
	assert.Equal(t, flaggerv1.CanaryPhaseSucceeded, c.Status.Phase)
}

func TestScheduler_DeploymentPortDiscovery(t *testing.T) {
	mocks := newDeploymentFixture(nil)

	// enable port discovery
	cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
	require.NoError(t, err)
	cd.Spec.Service.PortDiscovery = true
	_, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{})
	require.NoError(t, err)

	mocks.ctrl.advanceCanary("podinfo", "default")

	canarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get(context.TODO(), "podinfo-canary", metav1.GetOptions{})
	require.NoError(t, err)
	require.Len(t, canarySvc.Spec.Ports, 3)

	matchPorts := func(lookup string) bool {
		switch lookup {
		case
			"http 9898",
			"http-metrics 8080",
			"tcp-podinfo-2 8888":
			return true
		}
		return false
	}

	for _, port := range canarySvc.Spec.Ports {
		require.True(t, matchPorts(fmt.Sprintf("%s %v", port.Name, port.Port)))
	}
}

func TestScheduler_DeploymentTargetPortNumber(t *testing.T) {
	mocks := newDeploymentFixture(nil)

	cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
	require.NoError(t, err)
	cd.Spec.Service.Port = 80
	cd.Spec.Service.TargetPort = intstr.FromInt(9898)
	cd.Spec.Service.PortDiscovery = true
	_, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{})
	require.NoError(t, err)

	mocks.ctrl.advanceCanary("podinfo", "default")

	canarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get(context.TODO(), "podinfo-canary", metav1.GetOptions{})
	require.NoError(t, err)
	require.Len(t, canarySvc.Spec.Ports, 3)

	matchPorts := func(lookup string) bool {
		switch lookup {
		case
			"http 80",
			"http-metrics 8080",
			"tcp-podinfo-2 8888":
			return true
		}
		return false
	}

	for _, port := range canarySvc.Spec.Ports {
		require.True(t, matchPorts(fmt.Sprintf("%s %v", port.Name, port.Port)))
	}
}

func TestScheduler_DeploymentTargetPortName(t *testing.T) {
	mocks := newDeploymentFixture(nil)

	cd, err := mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Get(context.TODO(), "podinfo", metav1.GetOptions{})
	require.NoError(t, err)
	cd.Spec.Service.Port = 8080
	cd.Spec.Service.TargetPort = intstr.FromString("http")
	cd.Spec.Service.PortDiscovery = true
	_, err = mocks.flaggerClient.FlaggerV1beta1().Canaries("default").Update(context.TODO(), cd, metav1.UpdateOptions{})
	require.NoError(t, err)

	mocks.ctrl.advanceCanary("podinfo", "default")

	canarySvc, err := mocks.kubeClient.CoreV1().Services("default").Get(context.TODO(), "podinfo-canary", metav1.GetOptions{})
	require.NoError(t, err)
	require.Len(t, canarySvc.Spec.Ports, 3)

	matchPorts := func(lookup string) bool {
		switch lookup {
		case
			"http 8080",
			"http-metrics 8080",
			"tcp-podinfo-2 8888":
			return true
		}
		return false
	}

	for _, port := range canarySvc.Spec.Ports {
		require.True(t, matchPorts(fmt.Sprintf("%s %v", port.Name, port.Port)))
	}
}

func TestScheduler_DeploymentAlerts(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		b, err := ioutil.ReadAll(r.Body)
		require.NoError(t, err)
		var payload = notifier.SlackPayload{}
		err = json.Unmarshal(b, &payload)
		require.NoError(t, err)
		require.Equal(t, "podinfo.default", payload.Attachments[0].AuthorName)
	}))
	defer ts.Close()

	canary := newDeploymentTestCanary()
	canary.Spec.Analysis.Alerts = []flaggerv1.CanaryAlert{
		{
			Name:     "slack-dev",
			Severity: "info",
			ProviderRef: flaggerv1.CrossNamespaceObjectReference{
				Name:      "slack",
				Namespace: "default",
			},
		},
		{
			Name:     "slack-prod",
			Severity: "info",
			ProviderRef: flaggerv1.CrossNamespaceObjectReference{
				Name: "slack",
			},
		},
	}
	mocks := newDeploymentFixture(canary)

	secret := newDeploymentTestAlertProviderSecret()
	secret.Data = map[string][]byte{
		"address": []byte(ts.URL),
	}
	_, err := mocks.kubeClient.CoreV1().Secrets("default").Update(context.TODO(), secret, metav1.UpdateOptions{})
	require.NoError(t, err)

	// init canary
	mocks.ctrl.advanceCanary("podinfo", "default")

	// make primary ready
	mocks.makePrimaryReady(t)

	// initialization done - now send alert
	mocks.ctrl.advanceCanary("podinfo", "default")
}