From fda733ea5a5f65baa80c6a0eeb0e3d271a5f9450 Mon Sep 17 00:00:00 2001 From: Vlad Losev Date: Tue, 2 Jun 2020 17:20:12 -0700 Subject: [PATCH 01/11] Adds support for auto-reloading secrets and configmaps by annotation. --- internal/pkg/cmd/reloader.go | 6 +- internal/pkg/handler/upgrade.go | 17 ++- internal/pkg/handler/upgrade_test.go | 177 ++++++++++++++++++++++++--- internal/pkg/options/flags.go | 12 +- internal/pkg/testutil/kube.go | 29 ++--- internal/pkg/util/config.go | 36 +++--- 6 files changed, 224 insertions(+), 53 deletions(-) diff --git a/internal/pkg/cmd/reloader.go b/internal/pkg/cmd/reloader.go index b92dd266..17d8d42f 100644 --- a/internal/pkg/cmd/reloader.go +++ b/internal/pkg/cmd/reloader.go @@ -24,9 +24,11 @@ func NewReloaderCommand() *cobra.Command { } // options - cmd.PersistentFlags().StringVar(&options.ConfigmapUpdateOnChangeAnnotation, "configmap-annotation", "configmap.reloader.stakater.com/reload", "annotation to detect changes in configmaps") - cmd.PersistentFlags().StringVar(&options.SecretUpdateOnChangeAnnotation, "secret-annotation", "secret.reloader.stakater.com/reload", "annotation to detect changes in secrets") + cmd.PersistentFlags().StringVar(&options.ConfigmapUpdateOnChangeAnnotation, "configmap-annotation", "configmap.reloader.stakater.com/reload", "annotation to detect changes in configmaps, specified by name") + cmd.PersistentFlags().StringVar(&options.SecretUpdateOnChangeAnnotation, "secret-annotation", "secret.reloader.stakater.com/reload", "annotation to detect changes in secrets, specified by name") cmd.PersistentFlags().StringVar(&options.ReloaderAutoAnnotation, "auto-annotation", "reloader.stakater.com/auto", "annotation to detect changes in secrets") + cmd.PersistentFlags().StringVar(&options.ConfigmapUpdateAutoSearchAnnotation, "configmap-auto-by-annotation", "configmap.reloader.stakater.com/reload-by-annotation", "annotation to detect changes in configmaps, searched by annotation") + cmd.PersistentFlags().StringVar(&options.SecretUpdateAutoSearchAnnotation, "secret-auto-by-annotation", "secret.reloader.stakater.com/reload-by-annotation", "annotation to detect changes in secret, searched by annotations") cmd.PersistentFlags().StringVar(&options.LogFormat, "log-format", "", "Log format to use (empty string for text, or JSON") cmd.PersistentFlags().StringSlice("resources-to-ignore", []string{}, "list of resources to ignore (valid options 'configMaps' or 'secrets')") cmd.PersistentFlags().StringSlice("namespaces-to-ignore", []string{}, "list of namespaces to ignore") diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index bb7d6bae..38baf2eb 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -99,10 +99,12 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc // find correct annotation and update the resource annotations := upgradeFuncs.AnnotationsFunc(i) annotationValue, found := annotations[config.Annotation] + searchAnnotationValue, foundSearchAnn := annotations[config.SearchAnnotation] reloaderEnabledValue, foundAuto := annotations[options.ReloaderAutoAnnotation] - if !found && !foundAuto { + if !found && !foundAuto && !foundSearchAnn { annotations = upgradeFuncs.PodAnnotationsFunc(i) annotationValue = annotations[config.Annotation] + searchAnnotationValue = annotations[config.SearchAnnotation] reloaderEnabledValue = annotations[options.ReloaderAutoAnnotation] } result := constants.NotUpdated @@ -123,6 +125,19 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc } } + if result != constants.Updated && searchAnnotationValue != "" { + keyValue := strings.Split(searchAnnotationValue, "=") + key := keyValue[0] + searchValue := "" + if len(keyValue) > 1 { + searchValue = keyValue[1] + } + value, found := config.ResourceAnnotations[key] + if found && searchValue == "" || value == searchValue { + result = updateContainers(upgradeFuncs, i, config, true) + } + } + if result == constants.Updated { err = upgradeFuncs.UpdateFunc(clients, config.Namespace, i) resourceName := util.ToObjectMeta(i).Name diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index 6c8098cf..6bc8e313 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -15,6 +15,7 @@ import ( "github.com/stakater/Reloader/internal/pkg/testutil" "github.com/stakater/Reloader/internal/pkg/util" "github.com/stakater/Reloader/pkg/kube" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" testclient "k8s.io/client-go/kubernetes/fake" ) @@ -622,7 +623,6 @@ func TestRollingUpgradeForDeploymentWithConfigmapInProjectedVolume(t *testing.T) collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Configmap in projected volume") } @@ -638,6 +638,166 @@ func TestRollingUpgradeForDeploymentWithConfigmapInProjectedVolume(t *testing.T) } } +func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotation(t *testing.T) { + annotatedConfigmapName := "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) + configmapObj := testutil.GetConfigmap(namespace, annotatedConfigmapName, "www.google.com") + configmapObj.Annotations = map[string]string{"test-annotation": "test"} + configmap, err := clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Create(configmapObj) + if err != nil { + t.Errorf("Failed to create config map with annotation.") + } + defer clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Delete(configmap.Name, &v1.DeleteOptions{}) + deploymentObj := testutil.GetDeploymentWithEnvVars(namespace, annotatedConfigmapName) + deploymentObj.Annotations = map[string]string{options.ConfigmapUpdateAutoSearchAnnotation: "test-annotation=test"} + deployment, err := clients.KubernetesClient.AppsV1().Deployments(namespace).Create(deploymentObj) + if err != nil { + t.Errorf("Failed to create deployment with search annotation.") + } + defer clients.KubernetesClient.AppsV1().Deployments(namespace).Delete(deployment.Name, &v1.DeleteOptions{}) + + shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, annotatedConfigmapName, "www.stakater.com") + config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, annotatedConfigmapName, shaData, "") + config.SearchAnnotation = options.ConfigmapUpdateAutoSearchAnnotation + config.ResourceAnnotations = configmap.Annotations + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err = PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with Configmap") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceUpdate(clients, config, constants.ConfigmapEnvVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } +} + +func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNoValue(t *testing.T) { + annotatedConfigmapName := "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) + configmapObj := testutil.GetConfigmap(namespace, annotatedConfigmapName, "www.google.com") + configmapObj.Annotations = map[string]string{"test-annotation": "test"} + configmap, err := clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Create(configmapObj) + if err != nil { + t.Errorf("Failed to create config map with annotation.") + } + defer clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Delete(configmap.Name, &v1.DeleteOptions{}) + deploymentObj := testutil.GetDeploymentWithEnvVars(namespace, annotatedConfigmapName) + deploymentObj.Annotations = map[string]string{options.ConfigmapUpdateAutoSearchAnnotation: "test-annotation"} + deployment, err := clients.KubernetesClient.AppsV1().Deployments(namespace).Create(deploymentObj) + if err != nil { + t.Errorf("Failed to create deployment with search annotation.") + } + defer clients.KubernetesClient.AppsV1().Deployments(namespace).Delete(deployment.Name, &v1.DeleteOptions{}) + + shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, annotatedConfigmapName, "www.stakater.com") + config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, annotatedConfigmapName, shaData, "") + config.SearchAnnotation = options.ConfigmapUpdateAutoSearchAnnotation + config.ResourceAnnotations = configmap.Annotations + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err = PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with Configmap") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceUpdate(clients, config, constants.ConfigmapEnvVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } +} + +func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNotFound(t *testing.T) { + annotatedConfigmapName := "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) + configmapObj := testutil.GetConfigmap(namespace, annotatedConfigmapName, "www.google.com") + configmapObj.Annotations = map[string]string{"test-annotation": "test"} + configmap, err := clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Create(configmapObj) + if err != nil { + t.Errorf("Failed to create config map with annotation.") + } + defer clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Delete(configmap.Name, &v1.DeleteOptions{}) + deploymentObj := testutil.GetDeploymentWithEnvVars(namespace, annotatedConfigmapName) + deploymentObj.Annotations = map[string]string{options.ConfigmapUpdateAutoSearchAnnotation: "test-annotation=not-found"} + deployment, err := clients.KubernetesClient.AppsV1().Deployments(namespace).Create(deploymentObj) + if err != nil { + t.Errorf("Failed to create deployment with search annotation.") + } + defer clients.KubernetesClient.AppsV1().Deployments(namespace).Delete(deployment.Name, &v1.DeleteOptions{}) + + shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, annotatedConfigmapName, "www.stakater.com") + config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, annotatedConfigmapName, shaData, "") + config.SearchAnnotation = options.ConfigmapUpdateAutoSearchAnnotation + config.ResourceAnnotations = configmap.Annotations + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err = PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with Configmap") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceUpdate(clients, config, constants.ConfigmapEnvVarPostfix, deploymentFuncs) + if updated { + t.Errorf("Deployment was updated unexpectedly") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) > 0 { + t.Errorf("Counter was increased unexpectedly") + } +} + +func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNotMapped(t *testing.T) { + annotatedConfigmapName := "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) + configmapObj := testutil.GetConfigmap(namespace, annotatedConfigmapName, "www.google.com") + configmapObj.Annotations = map[string]string{"test-annotation": "test"} + configmap, err := clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Create(configmapObj) + if err != nil { + t.Errorf("Failed to create config map with annotation.") + } + defer clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Delete(configmap.Name, &v1.DeleteOptions{}) + deploymentObj := testutil.GetDeploymentWithEnvVars(namespace, annotatedConfigmapName+"-different") + deploymentObj.Annotations = map[string]string{options.ConfigmapUpdateAutoSearchAnnotation: "test-annotation=test"} + deployment, err := clients.KubernetesClient.AppsV1().Deployments(namespace).Create(deploymentObj) + if err != nil { + t.Errorf("Failed to create deployment with search annotation.") + } + defer clients.KubernetesClient.AppsV1().Deployments(namespace).Delete(deployment.Name, &v1.DeleteOptions{}) + + shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, annotatedConfigmapName, "www.stakater.com") + config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, annotatedConfigmapName, shaData, "") + config.SearchAnnotation = options.ConfigmapUpdateAutoSearchAnnotation + config.ResourceAnnotations = configmap.Annotations + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err = PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with Configmap") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceUpdate(clients, config, constants.ConfigmapEnvVarPostfix, deploymentFuncs) + if updated { + t.Errorf("Deployment was updated unexpectedly") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) > 0 { + t.Errorf("Counter was increased unexpectedly") + } +} + func TestRollingUpgradeForDeploymentWithConfigmapInInitContainer(t *testing.T) { shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, configmapWithInitContainer, "www.stakater.com") config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, configmapWithInitContainer, shaData, options.ConfigmapUpdateOnChangeAnnotation) @@ -645,7 +805,6 @@ func TestRollingUpgradeForDeploymentWithConfigmapInInitContainer(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Configmap") } @@ -691,7 +850,6 @@ func TestRollingUpgradeForDeploymentWithConfigmapAsEnvVar(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Configmap used as env var") } @@ -714,7 +872,6 @@ func TestRollingUpgradeForDeploymentWithConfigmapAsEnvVarInInitContainer(t *test collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Configmap used as env var") } @@ -737,7 +894,6 @@ func TestRollingUpgradeForDeploymentWithConfigmapAsEnvVarFrom(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Configmap used as env var") } @@ -760,7 +916,6 @@ func TestRollingUpgradeForDeploymentWithSecret(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Secret") } @@ -806,7 +961,6 @@ func TestRollingUpgradeForDeploymentWithSecretinInitContainer(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Secret") } @@ -852,7 +1006,6 @@ func TestRollingUpgradeForDeploymentWithSecretAsEnvVar(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Secret") } @@ -875,7 +1028,6 @@ func TestRollingUpgradeForDeploymentWithSecretAsEnvVarFrom(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Secret") } @@ -898,7 +1050,6 @@ func TestRollingUpgradeForDeploymentWithSecretAsEnvVarInInitContainer(t *testing collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Secret") } @@ -921,7 +1072,6 @@ func TestRollingUpgradeForDaemonSetWithConfigmap(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, daemonSetFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for DaemonSet with configmap") } @@ -967,7 +1117,6 @@ func TestRollingUpgradeForDaemonSetWithConfigmapAsEnvVar(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, daemonSetFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for DaemonSet with configmap used as env var") } @@ -990,7 +1139,6 @@ func TestRollingUpgradeForDaemonSetWithSecret(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, daemonSetFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for DaemonSet with secret") } @@ -1036,7 +1184,6 @@ func TestRollingUpgradeForStatefulSetWithConfigmap(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, statefulSetFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for StatefulSet with configmap") } @@ -1082,7 +1229,6 @@ func TestRollingUpgradeForStatefulSetWithSecret(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, statefulSetFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for StatefulSet with secret") } @@ -1128,7 +1274,6 @@ func TestRollingUpgradeForDeploymentWithPodAnnotations(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with pod annotations") } diff --git a/internal/pkg/options/flags.go b/internal/pkg/options/flags.go index 41c2daba..c6be7ee4 100644 --- a/internal/pkg/options/flags.go +++ b/internal/pkg/options/flags.go @@ -1,12 +1,20 @@ package options var ( - // ConfigmapUpdateOnChangeAnnotation is an annotation to detect changes in configmaps + // ConfigmapUpdateOnChangeAnnotation is an annotation to detect changes in + // configmaps specified by name ConfigmapUpdateOnChangeAnnotation = "configmap.reloader.stakater.com/reload" - // SecretUpdateOnChangeAnnotation is an annotation to detect changes in secrets + // SecretUpdateOnChangeAnnotation is an annotation to detect changes in + // secrets specified by name SecretUpdateOnChangeAnnotation = "secret.reloader.stakater.com/reload" // ReloaderAutoAnnotation is an annotation to detect changes in secrets ReloaderAutoAnnotation = "reloader.stakater.com/auto" + // ConfigmapUpdateAutoSearchAnnotation is an annotation to detect changes in + // configmaps searched by annotation + ConfigmapUpdateAutoSearchAnnotation = "configmap.reloader.stakater.com/auto-by-annotation" + // SecretUpdateAutoSearchAnnotation is an annotation to detect changes in + // secrets searched by annotation + SecretUpdateAutoSearchAnnotation = "secret.reloader.stakater.com/auto-by-annotation" // LogFormat is the log format to use (json, or empty string for default) LogFormat = "" ) diff --git a/internal/pkg/testutil/kube.go b/internal/pkg/testutil/kube.go index 479f3be8..c6ad67ce 100644 --- a/internal/pkg/testutil/kube.go +++ b/internal/pkg/testutil/kube.go @@ -598,7 +598,6 @@ func CreateConfigMap(client kubernetes.Interface, namespace string, configmapNam logrus.Infof("Creating configmap") configmapClient := client.CoreV1().ConfigMaps(namespace) _, err := configmapClient.Create(GetConfigmap(namespace, configmapName, data)) - time.Sleep(3 * time.Second) return configmapClient, err } @@ -607,7 +606,6 @@ func CreateSecret(client kubernetes.Interface, namespace string, secretName stri logrus.Infof("Creating secret") secretClient := client.CoreV1().Secrets(namespace) _, err := secretClient.Create(GetSecret(namespace, secretName, data)) - time.Sleep(3 * time.Second) return secretClient, err } // CreateDeployment creates a deployment in given namespace and returns the Deployment @@ -621,7 +619,6 @@ func CreateDeployment(client kubernetes.Interface, deploymentName string, namesp deploymentObj = GetDeploymentWithEnvVars(namespace, deploymentName) } deployment, err := deploymentClient.Create(deploymentObj) - time.Sleep(3 * time.Second) return deployment, err } @@ -636,7 +633,6 @@ func CreateDeploymentConfig(client appsclient.Interface, deploymentName string, deploymentConfigObj = GetDeploymentConfigWithEnvVars(namespace, deploymentName) } deploymentConfig, err := deploymentConfigsClient.Create(deploymentConfigObj) - time.Sleep(5 * time.Second) return deploymentConfig, err } @@ -651,7 +647,6 @@ func CreateDeploymentWithInitContainer(client kubernetes.Interface, deploymentNa deploymentObj = GetDeploymentWithInitContainerAndEnv(namespace, deploymentName) } deployment, err := deploymentClient.Create(deploymentObj) - time.Sleep(3 * time.Second) return deployment, err } @@ -661,7 +656,6 @@ func CreateDeploymentWithEnvVarSource(client kubernetes.Interface, deploymentNam deploymentClient := client.AppsV1().Deployments(namespace) deploymentObj := GetDeploymentWithEnvVarSources(namespace, deploymentName) deployment, err := deploymentClient.Create(deploymentObj) - time.Sleep(3 * time.Second) return deployment, err } @@ -671,7 +665,6 @@ func CreateDeploymentWithPodAnnotations(client kubernetes.Interface, deploymentN deploymentClient := client.AppsV1().Deployments(namespace) deploymentObj := GetDeploymentWithPodAnnotations(namespace, deploymentName, both) deployment, err := deploymentClient.Create(deploymentObj) - time.Sleep(3 * time.Second) return deployment, err } @@ -686,7 +679,6 @@ func CreateDaemonSet(client kubernetes.Interface, daemonsetName string, namespac daemonsetObj = GetDaemonSetWithEnvVars(namespace, daemonsetName) } daemonset, err := daemonsetClient.Create(daemonsetObj) - time.Sleep(3 * time.Second) return daemonset, err } @@ -701,7 +693,6 @@ func CreateStatefulSet(client kubernetes.Interface, statefulsetName string, name statefulsetObj = GetStatefulSetWithEnvVar(namespace, statefulsetName) } statefulset, err := statefulsetClient.Create(statefulsetObj) - time.Sleep(3 * time.Second) return statefulset, err } @@ -709,7 +700,6 @@ func CreateStatefulSet(client kubernetes.Interface, statefulsetName string, name func DeleteDeployment(client kubernetes.Interface, namespace string, deploymentName string) error { logrus.Infof("Deleting Deployment") deploymentError := client.AppsV1().Deployments(namespace).Delete(deploymentName, &metav1.DeleteOptions{}) - time.Sleep(3 * time.Second) return deploymentError } @@ -717,7 +707,6 @@ func DeleteDeployment(client kubernetes.Interface, namespace string, deploymentN func DeleteDeploymentConfig(client appsclient.Interface, namespace string, deploymentConfigName string) error { logrus.Infof("Deleting DeploymentConfig") deploymentConfigError := client.AppsV1().DeploymentConfigs(namespace).Delete(deploymentConfigName, &metav1.DeleteOptions{}) - time.Sleep(3 * time.Second) return deploymentConfigError } @@ -725,7 +714,6 @@ func DeleteDeploymentConfig(client appsclient.Interface, namespace string, deplo func DeleteDaemonSet(client kubernetes.Interface, namespace string, daemonsetName string) error { logrus.Infof("Deleting DaemonSet %s", daemonsetName) daemonsetError := client.AppsV1().DaemonSets(namespace).Delete(daemonsetName, &metav1.DeleteOptions{}) - time.Sleep(3 * time.Second) return daemonsetError } @@ -733,7 +721,6 @@ func DeleteDaemonSet(client kubernetes.Interface, namespace string, daemonsetNam func DeleteStatefulSet(client kubernetes.Interface, namespace string, statefulsetName string) error { logrus.Infof("Deleting StatefulSet %s", statefulsetName) statefulsetError := client.AppsV1().StatefulSets(namespace).Delete(statefulsetName, &metav1.DeleteOptions{}) - time.Sleep(3 * time.Second) return statefulsetError } @@ -747,7 +734,6 @@ func UpdateConfigMap(configmapClient core_v1.ConfigMapInterface, namespace strin configmap = GetConfigmap(namespace, configmapName, data) } _, updateErr := configmapClient.Update(configmap) - time.Sleep(3 * time.Second) return updateErr } @@ -761,7 +747,6 @@ func UpdateSecret(secretClient core_v1.SecretInterface, namespace string, secret secret = GetSecret(namespace, secretName, data) } _, updateErr := secretClient.Update(secret) - time.Sleep(3 * time.Second) return updateErr } @@ -769,7 +754,6 @@ func UpdateSecret(secretClient core_v1.SecretInterface, namespace string, secret func DeleteConfigMap(client kubernetes.Interface, namespace string, configmapName string) error { logrus.Infof("Deleting configmap %q.\n", configmapName) err := client.CoreV1().ConfigMaps(namespace).Delete(configmapName, &metav1.DeleteOptions{}) - time.Sleep(3 * time.Second) return err } @@ -777,7 +761,6 @@ func DeleteConfigMap(client kubernetes.Interface, namespace string, configmapNam func DeleteSecret(client kubernetes.Interface, namespace string, secretName string) error { logrus.Infof("Deleting secret %q.\n", secretName) err := client.CoreV1().Secrets(namespace).Delete(secretName, &metav1.DeleteOptions{}) - time.Sleep(3 * time.Second) return err } @@ -798,6 +781,7 @@ func VerifyResourceUpdate(clients kube.Clients, config util.Config, envVarPostfi containers := upgradeFuncs.ContainersFunc(i) // match statefulsets with the correct annotation annotationValue := util.ToObjectMeta(i).Annotations[config.Annotation] + searchAnnotationValue := util.ToObjectMeta(i).Annotations[config.SearchAnnotation] reloaderEnabledValue := util.ToObjectMeta(i).Annotations[options.ReloaderAutoAnnotation] reloaderEnabled, err := strconv.ParseBool(reloaderEnabledValue) matches := false @@ -811,6 +795,17 @@ func VerifyResourceUpdate(clients kube.Clients, config util.Config, envVarPostfi break } } + } else if searchAnnotationValue != "" { + keyValue := strings.Split(searchAnnotationValue, "=") + key := keyValue[0] + valueToSearch := "" + if len(keyValue) > 1 { + valueToSearch = keyValue[1] + } + value, found := config.ResourceAnnotations[key] + if found && (valueToSearch == "" || value == valueToSearch) { + matches = true + } } if matches { diff --git a/internal/pkg/util/config.go b/internal/pkg/util/config.go index e8de37e2..51354214 100644 --- a/internal/pkg/util/config.go +++ b/internal/pkg/util/config.go @@ -8,31 +8,37 @@ import ( //Config contains rolling upgrade configuration parameters type Config struct { - Namespace string - ResourceName string - Annotation string - SHAValue string - Type string + Namespace string + ResourceName string + ResourceAnnotations map[string]string + Annotation string + SearchAnnotation string + SHAValue string + Type string } // GetConfigmapConfig provides utility config for configmap func GetConfigmapConfig(configmap *v1.ConfigMap) Config { return Config{ - Namespace: configmap.Namespace, - ResourceName: configmap.Name, - Annotation: options.ConfigmapUpdateOnChangeAnnotation, - SHAValue: GetSHAfromConfigmap(configmap.Data), - Type: constants.ConfigmapEnvVarPostfix, + Namespace: configmap.Namespace, + ResourceName: configmap.Name, + ResourceAnnotations: configmap.Annotations, + Annotation: options.ConfigmapUpdateOnChangeAnnotation, + SearchAnnotation: options.ConfigmapUpdateAutoSearchAnnotation, + SHAValue: GetSHAfromConfigmap(configmap.Data), + Type: constants.ConfigmapEnvVarPostfix, } } // GetSecretConfig provides utility config for secret func GetSecretConfig(secret *v1.Secret) Config { return Config{ - Namespace: secret.Namespace, - ResourceName: secret.Name, - Annotation: options.SecretUpdateOnChangeAnnotation, - SHAValue: GetSHAfromSecret(secret.Data), - Type: constants.SecretEnvVarPostfix, + Namespace: secret.Namespace, + ResourceName: secret.Name, + ResourceAnnotations: secret.Annotations, + Annotation: options.SecretUpdateOnChangeAnnotation, + SearchAnnotation: options.SecretUpdateAutoSearchAnnotation, + SHAValue: GetSHAfromSecret(secret.Data), + Type: constants.SecretEnvVarPostfix, } } From 5980c91560417d24a5033ca85221d54e54fddd82 Mon Sep 17 00:00:00 2001 From: Vlad Losev Date: Tue, 2 Jun 2020 19:28:05 -0700 Subject: [PATCH 02/11] Abstracts out configmap and deployment creation. --- internal/pkg/handler/upgrade_test.go | 61 ++++++++++++++++------------ internal/pkg/testutil/kube.go | 13 ++++-- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index 6bc8e313..b452c233 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -15,6 +15,7 @@ import ( "github.com/stakater/Reloader/internal/pkg/testutil" "github.com/stakater/Reloader/internal/pkg/util" "github.com/stakater/Reloader/pkg/kube" + core_v1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" testclient "k8s.io/client-go/kubernetes/fake" ) @@ -638,18 +639,25 @@ func TestRollingUpgradeForDeploymentWithConfigmapInProjectedVolume(t *testing.T) } } +func createConfigMap(clients *kube.Clients, namespace, name string, annotations map[string]string) (*core_v1.ConfigMap, error) { + configmapObj := testutil.GetConfigmap(namespace, name, "www.google.com") + configmapObj.Annotations = annotations + return clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Create(configmapObj) +} + func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotation(t *testing.T) { annotatedConfigmapName := "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) - configmapObj := testutil.GetConfigmap(namespace, annotatedConfigmapName, "www.google.com") - configmapObj.Annotations = map[string]string{"test-annotation": "test"} - configmap, err := clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Create(configmapObj) + configmap, err := createConfigMap(&clients, namespace, annotatedConfigmapName, map[string]string{"test-annotation": "test"}) if err != nil { t.Errorf("Failed to create config map with annotation.") } defer clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Delete(configmap.Name, &v1.DeleteOptions{}) - deploymentObj := testutil.GetDeploymentWithEnvVars(namespace, annotatedConfigmapName) - deploymentObj.Annotations = map[string]string{options.ConfigmapUpdateAutoSearchAnnotation: "test-annotation=test"} - deployment, err := clients.KubernetesClient.AppsV1().Deployments(namespace).Create(deploymentObj) + deployment, err := testutil.CreateDeploymentWithEnvVarSourceAndAnnotations( + clients.KubernetesClient, + annotatedConfigmapName, + namespace, + map[string]string{options.ConfigmapUpdateAutoSearchAnnotation: "test-annotation=test"}, + ) if err != nil { t.Errorf("Failed to create deployment with search annotation.") } @@ -680,16 +688,16 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotation(t *testing. func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNoValue(t *testing.T) { annotatedConfigmapName := "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) - configmapObj := testutil.GetConfigmap(namespace, annotatedConfigmapName, "www.google.com") - configmapObj.Annotations = map[string]string{"test-annotation": "test"} - configmap, err := clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Create(configmapObj) + configmap, err := createConfigMap(&clients, namespace, annotatedConfigmapName, map[string]string{"test-annotation": "test"}) if err != nil { t.Errorf("Failed to create config map with annotation.") } - defer clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Delete(configmap.Name, &v1.DeleteOptions{}) - deploymentObj := testutil.GetDeploymentWithEnvVars(namespace, annotatedConfigmapName) - deploymentObj.Annotations = map[string]string{options.ConfigmapUpdateAutoSearchAnnotation: "test-annotation"} - deployment, err := clients.KubernetesClient.AppsV1().Deployments(namespace).Create(deploymentObj) + deployment, err := testutil.CreateDeploymentWithEnvVarSourceAndAnnotations( + clients.KubernetesClient, + annotatedConfigmapName, + namespace, + map[string]string{options.ConfigmapUpdateAutoSearchAnnotation: "test-annotation"}, + ) if err != nil { t.Errorf("Failed to create deployment with search annotation.") } @@ -720,21 +728,21 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNoValue(t *t func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNotFound(t *testing.T) { annotatedConfigmapName := "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) - configmapObj := testutil.GetConfigmap(namespace, annotatedConfigmapName, "www.google.com") - configmapObj.Annotations = map[string]string{"test-annotation": "test"} - configmap, err := clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Create(configmapObj) + configmap, err := createConfigMap(&clients, namespace, annotatedConfigmapName, map[string]string{"test-annotation": "not-found"}) if err != nil { t.Errorf("Failed to create config map with annotation.") } defer clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Delete(configmap.Name, &v1.DeleteOptions{}) - deploymentObj := testutil.GetDeploymentWithEnvVars(namespace, annotatedConfigmapName) - deploymentObj.Annotations = map[string]string{options.ConfigmapUpdateAutoSearchAnnotation: "test-annotation=not-found"} - deployment, err := clients.KubernetesClient.AppsV1().Deployments(namespace).Create(deploymentObj) + deployment, err := testutil.CreateDeploymentWithEnvVarSourceAndAnnotations( + clients.KubernetesClient, + annotatedConfigmapName, + namespace, + map[string]string{options.ConfigmapUpdateAutoSearchAnnotation: "test-annotation=test"}, + ) if err != nil { t.Errorf("Failed to create deployment with search annotation.") } defer clients.KubernetesClient.AppsV1().Deployments(namespace).Delete(deployment.Name, &v1.DeleteOptions{}) - shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, annotatedConfigmapName, "www.stakater.com") config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, annotatedConfigmapName, shaData, "") config.SearchAnnotation = options.ConfigmapUpdateAutoSearchAnnotation @@ -760,16 +768,17 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNotFound(t * func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNotMapped(t *testing.T) { annotatedConfigmapName := "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) - configmapObj := testutil.GetConfigmap(namespace, annotatedConfigmapName, "www.google.com") - configmapObj.Annotations = map[string]string{"test-annotation": "test"} - configmap, err := clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Create(configmapObj) + configmap, err := createConfigMap(&clients, namespace, annotatedConfigmapName, map[string]string{"test-annotation": "test"}) if err != nil { t.Errorf("Failed to create config map with annotation.") } defer clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Delete(configmap.Name, &v1.DeleteOptions{}) - deploymentObj := testutil.GetDeploymentWithEnvVars(namespace, annotatedConfigmapName+"-different") - deploymentObj.Annotations = map[string]string{options.ConfigmapUpdateAutoSearchAnnotation: "test-annotation=test"} - deployment, err := clients.KubernetesClient.AppsV1().Deployments(namespace).Create(deploymentObj) + deployment, err := testutil.CreateDeploymentWithEnvVarSourceAndAnnotations( + clients.KubernetesClient, + annotatedConfigmapName+"-different", + namespace, + map[string]string{options.ConfigmapUpdateAutoSearchAnnotation: "test-annotation=test"}, + ) if err != nil { t.Errorf("Failed to create deployment with search annotation.") } diff --git a/internal/pkg/testutil/kube.go b/internal/pkg/testutil/kube.go index c6ad67ce..2a69ef87 100644 --- a/internal/pkg/testutil/kube.go +++ b/internal/pkg/testutil/kube.go @@ -650,16 +650,23 @@ func CreateDeploymentWithInitContainer(client kubernetes.Interface, deploymentNa return deployment, err } -// CreateDeploymentWithEnvVarSource creates a deployment in given namespace and returns the Deployment -func CreateDeploymentWithEnvVarSource(client kubernetes.Interface, deploymentName string, namespace string) (*appsv1.Deployment, error) { +// CreateDeploymentWithEnvVarSourceAndAnnotations returns a deployment in given +// namespace with given annotations. +func CreateDeploymentWithEnvVarSourceAndAnnotations(client kubernetes.Interface, deploymentName string, namespace string, annotations map[string]string) (*appsv1.Deployment, error) { logrus.Infof("Creating Deployment") deploymentClient := client.AppsV1().Deployments(namespace) deploymentObj := GetDeploymentWithEnvVarSources(namespace, deploymentName) + deploymentObj.Annotations = annotations deployment, err := deploymentClient.Create(deploymentObj) return deployment, err } -// CreateDeployment creates a deployment in given namespace and returns the Deployment +// CreateDeploymentWithEnvVarSource creates a deployment in given namespace and returns the Deployment +func CreateDeploymentWithEnvVarSource(client kubernetes.Interface, deploymentName string, namespace string) (*appsv1.Deployment, error) { + return CreateDeploymentWithEnvVarSourceAndAnnotations(client, deploymentName, namespace, map[string]string{}) +} + +// CreateDeploymentWithPodAnnotations creates a deployment in given namespace and returns the Deployment func CreateDeploymentWithPodAnnotations(client kubernetes.Interface, deploymentName string, namespace string, both bool) (*appsv1.Deployment, error) { logrus.Infof("Creating Deployment") deploymentClient := client.AppsV1().Deployments(namespace) From 17f8b8111081a48327fcfec52201c61a4621ad0d Mon Sep 17 00:00:00 2001 From: Vlad Losev Date: Wed, 3 Jun 2020 17:40:57 -0700 Subject: [PATCH 03/11] Simplifies annotations for searching secrets for reload. --- internal/pkg/cmd/reloader.go | 4 +- internal/pkg/handler/upgrade.go | 16 +++----- internal/pkg/handler/upgrade_test.go | 59 ++++------------------------ internal/pkg/options/flags.go | 12 +++--- internal/pkg/testutil/kube.go | 13 ++---- internal/pkg/util/config.go | 3 -- 6 files changed, 24 insertions(+), 83 deletions(-) diff --git a/internal/pkg/cmd/reloader.go b/internal/pkg/cmd/reloader.go index 17d8d42f..0edaf5b7 100644 --- a/internal/pkg/cmd/reloader.go +++ b/internal/pkg/cmd/reloader.go @@ -27,8 +27,8 @@ func NewReloaderCommand() *cobra.Command { cmd.PersistentFlags().StringVar(&options.ConfigmapUpdateOnChangeAnnotation, "configmap-annotation", "configmap.reloader.stakater.com/reload", "annotation to detect changes in configmaps, specified by name") cmd.PersistentFlags().StringVar(&options.SecretUpdateOnChangeAnnotation, "secret-annotation", "secret.reloader.stakater.com/reload", "annotation to detect changes in secrets, specified by name") cmd.PersistentFlags().StringVar(&options.ReloaderAutoAnnotation, "auto-annotation", "reloader.stakater.com/auto", "annotation to detect changes in secrets") - cmd.PersistentFlags().StringVar(&options.ConfigmapUpdateAutoSearchAnnotation, "configmap-auto-by-annotation", "configmap.reloader.stakater.com/reload-by-annotation", "annotation to detect changes in configmaps, searched by annotation") - cmd.PersistentFlags().StringVar(&options.SecretUpdateAutoSearchAnnotation, "secret-auto-by-annotation", "secret.reloader.stakater.com/reload-by-annotation", "annotation to detect changes in secret, searched by annotations") + cmd.PersistentFlags().StringVar(&options.AutoSearchAnnotation, "auto-search-annotation", "reloader.stakater.com/annotated", "annotation to detect changes in configmaps or secrets tagged with special match annotation") + cmd.PersistentFlags().StringVar(&options.SearchMatchAnnotation, "search-match-annotation", "reloader.stakater.com/match", "annotation to mark secrets or configmapts to match the search") cmd.PersistentFlags().StringVar(&options.LogFormat, "log-format", "", "Log format to use (empty string for text, or JSON") cmd.PersistentFlags().StringSlice("resources-to-ignore", []string{}, "list of resources to ignore (valid options 'configMaps' or 'secrets')") cmd.PersistentFlags().StringSlice("namespaces-to-ignore", []string{}, "list of namespaces to ignore") diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index 38baf2eb..508481b9 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -99,12 +99,12 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc // find correct annotation and update the resource annotations := upgradeFuncs.AnnotationsFunc(i) annotationValue, found := annotations[config.Annotation] - searchAnnotationValue, foundSearchAnn := annotations[config.SearchAnnotation] + searchAnnotationValue, foundSearchAnn := annotations[options.AutoSearchAnnotation] reloaderEnabledValue, foundAuto := annotations[options.ReloaderAutoAnnotation] if !found && !foundAuto && !foundSearchAnn { annotations = upgradeFuncs.PodAnnotationsFunc(i) annotationValue = annotations[config.Annotation] - searchAnnotationValue = annotations[config.SearchAnnotation] + searchAnnotationValue = annotations[options.AutoSearchAnnotation] reloaderEnabledValue = annotations[options.ReloaderAutoAnnotation] } result := constants.NotUpdated @@ -125,15 +125,9 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc } } - if result != constants.Updated && searchAnnotationValue != "" { - keyValue := strings.Split(searchAnnotationValue, "=") - key := keyValue[0] - searchValue := "" - if len(keyValue) > 1 { - searchValue = keyValue[1] - } - value, found := config.ResourceAnnotations[key] - if found && searchValue == "" || value == searchValue { + if result != constants.Updated && searchAnnotationValue == "true" { + matchAnnotationValue := config.ResourceAnnotations[options.SearchMatchAnnotation] + if matchAnnotationValue == "true" { result = updateContainers(upgradeFuncs, i, config, true) } } diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index b452c233..ee3a670b 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -646,8 +646,8 @@ func createConfigMap(clients *kube.Clients, namespace, name string, annotations } func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotation(t *testing.T) { - annotatedConfigmapName := "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) - configmap, err := createConfigMap(&clients, namespace, annotatedConfigmapName, map[string]string{"test-annotation": "test"}) + annotatedConfigmapName := "testconfigmapAnnotated-handler" + configmap, err := createConfigMap(&clients, namespace, annotatedConfigmapName, map[string]string{"reloader.stakater.com/match": "true"}) if err != nil { t.Errorf("Failed to create config map with annotation.") } @@ -656,7 +656,7 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotation(t *testing. clients.KubernetesClient, annotatedConfigmapName, namespace, - map[string]string{options.ConfigmapUpdateAutoSearchAnnotation: "test-annotation=test"}, + map[string]string{"reloader.stakater.com/annotated": "true"}, ) if err != nil { t.Errorf("Failed to create deployment with search annotation.") @@ -665,7 +665,6 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotation(t *testing. shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, annotatedConfigmapName, "www.stakater.com") config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, annotatedConfigmapName, shaData, "") - config.SearchAnnotation = options.ConfigmapUpdateAutoSearchAnnotation config.ResourceAnnotations = configmap.Annotations deploymentFuncs := GetDeploymentRollingUpgradeFuncs() collectors := getCollectors() @@ -686,9 +685,9 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotation(t *testing. } } -func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNoValue(t *testing.T) { +func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNoTriggers(t *testing.T) { annotatedConfigmapName := "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) - configmap, err := createConfigMap(&clients, namespace, annotatedConfigmapName, map[string]string{"test-annotation": "test"}) + configmap, err := createConfigMap(&clients, namespace, annotatedConfigmapName, map[string]string{"reloader.stakater.com/match": "false"}) if err != nil { t.Errorf("Failed to create config map with annotation.") } @@ -696,7 +695,7 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNoValue(t *t clients.KubernetesClient, annotatedConfigmapName, namespace, - map[string]string{options.ConfigmapUpdateAutoSearchAnnotation: "test-annotation"}, + map[string]string{"reloader.stakater.com/annotated": "true"}, ) if err != nil { t.Errorf("Failed to create deployment with search annotation.") @@ -705,47 +704,6 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNoValue(t *t shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, annotatedConfigmapName, "www.stakater.com") config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, annotatedConfigmapName, shaData, "") - config.SearchAnnotation = options.ConfigmapUpdateAutoSearchAnnotation - config.ResourceAnnotations = configmap.Annotations - deploymentFuncs := GetDeploymentRollingUpgradeFuncs() - collectors := getCollectors() - - err = PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) - if err != nil { - t.Errorf("Rolling upgrade failed for Deployment with Configmap") - } - - logrus.Infof("Verifying deployment update") - updated := testutil.VerifyResourceUpdate(clients, config, constants.ConfigmapEnvVarPostfix, deploymentFuncs) - if !updated { - t.Errorf("Deployment was not updated") - } - - if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { - t.Errorf("Counter was not increased") - } -} - -func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNotFound(t *testing.T) { - annotatedConfigmapName := "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) - configmap, err := createConfigMap(&clients, namespace, annotatedConfigmapName, map[string]string{"test-annotation": "not-found"}) - if err != nil { - t.Errorf("Failed to create config map with annotation.") - } - defer clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Delete(configmap.Name, &v1.DeleteOptions{}) - deployment, err := testutil.CreateDeploymentWithEnvVarSourceAndAnnotations( - clients.KubernetesClient, - annotatedConfigmapName, - namespace, - map[string]string{options.ConfigmapUpdateAutoSearchAnnotation: "test-annotation=test"}, - ) - if err != nil { - t.Errorf("Failed to create deployment with search annotation.") - } - defer clients.KubernetesClient.AppsV1().Deployments(namespace).Delete(deployment.Name, &v1.DeleteOptions{}) - shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, annotatedConfigmapName, "www.stakater.com") - config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, annotatedConfigmapName, shaData, "") - config.SearchAnnotation = options.ConfigmapUpdateAutoSearchAnnotation config.ResourceAnnotations = configmap.Annotations deploymentFuncs := GetDeploymentRollingUpgradeFuncs() collectors := getCollectors() @@ -768,7 +726,7 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNotFound(t * func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNotMapped(t *testing.T) { annotatedConfigmapName := "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) - configmap, err := createConfigMap(&clients, namespace, annotatedConfigmapName, map[string]string{"test-annotation": "test"}) + configmap, err := createConfigMap(&clients, namespace, annotatedConfigmapName, map[string]string{"reloader.stakater.com/match": "true"}) if err != nil { t.Errorf("Failed to create config map with annotation.") } @@ -777,7 +735,7 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNotMapped(t clients.KubernetesClient, annotatedConfigmapName+"-different", namespace, - map[string]string{options.ConfigmapUpdateAutoSearchAnnotation: "test-annotation=test"}, + map[string]string{"reloader.stakater.com/annotated": "true"}, ) if err != nil { t.Errorf("Failed to create deployment with search annotation.") @@ -786,7 +744,6 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNotMapped(t shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, annotatedConfigmapName, "www.stakater.com") config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, annotatedConfigmapName, shaData, "") - config.SearchAnnotation = options.ConfigmapUpdateAutoSearchAnnotation config.ResourceAnnotations = configmap.Annotations deploymentFuncs := GetDeploymentRollingUpgradeFuncs() collectors := getCollectors() diff --git a/internal/pkg/options/flags.go b/internal/pkg/options/flags.go index c6be7ee4..28f9d05b 100644 --- a/internal/pkg/options/flags.go +++ b/internal/pkg/options/flags.go @@ -9,12 +9,12 @@ var ( SecretUpdateOnChangeAnnotation = "secret.reloader.stakater.com/reload" // ReloaderAutoAnnotation is an annotation to detect changes in secrets ReloaderAutoAnnotation = "reloader.stakater.com/auto" - // ConfigmapUpdateAutoSearchAnnotation is an annotation to detect changes in - // configmaps searched by annotation - ConfigmapUpdateAutoSearchAnnotation = "configmap.reloader.stakater.com/auto-by-annotation" - // SecretUpdateAutoSearchAnnotation is an annotation to detect changes in - // secrets searched by annotation - SecretUpdateAutoSearchAnnotation = "secret.reloader.stakater.com/auto-by-annotation" + // AutoSearchAnnotation is an annotation to detect changes in + // configmaps or triggers with the SearchMatchAnnotation + AutoSearchAnnotation = "reloader.stakater.com/annotated" + // SearchMatchAnnotation is an annotation to tag secrets to be found with + // AutoSearchAnnotation + SearchMatchAnnotation = "reloader.stakater.com/match" // LogFormat is the log format to use (json, or empty string for default) LogFormat = "" ) diff --git a/internal/pkg/testutil/kube.go b/internal/pkg/testutil/kube.go index 2a69ef87..6d2d9938 100644 --- a/internal/pkg/testutil/kube.go +++ b/internal/pkg/testutil/kube.go @@ -788,7 +788,7 @@ func VerifyResourceUpdate(clients kube.Clients, config util.Config, envVarPostfi containers := upgradeFuncs.ContainersFunc(i) // match statefulsets with the correct annotation annotationValue := util.ToObjectMeta(i).Annotations[config.Annotation] - searchAnnotationValue := util.ToObjectMeta(i).Annotations[config.SearchAnnotation] + searchAnnotationValue := util.ToObjectMeta(i).Annotations[options.AutoSearchAnnotation] reloaderEnabledValue := util.ToObjectMeta(i).Annotations[options.ReloaderAutoAnnotation] reloaderEnabled, err := strconv.ParseBool(reloaderEnabledValue) matches := false @@ -802,15 +802,8 @@ func VerifyResourceUpdate(clients kube.Clients, config util.Config, envVarPostfi break } } - } else if searchAnnotationValue != "" { - keyValue := strings.Split(searchAnnotationValue, "=") - key := keyValue[0] - valueToSearch := "" - if len(keyValue) > 1 { - valueToSearch = keyValue[1] - } - value, found := config.ResourceAnnotations[key] - if found && (valueToSearch == "" || value == valueToSearch) { + } else if searchAnnotationValue == "true" { + if config.ResourceAnnotations[options.SearchMatchAnnotation] == "true" { matches = true } } diff --git a/internal/pkg/util/config.go b/internal/pkg/util/config.go index 51354214..2244328d 100644 --- a/internal/pkg/util/config.go +++ b/internal/pkg/util/config.go @@ -12,7 +12,6 @@ type Config struct { ResourceName string ResourceAnnotations map[string]string Annotation string - SearchAnnotation string SHAValue string Type string } @@ -24,7 +23,6 @@ func GetConfigmapConfig(configmap *v1.ConfigMap) Config { ResourceName: configmap.Name, ResourceAnnotations: configmap.Annotations, Annotation: options.ConfigmapUpdateOnChangeAnnotation, - SearchAnnotation: options.ConfigmapUpdateAutoSearchAnnotation, SHAValue: GetSHAfromConfigmap(configmap.Data), Type: constants.ConfigmapEnvVarPostfix, } @@ -37,7 +35,6 @@ func GetSecretConfig(secret *v1.Secret) Config { ResourceName: secret.Name, ResourceAnnotations: secret.Annotations, Annotation: options.SecretUpdateOnChangeAnnotation, - SearchAnnotation: options.SecretUpdateAutoSearchAnnotation, SHAValue: GetSHAfromSecret(secret.Data), Type: constants.SecretEnvVarPostfix, } From e81b49d81b2d4edfb7b5a9a90d9c7ac25e49a9df Mon Sep 17 00:00:00 2001 From: Vlad Losev Date: Wed, 3 Jun 2020 18:11:38 -0700 Subject: [PATCH 04/11] Renames search annotation. --- internal/pkg/cmd/reloader.go | 2 +- internal/pkg/handler/upgrade_test.go | 6 +++--- internal/pkg/options/flags.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/pkg/cmd/reloader.go b/internal/pkg/cmd/reloader.go index 0edaf5b7..1851b227 100644 --- a/internal/pkg/cmd/reloader.go +++ b/internal/pkg/cmd/reloader.go @@ -27,7 +27,7 @@ func NewReloaderCommand() *cobra.Command { cmd.PersistentFlags().StringVar(&options.ConfigmapUpdateOnChangeAnnotation, "configmap-annotation", "configmap.reloader.stakater.com/reload", "annotation to detect changes in configmaps, specified by name") cmd.PersistentFlags().StringVar(&options.SecretUpdateOnChangeAnnotation, "secret-annotation", "secret.reloader.stakater.com/reload", "annotation to detect changes in secrets, specified by name") cmd.PersistentFlags().StringVar(&options.ReloaderAutoAnnotation, "auto-annotation", "reloader.stakater.com/auto", "annotation to detect changes in secrets") - cmd.PersistentFlags().StringVar(&options.AutoSearchAnnotation, "auto-search-annotation", "reloader.stakater.com/annotated", "annotation to detect changes in configmaps or secrets tagged with special match annotation") + cmd.PersistentFlags().StringVar(&options.AutoSearchAnnotation, "auto-search-annotation", "reloader.stakater.com/search", "annotation to detect changes in configmaps or secrets tagged with special match annotation") cmd.PersistentFlags().StringVar(&options.SearchMatchAnnotation, "search-match-annotation", "reloader.stakater.com/match", "annotation to mark secrets or configmapts to match the search") cmd.PersistentFlags().StringVar(&options.LogFormat, "log-format", "", "Log format to use (empty string for text, or JSON") cmd.PersistentFlags().StringSlice("resources-to-ignore", []string{}, "list of resources to ignore (valid options 'configMaps' or 'secrets')") diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index ee3a670b..578757bb 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -656,7 +656,7 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotation(t *testing. clients.KubernetesClient, annotatedConfigmapName, namespace, - map[string]string{"reloader.stakater.com/annotated": "true"}, + map[string]string{"reloader.stakater.com/search": "true"}, ) if err != nil { t.Errorf("Failed to create deployment with search annotation.") @@ -695,7 +695,7 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNoTriggers(t clients.KubernetesClient, annotatedConfigmapName, namespace, - map[string]string{"reloader.stakater.com/annotated": "true"}, + map[string]string{"reloader.stakater.com/search": "true"}, ) if err != nil { t.Errorf("Failed to create deployment with search annotation.") @@ -735,7 +735,7 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNotMapped(t clients.KubernetesClient, annotatedConfigmapName+"-different", namespace, - map[string]string{"reloader.stakater.com/annotated": "true"}, + map[string]string{"reloader.stakater.com/search": "true"}, ) if err != nil { t.Errorf("Failed to create deployment with search annotation.") diff --git a/internal/pkg/options/flags.go b/internal/pkg/options/flags.go index 28f9d05b..1bf681ed 100644 --- a/internal/pkg/options/flags.go +++ b/internal/pkg/options/flags.go @@ -11,7 +11,7 @@ var ( ReloaderAutoAnnotation = "reloader.stakater.com/auto" // AutoSearchAnnotation is an annotation to detect changes in // configmaps or triggers with the SearchMatchAnnotation - AutoSearchAnnotation = "reloader.stakater.com/annotated" + AutoSearchAnnotation = "reloader.stakater.com/search" // SearchMatchAnnotation is an annotation to tag secrets to be found with // AutoSearchAnnotation SearchMatchAnnotation = "reloader.stakater.com/match" From 965cacf1bacc8c1e88d5ed072d3447e5824536b6 Mon Sep 17 00:00:00 2001 From: Vlad Losev Date: Wed, 3 Jun 2020 18:13:26 -0700 Subject: [PATCH 05/11] Updates documentation. --- README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/README.md b/README.md index c0039789..e1f2acd2 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,33 @@ spec: This will discover deployments/daemonsets/statefulset automatically where `foo-configmap` or `foo-secret` is being used either via environment variable or from volume mount. And it will perform rolling upgrade on related pods when `foo-configmap` or `foo-secret`are updated. +You can restrict this discovery to only `ConfigMap` or `Secret` objects that +are tagged with a special annotation. To take advantage of that, annotate +your deployment/daemonset/statefulset like this: + +```yaml +kind: Deployment +metadata: + annotations: + reloader.stakater.com/search: "true" +spec: + template: +``` + +and Reloader will trigger the rolling upgrade when any `ConfigMap` or `Secret` annotated like this: +and then annotate your `ConfigMap` or `Secret` so: + +```yaml +kind: ConfigMap +metadata: + annotations: + reloader.stakater.com/match: "true" +data: + key: value +``` + +is modified, provided it is being used in an environment variable or a volume mount. + We can also specify a specific configmap or secret which would trigger rolling upgrade only upon change in our specified configmap or secret, this way, it will not trigger rolling upgrade upon changes in all configmaps or secrets used in a deployment, daemonset or statefulset. To do this either set the auto annotation to `"false"` (`reloader.stakater.com/auto: "false"`) or remove it altogether, and use annotations mentioned [here](#Configmap) or [here](#Secret) @@ -99,6 +126,8 @@ spec: - `reloader.stakater.com/auto: "true"` will only reload the pod, if the configmap or secret is used (as a volume mount or as an env) in `DeploymentConfigs/Deployment/Daemonsets/Statefulsets` - `secret.reloader.stakater.com/reload` or `configmap.reloader.stakater.com/reload` annotation will reload the pod upon changes in specified configmap or secret, irrespective of the usage of configmap or secret. - you may override the auto annotation with the `--auto-annotation` flag +- you may override the search annotation with the `--auto-search-annotation` flag + and the match annotation with the `--search-match-annotation` flag - you may override the configmap annotation with the `--configmap-annotation` flag - you may override the secret annotation with the `--secret-annotation` flag - you may want to prevent watching certain namespaces with the `--namespaces-to-ignore` flag From 3131116ed665b5686df8443263908c433a213a13 Mon Sep 17 00:00:00 2001 From: Vlad Losev Date: Thu, 4 Jun 2020 10:05:44 -0700 Subject: [PATCH 06/11] Re-adds sleep statements. --- internal/pkg/handler/upgrade_test.go | 16 ++++++++++++++++ internal/pkg/testutil/kube.go | 17 +++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index 578757bb..565cd9e8 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -715,6 +715,7 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNoTriggers(t logrus.Infof("Verifying deployment update") updated := testutil.VerifyResourceUpdate(clients, config, constants.ConfigmapEnvVarPostfix, deploymentFuncs) + time.Sleep(5 * time.Second) if updated { t.Errorf("Deployment was updated unexpectedly") } @@ -771,6 +772,7 @@ func TestRollingUpgradeForDeploymentWithConfigmapInInitContainer(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Configmap") } @@ -816,6 +818,7 @@ func TestRollingUpgradeForDeploymentWithConfigmapAsEnvVar(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Configmap used as env var") } @@ -838,6 +841,7 @@ func TestRollingUpgradeForDeploymentWithConfigmapAsEnvVarInInitContainer(t *test collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Configmap used as env var") } @@ -860,6 +864,7 @@ func TestRollingUpgradeForDeploymentWithConfigmapAsEnvVarFrom(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Configmap used as env var") } @@ -882,6 +887,7 @@ func TestRollingUpgradeForDeploymentWithSecret(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Secret") } @@ -927,6 +933,7 @@ func TestRollingUpgradeForDeploymentWithSecretinInitContainer(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Secret") } @@ -972,6 +979,7 @@ func TestRollingUpgradeForDeploymentWithSecretAsEnvVar(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Secret") } @@ -994,6 +1002,7 @@ func TestRollingUpgradeForDeploymentWithSecretAsEnvVarFrom(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Secret") } @@ -1016,6 +1025,7 @@ func TestRollingUpgradeForDeploymentWithSecretAsEnvVarInInitContainer(t *testing collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Secret") } @@ -1038,6 +1048,7 @@ func TestRollingUpgradeForDaemonSetWithConfigmap(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, daemonSetFuncs, collectors) + time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for DaemonSet with configmap") } @@ -1083,6 +1094,7 @@ func TestRollingUpgradeForDaemonSetWithConfigmapAsEnvVar(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, daemonSetFuncs, collectors) + time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for DaemonSet with configmap used as env var") } @@ -1105,6 +1117,7 @@ func TestRollingUpgradeForDaemonSetWithSecret(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, daemonSetFuncs, collectors) + time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for DaemonSet with secret") } @@ -1150,6 +1163,7 @@ func TestRollingUpgradeForStatefulSetWithConfigmap(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, statefulSetFuncs, collectors) + time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for StatefulSet with configmap") } @@ -1195,6 +1209,7 @@ func TestRollingUpgradeForStatefulSetWithSecret(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, statefulSetFuncs, collectors) + time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for StatefulSet with secret") } @@ -1240,6 +1255,7 @@ func TestRollingUpgradeForDeploymentWithPodAnnotations(t *testing.T) { collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with pod annotations") } diff --git a/internal/pkg/testutil/kube.go b/internal/pkg/testutil/kube.go index 6d2d9938..07acd05d 100644 --- a/internal/pkg/testutil/kube.go +++ b/internal/pkg/testutil/kube.go @@ -598,6 +598,7 @@ func CreateConfigMap(client kubernetes.Interface, namespace string, configmapNam logrus.Infof("Creating configmap") configmapClient := client.CoreV1().ConfigMaps(namespace) _, err := configmapClient.Create(GetConfigmap(namespace, configmapName, data)) + time.Sleep(3 * time.Second) return configmapClient, err } @@ -606,6 +607,7 @@ func CreateSecret(client kubernetes.Interface, namespace string, secretName stri logrus.Infof("Creating secret") secretClient := client.CoreV1().Secrets(namespace) _, err := secretClient.Create(GetSecret(namespace, secretName, data)) + time.Sleep(3 * time.Second) return secretClient, err } // CreateDeployment creates a deployment in given namespace and returns the Deployment @@ -619,6 +621,7 @@ func CreateDeployment(client kubernetes.Interface, deploymentName string, namesp deploymentObj = GetDeploymentWithEnvVars(namespace, deploymentName) } deployment, err := deploymentClient.Create(deploymentObj) + time.Sleep(3 * time.Second) return deployment, err } @@ -633,6 +636,7 @@ func CreateDeploymentConfig(client appsclient.Interface, deploymentName string, deploymentConfigObj = GetDeploymentConfigWithEnvVars(namespace, deploymentName) } deploymentConfig, err := deploymentConfigsClient.Create(deploymentConfigObj) + time.Sleep(5 * time.Second) return deploymentConfig, err } @@ -647,6 +651,7 @@ func CreateDeploymentWithInitContainer(client kubernetes.Interface, deploymentNa deploymentObj = GetDeploymentWithInitContainerAndEnv(namespace, deploymentName) } deployment, err := deploymentClient.Create(deploymentObj) + time.Sleep(3 * time.Second) return deployment, err } @@ -658,6 +663,7 @@ func CreateDeploymentWithEnvVarSourceAndAnnotations(client kubernetes.Interface, deploymentObj := GetDeploymentWithEnvVarSources(namespace, deploymentName) deploymentObj.Annotations = annotations deployment, err := deploymentClient.Create(deploymentObj) + time.Sleep(3 * time.Second) return deployment, err } @@ -672,6 +678,7 @@ func CreateDeploymentWithPodAnnotations(client kubernetes.Interface, deploymentN deploymentClient := client.AppsV1().Deployments(namespace) deploymentObj := GetDeploymentWithPodAnnotations(namespace, deploymentName, both) deployment, err := deploymentClient.Create(deploymentObj) + time.Sleep(3 * time.Second) return deployment, err } @@ -686,6 +693,7 @@ func CreateDaemonSet(client kubernetes.Interface, daemonsetName string, namespac daemonsetObj = GetDaemonSetWithEnvVars(namespace, daemonsetName) } daemonset, err := daemonsetClient.Create(daemonsetObj) + time.Sleep(3 * time.Second) return daemonset, err } @@ -700,6 +708,7 @@ func CreateStatefulSet(client kubernetes.Interface, statefulsetName string, name statefulsetObj = GetStatefulSetWithEnvVar(namespace, statefulsetName) } statefulset, err := statefulsetClient.Create(statefulsetObj) + time.Sleep(3 * time.Second) return statefulset, err } @@ -707,6 +716,7 @@ func CreateStatefulSet(client kubernetes.Interface, statefulsetName string, name func DeleteDeployment(client kubernetes.Interface, namespace string, deploymentName string) error { logrus.Infof("Deleting Deployment") deploymentError := client.AppsV1().Deployments(namespace).Delete(deploymentName, &metav1.DeleteOptions{}) + time.Sleep(3 * time.Second) return deploymentError } @@ -714,6 +724,7 @@ func DeleteDeployment(client kubernetes.Interface, namespace string, deploymentN func DeleteDeploymentConfig(client appsclient.Interface, namespace string, deploymentConfigName string) error { logrus.Infof("Deleting DeploymentConfig") deploymentConfigError := client.AppsV1().DeploymentConfigs(namespace).Delete(deploymentConfigName, &metav1.DeleteOptions{}) + time.Sleep(3 * time.Second) return deploymentConfigError } @@ -721,6 +732,7 @@ func DeleteDeploymentConfig(client appsclient.Interface, namespace string, deplo func DeleteDaemonSet(client kubernetes.Interface, namespace string, daemonsetName string) error { logrus.Infof("Deleting DaemonSet %s", daemonsetName) daemonsetError := client.AppsV1().DaemonSets(namespace).Delete(daemonsetName, &metav1.DeleteOptions{}) + time.Sleep(3 * time.Second) return daemonsetError } @@ -728,6 +740,7 @@ func DeleteDaemonSet(client kubernetes.Interface, namespace string, daemonsetNam func DeleteStatefulSet(client kubernetes.Interface, namespace string, statefulsetName string) error { logrus.Infof("Deleting StatefulSet %s", statefulsetName) statefulsetError := client.AppsV1().StatefulSets(namespace).Delete(statefulsetName, &metav1.DeleteOptions{}) + time.Sleep(3 * time.Second) return statefulsetError } @@ -741,6 +754,7 @@ func UpdateConfigMap(configmapClient core_v1.ConfigMapInterface, namespace strin configmap = GetConfigmap(namespace, configmapName, data) } _, updateErr := configmapClient.Update(configmap) + time.Sleep(3 * time.Second) return updateErr } @@ -754,6 +768,7 @@ func UpdateSecret(secretClient core_v1.SecretInterface, namespace string, secret secret = GetSecret(namespace, secretName, data) } _, updateErr := secretClient.Update(secret) + time.Sleep(3 * time.Second) return updateErr } @@ -761,6 +776,7 @@ func UpdateSecret(secretClient core_v1.SecretInterface, namespace string, secret func DeleteConfigMap(client kubernetes.Interface, namespace string, configmapName string) error { logrus.Infof("Deleting configmap %q.\n", configmapName) err := client.CoreV1().ConfigMaps(namespace).Delete(configmapName, &metav1.DeleteOptions{}) + time.Sleep(3 * time.Second) return err } @@ -768,6 +784,7 @@ func DeleteConfigMap(client kubernetes.Interface, namespace string, configmapNam func DeleteSecret(client kubernetes.Interface, namespace string, secretName string) error { logrus.Infof("Deleting secret %q.\n", secretName) err := client.CoreV1().Secrets(namespace).Delete(secretName, &metav1.DeleteOptions{}) + time.Sleep(3 * time.Second) return err } From be7d454504087284278a6f14b947a8ae4e648cf0 Mon Sep 17 00:00:00 2001 From: Vlad Losev Date: Thu, 4 Jun 2020 14:35:37 -0700 Subject: [PATCH 07/11] Fixes test using auto annotations. --- internal/pkg/testutil/kube.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/internal/pkg/testutil/kube.go b/internal/pkg/testutil/kube.go index 07acd05d..0ca20765 100644 --- a/internal/pkg/testutil/kube.go +++ b/internal/pkg/testutil/kube.go @@ -655,21 +655,15 @@ func CreateDeploymentWithInitContainer(client kubernetes.Interface, deploymentNa return deployment, err } -// CreateDeploymentWithEnvVarSourceAndAnnotations returns a deployment in given -// namespace with given annotations. -func CreateDeploymentWithEnvVarSourceAndAnnotations(client kubernetes.Interface, deploymentName string, namespace string, annotations map[string]string) (*appsv1.Deployment, error) { +// CreateDeploymentWithEnvVarSource creates a deployment in given namespace and returns the Deployment +func CreateDeploymentWithEnvVarSource(client kubernetes.Interface, deploymentName string, namespace string) (*appsv1.Deployment, error) { logrus.Infof("Creating Deployment") deploymentClient := client.AppsV1().Deployments(namespace) deploymentObj := GetDeploymentWithEnvVarSources(namespace, deploymentName) - deploymentObj.Annotations = annotations deployment, err := deploymentClient.Create(deploymentObj) time.Sleep(3 * time.Second) return deployment, err -} -// CreateDeploymentWithEnvVarSource creates a deployment in given namespace and returns the Deployment -func CreateDeploymentWithEnvVarSource(client kubernetes.Interface, deploymentName string, namespace string) (*appsv1.Deployment, error) { - return CreateDeploymentWithEnvVarSourceAndAnnotations(client, deploymentName, namespace, map[string]string{}) } // CreateDeploymentWithPodAnnotations creates a deployment in given namespace and returns the Deployment @@ -682,6 +676,18 @@ func CreateDeploymentWithPodAnnotations(client kubernetes.Interface, deploymentN return deployment, err } +// CreateDeploymentWithEnvVarSourceAndAnnotations returns a deployment in given +// namespace with given annotations. +func CreateDeploymentWithEnvVarSourceAndAnnotations(client kubernetes.Interface, deploymentName string, namespace string, annotations map[string]string) (*appsv1.Deployment, error) { + logrus.Infof("Creating Deployment") + deploymentClient := client.AppsV1().Deployments(namespace) + deploymentObj := GetDeploymentWithEnvVarSources(namespace, deploymentName) + deploymentObj.Annotations = annotations + deployment, err := deploymentClient.Create(deploymentObj) + time.Sleep(3 * time.Second) + return deployment, err +} + // CreateDaemonSet creates a deployment in given namespace and returns the DaemonSet func CreateDaemonSet(client kubernetes.Interface, daemonsetName string, namespace string, volumeMount bool) (*appsv1.DaemonSet, error) { logrus.Infof("Creating DaemonSet") From ff27cc0f519e529e70876e930ecf95e73c71e597 Mon Sep 17 00:00:00 2001 From: Vlad Losev Date: Thu, 4 Jun 2020 16:41:35 -0700 Subject: [PATCH 08/11] Simplifies test. --- internal/pkg/handler/upgrade_test.go | 81 +++++++++++----------------- 1 file changed, 30 insertions(+), 51 deletions(-) diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index 565cd9e8..bd4e7c34 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -39,6 +39,7 @@ var ( secretWithEnvFromName = "testsecretWithEnvFrom-handler-" + testutil.RandSeq(5) configmapWithPodAnnotations = "testconfigmapPodAnnotations-handler-" + testutil.RandSeq(5) configmapWithBothAnnotations = "testconfigmapBothAnnotations-handler-" + testutil.RandSeq(5) + configmapAnnotated = "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) ) func TestMain(m *testing.M) { @@ -229,6 +230,17 @@ func setup() { logrus.Errorf("Error in Deployment with secret configmap as envFrom source creation: %v", err) } + // Creating Deployment with envFrom source as secret + _, err = testutil.CreateDeploymentWithEnvVarSourceAndAnnotations( + clients.KubernetesClient, + configmapAnnotated, + namespace, + map[string]string{"reloader.stakater.com/search": "true"}, + ) + if err != nil { + logrus.Errorf("Error in Deployment with secret configmap as envFrom source creation: %v", err) + } + // Creating DaemonSet with configmap _, err = testutil.CreateDaemonSet(clients.KubernetesClient, configmapName, namespace, true) if err != nil { @@ -411,6 +423,12 @@ func teardown() { logrus.Errorf("Error while deleting deployment with both annotations %v", deploymentError) } + // Deleting Deployment with search annotation + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, namespace, configmapAnnotated) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with search annotation %v", deploymentError) + } + // Deleting DaemonSet with configmap daemonSetError := testutil.DeleteDaemonSet(clients.KubernetesClient, namespace, configmapName) if daemonSetError != nil { @@ -646,30 +664,13 @@ func createConfigMap(clients *kube.Clients, namespace, name string, annotations } func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotation(t *testing.T) { - annotatedConfigmapName := "testconfigmapAnnotated-handler" - configmap, err := createConfigMap(&clients, namespace, annotatedConfigmapName, map[string]string{"reloader.stakater.com/match": "true"}) - if err != nil { - t.Errorf("Failed to create config map with annotation.") - } - defer clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Delete(configmap.Name, &v1.DeleteOptions{}) - deployment, err := testutil.CreateDeploymentWithEnvVarSourceAndAnnotations( - clients.KubernetesClient, - annotatedConfigmapName, - namespace, - map[string]string{"reloader.stakater.com/search": "true"}, - ) - if err != nil { - t.Errorf("Failed to create deployment with search annotation.") - } - defer clients.KubernetesClient.AppsV1().Deployments(namespace).Delete(deployment.Name, &v1.DeleteOptions{}) - - shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, annotatedConfigmapName, "www.stakater.com") - config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, annotatedConfigmapName, shaData, "") - config.ResourceAnnotations = configmap.Annotations + shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, configmapAnnotated, "www.stakater.com") + config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, configmapAnnotated, shaData, "") + config.ResourceAnnotations = map[string]string{"reloader.stakater.com/match": "true"} deploymentFuncs := GetDeploymentRollingUpgradeFuncs() collectors := getCollectors() - err = PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Configmap") } @@ -686,29 +687,13 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotation(t *testing. } func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNoTriggers(t *testing.T) { - annotatedConfigmapName := "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) - configmap, err := createConfigMap(&clients, namespace, annotatedConfigmapName, map[string]string{"reloader.stakater.com/match": "false"}) - if err != nil { - t.Errorf("Failed to create config map with annotation.") - } - deployment, err := testutil.CreateDeploymentWithEnvVarSourceAndAnnotations( - clients.KubernetesClient, - annotatedConfigmapName, - namespace, - map[string]string{"reloader.stakater.com/search": "true"}, - ) - if err != nil { - t.Errorf("Failed to create deployment with search annotation.") - } - defer clients.KubernetesClient.AppsV1().Deployments(namespace).Delete(deployment.Name, &v1.DeleteOptions{}) - - shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, annotatedConfigmapName, "www.stakater.com") - config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, annotatedConfigmapName, shaData, "") - config.ResourceAnnotations = configmap.Annotations + shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, configmapAnnotated, "www.stakater.com") + config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, configmapAnnotated, shaData, "") + config.ResourceAnnotations = map[string]string{"reloader.stakater.com/match": "false"} deploymentFuncs := GetDeploymentRollingUpgradeFuncs() collectors := getCollectors() - err = PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Configmap") } @@ -726,15 +711,9 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNoTriggers(t } func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNotMapped(t *testing.T) { - annotatedConfigmapName := "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) - configmap, err := createConfigMap(&clients, namespace, annotatedConfigmapName, map[string]string{"reloader.stakater.com/match": "true"}) - if err != nil { - t.Errorf("Failed to create config map with annotation.") - } - defer clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Delete(configmap.Name, &v1.DeleteOptions{}) deployment, err := testutil.CreateDeploymentWithEnvVarSourceAndAnnotations( clients.KubernetesClient, - annotatedConfigmapName+"-different", + configmapAnnotated+"-different", namespace, map[string]string{"reloader.stakater.com/search": "true"}, ) @@ -743,9 +722,9 @@ func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNotMapped(t } defer clients.KubernetesClient.AppsV1().Deployments(namespace).Delete(deployment.Name, &v1.DeleteOptions{}) - shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, annotatedConfigmapName, "www.stakater.com") - config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, annotatedConfigmapName, shaData, "") - config.ResourceAnnotations = configmap.Annotations + shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, configmapAnnotated, "www.stakater.com") + config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, configmapAnnotated, shaData, "") + config.ResourceAnnotations = map[string]string{"reloader.stakater.com/match": "false"} deploymentFuncs := GetDeploymentRollingUpgradeFuncs() collectors := getCollectors() From 0988e8947f0e51af270c3945e0f907fcccc8b971 Mon Sep 17 00:00:00 2001 From: Vlad Losev Date: Thu, 4 Jun 2020 17:24:03 -0700 Subject: [PATCH 09/11] Removes unused import. --- internal/pkg/handler/upgrade_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index bd4e7c34..b65bbda9 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -15,7 +15,6 @@ import ( "github.com/stakater/Reloader/internal/pkg/testutil" "github.com/stakater/Reloader/internal/pkg/util" "github.com/stakater/Reloader/pkg/kube" - core_v1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" testclient "k8s.io/client-go/kubernetes/fake" ) From 6bcec06052bf9a90d7639e1068db2f913355dbd2 Mon Sep 17 00:00:00 2001 From: Vlad Losev Date: Tue, 9 Jun 2020 19:45:12 -0700 Subject: [PATCH 10/11] Fixes missing import. --- internal/pkg/handler/upgrade_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index b65bbda9..bd4e7c34 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -15,6 +15,7 @@ import ( "github.com/stakater/Reloader/internal/pkg/testutil" "github.com/stakater/Reloader/internal/pkg/util" "github.com/stakater/Reloader/pkg/kube" + core_v1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" testclient "k8s.io/client-go/kubernetes/fake" ) From e61f9a6bdbed2818533d37ea1fa6d5ab91488110 Mon Sep 17 00:00:00 2001 From: Vlad Losev Date: Sun, 21 Jun 2020 14:15:58 -0700 Subject: [PATCH 11/11] Adds note on incompatibility with 'reloader.stakater.com/auto'. --- README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e1f2acd2..180654da 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,8 @@ spec: template: ``` -and Reloader will trigger the rolling upgrade when any `ConfigMap` or `Secret` annotated like this: -and then annotate your `ConfigMap` or `Secret` so: +and Reloader will trigger the rolling upgrade upon modification of any +`ConfigMap` or `Secret` annotated like this: ```yaml kind: ConfigMap @@ -63,7 +63,15 @@ data: key: value ``` -is modified, provided it is being used in an environment variable or a volume mount. +provided the secret/configmap is being used in an environment variable or a +volume mount. + +Please note that `reloader.stakater.com/search` and +`reloader.stakater.com/auto` do not work together. If you have the +`reloader.stakater.com/auto: "true"` annotation on your deployment, then it +will always restart upon a change in configmaps or secrets it uses, regardless +of whether they have the `reloader.stakater.com/match: "true"` annotation or +not. We can also specify a specific configmap or secret which would trigger rolling upgrade only upon change in our specified configmap or secret, this way, it will not trigger rolling upgrade upon changes in all configmaps or secrets used in a deployment, daemonset or statefulset. To do this either set the auto annotation to `"false"` (`reloader.stakater.com/auto: "false"`) or remove it altogether, and use annotations mentioned [here](#Configmap) or [here](#Secret)