diff --git a/README.md b/README.md index c0039789..180654da 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,41 @@ spec: This will discover deployments/daemonsets/statefulset automatically where `foo-configmap` or `foo-secret` is being used either via environment variable or from volume mount. And it will perform rolling upgrade on related pods when `foo-configmap` or `foo-secret`are updated. +You can restrict this discovery to only `ConfigMap` or `Secret` objects that +are tagged with a special annotation. To take advantage of that, annotate +your deployment/daemonset/statefulset like this: + +```yaml +kind: Deployment +metadata: + annotations: + reloader.stakater.com/search: "true" +spec: + template: +``` + +and Reloader will trigger the rolling upgrade upon modification of any +`ConfigMap` or `Secret` annotated like this: + +```yaml +kind: ConfigMap +metadata: + annotations: + reloader.stakater.com/match: "true" +data: + key: value +``` + +provided the secret/configmap is being used in an environment variable or a +volume mount. + +Please note that `reloader.stakater.com/search` and +`reloader.stakater.com/auto` do not work together. If you have the +`reloader.stakater.com/auto: "true"` annotation on your deployment, then it +will always restart upon a change in configmaps or secrets it uses, regardless +of whether they have the `reloader.stakater.com/match: "true"` annotation or +not. + We can also specify a specific configmap or secret which would trigger rolling upgrade only upon change in our specified configmap or secret, this way, it will not trigger rolling upgrade upon changes in all configmaps or secrets used in a deployment, daemonset or statefulset. To do this either set the auto annotation to `"false"` (`reloader.stakater.com/auto: "false"`) or remove it altogether, and use annotations mentioned [here](#Configmap) or [here](#Secret) @@ -99,6 +134,8 @@ spec: - `reloader.stakater.com/auto: "true"` will only reload the pod, if the configmap or secret is used (as a volume mount or as an env) in `DeploymentConfigs/Deployment/Daemonsets/Statefulsets` - `secret.reloader.stakater.com/reload` or `configmap.reloader.stakater.com/reload` annotation will reload the pod upon changes in specified configmap or secret, irrespective of the usage of configmap or secret. - you may override the auto annotation with the `--auto-annotation` flag +- you may override the search annotation with the `--auto-search-annotation` flag + and the match annotation with the `--search-match-annotation` flag - you may override the configmap annotation with the `--configmap-annotation` flag - you may override the secret annotation with the `--secret-annotation` flag - you may want to prevent watching certain namespaces with the `--namespaces-to-ignore` flag diff --git a/internal/pkg/cmd/reloader.go b/internal/pkg/cmd/reloader.go index b92dd266..1851b227 100644 --- a/internal/pkg/cmd/reloader.go +++ b/internal/pkg/cmd/reloader.go @@ -24,9 +24,11 @@ func NewReloaderCommand() *cobra.Command { } // options - cmd.PersistentFlags().StringVar(&options.ConfigmapUpdateOnChangeAnnotation, "configmap-annotation", "configmap.reloader.stakater.com/reload", "annotation to detect changes in configmaps") - cmd.PersistentFlags().StringVar(&options.SecretUpdateOnChangeAnnotation, "secret-annotation", "secret.reloader.stakater.com/reload", "annotation to detect changes in secrets") + cmd.PersistentFlags().StringVar(&options.ConfigmapUpdateOnChangeAnnotation, "configmap-annotation", "configmap.reloader.stakater.com/reload", "annotation to detect changes in configmaps, specified by name") + cmd.PersistentFlags().StringVar(&options.SecretUpdateOnChangeAnnotation, "secret-annotation", "secret.reloader.stakater.com/reload", "annotation to detect changes in secrets, specified by name") cmd.PersistentFlags().StringVar(&options.ReloaderAutoAnnotation, "auto-annotation", "reloader.stakater.com/auto", "annotation to detect changes in secrets") + cmd.PersistentFlags().StringVar(&options.AutoSearchAnnotation, "auto-search-annotation", "reloader.stakater.com/search", "annotation to detect changes in configmaps or secrets tagged with special match annotation") + cmd.PersistentFlags().StringVar(&options.SearchMatchAnnotation, "search-match-annotation", "reloader.stakater.com/match", "annotation to mark secrets or configmapts to match the search") cmd.PersistentFlags().StringVar(&options.LogFormat, "log-format", "", "Log format to use (empty string for text, or JSON") cmd.PersistentFlags().StringSlice("resources-to-ignore", []string{}, "list of resources to ignore (valid options 'configMaps' or 'secrets')") cmd.PersistentFlags().StringSlice("namespaces-to-ignore", []string{}, "list of namespaces to ignore") diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index bb7d6bae..508481b9 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -99,10 +99,12 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc // find correct annotation and update the resource annotations := upgradeFuncs.AnnotationsFunc(i) annotationValue, found := annotations[config.Annotation] + searchAnnotationValue, foundSearchAnn := annotations[options.AutoSearchAnnotation] reloaderEnabledValue, foundAuto := annotations[options.ReloaderAutoAnnotation] - if !found && !foundAuto { + if !found && !foundAuto && !foundSearchAnn { annotations = upgradeFuncs.PodAnnotationsFunc(i) annotationValue = annotations[config.Annotation] + searchAnnotationValue = annotations[options.AutoSearchAnnotation] reloaderEnabledValue = annotations[options.ReloaderAutoAnnotation] } result := constants.NotUpdated @@ -123,6 +125,13 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc } } + if result != constants.Updated && searchAnnotationValue == "true" { + matchAnnotationValue := config.ResourceAnnotations[options.SearchMatchAnnotation] + if matchAnnotationValue == "true" { + result = updateContainers(upgradeFuncs, i, config, true) + } + } + if result == constants.Updated { err = upgradeFuncs.UpdateFunc(clients, config.Namespace, i) resourceName := util.ToObjectMeta(i).Name diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index 6c8098cf..bd4e7c34 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -15,6 +15,8 @@ import ( "github.com/stakater/Reloader/internal/pkg/testutil" "github.com/stakater/Reloader/internal/pkg/util" "github.com/stakater/Reloader/pkg/kube" + core_v1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" testclient "k8s.io/client-go/kubernetes/fake" ) @@ -37,6 +39,7 @@ var ( secretWithEnvFromName = "testsecretWithEnvFrom-handler-" + testutil.RandSeq(5) configmapWithPodAnnotations = "testconfigmapPodAnnotations-handler-" + testutil.RandSeq(5) configmapWithBothAnnotations = "testconfigmapBothAnnotations-handler-" + testutil.RandSeq(5) + configmapAnnotated = "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) ) func TestMain(m *testing.M) { @@ -227,6 +230,17 @@ func setup() { logrus.Errorf("Error in Deployment with secret configmap as envFrom source creation: %v", err) } + // Creating Deployment with envFrom source as secret + _, err = testutil.CreateDeploymentWithEnvVarSourceAndAnnotations( + clients.KubernetesClient, + configmapAnnotated, + namespace, + map[string]string{"reloader.stakater.com/search": "true"}, + ) + if err != nil { + logrus.Errorf("Error in Deployment with secret configmap as envFrom source creation: %v", err) + } + // Creating DaemonSet with configmap _, err = testutil.CreateDaemonSet(clients.KubernetesClient, configmapName, namespace, true) if err != nil { @@ -409,6 +423,12 @@ func teardown() { logrus.Errorf("Error while deleting deployment with both annotations %v", deploymentError) } + // Deleting Deployment with search annotation + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, namespace, configmapAnnotated) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with search annotation %v", deploymentError) + } + // Deleting DaemonSet with configmap daemonSetError := testutil.DeleteDaemonSet(clients.KubernetesClient, namespace, configmapName) if daemonSetError != nil { @@ -622,7 +642,6 @@ func TestRollingUpgradeForDeploymentWithConfigmapInProjectedVolume(t *testing.T) collectors := getCollectors() err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) - time.Sleep(5 * time.Second) if err != nil { t.Errorf("Rolling upgrade failed for Deployment with Configmap in projected volume") } @@ -638,6 +657,93 @@ func TestRollingUpgradeForDeploymentWithConfigmapInProjectedVolume(t *testing.T) } } +func createConfigMap(clients *kube.Clients, namespace, name string, annotations map[string]string) (*core_v1.ConfigMap, error) { + configmapObj := testutil.GetConfigmap(namespace, name, "www.google.com") + configmapObj.Annotations = annotations + return clients.KubernetesClient.CoreV1().ConfigMaps(namespace).Create(configmapObj) +} + +func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotation(t *testing.T) { + shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, configmapAnnotated, "www.stakater.com") + config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, configmapAnnotated, shaData, "") + config.ResourceAnnotations = map[string]string{"reloader.stakater.com/match": "true"} + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with Configmap") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceUpdate(clients, config, constants.ConfigmapEnvVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } +} + +func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNoTriggers(t *testing.T) { + shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, configmapAnnotated, "www.stakater.com") + config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, configmapAnnotated, shaData, "") + config.ResourceAnnotations = map[string]string{"reloader.stakater.com/match": "false"} + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with Configmap") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceUpdate(clients, config, constants.ConfigmapEnvVarPostfix, deploymentFuncs) + time.Sleep(5 * time.Second) + if updated { + t.Errorf("Deployment was updated unexpectedly") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) > 0 { + t.Errorf("Counter was increased unexpectedly") + } +} + +func TestRollingUpgradeForDeploymentWithConfigmapViaSearchAnnotationNotMapped(t *testing.T) { + deployment, err := testutil.CreateDeploymentWithEnvVarSourceAndAnnotations( + clients.KubernetesClient, + configmapAnnotated+"-different", + namespace, + map[string]string{"reloader.stakater.com/search": "true"}, + ) + if err != nil { + t.Errorf("Failed to create deployment with search annotation.") + } + defer clients.KubernetesClient.AppsV1().Deployments(namespace).Delete(deployment.Name, &v1.DeleteOptions{}) + + shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, configmapAnnotated, "www.stakater.com") + config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, configmapAnnotated, shaData, "") + config.ResourceAnnotations = map[string]string{"reloader.stakater.com/match": "false"} + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err = PerformRollingUpgrade(clients, config, deploymentFuncs, collectors) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with Configmap") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceUpdate(clients, config, constants.ConfigmapEnvVarPostfix, deploymentFuncs) + if updated { + t.Errorf("Deployment was updated unexpectedly") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) > 0 { + t.Errorf("Counter was increased unexpectedly") + } +} + func TestRollingUpgradeForDeploymentWithConfigmapInInitContainer(t *testing.T) { shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, configmapWithInitContainer, "www.stakater.com") config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, configmapWithInitContainer, shaData, options.ConfigmapUpdateOnChangeAnnotation) diff --git a/internal/pkg/options/flags.go b/internal/pkg/options/flags.go index 41c2daba..1bf681ed 100644 --- a/internal/pkg/options/flags.go +++ b/internal/pkg/options/flags.go @@ -1,12 +1,20 @@ package options var ( - // ConfigmapUpdateOnChangeAnnotation is an annotation to detect changes in configmaps + // ConfigmapUpdateOnChangeAnnotation is an annotation to detect changes in + // configmaps specified by name ConfigmapUpdateOnChangeAnnotation = "configmap.reloader.stakater.com/reload" - // SecretUpdateOnChangeAnnotation is an annotation to detect changes in secrets + // SecretUpdateOnChangeAnnotation is an annotation to detect changes in + // secrets specified by name SecretUpdateOnChangeAnnotation = "secret.reloader.stakater.com/reload" // ReloaderAutoAnnotation is an annotation to detect changes in secrets ReloaderAutoAnnotation = "reloader.stakater.com/auto" + // AutoSearchAnnotation is an annotation to detect changes in + // configmaps or triggers with the SearchMatchAnnotation + AutoSearchAnnotation = "reloader.stakater.com/search" + // SearchMatchAnnotation is an annotation to tag secrets to be found with + // AutoSearchAnnotation + SearchMatchAnnotation = "reloader.stakater.com/match" // LogFormat is the log format to use (json, or empty string for default) LogFormat = "" ) diff --git a/internal/pkg/testutil/kube.go b/internal/pkg/testutil/kube.go index 479f3be8..0ca20765 100644 --- a/internal/pkg/testutil/kube.go +++ b/internal/pkg/testutil/kube.go @@ -663,9 +663,10 @@ func CreateDeploymentWithEnvVarSource(client kubernetes.Interface, deploymentNam deployment, err := deploymentClient.Create(deploymentObj) time.Sleep(3 * time.Second) return deployment, err + } -// CreateDeployment creates a deployment in given namespace and returns the Deployment +// CreateDeploymentWithPodAnnotations creates a deployment in given namespace and returns the Deployment func CreateDeploymentWithPodAnnotations(client kubernetes.Interface, deploymentName string, namespace string, both bool) (*appsv1.Deployment, error) { logrus.Infof("Creating Deployment") deploymentClient := client.AppsV1().Deployments(namespace) @@ -675,6 +676,18 @@ func CreateDeploymentWithPodAnnotations(client kubernetes.Interface, deploymentN return deployment, err } +// CreateDeploymentWithEnvVarSourceAndAnnotations returns a deployment in given +// namespace with given annotations. +func CreateDeploymentWithEnvVarSourceAndAnnotations(client kubernetes.Interface, deploymentName string, namespace string, annotations map[string]string) (*appsv1.Deployment, error) { + logrus.Infof("Creating Deployment") + deploymentClient := client.AppsV1().Deployments(namespace) + deploymentObj := GetDeploymentWithEnvVarSources(namespace, deploymentName) + deploymentObj.Annotations = annotations + deployment, err := deploymentClient.Create(deploymentObj) + time.Sleep(3 * time.Second) + return deployment, err +} + // CreateDaemonSet creates a deployment in given namespace and returns the DaemonSet func CreateDaemonSet(client kubernetes.Interface, daemonsetName string, namespace string, volumeMount bool) (*appsv1.DaemonSet, error) { logrus.Infof("Creating DaemonSet") @@ -798,6 +811,7 @@ func VerifyResourceUpdate(clients kube.Clients, config util.Config, envVarPostfi containers := upgradeFuncs.ContainersFunc(i) // match statefulsets with the correct annotation annotationValue := util.ToObjectMeta(i).Annotations[config.Annotation] + searchAnnotationValue := util.ToObjectMeta(i).Annotations[options.AutoSearchAnnotation] reloaderEnabledValue := util.ToObjectMeta(i).Annotations[options.ReloaderAutoAnnotation] reloaderEnabled, err := strconv.ParseBool(reloaderEnabledValue) matches := false @@ -811,6 +825,10 @@ func VerifyResourceUpdate(clients kube.Clients, config util.Config, envVarPostfi break } } + } else if searchAnnotationValue == "true" { + if config.ResourceAnnotations[options.SearchMatchAnnotation] == "true" { + matches = true + } } if matches { diff --git a/internal/pkg/util/config.go b/internal/pkg/util/config.go index e8de37e2..2244328d 100644 --- a/internal/pkg/util/config.go +++ b/internal/pkg/util/config.go @@ -8,31 +8,34 @@ import ( //Config contains rolling upgrade configuration parameters type Config struct { - Namespace string - ResourceName string - Annotation string - SHAValue string - Type string + Namespace string + ResourceName string + ResourceAnnotations map[string]string + Annotation string + SHAValue string + Type string } // GetConfigmapConfig provides utility config for configmap func GetConfigmapConfig(configmap *v1.ConfigMap) Config { return Config{ - Namespace: configmap.Namespace, - ResourceName: configmap.Name, - Annotation: options.ConfigmapUpdateOnChangeAnnotation, - SHAValue: GetSHAfromConfigmap(configmap.Data), - Type: constants.ConfigmapEnvVarPostfix, + Namespace: configmap.Namespace, + ResourceName: configmap.Name, + ResourceAnnotations: configmap.Annotations, + Annotation: options.ConfigmapUpdateOnChangeAnnotation, + SHAValue: GetSHAfromConfigmap(configmap.Data), + Type: constants.ConfigmapEnvVarPostfix, } } // GetSecretConfig provides utility config for secret func GetSecretConfig(secret *v1.Secret) Config { return Config{ - Namespace: secret.Namespace, - ResourceName: secret.Name, - Annotation: options.SecretUpdateOnChangeAnnotation, - SHAValue: GetSHAfromSecret(secret.Data), - Type: constants.SecretEnvVarPostfix, + Namespace: secret.Namespace, + ResourceName: secret.Name, + ResourceAnnotations: secret.Annotations, + Annotation: options.SecretUpdateOnChangeAnnotation, + SHAValue: GetSHAfromSecret(secret.Data), + Type: constants.SecretEnvVarPostfix, } }