176 - Add reload strategies to support pod annotation templates

.gitignore

@@ -8,4 +8,5 @@ _gopath/
 .DS_Store
 .vscode
 vendor
-dist
+dist
+Reloader

README.md

@@ -145,6 +145,19 @@ spec:
 - you may want to prevent watching certain namespaces with the `--namespaces-to-ignore` flag
 - you may want to prevent watching certain resources with the `--resources-to-ignore` flag
 - you can configure logging in JSON format with the `--log-format=json` option
+- you can configure the "reload strategy" with the `--reload-strategy=<strategy-name>` option (details below)
+
+## Reload Strategies
+Reloader supports multiple "reload" strategies for performing rolling upgrades to resources. The following list describes them:
+- **env-vars**: When a tracked `configMap`/`secret` is updated, this strategy attaches a Reloader specific environment variable to any containers 
+  referencing the changed `configMap` or `secret` on the owning resource (e.g., `Deployment`, `StatefulSet`, etc.).
+  This strategy can be specified with the `--reload-strategy=env-vars` argument. Note: This is the default reload strategy.
+- **annotations**: When a tracked `configMap`/`secret` is updated, this strategy attaches a `reloader.stakater.com/last-reloaded-from` pod template annotation
+  on the owning resource (e.g., `Deployment`, `StatefulSet`, etc.). This strategy is useful when using resource syncing tools like ArgoCD, since it will not cause these tools
+  to detect configuration drift after a resource is reloaded. Note: Since the attached pod template annotation only tracks the last reload source, this strategy will reload any tracked resource should its 
+  `configMap` or `secret` be deleted and recreated.
+  This strategy can be specified with the `--reload-strategy=annotations` argument.
+  
 
 ## Deploying to Kubernetes
 

internal/pkg/callbacks/rolling_upgrade.go

@@ -52,6 +52,15 @@ func GetDeploymentItems(clients kube.Clients, namespace string) []interface{} {
 	if err != nil {
 		logrus.Errorf("Failed to list deployments %v", err)
 	}
+
+	// Ensure we always have pod annotations to add to
+	for i, v := range deployments.Items {
+		if v.Spec.Template.ObjectMeta.Annotations == nil {
+			annotations := make(map[string]string)
+			deployments.Items[i].Spec.Template.ObjectMeta.Annotations = annotations
+		}
+	}
+
 	return util.InterfaceSlice(deployments.Items)
 }
 
@@ -61,6 +70,14 @@ func GetDaemonSetItems(clients kube.Clients, namespace string) []interface{} {
 	if err != nil {
 		logrus.Errorf("Failed to list daemonSets %v", err)
 	}
+
+	// Ensure we always have pod annotations to add to
+	for i, v := range daemonSets.Items {
+		if v.Spec.Template.ObjectMeta.Annotations == nil {
+			daemonSets.Items[i].Spec.Template.ObjectMeta.Annotations = make(map[string]string)
+		}
+	}
+
 	return util.InterfaceSlice(daemonSets.Items)
 }
 
@@ -70,6 +87,14 @@ func GetStatefulSetItems(clients kube.Clients, namespace string) []interface{} {
 	if err != nil {
 		logrus.Errorf("Failed to list statefulSets %v", err)
 	}
+
+	// Ensure we always have pod annotations to add to
+	for i, v := range statefulSets.Items {
+		if v.Spec.Template.ObjectMeta.Annotations == nil {
+			statefulSets.Items[i].Spec.Template.ObjectMeta.Annotations = make(map[string]string)
+		}
+	}
+
 	return util.InterfaceSlice(statefulSets.Items)
 }
 
@@ -79,6 +104,14 @@ func GetDeploymentConfigItems(clients kube.Clients, namespace string) []interfac
 	if err != nil {
 		logrus.Errorf("Failed to list deploymentConfigs %v", err)
 	}
+
+	// Ensure we always have pod annotations to add to
+	for i, v := range deploymentConfigs.Items {
+		if v.Spec.Template.ObjectMeta.Annotations == nil {
+			deploymentConfigs.Items[i].Spec.Template.ObjectMeta.Annotations = make(map[string]string)
+		}
+	}
+
 	return util.InterfaceSlice(deploymentConfigs.Items)
 }
 
@@ -88,6 +121,14 @@ func GetRolloutItems(clients kube.Clients, namespace string) []interface{} {
 	if err != nil {
 		logrus.Errorf("Failed to list Rollouts %v", err)
 	}
+
+	// Ensure we always have pod annotations to add to
+	for i, v := range rollouts.Items {
+		if v.Spec.Template.ObjectMeta.Annotations == nil {
+			rollouts.Items[i].Spec.Template.ObjectMeta.Annotations = make(map[string]string)
+		}
+	}
+
 	return util.InterfaceSlice(rollouts.Items)
 }
 

internal/pkg/cmd/reloader.go

@@ -3,7 +3,9 @@ package cmd
 import (
 	"errors"
 	"fmt"
+	"github.com/stakater/Reloader/internal/pkg/constants"
 	"os"
+	"strings"
 
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -18,9 +20,10 @@ import (
 // NewReloaderCommand starts the reloader controller
 func NewReloaderCommand() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "reloader",
+		Use:     "reloader",
-		Short: "A watcher for your Kubernetes cluster",
+		Short:   "A watcher for your Kubernetes cluster",
-		Run:   startReloader,
+		PreRunE: validateFlags,
+		Run:     startReloader,
 	}
 
 	// options
@@ -33,9 +36,24 @@ func NewReloaderCommand() *cobra.Command {
 	cmd.PersistentFlags().StringSlice("resources-to-ignore", []string{}, "list of resources to ignore (valid options 'configMaps' or 'secrets')")
 	cmd.PersistentFlags().StringSlice("namespaces-to-ignore", []string{}, "list of namespaces to ignore")
 	cmd.PersistentFlags().StringVar(&options.IsArgoRollouts, "is-Argo-Rollouts", "false", "Add support for argo rollouts")
+	cmd.PersistentFlags().StringVar(&options.ReloadStrategy, constants.ReloadStrategyFlag, constants.EnvVarsReloadStrategy, "Specifies the desired reload strategy")
+
 	return cmd
 }
 
+func validateFlags(*cobra.Command, []string) error {
+	// Ensure the reload strategy is one of the following...
+	valid := []string{constants.EnvVarsReloadStrategy, constants.AnnotationsReloadStrategy}
+	for _, s := range valid {
+		if s == options.ReloadStrategy {
+			return nil
+		}
+	}
+
+	err := fmt.Sprintf("%s must be one of: %s", constants.ReloadStrategyFlag, strings.Join(valid, ", "))
+	return errors.New(err)
+}
+
 func configureLogging(logFormat string) error {
 	switch logFormat {
 	case "json":

internal/pkg/constants/constants.go

@@ -7,4 +7,16 @@ const (
 	SecretEnvVarPostfix = "SECRET"
 	// EnvVarPrefix is a Prefix for environment variable
 	EnvVarPrefix = "STAKATER_"
+
+	// ReloaderAnnotationPrefix is a Prefix for all reloader annotations
+	ReloaderAnnotationPrefix = "reloader.stakater.com"
+	// LastReloadedFromAnnotation is an annotation used to describe the last resource that triggered a reload
+	LastReloadedFromAnnotation = "last-reloaded-from"
+
+	// 	ReloadStrategyFlag The reload strategy flag name
+	ReloadStrategyFlag = "reload-strategy"
+	// EnvVarsReloadStrategy instructs Reloader to add container environment variables to facilitate a restart
+	EnvVarsReloadStrategy = "env-vars"
+	// AnnotationsReloadStrategy instructs Reloader to add pod template annotations to facilitate a restart
+	AnnotationsReloadStrategy = "annotations"
 )

internal/pkg/handler/upgrade.go

@@ -1,9 +1,9 @@
 package handler
 
 import (
-	"strconv"
+	"encoding/json"
-	"strings"
+	"errors"
-
+	"fmt"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 	"github.com/stakater/Reloader/internal/pkg/callbacks"
@@ -13,6 +13,8 @@ import (
 	"github.com/stakater/Reloader/internal/pkg/util"
 	"github.com/stakater/Reloader/pkg/kube"
 	v1 "k8s.io/api/core/v1"
+	"strconv"
+	"strings"
 )
 
 // GetDeploymentRollingUpgradeFuncs returns all callback funcs for a deployment
@@ -146,7 +148,7 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
 		result := constants.NotUpdated
 		reloaderEnabled, err := strconv.ParseBool(reloaderEnabledValue)
 		if err == nil && reloaderEnabled {
-			result = updateContainers(upgradeFuncs, i, config, true)
+			result = invokeReloadStrategy(upgradeFuncs, i, config, true)
 		}
 
 		if result != constants.Updated && annotationValue != "" {
@@ -154,7 +156,7 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
 			for _, value := range values {
 				value = strings.Trim(value, " ")
 				if value == config.ResourceName {
-					result = updateContainers(upgradeFuncs, i, config, false)
+					result = invokeReloadStrategy(upgradeFuncs, i, config, false)
 					if result == constants.Updated {
 						break
 					}
@@ -165,7 +167,7 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
 		if result != constants.Updated && searchAnnotationValue == "true" {
 			matchAnnotationValue := config.ResourceAnnotations[options.SearchMatchAnnotation]
 			if matchAnnotationValue == "true" {
-				result = updateContainers(upgradeFuncs, i, config, true)
+				result = invokeReloadStrategy(upgradeFuncs, i, config, true)
 			}
 		}
 
@@ -257,7 +259,7 @@ func getContainerWithEnvReference(containers []v1.Container, resourceName string
 	return nil
 }
 
-func getContainerToUpdate(upgradeFuncs callbacks.RollingUpgradeFuncs, item interface{}, config util.Config, autoReload bool) *v1.Container {
+func getContainerUsingResource(upgradeFuncs callbacks.RollingUpgradeFuncs, item interface{}, config util.Config, autoReload bool) *v1.Container {
 	volumes := upgradeFuncs.VolumesFunc(item)
 	containers := upgradeFuncs.ContainersFunc(item)
 	initContainers := upgradeFuncs.InitContainersFunc(item)
@@ -296,10 +298,70 @@ func getContainerToUpdate(upgradeFuncs callbacks.RollingUpgradeFuncs, item inter
 	return container
 }
 
-func updateContainers(upgradeFuncs callbacks.RollingUpgradeFuncs, item interface{}, config util.Config, autoReload bool) constants.Result {
+func invokeReloadStrategy(upgradeFuncs callbacks.RollingUpgradeFuncs, item interface{}, config util.Config, autoReload bool) constants.Result {
+	if options.ReloadStrategy == constants.AnnotationsReloadStrategy {
+		return updatePodAnnotations(upgradeFuncs, item, config, autoReload)
+	}
+
+	return updateContainerEnvVars(upgradeFuncs, item, config, autoReload)
+}
+
+func updatePodAnnotations(upgradeFuncs callbacks.RollingUpgradeFuncs, item interface{}, config util.Config, autoReload bool) constants.Result {
+	container := getContainerUsingResource(upgradeFuncs, item, config, autoReload)
+	if container == nil {
+		return constants.NoContainerFound
+	}
+
+	// Generate reloaded annotations. Attaching this to the item's annotation will trigger a rollout
+	// Note: the data on this struct is purely informational and is not used for future updates
+	reloadSource := util.NewReloadSourceFromConfig(config, []string{container.Name})
+	annotations, err := createReloadedAnnotations(&reloadSource)
+	if err != nil {
+		logrus.Errorf("Failed to create reloaded annotations for %s! error = %v", config.ResourceName, err)
+		return constants.NotUpdated
+	}
+
+	// Copy the all annotations to the item's annotations
+	pa := upgradeFuncs.PodAnnotationsFunc(item)
+	if pa == nil {
+		return constants.NotUpdated
+	}
+
+	for k, v := range annotations {
+		pa[k] = v
+	}
+
+	return constants.Updated
+}
+
+func createReloadedAnnotations(target *util.ReloadSource) (map[string]string, error) {
+	if target == nil {
+		return nil, errors.New("target is required")
+	}
+
+	// Create a single "last-invokeReloadStrategy-from" annotation that stores metadata about the
+	// resource that caused the last invokeReloadStrategy.
+	// Intentionally only storing the last item in order to keep
+	// the generated annotations as small as possible.
+	annotations := make(map[string]string)
+	lastReloadedResourceName := fmt.Sprintf("%s/%s",
+		constants.ReloaderAnnotationPrefix,
+		constants.LastReloadedFromAnnotation,
+	)
+
+	lastReloadedResource, err := json.Marshal(target)
+	if err != nil {
+		return nil, err
+	}
+
+	annotations[lastReloadedResourceName] = string(lastReloadedResource)
+	return annotations, nil
+}
+
+func updateContainerEnvVars(upgradeFuncs callbacks.RollingUpgradeFuncs, item interface{}, config util.Config, autoReload bool) constants.Result {
 	var result constants.Result
 	envVar := constants.EnvVarPrefix + util.ConvertToEnvVarName(config.ResourceName) + "_" + config.Type
-	container := getContainerToUpdate(upgradeFuncs, item, config, autoReload)
+	container := getContainerUsingResource(upgradeFuncs, item, config, autoReload)
 
 	if container == nil {
 		return constants.NoContainerFound

internal/pkg/options/flags.go

@@ -1,5 +1,7 @@
 package options
 
+import "github.com/stakater/Reloader/internal/pkg/constants"
+
 var (
 	// ConfigmapUpdateOnChangeAnnotation is an annotation to detect changes in
 	// configmaps specified by name
@@ -17,6 +19,8 @@ var (
 	SearchMatchAnnotation = "reloader.stakater.com/match"
 	// LogFormat is the log format to use (json, or empty string for default)
 	LogFormat = ""
-	// Adds support for argo rollouts
+	// IsArgoRollouts Adds support for argo rollouts
 	IsArgoRollouts = "false"
+	// ReloadStrategy Specify the update strategy
+	ReloadStrategy = constants.EnvVarsReloadStrategy
 )

internal/pkg/testutil/kube.go

@@ -2,6 +2,8 @@ package testutil
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 	"math/rand"
 	"sort"
 	"strconv"
@@ -563,8 +565,8 @@ func GetSecretWithUpdatedLabel(namespace string, secretName string, label string
 	}
 }
 
-// GetResourceSHA returns the SHA value of given environment variable
+// GetResourceSHAFromEnvVar returns the SHA value of given environment variable
-func GetResourceSHA(containers []v1.Container, envVar string) string {
+func GetResourceSHAFromEnvVar(containers []v1.Container, envVar string) string {
 	for i := range containers {
 		envs := containers[i].Env
 		for j := range envs {
@@ -576,6 +578,28 @@ func GetResourceSHA(containers []v1.Container, envVar string) string {
 	return ""
 }
 
+// GetResourceSHAFromAnnotation returns the SHA value of given environment variable
+func GetResourceSHAFromAnnotation(podAnnotations map[string]string) string {
+	lastReloadedResourceName := fmt.Sprintf("%s/%s",
+		constants.ReloaderAnnotationPrefix,
+		constants.LastReloadedFromAnnotation,
+	)
+
+	annotationJson, ok := podAnnotations[lastReloadedResourceName]
+	if !ok {
+		return ""
+	}
+
+	var last util.ReloadSource
+	bytes := []byte(annotationJson)
+	err := json.Unmarshal(bytes, &last)
+	if err != nil {
+		return ""
+	}
+
+	return last.Hash
+}
+
 //ConvertResourceToSHA generates SHA from secret or configmap data
 func ConvertResourceToSHA(resourceType string, namespace string, resourceName string, data string) string {
 	values := []string{}
@@ -806,8 +830,8 @@ func RandSeq(n int) string {
 	return string(b)
 }
 
-// VerifyResourceUpdate verifies whether the rolling upgrade happened or not
+// VerifyResourceEnvVarUpdate verifies whether the rolling upgrade happened or not
-func VerifyResourceUpdate(clients kube.Clients, config util.Config, envVarPostfix string, upgradeFuncs callbacks.RollingUpgradeFuncs) bool {
+func VerifyResourceEnvVarUpdate(clients kube.Clients, config util.Config, envVarPostfix string, upgradeFuncs callbacks.RollingUpgradeFuncs) bool {
 	items := upgradeFuncs.ItemsFunc(clients, config.Namespace)
 	for _, i := range items {
 		containers := upgradeFuncs.ContainersFunc(i)
@@ -836,7 +860,45 @@ func VerifyResourceUpdate(clients kube.Clients, config util.Config, envVarPostfi
 
 		if matches {
 			envName := constants.EnvVarPrefix + util.ConvertToEnvVarName(config.ResourceName) + "_" + envVarPostfix
-			updated := GetResourceSHA(containers, envName)
+			updated := GetResourceSHAFromEnvVar(containers, envName)
+			if updated == config.SHAValue {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// VerifyResourceAnnotationUpdate verifies whether the rolling upgrade happened or not
+func VerifyResourceAnnotationUpdate(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs) bool {
+	items := upgradeFuncs.ItemsFunc(clients, config.Namespace)
+	for _, i := range items {
+		podAnnotations := upgradeFuncs.PodAnnotationsFunc(i)
+		// match statefulsets with the correct annotation
+		annotationValue := util.ToObjectMeta(i).Annotations[config.Annotation]
+		searchAnnotationValue := util.ToObjectMeta(i).Annotations[options.AutoSearchAnnotation]
+		reloaderEnabledValue := util.ToObjectMeta(i).Annotations[options.ReloaderAutoAnnotation]
+		reloaderEnabled, err := strconv.ParseBool(reloaderEnabledValue)
+		matches := false
+		if err == nil && reloaderEnabled {
+			matches = true
+		} else if annotationValue != "" {
+			values := strings.Split(annotationValue, ",")
+			for _, value := range values {
+				value = strings.Trim(value, " ")
+				if value == config.ResourceName {
+					matches = true
+					break
+				}
+			}
+		} else if searchAnnotationValue == "true" {
+			if config.ResourceAnnotations[options.SearchMatchAnnotation] == "true" {
+				matches = true
+			}
+		}
+
+		if matches {
+			updated := GetResourceSHAFromAnnotation(podAnnotations)
 			if updated == config.SHAValue {
 				return true
 			}

internal/pkg/util/reload_source.go

@@ -0,0 +1,39 @@
+package util
+
+import "time"
+
+type ReloadSource struct {
+	Type          string   `json:"type"`
+	Name          string   `json:"name"`
+	Namespace     string   `json:"namespace"`
+	Hash          string   `json:"hash"`
+	ContainerRefs []string `json:"containerRefs"`
+	ObservedAt    int64    `json:"observedAt"`
+}
+
+func NewReloadSource(
+	resourceName string,
+	resourceNamespace string,
+	resourceType string,
+	resourceHash string,
+	containerRefs []string,
+) ReloadSource {
+	return ReloadSource{
+		ObservedAt:    time.Now().Unix(),
+		Name:          resourceName,
+		Namespace:     resourceNamespace,
+		Type:          resourceType,
+		Hash:          resourceHash,
+		ContainerRefs: containerRefs,
+	}
+}
+
+func NewReloadSourceFromConfig(config Config, containerRefs []string) ReloadSource {
+	return NewReloadSource(
+		config.ResourceName,
+		config.Namespace,
+		config.Type,
+		config.SHAValue,
+		containerRefs,
+	)
+}