Merge pull request #695 from deschmih/master

feature reload-on-delete implemented, test cases enhanced

deployments/kubernetes/chart/reloader/templates/deployment.yaml

@@ -175,7 +175,7 @@ spec:
           {{- . | toYaml | nindent 10 }}
           {{- end }}
       {{- end }}
-      {{- if or (.Values.reloader.logFormat) (.Values.reloader.ignoreSecrets) (.Values.reloader.ignoreNamespaces) (.Values.reloader.namespaceSelector) (.Values.reloader.resourceLabelSelector) (.Values.reloader.ignoreConfigMaps) (.Values.reloader.custom_annotations) (eq .Values.reloader.isArgoRollouts true) (eq .Values.reloader.reloadOnCreate true) (ne .Values.reloader.reloadStrategy "default") (.Values.reloader.enableHA) (.Values.reloader.autoReloadAll)}}
+      {{- if or (.Values.reloader.logFormat) (.Values.reloader.ignoreSecrets) (.Values.reloader.ignoreNamespaces) (.Values.reloader.namespaceSelector) (.Values.reloader.resourceLabelSelector) (.Values.reloader.ignoreConfigMaps) (.Values.reloader.custom_annotations) (eq .Values.reloader.isArgoRollouts true) (eq .Values.reloader.reloadOnCreate true) (eq .Values.reloader.reloadOnDelete true) (ne .Values.reloader.reloadStrategy "default") (.Values.reloader.enableHA) (.Values.reloader.autoReloadAll)}}
         args:
           {{- if .Values.reloader.logFormat }}
           - "--log-format={{ .Values.reloader.logFormat }}"
@@ -235,6 +235,9 @@ spec:
           {{- if eq .Values.reloader.reloadOnCreate true }}
           - "--reload-on-create={{ .Values.reloader.reloadOnCreate }}"
           {{- end }}
+          {{- if eq .Values.reloader.reloadOnDelete true }}
+          - "--reload-on-delete={{ .Values.reloader.reloadOnDelete }}"
+          {{- end }}
           {{- if eq .Values.reloader.syncAfterRestart true }}
           - "--sync-after-restart={{ .Values.reloader.syncAfterRestart }}"
           {{- end }}

deployments/kubernetes/chart/reloader/values.yaml

@@ -18,6 +18,7 @@ reloader:
   ignoreSecrets: false
   ignoreConfigMaps: false
   reloadOnCreate: false
+  reloadOnDelete: false
   syncAfterRestart: false
   reloadStrategy: default # Set to default, env-vars or annotations
   ignoreNamespaces: "" # Comma separated list of namespaces to ignore

internal/pkg/cmd/reloader.go

@@ -51,6 +51,7 @@ func NewReloaderCommand() *cobra.Command {
 	cmd.PersistentFlags().StringVar(&options.IsArgoRollouts, "is-Argo-Rollouts", "false", "Add support for argo rollouts")
 	cmd.PersistentFlags().StringVar(&options.ReloadStrategy, constants.ReloadStrategyFlag, constants.EnvVarsReloadStrategy, "Specifies the desired reload strategy")
 	cmd.PersistentFlags().StringVar(&options.ReloadOnCreate, "reload-on-create", "false", "Add support to watch create events")
+	cmd.PersistentFlags().StringVar(&options.ReloadOnDelete, "reload-on-delete", "false", "Add support to watch delete events")
 	cmd.PersistentFlags().BoolVar(&options.EnableHA, "enable-ha", false, "Adds support for running multiple replicas via leadership election")
 	cmd.PersistentFlags().BoolVar(&options.SyncAfterRestart, "sync-after-restart", false, "Sync add events after reloader restarts")
 

internal/pkg/controller/controller.go

@@ -178,13 +178,22 @@ func (c *Controller) Update(old interface{}, new interface{}) {
 
 // Delete function to add an object to the queue in case of deleting a resource
 func (c *Controller) Delete(old interface{}) {
+
+	if options.ReloadOnDelete == "true" {
+		if !c.resourceInIgnoredNamespace(old) && c.resourceInSelectedNamespaces(old) && secretControllerInitialized && configmapControllerInitialized {
+			c.queue.Add(handler.ResourceDeleteHandler{
+				Resource:   old,
+				Collectors: c.collectors,
+				Recorder:   c.recorder,
+			})
+		}
+	}
+
 	switch object := old.(type) {
 	case *v1.Namespace:
 		c.removeSelectedNamespaceFromCache(*object)
 		return
 	}
-
-	// Todo: Any future delete event can be handled here
 }
 
 // Run function for controller which handles the queue

internal/pkg/handler/create.go

@@ -27,7 +27,7 @@ func (r ResourceCreatedHandler) Handle() error {
 			return sendUpgradeWebhook(config, options.WebhookUrl)
 		}
 		// process resource based on its type
-		return doRollingUpgrade(config, r.Collectors, r.Recorder)
+		return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy)
 	}
 	return nil
 }

internal/pkg/handler/delete.go

@@ -0,0 +1,92 @@
+package handler
+
+import (
+	"github.com/sirupsen/logrus"
+	"github.com/stakater/Reloader/internal/pkg/callbacks"
+	"github.com/stakater/Reloader/internal/pkg/constants"
+	"github.com/stakater/Reloader/internal/pkg/metrics"
+	"github.com/stakater/Reloader/internal/pkg/options"
+	"github.com/stakater/Reloader/internal/pkg/testutil"
+	"github.com/stakater/Reloader/internal/pkg/util"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/record"
+)
+
+// ResourceDeleteHandler contains new objects
+type ResourceDeleteHandler struct {
+	Resource   interface{}
+	Collectors metrics.Collectors
+	Recorder   record.EventRecorder
+}
+
+// Handle processes resources being deleted
+func (r ResourceDeleteHandler) Handle() error {
+	if r.Resource == nil {
+		logrus.Errorf("Resource delete handler received nil resource")
+	} else {
+		config, _ := r.GetConfig()
+		// Send webhook
+		if options.WebhookUrl != "" {
+			return sendUpgradeWebhook(config, options.WebhookUrl)
+		}
+		// process resource based on its type
+		return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeDeleteStrategy)
+	}
+	return nil
+}
+
+// GetConfig gets configurations containing SHA, annotations, namespace and resource name
+func (r ResourceDeleteHandler) GetConfig() (util.Config, string) {
+	var oldSHAData string
+	var config util.Config
+	if _, ok := r.Resource.(*v1.ConfigMap); ok {
+		config = util.GetConfigmapConfig(r.Resource.(*v1.ConfigMap))
+	} else if _, ok := r.Resource.(*v1.Secret); ok {
+		config = util.GetSecretConfig(r.Resource.(*v1.Secret))
+	} else {
+		logrus.Warnf("Invalid resource: Resource should be 'Secret' or 'Configmap' but found, %v", r.Resource)
+	}
+	return config, oldSHAData
+}
+
+func invokeDeleteStrategy(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result {
+	if options.ReloadStrategy == constants.AnnotationsReloadStrategy {
+		return removePodAnnotations(upgradeFuncs, item, config, autoReload)
+	}
+
+	return removeContainerEnvVars(upgradeFuncs, item, config, autoReload)
+}
+
+func removePodAnnotations(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result {
+	config.SHAValue = testutil.GetSHAfromEmptyData()
+	return updatePodAnnotations(upgradeFuncs, item, config, autoReload)
+}
+
+func removeContainerEnvVars(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result {
+	envVar := getEnvVarName(config.ResourceName, config.Type)
+	container := getContainerUsingResource(upgradeFuncs, item, config, autoReload)
+
+	if container == nil {
+		return constants.NoContainerFound
+	}
+
+	//remove if env var exists
+	containers := upgradeFuncs.ContainersFunc(item)
+	for i := range containers {
+		envs := containers[i].Env
+		index := -1
+		for j := range envs {
+			if envs[j].Name == envVar {
+				index = j
+				break
+			}
+		}
+		if index != -1 {
+			containers[i].Env = append(containers[i].Env[:index], containers[i].Env[index+1:]...)
+			return constants.Updated
+		}
+	}
+
+	return constants.NotUpdated
+}

internal/pkg/handler/update.go

@@ -29,7 +29,7 @@ func (r ResourceUpdatedHandler) Handle() error {
 				return sendUpgradeWebhook(config, options.WebhookUrl)
 			}
 			// process resource based on its type
-			return doRollingUpgrade(config, r.Collectors, r.Recorder)
+			return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy)
 		}
 	}
 	return nil

internal/pkg/handler/upgrade.go

@@ -142,35 +142,35 @@ func sendWebhook(url string) (string, []error) {
 	return buffer.String(), nil
 }
 
-func doRollingUpgrade(config util.Config, collectors metrics.Collectors, recorder record.EventRecorder) error {
+func doRollingUpgrade(config util.Config, collectors metrics.Collectors, recorder record.EventRecorder, invoke invokeStrategy) error {
 	clients := kube.GetClients()
 
-	err := rollingUpgrade(clients, config, GetDeploymentRollingUpgradeFuncs(), collectors, recorder)
+	err := rollingUpgrade(clients, config, GetDeploymentRollingUpgradeFuncs(), collectors, recorder, invoke)
 	if err != nil {
 		return err
 	}
-	err = rollingUpgrade(clients, config, GetCronJobCreateJobFuncs(), collectors, recorder)
+	err = rollingUpgrade(clients, config, GetCronJobCreateJobFuncs(), collectors, recorder, invoke)
 	if err != nil {
 		return err
 	}
-	err = rollingUpgrade(clients, config, GetDaemonSetRollingUpgradeFuncs(), collectors, recorder)
+	err = rollingUpgrade(clients, config, GetDaemonSetRollingUpgradeFuncs(), collectors, recorder, invoke)
 	if err != nil {
 		return err
 	}
-	err = rollingUpgrade(clients, config, GetStatefulSetRollingUpgradeFuncs(), collectors, recorder)
+	err = rollingUpgrade(clients, config, GetStatefulSetRollingUpgradeFuncs(), collectors, recorder, invoke)
 	if err != nil {
 		return err
 	}
 
 	if kube.IsOpenshift {
-		err = rollingUpgrade(clients, config, GetDeploymentConfigRollingUpgradeFuncs(), collectors, recorder)
+		err = rollingUpgrade(clients, config, GetDeploymentConfigRollingUpgradeFuncs(), collectors, recorder, invoke)
 		if err != nil {
 			return err
 		}
 	}
 
 	if options.IsArgoRollouts == "true" {
-		err = rollingUpgrade(clients, config, GetArgoRolloutRollingUpgradeFuncs(), collectors, recorder)
+		err = rollingUpgrade(clients, config, GetArgoRolloutRollingUpgradeFuncs(), collectors, recorder, invoke)
 		if err != nil {
 			return err
 		}
@@ -179,17 +179,17 @@ func doRollingUpgrade(config util.Config, collectors metrics.Collectors, recorde
 	return nil
 }
 
-func rollingUpgrade(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder) error {
+func rollingUpgrade(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy) error {
 
-	err := PerformRollingUpgrade(clients, config, upgradeFuncs, collectors, recorder)
+	err := PerformAction(clients, config, upgradeFuncs, collectors, recorder, strategy)
 	if err != nil {
 		logrus.Errorf("Rolling upgrade for '%s' failed with error = %v", config.ResourceName, err)
 	}
 	return err
 }
 
-// PerformRollingUpgrade upgrades the deployment if there is any change in configmap or secret data
-func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder) error {
+// PerformAction invokes the deployment if there is any change in configmap or secret data
+func PerformAction(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy) error {
 	items := upgradeFuncs.ItemsFunc(clients, config.Namespace)
 
 	for _, i := range items {
@@ -210,7 +210,7 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
 		reloaderEnabled, _ := strconv.ParseBool(reloaderEnabledValue)
 		typedAutoAnnotationEnabled, _ := strconv.ParseBool(typedAutoAnnotationEnabledValue)
 		if reloaderEnabled || typedAutoAnnotationEnabled || reloaderEnabledValue == "" && typedAutoAnnotationEnabledValue == "" && options.AutoReloadAll {
-			result = invokeReloadStrategy(upgradeFuncs, i, config, true)
+			result = strategy(upgradeFuncs, i, config, true)
 		}
 
 		if result != constants.Updated && annotationValue != "" {
@@ -219,7 +219,7 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
 				value = strings.TrimSpace(value)
 				re := regexp.MustCompile("^" + value + "$")
 				if re.Match([]byte(config.ResourceName)) {
-					result = invokeReloadStrategy(upgradeFuncs, i, config, false)
+					result = strategy(upgradeFuncs, i, config, false)
 					if result == constants.Updated {
 						break
 					}
@@ -230,7 +230,7 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
 		if result != constants.Updated && searchAnnotationValue == "true" {
 			matchAnnotationValue := config.ResourceAnnotations[options.SearchMatchAnnotation]
 			if matchAnnotationValue == "true" {
-				result = invokeReloadStrategy(upgradeFuncs, i, config, true)
+				result = strategy(upgradeFuncs, i, config, true)
 			}
 		}
 
@@ -382,6 +382,8 @@ func getContainerUsingResource(upgradeFuncs callbacks.RollingUpgradeFuncs, item
 	return container
 }
 
+type invokeStrategy func(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result
+
 func invokeReloadStrategy(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result {
 	if options.ReloadStrategy == constants.AnnotationsReloadStrategy {
 		return updatePodAnnotations(upgradeFuncs, item, config, autoReload)
@@ -418,6 +420,13 @@ func updatePodAnnotations(upgradeFuncs callbacks.RollingUpgradeFuncs, item runti
 	return constants.Updated
 }
 
+func getReloaderAnnotationKey() string {
+	return fmt.Sprintf("%s/%s",
+		constants.ReloaderAnnotationPrefix,
+		constants.LastReloadedFromAnnotation,
+	)
+}
+
 func createReloadedAnnotations(target *util.ReloadSource) (map[string]string, error) {
 	if target == nil {
 		return nil, errors.New("target is required")
@@ -428,10 +437,7 @@ func createReloadedAnnotations(target *util.ReloadSource) (map[string]string, er
 	// Intentionally only storing the last item in order to keep
 	// the generated annotations as small as possible.
 	annotations := make(map[string]string)
-	lastReloadedResourceName := fmt.Sprintf("%s/%s",
-		constants.ReloaderAnnotationPrefix,
-		constants.LastReloadedFromAnnotation,
-	)
+	lastReloadedResourceName := getReloaderAnnotationKey()
 
 	lastReloadedResource, err := json.Marshal(target)
 	if err != nil {
@@ -442,9 +448,13 @@ func createReloadedAnnotations(target *util.ReloadSource) (map[string]string, er
 	return annotations, nil
 }
 
+func getEnvVarName(resourceName string, typeName string) string {
+	return constants.EnvVarPrefix + util.ConvertToEnvVarName(resourceName) + "_" + typeName
+}
+
 func updateContainerEnvVars(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result {
 	var result constants.Result
-	envVar := constants.EnvVarPrefix + util.ConvertToEnvVarName(config.ResourceName) + "_" + config.Type
+	envVar := getEnvVarName(config.ResourceName, config.Type)
 	container := getContainerUsingResource(upgradeFuncs, item, config, autoReload)
 
 	if container == nil {

internal/pkg/options/flags.go

@@ -32,7 +32,9 @@ var (
 	// ReloadStrategy Specify the update strategy
 	ReloadStrategy = constants.EnvVarsReloadStrategy
 	// ReloadOnCreate Adds support to watch create events
-	ReloadOnCreate   = "false"
+	ReloadOnCreate = "false"
+	// ReloadOnDelete Adds support to watch delete events
+	ReloadOnDelete   = "false"
 	SyncAfterRestart = false
 	// EnableHA adds support for running multiple replicas via leadership election
 	EnableHA = false

internal/pkg/testutil/kube.go

@@ -934,6 +934,55 @@ func VerifyResourceEnvVarUpdate(clients kube.Clients, config util.Config, envVar
 	return false
 }
 
+// VerifyResourceEnvVarRemoved verifies whether the rolling upgrade happened or not and all Envvars SKAKATER_name_CONFIGMAP/SECRET are removed
+func VerifyResourceEnvVarRemoved(clients kube.Clients, config util.Config, envVarPostfix string, upgradeFuncs callbacks.RollingUpgradeFuncs) bool {
+	items := upgradeFuncs.ItemsFunc(clients, config.Namespace)
+	for _, i := range items {
+		containers := upgradeFuncs.ContainersFunc(i)
+		accessor, err := meta.Accessor(i)
+		if err != nil {
+			return false
+		}
+
+		annotations := accessor.GetAnnotations()
+		// match statefulsets with the correct annotation
+
+		annotationValue := annotations[config.Annotation]
+		searchAnnotationValue := annotations[options.AutoSearchAnnotation]
+		reloaderEnabledValue := annotations[options.ReloaderAutoAnnotation]
+		typedAutoAnnotationEnabledValue := annotations[config.TypedAutoAnnotation]
+		reloaderEnabled, err := strconv.ParseBool(reloaderEnabledValue)
+		typedAutoAnnotationEnabled, errTyped := strconv.ParseBool(typedAutoAnnotationEnabledValue)
+
+		matches := false
+		if err == nil && reloaderEnabled || errTyped == nil && typedAutoAnnotationEnabled {
+			matches = true
+		} else if annotationValue != "" {
+			values := strings.Split(annotationValue, ",")
+			for _, value := range values {
+				value = strings.Trim(value, " ")
+				if value == config.ResourceName {
+					matches = true
+					break
+				}
+			}
+		} else if searchAnnotationValue == "true" {
+			if config.ResourceAnnotations[options.SearchMatchAnnotation] == "true" {
+				matches = true
+			}
+		}
+
+		if matches {
+			envName := constants.EnvVarPrefix + util.ConvertToEnvVarName(config.ResourceName) + "_" + envVarPostfix
+			value := GetResourceSHAFromEnvVar(containers, envName)
+			if value == "" {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // VerifyResourceAnnotationUpdate verifies whether the rolling upgrade happened or not
 func VerifyResourceAnnotationUpdate(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs) bool {
 	items := upgradeFuncs.ItemsFunc(clients, config.Namespace)
@@ -978,3 +1027,7 @@ func VerifyResourceAnnotationUpdate(clients kube.Clients, config util.Config, up
 	}
 	return false
 }
+
+func GetSHAfromEmptyData() string {
+	return crypto.GenerateSHA("")
+}