Merge master into feat/e2e-test

internal/pkg/handler/create.go

@@ -1,6 +1,8 @@
 package handler
 
 import (
+	"time"
+
 	"github.com/sirupsen/logrus"
 	"github.com/stakater/Reloader/internal/pkg/metrics"
 	"github.com/stakater/Reloader/internal/pkg/options"
@@ -11,25 +13,46 @@ import (
 
 // ResourceCreatedHandler contains new objects
 type ResourceCreatedHandler struct {
-	Resource   interface{}
-	Collectors metrics.Collectors
-	Recorder   record.EventRecorder
+	Resource    interface{}
+	Collectors  metrics.Collectors
+	Recorder    record.EventRecorder
+	EnqueueTime time.Time // Time when this handler was added to the queue
+}
+
+// GetEnqueueTime returns when this handler was enqueued
+func (r ResourceCreatedHandler) GetEnqueueTime() time.Time {
+	return r.EnqueueTime
 }
 
 // Handle processes the newly created resource
 func (r ResourceCreatedHandler) Handle() error {
+	startTime := time.Now()
+	result := "error"
+
+	defer func() {
+		r.Collectors.RecordReconcile(result, time.Since(startTime))
+	}()
+
 	if r.Resource == nil {
 		logrus.Errorf("Resource creation handler received nil resource")
-	} else {
-		config, _ := r.GetConfig()
-		// Send webhook
-		if options.WebhookUrl != "" {
-			return sendUpgradeWebhook(config, options.WebhookUrl)
-		}
-		// process resource based on its type
-		return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy)
+		return nil
 	}
-	return nil
+
+	config, _ := r.GetConfig()
+	// Send webhook
+	if options.WebhookUrl != "" {
+		err := sendUpgradeWebhook(config, options.WebhookUrl)
+		if err == nil {
+			result = "success"
+		}
+		return err
+	}
+	// process resource based on its type
+	err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy)
+	if err == nil {
+		result = "success"
+	}
+	return err
 }
 
 // GetConfig gets configurations containing SHA, annotations, namespace and resource name

internal/pkg/handler/delete.go

@@ -3,6 +3,7 @@ package handler
 import (
 	"fmt"
 	"slices"
+	"time"
 
 	"github.com/sirupsen/logrus"
 	"github.com/stakater/Reloader/internal/pkg/callbacks"
@@ -20,25 +21,46 @@ import (
 
 // ResourceDeleteHandler contains new objects
 type ResourceDeleteHandler struct {
-	Resource   interface{}
-	Collectors metrics.Collectors
-	Recorder   record.EventRecorder
+	Resource    interface{}
+	Collectors  metrics.Collectors
+	Recorder    record.EventRecorder
+	EnqueueTime time.Time // Time when this handler was added to the queue
+}
+
+// GetEnqueueTime returns when this handler was enqueued
+func (r ResourceDeleteHandler) GetEnqueueTime() time.Time {
+	return r.EnqueueTime
 }
 
 // Handle processes resources being deleted
 func (r ResourceDeleteHandler) Handle() error {
+	startTime := time.Now()
+	result := "error"
+
+	defer func() {
+		r.Collectors.RecordReconcile(result, time.Since(startTime))
+	}()
+
 	if r.Resource == nil {
 		logrus.Errorf("Resource delete handler received nil resource")
-	} else {
-		config, _ := r.GetConfig()
-		// Send webhook
-		if options.WebhookUrl != "" {
-			return sendUpgradeWebhook(config, options.WebhookUrl)
-		}
-		// process resource based on its type
-		return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeDeleteStrategy)
+		return nil
 	}
-	return nil
+
+	config, _ := r.GetConfig()
+	// Send webhook
+	if options.WebhookUrl != "" {
+		err := sendUpgradeWebhook(config, options.WebhookUrl)
+		if err == nil {
+			result = "success"
+		}
+		return err
+	}
+	// process resource based on its type
+	err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeDeleteStrategy)
+	if err == nil {
+		result = "success"
+	}
+	return err
 }
 
 // GetConfig gets configurations containing SHA, annotations, namespace and resource name

internal/pkg/handler/handler.go

@@ -1,9 +1,18 @@
 package handler
 
-import "github.com/stakater/Reloader/pkg/common"
+import (
+	"time"
+
+	"github.com/stakater/Reloader/pkg/common"
+)
 
 // ResourceHandler handles the creation and update of resources
 type ResourceHandler interface {
 	Handle() error
 	GetConfig() (common.Config, string)
 }
+
+// TimedHandler is a handler that tracks when it was enqueued
+type TimedHandler interface {
+	GetEnqueueTime() time.Time
+}

internal/pkg/handler/pause_deployment_test.go

@@ -244,7 +244,7 @@ func TestHandleMissingTimerSimple(t *testing.T) {
 		}()
 
 		t.Run(test.name, func(t *testing.T) {
-			fakeClient := testclient.NewSimpleClientset()
+			fakeClient := testclient.NewClientset()
 			clients := kube.Clients{
 				KubernetesClient: fakeClient,
 			}
@@ -337,7 +337,7 @@ func TestPauseDeployment(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			fakeClient := testclient.NewSimpleClientset()
+			fakeClient := testclient.NewClientset()
 			clients := kube.Clients{
 				KubernetesClient: fakeClient,
 			}

internal/pkg/handler/update.go

@@ -1,6 +1,8 @@
 package handler
 
 import (
+	"time"
+
 	"github.com/sirupsen/logrus"
 	"github.com/stakater/Reloader/internal/pkg/metrics"
 	"github.com/stakater/Reloader/internal/pkg/options"
@@ -8,6 +10,7 @@ import (
 	"github.com/stakater/Reloader/pkg/common"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/record"
+	csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1"
 )
 
 // ResourceUpdatedHandler contains updated objects
@@ -16,38 +19,79 @@ type ResourceUpdatedHandler struct {
 	OldResource interface{}
 	Collectors  metrics.Collectors
 	Recorder    record.EventRecorder
+	EnqueueTime time.Time // Time when this handler was added to the queue
+}
+
+// GetEnqueueTime returns when this handler was enqueued
+func (r ResourceUpdatedHandler) GetEnqueueTime() time.Time {
+	return r.EnqueueTime
 }
 
 // Handle processes the updated resource
 func (r ResourceUpdatedHandler) Handle() error {
+	startTime := time.Now()
+	result := "error"
+
+	defer func() {
+		r.Collectors.RecordReconcile(result, time.Since(startTime))
+	}()
+
 	if r.Resource == nil || r.OldResource == nil {
 		logrus.Errorf("Resource update handler received nil resource")
-	} else {
-		config, oldSHAData := r.GetConfig()
-		if config.SHAValue != oldSHAData {
-			// Send a webhook if update
-			if options.WebhookUrl != "" {
-				return sendUpgradeWebhook(config, options.WebhookUrl)
-			}
-			// process resource based on its type
-			return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy)
-		}
+		return nil
 	}
+
+	config, oldSHAData := r.GetConfig()
+	if config.SHAValue != oldSHAData {
+		// Send a webhook if update
+		if options.WebhookUrl != "" {
+			err := sendUpgradeWebhook(config, options.WebhookUrl)
+			if err == nil {
+				result = "success"
+			}
+			return err
+		}
+		// process resource based on its type
+		err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy)
+		if err == nil {
+			result = "success"
+		}
+		return err
+	}
+
+	// No data change - skip
+	result = "skipped"
+	r.Collectors.RecordSkipped("no_data_change")
 	return nil
 }
 
 // GetConfig gets configurations containing SHA, annotations, namespace and resource name
 func (r ResourceUpdatedHandler) GetConfig() (common.Config, string) {
-	var oldSHAData string
-	var config common.Config
-	if _, ok := r.Resource.(*v1.ConfigMap); ok {
-		oldSHAData = util.GetSHAfromConfigmap(r.OldResource.(*v1.ConfigMap))
-		config = common.GetConfigmapConfig(r.Resource.(*v1.ConfigMap))
-	} else if _, ok := r.Resource.(*v1.Secret); ok {
-		oldSHAData = util.GetSHAfromSecret(r.OldResource.(*v1.Secret).Data)
-		config = common.GetSecretConfig(r.Resource.(*v1.Secret))
-	} else {
-		logrus.Warnf("Invalid resource: Resource should be 'Secret' or 'Configmap' but found, %v", r.Resource)
+	var (
+		oldSHAData string
+		config     common.Config
+	)
+
+	switch res := r.Resource.(type) {
+	case *v1.ConfigMap:
+		if old, ok := r.OldResource.(*v1.ConfigMap); ok && old != nil {
+			oldSHAData = util.GetSHAfromConfigmap(old)
+		}
+		config = common.GetConfigmapConfig(res)
+
+	case *v1.Secret:
+		if old, ok := r.OldResource.(*v1.Secret); ok && old != nil {
+			oldSHAData = util.GetSHAfromSecret(old.Data)
+		}
+		config = common.GetSecretConfig(res)
+
+	case *csiv1.SecretProviderClassPodStatus:
+		if old, ok := r.OldResource.(*csiv1.SecretProviderClassPodStatus); ok && old != nil && old.Status.Objects != nil {
+			oldSHAData = util.GetSHAfromSecretProviderClassPodStatus(old.Status)
+		}
+		config = common.GetSecretProviderClassPodStatusConfig(res)
+	default:
+		logrus.Warnf("Invalid resource: Resource should be 'Secret', 'Configmap' or 'SecretProviderClassPodStatus' but found, %T", r.Resource)
 	}
 	return config, oldSHAData
 }

internal/pkg/handler/upgrade.go

@@ -2,11 +2,14 @@ package handler
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"os"
+	"strings"
+	"time"
 
 	"github.com/parnurzeal/gorequest"
 	"github.com/prometheus/client_golang/prometheus"
@@ -23,6 +26,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	patchtypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -236,23 +240,35 @@ func rollingUpgrade(clients kube.Clients, config common.Config, upgradeFuncs cal
 func PerformAction(clients kube.Clients, config common.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy) error {
 	items := upgradeFuncs.ItemsFunc(clients, config.Namespace)
 
+	// Record workloads scanned
+	collectors.RecordWorkloadsScanned(upgradeFuncs.ResourceType, len(items))
+
+	matchedCount := 0
 	for _, item := range items {
-		err := retryOnConflict(retry.DefaultRetry, func(fetchResource bool) error {
+		matched, err := retryOnConflict(retry.DefaultRetry, func(fetchResource bool) (bool, error) {
 			return upgradeResource(clients, config, upgradeFuncs, collectors, recorder, strategy, item, fetchResource)
 		})
 		if err != nil {
 			return err
 		}
+		if matched {
+			matchedCount++
+		}
 	}
 
+	// Record workloads matched
+	collectors.RecordWorkloadsMatched(upgradeFuncs.ResourceType, matchedCount)
+
 	return nil
 }
 
-func retryOnConflict(backoff wait.Backoff, fn func(_ bool) error) error {
+func retryOnConflict(backoff wait.Backoff, fn func(_ bool) (bool, error)) (bool, error) {
 	var lastError error
+	var matched bool
 	fetchResource := false // do not fetch resource on first attempt, already done by ItemsFunc
 	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
-		err := fn(fetchResource)
+		var err error
+		matched, err = fn(fetchResource)
 		fetchResource = true
 		switch {
 		case err == nil:
@@ -267,35 +283,42 @@ func retryOnConflict(backoff wait.Backoff, fn func(_ bool) error) error {
 	if wait.Interrupted(err) {
 		err = lastError
 	}
-	return err
+	return matched, err
 }
 
-func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy, resource runtime.Object, fetchResource bool) error {
+func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy, resource runtime.Object, fetchResource bool) (bool, error) {
+	actionStartTime := time.Now()
+
 	accessor, err := meta.Accessor(resource)
 	if err != nil {
-		return err
+		return false, err
 	}
 
 	resourceName := accessor.GetName()
 	if fetchResource {
 		resource, err = upgradeFuncs.ItemFunc(clients, resourceName, config.Namespace)
 		if err != nil {
-			return err
+			return false, err
 		}
 	}
+	if config.Type == constants.SecretProviderClassEnvVarPostfix {
+		populateAnnotationsFromSecretProviderClass(clients, &config)
+	}
+
 	annotations := upgradeFuncs.AnnotationsFunc(resource)
 	podAnnotations := upgradeFuncs.PodAnnotationsFunc(resource)
 	result := common.ShouldReload(config, upgradeFuncs.ResourceType, annotations, podAnnotations, common.GetCommandLineOptions())
 
 	if !result.ShouldReload {
 		logrus.Debugf("No changes detected in '%s' of type '%s' in namespace '%s'", config.ResourceName, config.Type, config.Namespace)
-		return nil
+		return false, nil
 	}
 
 	strategyResult := strategy(upgradeFuncs, resource, config, result.AutoReload)
 
 	if strategyResult.Result != constants.Updated {
-		return nil
+		collectors.RecordSkipped("strategy_not_updated")
+		return false, nil
 	}
 
 	// find correct annotation and update the resource
@@ -309,7 +332,7 @@ func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs ca
 			_, err = PauseDeployment(deployment, clients, config.Namespace, pauseInterval)
 			if err != nil {
 				logrus.Errorf("Failed to pause deployment '%s' in namespace '%s': %v", resourceName, config.Namespace, err)
-				return err
+				return true, err
 			}
 		}
 	}
@@ -320,16 +343,19 @@ func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs ca
 		err = upgradeFuncs.UpdateFunc(clients, config.Namespace, resource)
 	}
 
+	actionLatency := time.Since(actionStartTime)
+
 	if err != nil {
 		message := fmt.Sprintf("Update for '%s' of type '%s' in namespace '%s' failed with error %v", resourceName, upgradeFuncs.ResourceType, config.Namespace, err)
 		logrus.Errorf("Update for '%s' of type '%s' in namespace '%s' failed with error %v", resourceName, upgradeFuncs.ResourceType, config.Namespace, err)
 
 		collectors.Reloaded.With(prometheus.Labels{"success": "false"}).Inc()
 		collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "false", "namespace": config.Namespace}).Inc()
+		collectors.RecordAction(upgradeFuncs.ResourceType, "error", actionLatency)
 		if recorder != nil {
 			recorder.Event(resource, v1.EventTypeWarning, "ReloadFail", message)
 		}
-		return err
+		return true, err
 	} else {
 		message := fmt.Sprintf("Changes detected in '%s' of type '%s' in namespace '%s'", config.ResourceName, config.Type, config.Namespace)
 		message += fmt.Sprintf(", Updated '%s' of type '%s' in namespace '%s'", resourceName, upgradeFuncs.ResourceType, config.Namespace)
@@ -338,6 +364,7 @@ func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs ca
 
 		collectors.Reloaded.With(prometheus.Labels{"success": "true"}).Inc()
 		collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": config.Namespace}).Inc()
+		collectors.RecordAction(upgradeFuncs.ResourceType, "success", actionLatency)
 		alert_on_reload, ok := os.LookupEnv("ALERT_ON_RELOAD")
 		if recorder != nil {
 			recorder.Event(resource, v1.EventTypeNormal, "Reloaded", message)
@@ -350,7 +377,7 @@ func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs ca
 		}
 	}
 
-	return nil
+	return true, nil
 }
 
 func getVolumeMountName(volumes []v1.Volume, mountType string, volumeName string) string {
@@ -380,6 +407,10 @@ func getVolumeMountName(volumes []v1.Volume, mountType string, volumeName string
 					}
 				}
 			}
+		case constants.SecretProviderClassEnvVarPostfix:
+			if volumes[i].CSI != nil && volumes[i].CSI.VolumeAttributes["secretProviderClass"] == volumeName {
+				return volumes[i].Name
+			}
 		}
 	}
 
@@ -516,6 +547,10 @@ func updatePodAnnotations(upgradeFuncs callbacks.RollingUpgradeFuncs, item runti
 		return InvokeStrategyResult{constants.NotUpdated, nil}
 	}
 
+	if config.Type == constants.SecretProviderClassEnvVarPostfix && secretProviderClassAnnotationReloaded(pa, config) {
+		return InvokeStrategyResult{constants.NotUpdated, nil}
+	}
+
 	for k, v := range annotations {
 		pa[k] = v
 	}
@@ -523,6 +558,11 @@ func updatePodAnnotations(upgradeFuncs callbacks.RollingUpgradeFuncs, item runti
 	return InvokeStrategyResult{constants.Updated, &Patch{Type: patchtypes.StrategicMergePatchType, Bytes: patch}}
 }
 
+func secretProviderClassAnnotationReloaded(oldAnnotations map[string]string, newConfig common.Config) bool {
+	annotation := oldAnnotations[getReloaderAnnotationKey()]
+	return strings.Contains(annotation, newConfig.ResourceName) && strings.Contains(annotation, newConfig.SHAValue)
+}
+
 func getReloaderAnnotationKey() string {
 	return fmt.Sprintf("%s/%s",
 		constants.ReloaderAnnotationPrefix,
@@ -573,6 +613,10 @@ func updateContainerEnvVars(upgradeFuncs callbacks.RollingUpgradeFuncs, item run
 		return InvokeStrategyResult{constants.NoContainerFound, nil}
 	}
 
+	if config.Type == constants.SecretProviderClassEnvVarPostfix && secretProviderClassEnvReloaded(upgradeFuncs.ContainersFunc(item), envVar, config.SHAValue) {
+		return InvokeStrategyResult{constants.NotUpdated, nil}
+	}
+
 	//update if env var exists
 	updateResult := updateEnvVar(container, envVar, config.SHAValue)
 
@@ -609,6 +653,32 @@ func updateEnvVar(container *v1.Container, envVar string, shaData string) consta
 	return constants.NoEnvVarFound
 }
 
+func secretProviderClassEnvReloaded(containers []v1.Container, envVar string, shaData string) bool {
+	for _, container := range containers {
+		for _, env := range container.Env {
+			if env.Name == envVar {
+				return env.Value == shaData
+			}
+		}
+	}
+	return false
+}
+
+func populateAnnotationsFromSecretProviderClass(clients kube.Clients, config *common.Config) {
+	obj, err := clients.CSIClient.SecretsstoreV1().SecretProviderClasses(config.Namespace).Get(context.Background(), config.ResourceName, metav1.GetOptions{})
+	annotations := make(map[string]string)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			logrus.Warnf("SecretProviderClass '%s' not found in namespace '%s'", config.ResourceName, config.Namespace)
+		} else {
+			logrus.Errorf("Failed to get SecretProviderClass '%s' in namespace '%s': %v", config.ResourceName, config.Namespace, err)
+		}
+	} else if obj.Annotations != nil {
+		annotations = obj.Annotations
+	}
+	config.ResourceAnnotations = annotations
+}
+
 func jsonEscape(toEscape string) (string, error) {
 	bytes, err := json.Marshal(toEscape)
 	if err != nil {