From f6887b4d8afe4f7e0865ee59b1dfc3994d5abff3 Mon Sep 17 00:00:00 2001 From: Zanis <22601571+ZanisO@users.noreply.github.com> Date: Tue, 24 Dec 2024 22:45:19 +0000 Subject: [PATCH 01/45] Added support for CSI secret provider Signed-off-by: Zanis <22601571+ZanisO@users.noreply.github.com> --- go.mod | 1 + go.sum | 2 ++ internal/pkg/cmd/reloader.go | 5 +++++ internal/pkg/constants/constants.go | 2 ++ internal/pkg/controller/controller.go | 11 ++++++++++- internal/pkg/handler/update.go | 4 ++++ internal/pkg/handler/upgrade.go | 9 +++++++++ internal/pkg/options/flags.go | 4 ++++ internal/pkg/util/config.go | 13 ++++++++++++- internal/pkg/util/util.go | 11 +++++++++++ pkg/kube/client.go | 18 ++++++++++++++++++ pkg/kube/resourcemapper.go | 8 +++++--- 12 files changed, 83 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index a5e04f8..fcc49ae 100644 --- a/go.mod +++ b/go.mod @@ -69,6 +69,7 @@ require ( k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/secrets-store-csi-driver v1.4.7 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 995b99c..5de1fc0 100644 --- a/go.sum +++ b/go.sum @@ -403,6 +403,8 @@ k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/secrets-store-csi-driver v1.4.7 h1:AyuwmPTW2GoPD2RjyVD3OrH1J9cdPZx+0h2qJvzbGXs= +sigs.k8s.io/secrets-store-csi-driver v1.4.7/go.mod h1:0/wMVOv8qLx7YNVMGU+Sh7S4D6TH6GhyEpouo28OTUU= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/internal/pkg/cmd/reloader.go b/internal/pkg/cmd/reloader.go index f0aac83..fb7c012 100644 --- a/internal/pkg/cmd/reloader.go +++ b/internal/pkg/cmd/reloader.go @@ -54,6 +54,7 @@ func NewReloaderCommand() *cobra.Command { cmd.PersistentFlags().StringVar(&options.ReloadOnDelete, "reload-on-delete", "false", "Add support to watch delete events") cmd.PersistentFlags().BoolVar(&options.EnableHA, "enable-ha", false, "Adds support for running multiple replicas via leadership election") cmd.PersistentFlags().BoolVar(&options.SyncAfterRestart, "sync-after-restart", false, "Sync add events after reloader restarts") + cmd.PersistentFlags().BoolVar(&options.EnableCSIIntegration, "enable-csi-integration", false, "Watch SecretProviderClassPodStatus for changes") return cmd } @@ -176,6 +177,10 @@ func startReloader(cmd *cobra.Command, args []string) { var controllers []*controller.Controller for k := range kube.ResourceMap { + if k == "secretproviderclasspodstatuses" && !options.EnableCSIIntegration { + continue + } + if ignoredResourcesList.Contains(k) || (len(namespaceLabelSelector) == 0 && k == "namespaces") { continue } diff --git a/internal/pkg/constants/constants.go b/internal/pkg/constants/constants.go index 18d1cc7..6ad3bd5 100644 --- a/internal/pkg/constants/constants.go +++ b/internal/pkg/constants/constants.go @@ -8,6 +8,8 @@ const ( ConfigmapEnvVarPostfix = "CONFIGMAP" // SecretEnvVarPostfix is a postfix for secret envVar SecretEnvVarPostfix = "SECRET" + // SecretEnvVarSecretProviderClassPodStatus is a postfix for secretproviderclasspodstatus envVar + SecretProviderClassEnvVarPostfix = "SECRETPROVIDERCLASS" // EnvVarPrefix is a Prefix for environment variable EnvVarPrefix = "STAKATER_" diff --git a/internal/pkg/controller/controller.go b/internal/pkg/controller/controller.go index 7dc7664..bf8ea4b 100644 --- a/internal/pkg/controller/controller.go +++ b/internal/pkg/controller/controller.go @@ -79,7 +79,16 @@ func NewController( } } - listWatcher := cache.NewFilteredListWatchFromClient(client.CoreV1().RESTClient(), resource, namespace, optionsModifier) + getterRESTClient := client.CoreV1().RESTClient() + if resource == "secretproviderclasspodstatuses" { + csiClient, err := kube.GetCSIClient() + if err != nil { + logrus.Fatal(err) + } + getterRESTClient = csiClient.SecretsstoreV1().RESTClient() + } + + listWatcher := cache.NewFilteredListWatchFromClient(getterRESTClient, resource, namespace, optionsModifier) _, informer := cache.NewInformerWithOptions(cache.InformerOptions{ ListerWatcher: listWatcher, diff --git a/internal/pkg/handler/update.go b/internal/pkg/handler/update.go index 0575e19..6a0baac 100644 --- a/internal/pkg/handler/update.go +++ b/internal/pkg/handler/update.go @@ -7,6 +7,7 @@ import ( "github.com/stakater/Reloader/internal/pkg/util" v1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/record" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" ) // ResourceUpdatedHandler contains updated objects @@ -45,6 +46,9 @@ func (r ResourceUpdatedHandler) GetConfig() (util.Config, string) { } else if _, ok := r.Resource.(*v1.Secret); ok { oldSHAData = util.GetSHAfromSecret(r.OldResource.(*v1.Secret).Data) config = util.GetSecretConfig(r.Resource.(*v1.Secret)) + } else if _, ok := r.Resource.(*csiv1.SecretProviderClassPodStatus); ok { + oldSHAData = util.GetSHAfromSecretProviderClassPodStatus(r.OldResource.(*csiv1.SecretProviderClassPodStatus).Status) + config = util.GetSecretProviderClassPodStatusConfig(r.Resource.(*csiv1.SecretProviderClassPodStatus)) } else { logrus.Warnf("Invalid resource: Resource should be 'Secret' or 'Configmap' but found, %v", r.Resource) } diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index 8365fb5..3d04a2c 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -219,6 +219,7 @@ func PerformAction(clients kube.Clients, config util.Config, upgradeFuncs callba typedAutoAnnotationEnabledValue, foundTypedAuto := annotations[config.TypedAutoAnnotation] excludeConfigmapAnnotationValue, foundExcludeConfigmap := annotations[options.ConfigmapExcludeReloaderAnnotation] excludeSecretAnnotationValue, foundExcludeSecret := annotations[options.SecretExcludeReloaderAnnotation] + excludeSecretProviderClassProviderAnnotationValue, foundExcludeSecretProviderClass := annotations[options.SecretProviderClassExcludeReloaderAnnotation] if !found && !foundAuto && !foundTypedAuto && !foundSearchAnn { annotations = upgradeFuncs.PodAnnotationsFunc(i) @@ -239,6 +240,10 @@ func PerformAction(clients kube.Clients, config util.Config, upgradeFuncs callba if foundExcludeSecret { isResourceExcluded = checkIfResourceIsExcluded(config.ResourceName, excludeSecretAnnotationValue) } + case constants.SecretProviderClassEnvVarPostfix: + if foundExcludeSecretProviderClass { + isResourceExcluded = checkIfResourceIsExcluded(config.ResourceName, excludeSecretProviderClassProviderAnnotationValue) + } } if isResourceExcluded { @@ -355,6 +360,10 @@ func getVolumeMountName(volumes []v1.Volume, mountType string, volumeName string } } } + } else if mountType == constants.SecretProviderClassEnvVarPostfix { + if volumes[i].CSI != nil && volumes[i].CSI.VolumeAttributes["secretProviderClass"] == volumeName { + return volumes[i].Name + } } } diff --git a/internal/pkg/options/flags.go b/internal/pkg/options/flags.go index 081acc3..8a5e9b4 100644 --- a/internal/pkg/options/flags.go +++ b/internal/pkg/options/flags.go @@ -30,6 +30,8 @@ var ( ConfigmapExcludeReloaderAnnotation = "configmaps.exclude.reloader.stakater.com/reload" // SecretExcludeReloaderAnnotation is a comma separated list of secrets that excludes detecting changes on secrets SecretExcludeReloaderAnnotation = "secrets.exclude.reloader.stakater.com/reload" + // SecretProviderClassExcludeReloaderAnnotation is a comma separated list of secret provider classes that excludes detecting changes on secret provider class + SecretProviderClassExcludeReloaderAnnotation = "secretproviderclass.exclude.reloader.stakater.com/reload" // AutoSearchAnnotation is an annotation to detect changes in // configmaps or triggers with the SearchMatchAnnotation AutoSearchAnnotation = "reloader.stakater.com/search" @@ -55,6 +57,8 @@ var ( EnableHA = false // Url to send a request to instead of triggering a reload WebhookUrl = "" + // EnableCsiIntegration Adds support to watch SecretProviderClassPodStatus and restart deployment based on it + EnableCSIIntegration = false ) func ToArgoRolloutStrategy(s string) ArgoRolloutStrategy { diff --git a/internal/pkg/util/config.go b/internal/pkg/util/config.go index 184eb68..460fc24 100644 --- a/internal/pkg/util/config.go +++ b/internal/pkg/util/config.go @@ -4,9 +4,10 @@ import ( "github.com/stakater/Reloader/internal/pkg/constants" "github.com/stakater/Reloader/internal/pkg/options" v1 "k8s.io/api/core/v1" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" ) -//Config contains rolling upgrade configuration parameters +// Config contains rolling upgrade configuration parameters type Config struct { Namespace string ResourceName string @@ -42,3 +43,13 @@ func GetSecretConfig(secret *v1.Secret) Config { Type: constants.SecretEnvVarPostfix, } } + +func GetSecretProviderClassPodStatusConfig(podStatus *csiv1.SecretProviderClassPodStatus) Config { + return Config{ + Namespace: podStatus.Namespace, + ResourceName: podStatus.Status.SecretProviderClassName, + ResourceAnnotations: podStatus.Annotations, + SHAValue: GetSHAfromSecretProviderClassPodStatus(podStatus.Status), + Type: constants.SecretProviderClassEnvVarPostfix, + } +} diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index 1a2696d..f23094b 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -8,6 +8,7 @@ import ( "github.com/stakater/Reloader/internal/pkg/crypto" v1 "k8s.io/api/core/v1" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" ) // ConvertToEnvVarName converts the given text into a usable env var @@ -52,6 +53,16 @@ func GetSHAfromSecret(data map[string][]byte) string { return crypto.GenerateSHA(strings.Join(values, ";")) } +func GetSHAfromSecretProviderClassPodStatus(data csiv1.SecretProviderClassPodStatusStatus) string { + values := []string{} + for _, v := range data.Objects { + values = append(values, v.ID+"="+v.Version) + } + values = append(values, "SecretProviderClassName="+data.SecretProviderClassName) + sort.Strings(values) + return crypto.GenerateSHA(strings.Join(values, ";")) +} + type List []string type Map map[string]string diff --git a/pkg/kube/client.go b/pkg/kube/client.go index 4230063..140087d 100644 --- a/pkg/kube/client.go +++ b/pkg/kube/client.go @@ -11,6 +11,7 @@ import ( "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + csi "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned" ) // Clients struct exposes interfaces for kubernetes as well as openshift if available @@ -18,6 +19,7 @@ type Clients struct { KubernetesClient kubernetes.Interface OpenshiftAppsClient appsclient.Interface ArgoRolloutClient argorollout.Interface + CSIClient csi.Interface } var ( @@ -48,10 +50,18 @@ func GetClients() Clients { logrus.Warnf("Unable to create ArgoRollout client error = %v", err) } + var csiClient *csi.Clientset + + csiClient, err = GetCSIClient() + if err != nil { + logrus.Warnf("Unable to create CSI client error = %v", err) + } + return Clients{ KubernetesClient: client, OpenshiftAppsClient: appsClient, ArgoRolloutClient: rolloutClient, + CSIClient: csiClient, } } @@ -63,6 +73,14 @@ func GetArgoRolloutClient() (*argorollout.Clientset, error) { return argorollout.NewForConfig(config) } +func GetCSIClient() (*csi.Clientset, error) { + config, err := getConfig() + if err != nil { + return nil, err + } + return csi.NewForConfig(config) +} + func isOpenshift() bool { client, err := GetKubernetesClient() if err != nil { diff --git a/pkg/kube/resourcemapper.go b/pkg/kube/resourcemapper.go index fb42e61..595c35e 100644 --- a/pkg/kube/resourcemapper.go +++ b/pkg/kube/resourcemapper.go @@ -3,11 +3,13 @@ package kube import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" ) // ResourceMap are resources from where changes are going to be detected var ResourceMap = map[string]runtime.Object{ - "configMaps": &v1.ConfigMap{}, - "secrets": &v1.Secret{}, - "namespaces": &v1.Namespace{}, + "configMaps": &v1.ConfigMap{}, + "secrets": &v1.Secret{}, + "namespaces": &v1.Namespace{}, + "secretproviderclasspodstatuses": &csiv1.SecretProviderClassPodStatus{}, } From 6d1d017aa430f04cae4e8324c3b5a29d04ac90ff Mon Sep 17 00:00:00 2001 From: Zanis <22601571+ZanisO@users.noreply.github.com> Date: Thu, 2 Jan 2025 23:38:17 +0000 Subject: [PATCH 02/45] Don't reload existing config --- internal/pkg/handler/upgrade.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index 3d04a2c..865a0a6 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -479,6 +479,10 @@ func updatePodAnnotations(upgradeFuncs callbacks.RollingUpgradeFuncs, item runti return constants.NotUpdated } + if config.Type == constants.SecretProviderClassEnvVarPostfix && secretProviderClassAnnotationReloaded(pa, config) { + return constants.NotUpdated + } + for k, v := range annotations { pa[k] = v } @@ -493,6 +497,11 @@ func getReloaderAnnotationKey() string { ) } +func secretProviderClassAnnotationReloaded(oldAnnotations map[string]string, newConfig util.Config) bool { + annotaion := oldAnnotations[getReloaderAnnotationKey()] + return strings.Contains(annotaion, newConfig.ResourceName) && strings.Contains(annotaion, newConfig.SHAValue) +} + func createReloadedAnnotations(target *util.ReloadSource) (map[string]string, error) { if target == nil { return nil, errors.New("target is required") @@ -527,6 +536,10 @@ func updateContainerEnvVars(upgradeFuncs callbacks.RollingUpgradeFuncs, item run return constants.NoContainerFound } + if config.Type == constants.SecretProviderClassEnvVarPostfix && secretProviderClassEnvReloaded(upgradeFuncs.ContainersFunc(item), envVar, config.SHAValue) { + return constants.NotUpdated + } + //update if env var exists result = updateEnvVar(upgradeFuncs.ContainersFunc(item), envVar, config.SHAValue) @@ -557,3 +570,15 @@ func updateEnvVar(containers []v1.Container, envVar string, shaData string) cons } return constants.NoEnvVarFound } + +func secretProviderClassEnvReloaded(containers []v1.Container, envVar string, shaData string) bool { + for i := range containers { + envs := containers[i].Env + for j := range envs { + if envs[j].Name == envVar { + return envs[j].Value == shaData + } + } + } + return false +} From 3c39406ca981ec683c7de330c845f1da5b348e41 Mon Sep 17 00:00:00 2001 From: Zanis <22601571+ZanisO@users.noreply.github.com> Date: Thu, 9 Jan 2025 23:54:10 +0000 Subject: [PATCH 03/45] Added capability to use OnChangeAnnotation and TypeAutoAnnotation --- internal/pkg/cmd/reloader.go | 2 ++ internal/pkg/handler/upgrade.go | 17 +++++++++++++++++ internal/pkg/options/flags.go | 7 ++++++- internal/pkg/util/config.go | 5 ++++- pkg/kube/client.go | 10 +++++----- 5 files changed, 34 insertions(+), 7 deletions(-) diff --git a/internal/pkg/cmd/reloader.go b/internal/pkg/cmd/reloader.go index fb7c012..a1e2482 100644 --- a/internal/pkg/cmd/reloader.go +++ b/internal/pkg/cmd/reloader.go @@ -36,9 +36,11 @@ func NewReloaderCommand() *cobra.Command { cmd.PersistentFlags().BoolVar(&options.AutoReloadAll, "auto-reload-all", false, "Auto reload all resources") cmd.PersistentFlags().StringVar(&options.ConfigmapUpdateOnChangeAnnotation, "configmap-annotation", "configmap.reloader.stakater.com/reload", "annotation to detect changes in configmaps, specified by name") cmd.PersistentFlags().StringVar(&options.SecretUpdateOnChangeAnnotation, "secret-annotation", "secret.reloader.stakater.com/reload", "annotation to detect changes in secrets, specified by name") + cmd.PersistentFlags().StringVar(&options.SecretProviderClassUpdateOnChangeAnnotation, "spc-annotation", "secretproviderclass.reloader.stakater.com/reload", "annotation to detect changes in secretproviderclasses, specified by name") cmd.PersistentFlags().StringVar(&options.ReloaderAutoAnnotation, "auto-annotation", "reloader.stakater.com/auto", "annotation to detect changes in secrets/configmaps") cmd.PersistentFlags().StringVar(&options.ConfigmapReloaderAutoAnnotation, "configmap-auto-annotation", "configmap.reloader.stakater.com/auto", "annotation to detect changes in configmaps") cmd.PersistentFlags().StringVar(&options.SecretReloaderAutoAnnotation, "secret-auto-annotation", "secret.reloader.stakater.com/auto", "annotation to detect changes in secrets") + cmd.PersistentFlags().StringVar(&options.SecretProviderClassReloaderAutoAnnotation, "spc-auto-annotation", "secretproviderclass.reloader.stakater.com/auto", "annotation to detect changes in secretproviderclasses") cmd.PersistentFlags().StringVar(&options.AutoSearchAnnotation, "auto-search-annotation", "reloader.stakater.com/search", "annotation to detect changes in configmaps or secrets tagged with special match annotation") cmd.PersistentFlags().StringVar(&options.SearchMatchAnnotation, "search-match-annotation", "reloader.stakater.com/match", "annotation to mark secrets or configmaps to match the search") cmd.PersistentFlags().StringVar(&options.LogFormat, "log-format", "", "Log format to use (empty string for text, or JSON)") diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index 865a0a6..4542455 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -2,6 +2,7 @@ package handler import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -23,6 +24,7 @@ import ( "github.com/stakater/Reloader/pkg/kube" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" ) @@ -210,6 +212,10 @@ func rollingUpgrade(clients kube.Clients, config util.Config, upgradeFuncs callb func PerformAction(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy) error { items := upgradeFuncs.ItemsFunc(clients, config.Namespace) + if config.Type == constants.SecretProviderClassEnvVarPostfix { + populateAnnotationsFromSecretProviderClass(clients, &config) + } + for _, i := range items { // find correct annotation and update the resource annotations := upgradeFuncs.AnnotationsFunc(i) @@ -582,3 +588,14 @@ func secretProviderClassEnvReloaded(containers []v1.Container, envVar string, sh } return false } + +func populateAnnotationsFromSecretProviderClass(clients kube.Clients, config *util.Config) { + obj, err := clients.CSIClient.SecretsstoreV1().SecretProviderClasses(config.Namespace).Get(context.TODO(), config.ResourceName, metav1.GetOptions{}) + annotations := make(map[string]string) + if err != nil { + logrus.Infof("Couldn't find secretproviderclass '%s' in '%s' namespace for typed annotation", config.ResourceName, config.Namespace) + } else if obj.Annotations != nil { + annotations = obj.Annotations + } + config.ResourceAnnotations = annotations +} diff --git a/internal/pkg/options/flags.go b/internal/pkg/options/flags.go index 8a5e9b4..8267bed 100644 --- a/internal/pkg/options/flags.go +++ b/internal/pkg/options/flags.go @@ -20,18 +20,23 @@ var ( // SecretUpdateOnChangeAnnotation is an annotation to detect changes in // secrets specified by name SecretUpdateOnChangeAnnotation = "secret.reloader.stakater.com/reload" + // SecretProviderClassUpdateOnChangeAnnotation is an annotation to detect changes in + // secretproviderclasses specified by name + SecretProviderClassUpdateOnChangeAnnotation = "secretproviderclass.reloader.stakater.com/reload" // ReloaderAutoAnnotation is an annotation to detect changes in secrets/configmaps ReloaderAutoAnnotation = "reloader.stakater.com/auto" // ConfigmapReloaderAutoAnnotation is an annotation to detect changes in configmaps ConfigmapReloaderAutoAnnotation = "configmap.reloader.stakater.com/auto" // SecretReloaderAutoAnnotation is an annotation to detect changes in secrets SecretReloaderAutoAnnotation = "secret.reloader.stakater.com/auto" + // SecretProviderClassReloaderAutoAnnotation is an annotation to detect changes in secretproviderclasses + SecretProviderClassReloaderAutoAnnotation = "secretproviderclass.reloader.stakater.com/auto" // ConfigmapReloaderAutoAnnotation is a comma separated list of configmaps that excludes detecting changes on cms ConfigmapExcludeReloaderAnnotation = "configmaps.exclude.reloader.stakater.com/reload" // SecretExcludeReloaderAnnotation is a comma separated list of secrets that excludes detecting changes on secrets SecretExcludeReloaderAnnotation = "secrets.exclude.reloader.stakater.com/reload" // SecretProviderClassExcludeReloaderAnnotation is a comma separated list of secret provider classes that excludes detecting changes on secret provider class - SecretProviderClassExcludeReloaderAnnotation = "secretproviderclass.exclude.reloader.stakater.com/reload" + SecretProviderClassExcludeReloaderAnnotation = "secretproviderclasses.exclude.reloader.stakater.com/reload" // AutoSearchAnnotation is an annotation to detect changes in // configmaps or triggers with the SearchMatchAnnotation AutoSearchAnnotation = "reloader.stakater.com/search" diff --git a/internal/pkg/util/config.go b/internal/pkg/util/config.go index 460fc24..6d6ff21 100644 --- a/internal/pkg/util/config.go +++ b/internal/pkg/util/config.go @@ -45,10 +45,13 @@ func GetSecretConfig(secret *v1.Secret) Config { } func GetSecretProviderClassPodStatusConfig(podStatus *csiv1.SecretProviderClassPodStatus) Config { + // As csi injects SecretProviderClass, we will create config for it instead of SecretProviderClassPodStatus + // ResourceAnnotations will be retrieved during PerformAction call return Config{ Namespace: podStatus.Namespace, ResourceName: podStatus.Status.SecretProviderClassName, - ResourceAnnotations: podStatus.Annotations, + Annotation: options.SecretProviderClassUpdateOnChangeAnnotation, + TypedAutoAnnotation: options.SecretProviderClassReloaderAutoAnnotation, SHAValue: GetSHAfromSecretProviderClassPodStatus(podStatus.Status), Type: constants.SecretProviderClassEnvVarPostfix, } diff --git a/pkg/kube/client.go b/pkg/kube/client.go index 140087d..af67319 100644 --- a/pkg/kube/client.go +++ b/pkg/kube/client.go @@ -11,7 +11,7 @@ import ( "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - csi "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned" + csiclient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned" ) // Clients struct exposes interfaces for kubernetes as well as openshift if available @@ -19,7 +19,7 @@ type Clients struct { KubernetesClient kubernetes.Interface OpenshiftAppsClient appsclient.Interface ArgoRolloutClient argorollout.Interface - CSIClient csi.Interface + CSIClient csiclient.Interface } var ( @@ -50,7 +50,7 @@ func GetClients() Clients { logrus.Warnf("Unable to create ArgoRollout client error = %v", err) } - var csiClient *csi.Clientset + var csiClient *csiclient.Clientset csiClient, err = GetCSIClient() if err != nil { @@ -73,12 +73,12 @@ func GetArgoRolloutClient() (*argorollout.Clientset, error) { return argorollout.NewForConfig(config) } -func GetCSIClient() (*csi.Clientset, error) { +func GetCSIClient() (*csiclient.Clientset, error) { config, err := getConfig() if err != nil { return nil, err } - return csi.NewForConfig(config) + return csiclient.NewForConfig(config) } func isOpenshift() bool { From 75f9a23de30db042080531da3041486702dcd432 Mon Sep 17 00:00:00 2001 From: Zanis <22601571+ZanisO@users.noreply.github.com> Date: Tue, 14 Jan 2025 23:51:15 +0000 Subject: [PATCH 04/45] Added tests --- internal/pkg/handler/upgrade_test.go | 963 +++++++++++++++++++++++++-- internal/pkg/testutil/kube.go | 143 +++- 2 files changed, 1035 insertions(+), 71 deletions(-) diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index 2b71740..35acddb 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -21,57 +21,73 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" testclient "k8s.io/client-go/kubernetes/fake" + csitestclient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned/fake" ) var ( - clients = kube.Clients{KubernetesClient: testclient.NewSimpleClientset()} + clients = kube.Clients{ + KubernetesClient: testclient.NewSimpleClientset(), + CSIClient: csitestclient.NewSimpleClientset(), + } - arsNamespace = "test-handler-" + testutil.RandSeq(5) - arsConfigmapName = "testconfigmap-handler-" + testutil.RandSeq(5) - arsSecretName = "testsecret-handler-" + testutil.RandSeq(5) - arsProjectedConfigMapName = "testprojectedconfigmap-handler-" + testutil.RandSeq(5) - arsProjectedSecretName = "testprojectedsecret-handler-" + testutil.RandSeq(5) - arsConfigmapWithInitContainer = "testconfigmapInitContainerhandler-" + testutil.RandSeq(5) - arsSecretWithInitContainer = "testsecretWithInitContainer-handler-" + testutil.RandSeq(5) - arsProjectedConfigMapWithInitContainer = "testProjectedConfigMapWithInitContainer-handler" + testutil.RandSeq(5) - arsProjectedSecretWithInitContainer = "testProjectedSecretWithInitContainer-handler" + testutil.RandSeq(5) - arsConfigmapWithInitEnv = "configmapWithInitEnv-" + testutil.RandSeq(5) - arsSecretWithInitEnv = "secretWithInitEnv-handler-" + testutil.RandSeq(5) - arsConfigmapWithEnvName = "testconfigmapWithEnv-handler-" + testutil.RandSeq(5) - arsConfigmapWithEnvFromName = "testconfigmapWithEnvFrom-handler-" + testutil.RandSeq(5) - arsSecretWithEnvName = "testsecretWithEnv-handler-" + testutil.RandSeq(5) - arsSecretWithEnvFromName = "testsecretWithEnvFrom-handler-" + testutil.RandSeq(5) - arsConfigmapWithPodAnnotations = "testconfigmapPodAnnotations-handler-" + testutil.RandSeq(5) - arsConfigmapWithBothAnnotations = "testconfigmapBothAnnotations-handler-" + testutil.RandSeq(5) - arsConfigmapAnnotated = "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) - arsConfigMapWithNonAnnotatedDeployment = "testconfigmapNonAnnotatedDeployment-handler-" + testutil.RandSeq(5) - arsSecretWithSecretAutoAnnotation = "testsecretwithsecretautoannotationdeployment-handler-" + testutil.RandSeq(5) - arsConfigmapWithConfigMapAutoAnnotation = "testconfigmapwithconfigmapautoannotationdeployment-handler-" + testutil.RandSeq(5) - arsSecretWithExcludeSecretAnnotation = "testsecretwithsecretexcludeannotationdeployment-handler-" + testutil.RandSeq(5) - arsConfigmapWithExcludeConfigMapAnnotation = "testconfigmapwithconfigmapexcludeannotationdeployment-handler-" + testutil.RandSeq(5) + arsNamespace = "test-handler-" + testutil.RandSeq(5) + arsConfigmapName = "testconfigmap-handler-" + testutil.RandSeq(5) + arsSecretName = "testsecret-handler-" + testutil.RandSeq(5) + arsSecretProviderClassName = "testsecretproviderclass-handler-" + testutil.RandSeq(5) + arsProjectedConfigMapName = "testprojectedconfigmap-handler-" + testutil.RandSeq(5) + arsProjectedSecretName = "testprojectedsecret-handler-" + testutil.RandSeq(5) + arsConfigmapWithInitContainer = "testconfigmapInitContainerhandler-" + testutil.RandSeq(5) + arsSecretWithInitContainer = "testsecretWithInitContainer-handler-" + testutil.RandSeq(5) + arsSecretProviderClassWithInitContainer = "testsecretproviderclassWithInitContainer-handler-" + testutil.RandSeq(5) + arsProjectedConfigMapWithInitContainer = "testProjectedConfigMapWithInitContainer-handler" + testutil.RandSeq(5) + arsProjectedSecretWithInitContainer = "testProjectedSecretWithInitContainer-handler" + testutil.RandSeq(5) + arsConfigmapWithInitEnv = "configmapWithInitEnv-" + testutil.RandSeq(5) + arsSecretWithInitEnv = "secretWithInitEnv-handler-" + testutil.RandSeq(5) + arsConfigmapWithEnvName = "testconfigmapWithEnv-handler-" + testutil.RandSeq(5) + arsConfigmapWithEnvFromName = "testconfigmapWithEnvFrom-handler-" + testutil.RandSeq(5) + arsSecretWithEnvName = "testsecretWithEnv-handler-" + testutil.RandSeq(5) + arsSecretWithEnvFromName = "testsecretWithEnvFrom-handler-" + testutil.RandSeq(5) + arsConfigmapWithPodAnnotations = "testconfigmapPodAnnotations-handler-" + testutil.RandSeq(5) + arsConfigmapWithBothAnnotations = "testconfigmapBothAnnotations-handler-" + testutil.RandSeq(5) + arsConfigmapAnnotated = "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) + arsConfigMapWithNonAnnotatedDeployment = "testconfigmapNonAnnotatedDeployment-handler-" + testutil.RandSeq(5) + arsSecretWithSecretAutoAnnotation = "testsecretwithsecretautoannotationdeployment-handler-" + testutil.RandSeq(5) + arsConfigmapWithConfigMapAutoAnnotation = "testconfigmapwithconfigmapautoannotationdeployment-handler-" + testutil.RandSeq(5) + arsSecretProviderClassWithSPCAutoAnnotation = "testsecretproviderclasswithspcautoannotationdeployment-handler-" + testutil.RandSeq(5) + arsSecretWithExcludeSecretAnnotation = "testsecretwithsecretexcludeannotationdeployment-handler-" + testutil.RandSeq(5) + arsConfigmapWithExcludeConfigMapAnnotation = "testconfigmapwithconfigmapexcludeannotationdeployment-handler-" + testutil.RandSeq(5) + arsSecretProviderClassWithExcludeSPCAnnotation = "testsecretproviderclasswithspcexcludeannotationdeployment-handler-" + testutil.RandSeq(5) + arsSecretProviderClassReloadedWithSameConfig = "testsecretproviderclassreloadedwithsameconfig-handler-" + testutil.RandSeq(5) + arsSecretProviderClassReloadedWithDifferentConfig = "testsecretproviderclassreloadedwithdifferentconfig-handler-" + testutil.RandSeq(5) - ersNamespace = "test-handler-" + testutil.RandSeq(5) - ersConfigmapName = "testconfigmap-handler-" + testutil.RandSeq(5) - ersSecretName = "testsecret-handler-" + testutil.RandSeq(5) - ersProjectedConfigMapName = "testprojectedconfigmap-handler-" + testutil.RandSeq(5) - ersProjectedSecretName = "testprojectedsecret-handler-" + testutil.RandSeq(5) - ersConfigmapWithInitContainer = "testconfigmapInitContainerhandler-" + testutil.RandSeq(5) - ersSecretWithInitContainer = "testsecretWithInitContainer-handler-" + testutil.RandSeq(5) - ersProjectedConfigMapWithInitContainer = "testProjectedConfigMapWithInitContainer-handler" + testutil.RandSeq(5) - ersProjectedSecretWithInitContainer = "testProjectedSecretWithInitContainer-handler" + testutil.RandSeq(5) - ersConfigmapWithInitEnv = "configmapWithInitEnv-" + testutil.RandSeq(5) - ersSecretWithInitEnv = "secretWithInitEnv-handler-" + testutil.RandSeq(5) - ersConfigmapWithEnvName = "testconfigmapWithEnv-handler-" + testutil.RandSeq(5) - ersConfigmapWithEnvFromName = "testconfigmapWithEnvFrom-handler-" + testutil.RandSeq(5) - ersSecretWithEnvName = "testsecretWithEnv-handler-" + testutil.RandSeq(5) - ersSecretWithEnvFromName = "testsecretWithEnvFrom-handler-" + testutil.RandSeq(5) - ersConfigmapWithPodAnnotations = "testconfigmapPodAnnotations-handler-" + testutil.RandSeq(5) - ersConfigmapWithBothAnnotations = "testconfigmapBothAnnotations-handler-" + testutil.RandSeq(5) - ersConfigmapAnnotated = "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) - ersSecretWithSecretAutoAnnotation = "testsecretwithsecretautoannotationdeployment-handler-" + testutil.RandSeq(5) - ersConfigmapWithConfigMapAutoAnnotation = "testconfigmapwithconfigmapautoannotationdeployment-handler-" + testutil.RandSeq(5) - ersSecretWithSecretExcludeAnnotation = "testsecretwithsecretexcludeannotationdeployment-handler-" + testutil.RandSeq(5) - ersConfigmapWithConfigMapExcludeAnnotation = "testconfigmapwithconfigmapexcludeannotationdeployment-handler-" + testutil.RandSeq(5) + ersNamespace = "test-handler-" + testutil.RandSeq(5) + ersConfigmapName = "testconfigmap-handler-" + testutil.RandSeq(5) + ersSecretName = "testsecret-handler-" + testutil.RandSeq(5) + ersSecretProviderClassName = "testsecretproviderclass-handler-" + testutil.RandSeq(5) + ersProjectedConfigMapName = "testprojectedconfigmap-handler-" + testutil.RandSeq(5) + ersProjectedSecretName = "testprojectedsecret-handler-" + testutil.RandSeq(5) + ersConfigmapWithInitContainer = "testconfigmapInitContainerhandler-" + testutil.RandSeq(5) + ersSecretWithInitContainer = "testsecretWithInitContainer-handler-" + testutil.RandSeq(5) + ersSecretProviderClassWithInitContainer = "testsecretproviderclassWithInitContainer-handler-" + testutil.RandSeq(5) + ersProjectedConfigMapWithInitContainer = "testProjectedConfigMapWithInitContainer-handler" + testutil.RandSeq(5) + ersProjectedSecretWithInitContainer = "testProjectedSecretWithInitContainer-handler" + testutil.RandSeq(5) + ersConfigmapWithInitEnv = "configmapWithInitEnv-" + testutil.RandSeq(5) + ersSecretWithInitEnv = "secretWithInitEnv-handler-" + testutil.RandSeq(5) + ersConfigmapWithEnvName = "testconfigmapWithEnv-handler-" + testutil.RandSeq(5) + ersConfigmapWithEnvFromName = "testconfigmapWithEnvFrom-handler-" + testutil.RandSeq(5) + ersSecretWithEnvName = "testsecretWithEnv-handler-" + testutil.RandSeq(5) + ersSecretWithEnvFromName = "testsecretWithEnvFrom-handler-" + testutil.RandSeq(5) + ersConfigmapWithPodAnnotations = "testconfigmapPodAnnotations-handler-" + testutil.RandSeq(5) + ersConfigmapWithBothAnnotations = "testconfigmapBothAnnotations-handler-" + testutil.RandSeq(5) + ersConfigmapAnnotated = "testconfigmapAnnotated-handler-" + testutil.RandSeq(5) + ersSecretWithSecretAutoAnnotation = "testsecretwithsecretautoannotationdeployment-handler-" + testutil.RandSeq(5) + ersConfigmapWithConfigMapAutoAnnotation = "testconfigmapwithconfigmapautoannotationdeployment-handler-" + testutil.RandSeq(5) + ersSecretProviderClassWithSPCAutoAnnotation = "testsecretproviderclasswithspcautoannotationdeployment-handler-" + testutil.RandSeq(5) + ersSecretWithSecretExcludeAnnotation = "testsecretwithsecretexcludeannotationdeployment-handler-" + testutil.RandSeq(5) + ersConfigmapWithConfigMapExcludeAnnotation = "testconfigmapwithconfigmapexcludeannotationdeployment-handler-" + testutil.RandSeq(5) + ersSecretProviderClassWithExcludeSPCAnnotation = "testsecretproviderclasswithspcexcludeannotationdeployment-handler-" + testutil.RandSeq(5) + ersSecretProviderClassReloadedWithSameConfig = "testsecretproviderclassreloadedwithsameconfig-handler-" + testutil.RandSeq(5) + ersSecretProviderClassReloadedWithDifferentConfig = "testsecretproviderclassreloadedwithdifferentconfig-handler-" + testutil.RandSeq(5) ) func TestMain(m *testing.M) { @@ -110,6 +126,12 @@ func setupArs() { logrus.Errorf("Error in secret creation: %v", err) } + // Creating secretproviderclass + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassName, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + // Creating configmap will be used in projected volume _, err = testutil.CreateConfigMap(clients.KubernetesClient, arsNamespace, arsProjectedConfigMapName, "www.google.com") if err != nil { @@ -178,6 +200,12 @@ func setupArs() { logrus.Errorf("Error in secret creation: %v", err) } + // Creating secretproviderclass + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassWithInitContainer, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + _, err = testutil.CreateConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapWithPodAnnotations, "www.google.com") if err != nil { logrus.Errorf("Error in configmap creation: %v", err) @@ -194,6 +222,12 @@ func setupArs() { logrus.Errorf("Error in secret creation: %v", err) } + // Creating secretproviderclass used with secretproviderclass auto annotation + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassWithSPCAutoAnnotation, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + // Creating configmap used with configmap auto annotation _, err = testutil.CreateConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapWithConfigMapAutoAnnotation, "www.google.com") if err != nil { @@ -206,6 +240,24 @@ func setupArs() { logrus.Errorf("Error in secret creation: %v", err) } + // Creating secretproviderclass used with secret auto annotation + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassWithExcludeSPCAnnotation, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + + // Creating secretproviderclass to reload with same config + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassReloadedWithSameConfig, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + + // Creating secretproviderclass to reload with different config + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassReloadedWithDifferentConfig, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + // Creating configmap used with configmap auto annotation _, err = testutil.CreateConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapWithExcludeConfigMapAnnotation, "www.google.com") if err != nil { @@ -254,6 +306,12 @@ func setupArs() { logrus.Errorf("Error in Deployment with secret creation: %v", err) } + // Creating Deployment with secretproviderclass mounted in init container + _, err = testutil.CreateDeploymentWithInitContainer(clients.KubernetesClient, arsSecretProviderClassWithInitContainer, arsNamespace, true) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass creation: %v", err) + } + // Creating Deployment with configmap mounted as Env in init container _, err = testutil.CreateDeploymentWithInitContainer(clients.KubernetesClient, arsConfigmapWithInitEnv, arsNamespace, false) if err != nil { @@ -272,6 +330,12 @@ func setupArs() { logrus.Errorf("Error in Deployment with secret creation: %v", err) } + // Creating Deployment with secretproviderclass + _, err = testutil.CreateDeployment(clients.KubernetesClient, arsSecretProviderClassName, arsNamespace, true) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass creation: %v", err) + } + // Creating Deployment with env var source as configmap _, err = testutil.CreateDeployment(clients.KubernetesClient, arsConfigmapWithEnvName, arsNamespace, false) if err != nil { @@ -319,6 +383,12 @@ func setupArs() { logrus.Errorf("Error in Deployment with secret and with secret auto annotation: %v", err) } + // Creating Deployment with secretproviderclass and with secretproviderclass auto annotation + _, err = testutil.CreateDeploymentWithTypedAutoAnnotation(clients.KubernetesClient, arsSecretProviderClassWithSPCAutoAnnotation, arsNamespace, testutil.SecretProviderClassPodStatusResourceType) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass and with secretproviderclass auto annotation: %v", err) + } + // Creating Deployment with secret and with secret auto annotation _, err = testutil.CreateDeploymentWithTypedAutoAnnotation(clients.KubernetesClient, arsConfigmapWithConfigMapAutoAnnotation, arsNamespace, testutil.ConfigmapResourceType) if err != nil { @@ -326,11 +396,29 @@ func setupArs() { } // Creating Deployment with secret and exclude secret annotation - _, err = testutil.CreateDeploymentWithExcludeAnnotation(clients.KubernetesClient, arsSecretWithExcludeSecretAnnotation, arsNamespace, testutil.SecretResourceType) + _, err = testutil.CreateDeploymentWithExcludeAnnotation(clients.KubernetesClient, arsSecretWithExcludeSecretAnnotation, arsNamespace, testutil.ConfigmapResourceType) if err != nil { logrus.Errorf("Error in Deployment with secret and with secret exclude annotation: %v", err) } + // Creating Deployment with secretproviderclass and exclude secretproviderclass annotation + _, err = testutil.CreateDeploymentWithExcludeAnnotation(clients.KubernetesClient, arsSecretProviderClassWithExcludeSPCAnnotation, arsNamespace, testutil.SecretProviderClassPodStatusResourceType) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass and with secretproviderclass exclude annotation: %v", err) + } + + // Creating Deployment with secretproviderclass to reload with same config + _, err = testutil.CreateDeploymentWithTypedAutoAnnotation(clients.KubernetesClient, arsSecretProviderClassReloadedWithSameConfig, arsNamespace, testutil.SecretProviderClassPodStatusResourceType) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass to reload with same config: %v", err) + } + + // Creating Deployment with secretproviderclass to reload with different config + _, err = testutil.CreateDeploymentWithTypedAutoAnnotation(clients.KubernetesClient, arsSecretProviderClassReloadedWithDifferentConfig, arsNamespace, testutil.SecretProviderClassPodStatusResourceType) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass to reload with different config: %v", err) + } + // Creating Deployment with secret and exclude configmap annotation _, err = testutil.CreateDeploymentWithExcludeAnnotation(clients.KubernetesClient, arsConfigmapWithExcludeConfigMapAnnotation, arsNamespace, testutil.ConfigmapResourceType) if err != nil { @@ -349,6 +437,12 @@ func setupArs() { logrus.Errorf("Error in DaemonSet with secret creation: %v", err) } + // Creating DaemonSet with secretproviderclass + _, err = testutil.CreateDaemonSet(clients.KubernetesClient, arsSecretProviderClassName, arsNamespace, true) + if err != nil { + logrus.Errorf("Error in DaemonSet with secretproviderclass creation: %v", err) + } + // Creating DaemonSet with configmap in projected volume _, err = testutil.CreateDaemonSet(clients.KubernetesClient, arsProjectedConfigMapName, arsNamespace, true) if err != nil { @@ -385,6 +479,12 @@ func setupArs() { logrus.Errorf("Error in StatefulSet with secret creation: %v", err) } + // Creating StatefulSet with secretproviderclass + _, err = testutil.CreateStatefulSet(clients.KubernetesClient, arsSecretProviderClassName, arsNamespace, true) + if err != nil { + logrus.Errorf("Error in StatefulSet with secretproviderclass creation: %v", err) + } + // Creating StatefulSet with configmap in projected volume _, err = testutil.CreateStatefulSet(clients.KubernetesClient, arsProjectedConfigMapName, arsNamespace, true) if err != nil { @@ -436,6 +536,12 @@ func teardownArs() { logrus.Errorf("Error while deleting deployment with secret %v", deploymentError) } + // Deleting Deployment with secretproviderclass + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsSecretProviderClassName) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass %v", deploymentError) + } + // Deleting Deployment with configmap in projected volume deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsProjectedConfigMapName) if deploymentError != nil { @@ -484,6 +590,12 @@ func teardownArs() { logrus.Errorf("Error while deleting deployment with secret mounted in init container %v", deploymentError) } + // Deleting Deployment with secretproviderclass mounted in init container + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsSecretProviderClassWithInitContainer) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass mounted in init container %v", deploymentError) + } + // Deleting Deployment with configmap mounted as env in init container deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsConfigmapWithInitEnv) if deploymentError != nil { @@ -532,6 +644,12 @@ func teardownArs() { logrus.Errorf("Error while deleting deployment with secret auto annotation %v", deploymentError) } + // Deleting Deployment with secretproviderclass and secretproviderclass auto annotation + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsSecretProviderClassWithSPCAutoAnnotation) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass auto annotation %v", deploymentError) + } + // Deleting Deployment with configmap and configmap auto annotation deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsConfigmapWithConfigMapAutoAnnotation) if deploymentError != nil { @@ -544,6 +662,24 @@ func teardownArs() { logrus.Errorf("Error while deleting deployment with secret auto annotation %v", deploymentError) } + // Deleting Deployment with secretproviderclass and exclude secretproviderclass annotation + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsSecretProviderClassWithExcludeSPCAnnotation) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass auto annotation %v", deploymentError) + } + + // Deleting Deployment with secretproviderclass to reload with same config + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsSecretProviderClassReloadedWithSameConfig) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass to reload with same config %v", deploymentError) + } + + // Deleting Deployment with secretproviderclass to reload with different config + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsSecretProviderClassReloadedWithDifferentConfig) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass to reload with different config %v", deploymentError) + } + // Deleting Deployment with configmap and exclude configmap annotation deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsConfigmapWithExcludeConfigMapAnnotation) if deploymentError != nil { @@ -556,12 +692,18 @@ func teardownArs() { logrus.Errorf("Error while deleting daemonSet with configmap %v", daemonSetError) } - // Deleting Deployment with secret + // Deleting DeamonSet with secret daemonSetError = testutil.DeleteDaemonSet(clients.KubernetesClient, arsNamespace, arsSecretName) if daemonSetError != nil { logrus.Errorf("Error while deleting daemonSet with secret %v", daemonSetError) } + // Deleting DeamonSet with secretproviderclass + daemonSetError = testutil.DeleteDaemonSet(clients.KubernetesClient, arsNamespace, arsSecretProviderClassName) + if daemonSetError != nil { + logrus.Errorf("Error while deleting daemonSet with secretproviderclass %v", daemonSetError) + } + // Deleting DaemonSet with configmap in projected volume daemonSetError = testutil.DeleteDaemonSet(clients.KubernetesClient, arsNamespace, arsProjectedConfigMapName) if daemonSetError != nil { @@ -592,12 +734,18 @@ func teardownArs() { logrus.Errorf("Error while deleting statefulSet with configmap %v", statefulSetError) } - // Deleting Deployment with secret + // Deleting StatefulSet with secret statefulSetError = testutil.DeleteStatefulSet(clients.KubernetesClient, arsNamespace, arsSecretName) if statefulSetError != nil { logrus.Errorf("Error while deleting statefulSet with secret %v", statefulSetError) } + // Deleting StatefulSet with secretproviderclass + statefulSetError = testutil.DeleteStatefulSet(clients.KubernetesClient, arsNamespace, arsSecretProviderClassName) + if statefulSetError != nil { + logrus.Errorf("Error while deleting statefulSet with secretproviderclass %v", statefulSetError) + } + // Deleting StatefulSet with configmap in projected volume statefulSetError = testutil.DeleteStatefulSet(clients.KubernetesClient, arsNamespace, arsProjectedConfigMapName) if statefulSetError != nil { @@ -634,6 +782,12 @@ func teardownArs() { logrus.Errorf("Error while deleting the secret %v", err) } + // Deleting Secretproviderclass + err = testutil.DeleteSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass %v", err) + } + // Deleting configmap used in projected volume err = testutil.DeleteConfigMap(clients.KubernetesClient, arsNamespace, arsProjectedConfigMapName) if err != nil { @@ -682,6 +836,12 @@ func teardownArs() { logrus.Errorf("Error while deleting the secret used in init container %v", err) } + // Deleting Secretproviderclass used in init container + err = testutil.DeleteSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassWithInitContainer) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used in init container %v", err) + } + // Deleting Configmap used as env var source err = testutil.DeleteConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapWithEnvFromName) if err != nil { @@ -717,6 +877,12 @@ func teardownArs() { logrus.Errorf("Error while deleting the secret used with secret auto annotations: %v", err) } + // Deleting SecretProviderClass used with secretproviderclass auto annotation + err = testutil.DeleteSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassWithSPCAutoAnnotation) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used with secretproviderclass auto annotations: %v", err) + } + // Deleting ConfigMap used with configmap auto annotation err = testutil.DeleteConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapWithConfigMapAutoAnnotation) if err != nil { @@ -729,6 +895,24 @@ func teardownArs() { logrus.Errorf("Error while deleting the secret used with secret auto annotations: %v", err) } + // Deleting Secretproviderclass used with exclude secretproviderclass annotation + err = testutil.DeleteSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassWithExcludeSPCAnnotation) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used with secretproviderclass auto annotations: %v", err) + } + + // Deleting SecretProviderClass used with secretproviderclass to reload with same config + err = testutil.DeleteSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassReloadedWithSameConfig) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used with secretproviderclass to reload with same config: %v", err) + } + + // Deleting SecretProviderClass used with secretproviderclass to reload with different config + err = testutil.DeleteSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassReloadedWithDifferentConfig) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used with secretproviderclass to reload with different config: %v", err) + } + // Deleting ConfigMap used with exclude configmap annotation err = testutil.DeleteConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapWithExcludeConfigMapAnnotation) if err != nil { @@ -754,6 +938,12 @@ func setupErs() { logrus.Errorf("Error in secret creation: %v", err) } + // Creating secretproviderclass + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassName, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + // Creating configmap will be used in projected volume _, err = testutil.CreateConfigMap(clients.KubernetesClient, ersNamespace, ersProjectedConfigMapName, "www.google.com") if err != nil { @@ -822,6 +1012,12 @@ func setupErs() { logrus.Errorf("Error in secret creation: %v", err) } + // Creating secretproviderclass + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassWithInitContainer, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + _, err = testutil.CreateConfigMap(clients.KubernetesClient, ersNamespace, ersConfigmapWithPodAnnotations, "www.google.com") if err != nil { logrus.Errorf("Error in configmap creation: %v", err) @@ -839,6 +1035,12 @@ func setupErs() { logrus.Errorf("Error in configmap creation: %v", err) } + // Creating secretproviderclass used with secretproviderclass auto annotation + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassWithSPCAutoAnnotation, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + // Creating secret used with secret exclude annotation _, err = testutil.CreateSecret(clients.KubernetesClient, ersNamespace, ersSecretWithSecretExcludeAnnotation, data) if err != nil { @@ -851,6 +1053,24 @@ func setupErs() { logrus.Errorf("Error in configmap creation: %v", err) } + // Creating secretproviderclass used with secret exclude annotation + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassWithExcludeSPCAnnotation, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + + // Creating secretproviderclass to reload with same config + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassReloadedWithSameConfig, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + + // Creating secretproviderclass to reload with different config + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassReloadedWithDifferentConfig, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + // Creating Deployment with configmap _, err = testutil.CreateDeployment(clients.KubernetesClient, ersConfigmapName, ersNamespace, true) if err != nil { @@ -893,6 +1113,12 @@ func setupErs() { logrus.Errorf("Error in Deployment with secret creation: %v", err) } + // Creating Deployment with secretproviderclass mounted in init container + _, err = testutil.CreateDeploymentWithInitContainer(clients.KubernetesClient, ersSecretProviderClassWithInitContainer, ersNamespace, true) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass creation: %v", err) + } + // Creating Deployment with configmap mounted as Env in init container _, err = testutil.CreateDeploymentWithInitContainer(clients.KubernetesClient, ersConfigmapWithInitEnv, ersNamespace, false) if err != nil { @@ -911,6 +1137,12 @@ func setupErs() { logrus.Errorf("Error in Deployment with secret creation: %v", err) } + // Creating Deployment with secretproviderclass + _, err = testutil.CreateDeployment(clients.KubernetesClient, ersSecretProviderClassName, ersNamespace, true) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass creation: %v", err) + } + // Creating Deployment with env var source as configmap _, err = testutil.CreateDeployment(clients.KubernetesClient, ersConfigmapWithEnvName, ersNamespace, false) if err != nil { @@ -958,6 +1190,12 @@ func setupErs() { logrus.Errorf("Error in Deployment with configmap and with configmap auto annotation: %v", err) } + // Creating Deployment with secretproviderclass and with secretproviderclass auto annotation + _, err = testutil.CreateDeploymentWithTypedAutoAnnotation(clients.KubernetesClient, ersSecretProviderClassWithSPCAutoAnnotation, ersNamespace, testutil.SecretProviderClassPodStatusResourceType) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass and with secretproviderclass auto annotation: %v", err) + } + // Creating Deployment with secret and with secret exclude annotation _, err = testutil.CreateDeploymentWithExcludeAnnotation(clients.KubernetesClient, ersSecretWithSecretExcludeAnnotation, ersNamespace, testutil.SecretResourceType) if err != nil { @@ -970,6 +1208,12 @@ func setupErs() { logrus.Errorf("Error in Deployment with configmap and with configmap exclude annotation: %v", err) } + // Creating Deployment with secretproviderclass and with secretproviderclass exclude annotation + _, err = testutil.CreateDeploymentWithExcludeAnnotation(clients.KubernetesClient, ersSecretProviderClassWithExcludeSPCAnnotation, ersNamespace, testutil.SecretProviderClassPodStatusResourceType) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass and with secretproviderclass exclude annotation: %v", err) + } + // Creating DaemonSet with configmap _, err = testutil.CreateDaemonSet(clients.KubernetesClient, ersConfigmapName, ersNamespace, true) if err != nil { @@ -982,6 +1226,12 @@ func setupErs() { logrus.Errorf("Error in DaemonSet with secret creation: %v", err) } + // Creating DaemonSet with secretproviderclass + _, err = testutil.CreateDaemonSet(clients.KubernetesClient, ersSecretProviderClassName, ersNamespace, true) + if err != nil { + logrus.Errorf("Error in DaemonSet with secretproviderclass creation: %v", err) + } + // Creating DaemonSet with configmap in projected volume _, err = testutil.CreateDaemonSet(clients.KubernetesClient, ersProjectedConfigMapName, ersNamespace, true) if err != nil { @@ -1018,6 +1268,12 @@ func setupErs() { logrus.Errorf("Error in StatefulSet with secret creation: %v", err) } + // Creating StatefulSet with secretproviderclass + _, err = testutil.CreateStatefulSet(clients.KubernetesClient, ersSecretProviderClassName, ersNamespace, true) + if err != nil { + logrus.Errorf("Error in StatefulSet with secretproviderclass creation: %v", err) + } + // Creating StatefulSet with configmap in projected volume _, err = testutil.CreateStatefulSet(clients.KubernetesClient, ersProjectedConfigMapName, ersNamespace, true) if err != nil { @@ -1053,6 +1309,18 @@ func setupErs() { if err != nil { logrus.Errorf("Error in Deployment with both annotations: %v", err) } + + // Creating Deployment with secretproviderclass to reload with same config + _, err = testutil.CreateDeploymentWithTypedAutoAnnotation(clients.KubernetesClient, ersSecretProviderClassReloadedWithSameConfig, ersNamespace, testutil.SecretProviderClassPodStatusResourceType) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass to reload with same config: %v", err) + } + + // Creating Deployment with secretproviderclass to reload with different config + _, err = testutil.CreateDeploymentWithTypedAutoAnnotation(clients.KubernetesClient, ersSecretProviderClassReloadedWithDifferentConfig, ersNamespace, testutil.SecretProviderClassPodStatusResourceType) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass to reload with different config: %v", err) + } } func teardownErs() { @@ -1068,6 +1336,12 @@ func teardownErs() { logrus.Errorf("Error while deleting deployment with secret %v", deploymentError) } + // Deleting Deployment with secretproviderclass + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersSecretProviderClassName) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretprovider class %v", deploymentError) + } + // Deleting Deployment with configmap in projected volume deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersProjectedConfigMapName) if deploymentError != nil { @@ -1116,6 +1390,12 @@ func teardownErs() { logrus.Errorf("Error while deleting deployment with secret mounted in init container %v", deploymentError) } + // Deleting Deployment with secretproviderclass mounted in init container + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersSecretProviderClassWithInitContainer) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass mounted in init container %v", deploymentError) + } + // Deleting Deployment with configmap mounted as env in init container deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersConfigmapWithInitEnv) if deploymentError != nil { @@ -1170,6 +1450,12 @@ func teardownErs() { logrus.Errorf("Error while deleting deployment with configmap auto annotation %v", deploymentError) } + // Deleting Deployment with secretproviderclass and secretproviderclass auto annotation + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersSecretProviderClassWithSPCAutoAnnotation) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass auto annotation %v", deploymentError) + } + // Deleting Deployment with secret and secret exclude annotation deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersSecretWithSecretExcludeAnnotation) if deploymentError != nil { @@ -1182,18 +1468,42 @@ func teardownErs() { logrus.Errorf("Error while deleting deployment with configmap exclude annotation %v", deploymentError) } + // Deleting Deployment with secretproviderclass and secretproviderclass exclude annotation + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersSecretProviderClassWithExcludeSPCAnnotation) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass exclude annotation %v", deploymentError) + } + + // Deleting Deployment with secretproviderclass to reload with same config + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersSecretProviderClassReloadedWithSameConfig) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass to reload with same config %v", deploymentError) + } + + // Deleting Deployment with secretproviderclass to reload with different config + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersSecretProviderClassReloadedWithDifferentConfig) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass to reload with different config %v", deploymentError) + } + // Deleting DaemonSet with configmap daemonSetError := testutil.DeleteDaemonSet(clients.KubernetesClient, ersNamespace, ersConfigmapName) if daemonSetError != nil { logrus.Errorf("Error while deleting daemonSet with configmap %v", daemonSetError) } - // Deleting Deployment with secret + // Deleting DaemonSet with secret daemonSetError = testutil.DeleteDaemonSet(clients.KubernetesClient, ersNamespace, ersSecretName) if daemonSetError != nil { logrus.Errorf("Error while deleting daemonSet with secret %v", daemonSetError) } + // Deleting DaemonSet with secretproviderclass + daemonSetError = testutil.DeleteDaemonSet(clients.KubernetesClient, ersNamespace, ersSecretProviderClassName) + if daemonSetError != nil { + logrus.Errorf("Error while deleting daemonSet with secretproviderclass %v", daemonSetError) + } + // Deleting DaemonSet with configmap in projected volume daemonSetError = testutil.DeleteDaemonSet(clients.KubernetesClient, ersNamespace, ersProjectedConfigMapName) if daemonSetError != nil { @@ -1224,12 +1534,18 @@ func teardownErs() { logrus.Errorf("Error while deleting statefulSet with configmap %v", statefulSetError) } - // Deleting Deployment with secret + // Deleting StatefulSet with secret statefulSetError = testutil.DeleteStatefulSet(clients.KubernetesClient, ersNamespace, ersSecretName) if statefulSetError != nil { logrus.Errorf("Error while deleting statefulSet with secret %v", statefulSetError) } + // Deleting StatefulSet with secretproviderclass + statefulSetError = testutil.DeleteStatefulSet(clients.KubernetesClient, ersNamespace, ersSecretProviderClassName) + if statefulSetError != nil { + logrus.Errorf("Error while deleting statefulSet with secretproviderclass %v", statefulSetError) + } + // Deleting StatefulSet with configmap in projected volume statefulSetError = testutil.DeleteStatefulSet(clients.KubernetesClient, ersNamespace, ersProjectedConfigMapName) if statefulSetError != nil { @@ -1266,6 +1582,12 @@ func teardownErs() { logrus.Errorf("Error while deleting the secret %v", err) } + // Deleting SecretProviderClass + err = testutil.DeleteSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass %v", err) + } + // Deleting configmap used in projected volume err = testutil.DeleteConfigMap(clients.KubernetesClient, ersNamespace, ersProjectedConfigMapName) if err != nil { @@ -1314,6 +1636,12 @@ func teardownErs() { logrus.Errorf("Error while deleting the secret used in init container %v", err) } + // Deleting SecretProviderClass used in init container + err = testutil.DeleteSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassWithInitContainer) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used in init container %v", err) + } + // Deleting Configmap used as env var source err = testutil.DeleteConfigMap(clients.KubernetesClient, ersNamespace, ersConfigmapWithEnvFromName) if err != nil { @@ -1355,6 +1683,12 @@ func teardownErs() { logrus.Errorf("Error while deleting the configmap used with configmap auto annotation: %v", err) } + // Deleting SecretProviderClass used with secretproviderclass auto annotation + err = testutil.DeleteSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassWithSPCAutoAnnotation) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used with secretproviderclass auto annotation: %v", err) + } + // Deleting Secret used with secret exclude annotation err = testutil.DeleteSecret(clients.KubernetesClient, ersNamespace, ersSecretWithSecretExcludeAnnotation) if err != nil { @@ -1367,6 +1701,24 @@ func teardownErs() { logrus.Errorf("Error while deleting the configmap used with configmap exclude annotation: %v", err) } + // Deleting SecretProviderClass used with secretproviderclass exclude annotation + err = testutil.DeleteSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassWithExcludeSPCAnnotation) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used with secretproviderclass exclude annotation: %v", err) + } + + // Deleting SecretProviderClass used with secretproviderclass to reload with same config + err = testutil.DeleteSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassReloadedWithSameConfig) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used with secretproviderclass to reload with same config: %v", err) + } + + // Deleting SecretProviderClass used with secretproviderclass to reload with different config + err = testutil.DeleteSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassReloadedWithDifferentConfig) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used with secretproviderclass to reload with different config: %v", err) + } + // Deleting namespace testutil.DeleteNamespace(ersNamespace, clients.KubernetesClient) @@ -1838,6 +2190,38 @@ func TestRollingUpgradeForDeploymentWithSecretUsingArs(t *testing.T) { testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForDeploymentWithSecretProviderClassUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassName, "testing1") + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassName, shaData, options.SecretProviderClassUpdateOnChangeAnnotation, options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with SecretProviderClass") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": arsNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForDeploymentWithSecretInProjectedVolumeUsingArs(t *testing.T) { options.ReloadStrategy = constants.AnnotationsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -1902,6 +2286,38 @@ func TestRollingUpgradeForDeploymentWithSecretinInitContainerUsingArs(t *testing testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForDeploymentWithSecretproviderclassInInitContainerUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassWithInitContainer, "testing1") + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassWithInitContainer, shaData, options.SecretProviderClassUpdateOnChangeAnnotation, options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with SecretProviderClass") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": arsNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForDeploymentWithSecretInProjectedVolumeinInitContainerUsingArs(t *testing.T) { options.ReloadStrategy = constants.AnnotationsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -2050,6 +2466,100 @@ func TestRollingUpgradeForDeploymentWithSecretExcludeAnnotationUsingArs(t *testi } } +func TestRollingUpgradeForDeploymentWithSecretproviderclassExcludeAnnotationUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassWithExcludeSPCAnnotation, "testing1") + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassWithExcludeSPCAnnotation, shaData, "", options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with SecretProviderClass") + } + + logrus.Infof("Verifying deployment did not update") + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if updated { + t.Errorf("Deployment which had to be exluded was updated") + } +} + +func TestRollingUpgradeForDeploymentWithSecretProviderClassReloadedWithSameConfigUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassReloadedWithSameConfig, "testing1") + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassReloadedWithSameConfig, shaData, "", options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with same config") + } + + logrus.Infof("Verifying deployment did update") + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + logrus.Infof("Performing reload using same config") + err = PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Second rolling upgrade failed for Deployment with same config") + } + + logrus.Infof("Verifying second reload did not reload") + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 && + promtestutil.ToFloat64(collectors.Reloaded.With(labelFailed)) != 0 { + t.Errorf("Second reload with same config updated Deployment") + } +} + +func TestRollingUpgradeForDeploymentWithSecretProviderClassReloadedWithDifferentConfigUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassReloadedWithDifferentConfig, "testing1") + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassReloadedWithDifferentConfig, shaData, "", options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with different config") + } + + logrus.Infof("Verifying deployment did update") + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + logrus.Infof("Applying different config") + shaData = testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassReloadedWithDifferentConfig, "testing2") + config.SHAValue = shaData + + err = PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Second rolling upgrade failed for Deployment with different config") + } + + logrus.Infof("Verifying deployment did update") + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 2 && + promtestutil.ToFloat64(collectors.Reloaded.With(labelFailed)) != 0 { + t.Errorf("Second reload with different config did not update Deployment") + } +} + func TestRollingUpgradeForDeploymentWithSecretAutoAnnotationUsingArs(t *testing.T) { options.ReloadStrategy = constants.AnnotationsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -2082,6 +2592,38 @@ func TestRollingUpgradeForDeploymentWithSecretAutoAnnotationUsingArs(t *testing. testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForDeploymentWithSecretProviderClassAutoAnnotationUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassWithSPCAutoAnnotation, "testing1") + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassWithSPCAutoAnnotation, shaData, "", options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with SecretProviderClass") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": arsNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForDeploymentWithExcludeConfigMapAnnotationUsingArs(t *testing.T) { options.ReloadStrategy = constants.AnnotationsReloadStrategy envVarPostfix := constants.ConfigmapEnvVarPostfix @@ -2262,6 +2804,38 @@ func TestRollingUpgradeForDaemonSetWithSecretUsingArs(t *testing.T) { testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, daemonSetFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForDaemonSetWithSecretProviderClassUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassName, "testing1") + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassName, shaData, options.SecretUpdateOnChangeAnnotation, options.SecretReloaderAutoAnnotation) + daemonSetFuncs := GetDaemonSetRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, daemonSetFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for DaemonSet with SecretProviderClass") + } + + logrus.Infof("Verifying daemonSet update") + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, daemonSetFuncs) + if !updated { + t.Errorf("DaemonSet was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": arsNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, daemonSetFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForDaemonSetWithSecretInProjectedVolumeUsingArs(t *testing.T) { options.ReloadStrategy = constants.AnnotationsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -2390,6 +2964,38 @@ func TestRollingUpgradeForStatefulSetWithSecretUsingArs(t *testing.T) { testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, statefulSetFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForStatefulSetWithSecretProviderClassUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassName, "testing1") + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassName, shaData, options.SecretUpdateOnChangeAnnotation, options.SecretReloaderAutoAnnotation) + statefulSetFuncs := GetStatefulSetRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, statefulSetFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for StatefulSet with SecretProviderClass") + } + + logrus.Infof("Verifying statefulSet update") + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, statefulSetFuncs) + if !updated { + t.Errorf("StatefulSet was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": arsNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, statefulSetFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForStatefulSetWithSecretInProjectedVolumeUsingArs(t *testing.T) { options.ReloadStrategy = constants.AnnotationsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -2880,6 +3486,38 @@ func TestRollingUpgradeForDeploymentWithSecretUsingErs(t *testing.T) { testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForDeploymentWithSecretProviderClassUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassName, "testing1") + config := getConfigWithAnnotations(envVarPostfix, ersSecretProviderClassName, shaData, options.SecretProviderClassUpdateOnChangeAnnotation, options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with SecretProviderClass") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, envVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": ersNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForDeploymentWithSecretInProjectedVolumeUsingErs(t *testing.T) { options.ReloadStrategy = constants.EnvVarsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -2944,6 +3582,38 @@ func TestRollingUpgradeForDeploymentWithSecretinInitContainerUsingErs(t *testing testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForDeploymentWithSecretProviderClassinInitContainerUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassWithInitContainer, "testing1") + config := getConfigWithAnnotations(envVarPostfix, ersSecretProviderClassWithInitContainer, shaData, options.SecretProviderClassUpdateOnChangeAnnotation, options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with SecretProviderClass") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, envVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": ersNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForDeploymentWithSecretInProjectedVolumeinInitContainerUsingErs(t *testing.T) { options.ReloadStrategy = constants.EnvVarsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -3094,6 +3764,101 @@ func TestRollingUpgradeForDeploymentWithSecretExcludeAnnotationUsingErs(t *testi } } +func TestRollingUpgradeForDeploymentWithSecretProviderClassExcludeAnnotationUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassWithExcludeSPCAnnotation, "testing1") + config := getConfigWithAnnotations(envVarPostfix, ersSecretProviderClassWithExcludeSPCAnnotation, shaData, "", options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with exclude Secret") + } + + logrus.Infof("Verifying deployment did not update") + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, envVarPostfix, deploymentFuncs) + if updated { + t.Errorf("Deployment that had to be excluded was updated") + } +} + +func TestRollingUpgradeForDeploymentWithSecretProviderClassReloadedWithSameConfigUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassReloadedWithSameConfig, "testing1") + config := getConfigWithAnnotations(envVarPostfix, ersSecretProviderClassReloadedWithSameConfig, shaData, "", options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with same config") + } + + logrus.Infof("Verifying deployment did update") + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, envVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + logrus.Infof("Performing reload using same config") + err = PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Second rolling upgrade failed for Deployment with same config") + } + + logrus.Infof("Verifying second reload did not reload") + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 && + promtestutil.ToFloat64(collectors.Reloaded.With(labelFailed)) != 0 { + t.Errorf("Second reload with same config updated Deployment") + } +} + +func TestRollingUpgradeForDeploymentWithSecretProviderClassReloadedWithDifferentConfigUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassReloadedWithDifferentConfig, "testing1") + config := getConfigWithAnnotations(envVarPostfix, ersSecretProviderClassReloadedWithDifferentConfig, shaData, "", options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with different config") + } + + logrus.Infof("Verifying deployment did update") + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, envVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + logrus.Infof("Applying different config") + shaData = testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassReloadedWithDifferentConfig, "testing2") + config.SHAValue = shaData + + err = PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Second rolling upgrade failed for Deployment with different config") + } + + logrus.Infof("Verifying deployment did update") + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 2 && + promtestutil.ToFloat64(collectors.Reloaded.With(labelFailed)) != 0 { + t.Errorf("Second reload with different config did not update Deployment") + } +} + func TestRollingUpgradeForDeploymentWithSecretAutoAnnotationUsingErs(t *testing.T) { options.ReloadStrategy = constants.EnvVarsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -3126,6 +3891,38 @@ func TestRollingUpgradeForDeploymentWithSecretAutoAnnotationUsingErs(t *testing. testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForDeploymentWithSecretProviderClassAutoAnnotationUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassWithSPCAutoAnnotation, "testing1") + config := getConfigWithAnnotations(envVarPostfix, ersSecretProviderClassWithSPCAutoAnnotation, shaData, "", options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with SecretProviderClass") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, envVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": ersNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForDeploymentWithConfigMapExcludeAnnotationUsingErs(t *testing.T) { options.ReloadStrategy = constants.EnvVarsReloadStrategy envVarPostfix := constants.ConfigmapEnvVarPostfix @@ -3308,6 +4105,38 @@ func TestRollingUpgradeForDaemonSetWithSecretUsingErs(t *testing.T) { testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, daemonSetFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForDaemonSetWithSecretProviderClassUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassName, "testing1") + config := getConfigWithAnnotations(envVarPostfix, ersSecretProviderClassName, shaData, options.SecretProviderClassUpdateOnChangeAnnotation, options.SecretProviderClassReloaderAutoAnnotation) + daemonSetFuncs := GetDaemonSetRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, daemonSetFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for DaemonSet with SecretProviderClass") + } + + logrus.Infof("Verifying daemonSet update") + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, envVarPostfix, daemonSetFuncs) + if !updated { + t.Errorf("DaemonSet was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": ersNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, daemonSetFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForDaemonSetWithSecretInProjectedVolumeUsingErs(t *testing.T) { options.ReloadStrategy = constants.EnvVarsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -3436,6 +4265,38 @@ func TestRollingUpgradeForStatefulSetWithSecretUsingErs(t *testing.T) { testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, statefulSetFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForStatefulSetWithSecretProviderClassUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassName, "testing1") + config := getConfigWithAnnotations(envVarPostfix, ersSecretProviderClassName, shaData, options.SecretProviderClassUpdateOnChangeAnnotation, options.SecretProviderClassReloaderAutoAnnotation) + statefulSetFuncs := GetStatefulSetRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, statefulSetFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for StatefulSet with SecretProviderClass") + } + + logrus.Infof("Verifying statefulSet update") + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, envVarPostfix, statefulSetFuncs) + if !updated { + t.Errorf("StatefulSet was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": ersNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, statefulSetFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForStatefulSetWithSecretInProjectedVolumeUsingErs(t *testing.T) { options.ReloadStrategy = constants.EnvVarsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix diff --git a/internal/pkg/testutil/kube.go b/internal/pkg/testutil/kube.go index 1f779ab..29e4fd7 100644 --- a/internal/pkg/testutil/kube.go +++ b/internal/pkg/testutil/kube.go @@ -29,6 +29,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" core_v1 "k8s.io/client-go/kubernetes/typed/core/v1" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" + csiclient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned" + csiclient_v1 "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned/typed/apis/v1" ) var ( @@ -37,6 +40,8 @@ var ( ConfigmapResourceType = "configMaps" // SecretResourceType is a resource type which controller watches for changes SecretResourceType = "secrets" + // SecretproviderclasspodstatusResourceType is a resource type which controller watches for changes + SecretProviderClassPodStatusResourceType = "secretproviderclasspodstatuses" ) var ( @@ -72,16 +77,16 @@ func DeleteNamespace(namespace string, client kubernetes.Interface) { } } -func getObjectMeta(namespace string, name string, autoReload bool, secretAutoReload bool, configmapAutoReload bool, extraAnnotations map[string]string) metav1.ObjectMeta { +func getObjectMeta(namespace string, name string, autoReload bool, secretAutoReload bool, configmapAutoReload bool, secretproviderclass bool, extraAnnotations map[string]string) metav1.ObjectMeta { return metav1.ObjectMeta{ Name: name, Namespace: namespace, Labels: map[string]string{"firstLabel": "temp"}, - Annotations: getAnnotations(name, autoReload, secretAutoReload, configmapAutoReload, extraAnnotations), + Annotations: getAnnotations(name, autoReload, secretAutoReload, configmapAutoReload, secretproviderclass, extraAnnotations), } } -func getAnnotations(name string, autoReload bool, secretAutoReload bool, configmapAutoReload bool, extraAnnotations map[string]string) map[string]string { +func getAnnotations(name string, autoReload bool, secretAutoReload bool, configmapAutoReload bool, secretproviderclass bool, extraAnnotations map[string]string) map[string]string { annotations := make(map[string]string) if autoReload { annotations[options.ReloaderAutoAnnotation] = "true" @@ -96,7 +101,9 @@ func getAnnotations(name string, autoReload bool, secretAutoReload bool, configm if !(len(annotations) > 0) { annotations = map[string]string{ options.ConfigmapUpdateOnChangeAnnotation: name, - options.SecretUpdateOnChangeAnnotation: name} + options.SecretUpdateOnChangeAnnotation: name, + options.SecretProviderClassUpdateOnChangeAnnotation: name, + } } for k, v := range extraAnnotations { annotations[k] = v @@ -175,6 +182,15 @@ func getVolumes(name string) []v1.Volume { }, }, }, + { + Name: "secretproviderclass", + VolumeSource: v1.VolumeSource{ + CSI: &v1.CSIVolumeSource{ + Driver: "secrets-store.csi.k8s.io", + VolumeAttributes: map[string]string{"secretProviderClass": name}, + }, + }, + }, } } @@ -188,6 +204,10 @@ func getVolumeMounts() []v1.VolumeMount { MountPath: "etc/sec", Name: "secret", }, + { + MountPath: "etc/spc", + Name: "secretproviderclass", + }, { MountPath: "etc/projectedconfig", Name: "projectedconfigmap", @@ -347,7 +367,7 @@ func getPodTemplateSpecWithInitContainerAndEnv(name string) v1.PodTemplateSpec { func GetDeployment(namespace string, deploymentName string) *appsv1.Deployment { replicaset := int32(1) return &appsv1.Deployment{ - ObjectMeta: getObjectMeta(namespace, deploymentName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, deploymentName, false, false, false, false, map[string]string{}), Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -366,7 +386,7 @@ func GetDeploymentConfig(namespace string, deploymentConfigName string) *openshi replicaset := int32(1) podTemplateSpecWithVolume := getPodTemplateSpecWithVolumes(deploymentConfigName) return &openshiftv1.DeploymentConfig{ - ObjectMeta: getObjectMeta(namespace, deploymentConfigName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, deploymentConfigName, false, false, false, false, map[string]string{}), Spec: openshiftv1.DeploymentConfigSpec{ Replicas: replicaset, Strategy: openshiftv1.DeploymentStrategy{ @@ -381,7 +401,7 @@ func GetDeploymentConfig(namespace string, deploymentConfigName string) *openshi func GetDeploymentWithInitContainer(namespace string, deploymentName string) *appsv1.Deployment { replicaset := int32(1) return &appsv1.Deployment{ - ObjectMeta: getObjectMeta(namespace, deploymentName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, deploymentName, false, false, false, false, map[string]string{}), Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -399,7 +419,7 @@ func GetDeploymentWithInitContainer(namespace string, deploymentName string) *ap func GetDeploymentWithInitContainerAndEnv(namespace string, deploymentName string) *appsv1.Deployment { replicaset := int32(1) return &appsv1.Deployment{ - ObjectMeta: getObjectMeta(namespace, deploymentName, true, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, deploymentName, true, false, false, false, map[string]string{}), Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -416,7 +436,7 @@ func GetDeploymentWithInitContainerAndEnv(namespace string, deploymentName strin func GetDeploymentWithEnvVars(namespace string, deploymentName string) *appsv1.Deployment { replicaset := int32(1) return &appsv1.Deployment{ - ObjectMeta: getObjectMeta(namespace, deploymentName, true, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, deploymentName, true, false, false, false, map[string]string{}), Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -434,7 +454,7 @@ func GetDeploymentConfigWithEnvVars(namespace string, deploymentConfigName strin replicaset := int32(1) podTemplateSpecWithEnvVars := getPodTemplateSpecWithEnvVars(deploymentConfigName) return &openshiftv1.DeploymentConfig{ - ObjectMeta: getObjectMeta(namespace, deploymentConfigName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, deploymentConfigName, false, false, false, false, map[string]string{}), Spec: openshiftv1.DeploymentConfigSpec{ Replicas: replicaset, Strategy: openshiftv1.DeploymentStrategy{ @@ -448,7 +468,7 @@ func GetDeploymentConfigWithEnvVars(namespace string, deploymentConfigName strin func GetDeploymentWithEnvVarSources(namespace string, deploymentName string) *appsv1.Deployment { replicaset := int32(1) return &appsv1.Deployment{ - ObjectMeta: getObjectMeta(namespace, deploymentName, true, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, deploymentName, true, false, false, false, map[string]string{}), Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -465,7 +485,7 @@ func GetDeploymentWithEnvVarSources(namespace string, deploymentName string) *ap func GetDeploymentWithPodAnnotations(namespace string, deploymentName string, both bool) *appsv1.Deployment { replicaset := int32(1) deployment := &appsv1.Deployment{ - ObjectMeta: getObjectMeta(namespace, deploymentName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, deploymentName, false, false, false, false, map[string]string{}), Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -480,7 +500,7 @@ func GetDeploymentWithPodAnnotations(namespace string, deploymentName string, bo if !both { deployment.ObjectMeta.Annotations = nil } - deployment.Spec.Template.ObjectMeta.Annotations = getAnnotations(deploymentName, true, false, false, map[string]string{}) + deployment.Spec.Template.ObjectMeta.Annotations = getAnnotations(deploymentName, true, false, false, false, map[string]string{}) return deployment } @@ -488,9 +508,11 @@ func GetDeploymentWithTypedAutoAnnotation(namespace string, deploymentName strin replicaset := int32(1) var objectMeta metav1.ObjectMeta if resourceType == SecretResourceType { - objectMeta = getObjectMeta(namespace, deploymentName, false, true, false, map[string]string{}) + objectMeta = getObjectMeta(namespace, deploymentName, false, true, false, false, map[string]string{}) } else if resourceType == ConfigmapResourceType { - objectMeta = getObjectMeta(namespace, deploymentName, false, false, true, map[string]string{}) + objectMeta = getObjectMeta(namespace, deploymentName, false, false, true, false, map[string]string{}) + } else if resourceType == SecretProviderClassPodStatusResourceType { + objectMeta = getObjectMeta(namespace, deploymentName, false, false, false, true, map[string]string{}) } return &appsv1.Deployment{ @@ -517,6 +539,8 @@ func GetDeploymentWithExcludeAnnotation(namespace string, deploymentName string, annotation[options.SecretExcludeReloaderAnnotation] = deploymentName } else if resourceType == ConfigmapResourceType { annotation[options.ConfigmapExcludeReloaderAnnotation] = deploymentName + } else if resourceType == SecretProviderClassPodStatusResourceType { + annotation[options.SecretProviderClassExcludeReloaderAnnotation] = deploymentName } return &appsv1.Deployment{ @@ -542,7 +566,7 @@ func GetDeploymentWithExcludeAnnotation(namespace string, deploymentName string, // GetDaemonSet provides daemonset for testing func GetDaemonSet(namespace string, daemonsetName string) *appsv1.DaemonSet { return &appsv1.DaemonSet{ - ObjectMeta: getObjectMeta(namespace, daemonsetName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, daemonsetName, false, false, false, false, map[string]string{}), Spec: appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -557,7 +581,7 @@ func GetDaemonSet(namespace string, daemonsetName string) *appsv1.DaemonSet { func GetDaemonSetWithEnvVars(namespace string, daemonSetName string) *appsv1.DaemonSet { return &appsv1.DaemonSet{ - ObjectMeta: getObjectMeta(namespace, daemonSetName, true, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, daemonSetName, true, false, false, false, map[string]string{}), Spec: appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -573,7 +597,7 @@ func GetDaemonSetWithEnvVars(namespace string, daemonSetName string) *appsv1.Dae // GetStatefulSet provides statefulset for testing func GetStatefulSet(namespace string, statefulsetName string) *appsv1.StatefulSet { return &appsv1.StatefulSet{ - ObjectMeta: getObjectMeta(namespace, statefulsetName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, statefulsetName, false, false, false, false, map[string]string{}), Spec: appsv1.StatefulSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -589,7 +613,7 @@ func GetStatefulSet(namespace string, statefulsetName string) *appsv1.StatefulSe // GetStatefulSet provides statefulset for testing func GetStatefulSetWithEnvVar(namespace string, statefulsetName string) *appsv1.StatefulSet { return &appsv1.StatefulSet{ - ObjectMeta: getObjectMeta(namespace, statefulsetName, true, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, statefulsetName, true, false, false, false, map[string]string{}), Spec: appsv1.StatefulSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -614,6 +638,42 @@ func GetConfigmap(namespace string, configmapName string, testData string) *v1.C } } +func GetSecretProviderClass(namespace string, secretProviderClassName string, data string) *csiv1.SecretProviderClass { + return &csiv1.SecretProviderClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretProviderClassName, + Namespace: namespace, + }, + Spec: csiv1.SecretProviderClassSpec{ + Provider: "Test", + Parameters: map[string]string{ + "parameter1": data, + }, + }, + } +} + +func GetSecretProviderClassPodStatus(namespace string, secretProviderClassPodStatusName string, data string) *csiv1.SecretProviderClassPodStatus { + return &csiv1.SecretProviderClassPodStatus{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretProviderClassPodStatusName, + Namespace: namespace, + }, + Status: csiv1.SecretProviderClassPodStatusStatus{ + PodName: "test123", + SecretProviderClassName: secretProviderClassPodStatusName, + TargetPath: "/var/lib/kubelet/d8771ddf-935a-4199-a20b-f35f71c1d9e7/volumes/kubernetes.io~csi/secrets-store-inline/mount", + Mounted: true, + Objects: []csiv1.SecretProviderClassObject{ + { + ID: "parameter1", + Version: data, + }, + }, + }, + } +} + // GetConfigmapWithUpdatedLabel provides configmap for testing func GetConfigmapWithUpdatedLabel(namespace string, configmapName string, testLabel string, testData string) *v1.ConfigMap { return &v1.ConfigMap{ @@ -743,7 +803,7 @@ func GetResourceSHAFromAnnotation(podAnnotations map[string]string) string { return last.Hash } -// ConvertResourceToSHA generates SHA from secret or configmap data +// ConvertResourceToSHA generates SHA from secret, configmap or secretproviderclasspodstatus data func ConvertResourceToSHA(resourceType string, namespace string, resourceName string, data string) string { values := []string{} if resourceType == SecretResourceType { @@ -756,6 +816,12 @@ func ConvertResourceToSHA(resourceType string, namespace string, resourceName st for k, v := range configmap.Data { values = append(values, k+"="+v) } + } else if resourceType == SecretProviderClassPodStatusResourceType { + secretproviderclasspodstatus := GetSecretProviderClassPodStatus(namespace, resourceName, data) + for _, v := range secretproviderclasspodstatus.Status.Objects { + values = append(values, v.ID+"="+v.Version) + } + values = append(values, "SecretProviderClassName="+secretproviderclasspodstatus.Status.SecretProviderClassName) } sort.Strings(values) return crypto.GenerateSHA(strings.Join(values, ";")) @@ -770,6 +836,35 @@ func CreateConfigMap(client kubernetes.Interface, namespace string, configmapNam return configmapClient, err } +// CreateSecretProviderClass creates a SecretProviderClass in given namespace and returns the SecretProviderClassInterface +func CreateSecretProviderClass(client csiclient.Interface, namespace string, secretProviderClassName string, data string) (csiclient_v1.SecretProviderClassInterface, error) { + logrus.Infof("Creating SecretProviderClass") + secretProviderClassClient := client.SecretsstoreV1().SecretProviderClasses(namespace) + _, err := secretProviderClassClient.Create(context.TODO(), GetSecretProviderClass(namespace, secretProviderClassName, data), metav1.CreateOptions{}) + time.Sleep(3 * time.Second) + return secretProviderClassClient, err +} + +// CreateSecretProviderClass creates a SecretProviderClassPodStatus in given namespace and returns the SecretProviderClassInterface +func CreateSecretProviderClassPodStatus(client csiclient.Interface, namespace string, secretProviderClassPodStatusName string, data string) (csiclient_v1.SecretProviderClassPodStatusInterface, error) { + logrus.Infof("Creating SecretProviderClassPodStatus") + secretProviderClassPodStatusClient := client.SecretsstoreV1().SecretProviderClassPodStatuses(namespace) + secretProviderClassPodStatus := GetSecretProviderClassPodStatus(namespace, secretProviderClassPodStatusName, data) + _, err := secretProviderClassPodStatusClient.Create(context.TODO(), secretProviderClassPodStatus, metav1.CreateOptions{}) + time.Sleep(3 * time.Second) + return secretProviderClassPodStatusClient, err +} + +// CreateSecretProviderClassAndPodStatus creates a SecretProviderClass and SecretProviderClassPodStatus in given namespace +func CreateSecretProviderClassAndPodStatus(client csiclient.Interface, namespace string, name string, data string) error { + _, err := CreateSecretProviderClass(client, namespace, name, data) + if err != nil { + return err + } + _, err = CreateSecretProviderClassPodStatus(client, namespace, name, data) + return err +} + // CreateSecret creates a secret in given namespace and returns the SecretInterface func CreateSecret(client kubernetes.Interface, namespace string, secretName string, data string) (core_v1.SecretInterface, error) { logrus.Infof("Creating secret") @@ -1012,6 +1107,14 @@ func DeleteSecret(client kubernetes.Interface, namespace string, secretName stri return err } +// DeleteSecretProviderClass deletes a secretproviderclass in given namespace and returns the error if any +func DeleteSecretProviderClass(client csiclient.Interface, namespace string, secretProviderClassName string) error { + logrus.Infof("Deleting secretproviderclass %q.\n", secretProviderClassName) + err := client.SecretsstoreV1().SecretProviderClasses(namespace).Delete(context.TODO(), secretProviderClassName, metav1.DeleteOptions{}) + time.Sleep(3 * time.Second) + return err +} + // RandSeq generates a random sequence func RandSeq(n int) string { b := make([]rune, n) From 717291f1737d4e5d148acb9541b154f9ffc8f951 Mon Sep 17 00:00:00 2001 From: Zanis <22601571+ZanisO@users.noreply.github.com> Date: Sun, 19 Jan 2025 23:46:29 +0000 Subject: [PATCH 05/45] Added check to see if CSI CRDs are installed before running controller --- internal/pkg/cmd/reloader.go | 10 ++++++++-- pkg/kube/client.go | 24 +++++++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/internal/pkg/cmd/reloader.go b/internal/pkg/cmd/reloader.go index a1e2482..f17b2a2 100644 --- a/internal/pkg/cmd/reloader.go +++ b/internal/pkg/cmd/reloader.go @@ -179,8 +179,14 @@ func startReloader(cmd *cobra.Command, args []string) { var controllers []*controller.Controller for k := range kube.ResourceMap { - if k == "secretproviderclasspodstatuses" && !options.EnableCSIIntegration { - continue + if k == "secretproviderclasspodstatuses" { + if !options.EnableCSIIntegration { + continue + } + if !kube.IsCSIInstalled { + logrus.Infof("Can't run CSI controller as CSI CRDs are not installed") + continue + } } if ignoredResourcesList.Contains(k) || (len(namespaceLabelSelector) == 0 && k == "namespaces") { diff --git a/pkg/kube/client.go b/pkg/kube/client.go index af67319..9582929 100644 --- a/pkg/kube/client.go +++ b/pkg/kube/client.go @@ -25,6 +25,8 @@ type Clients struct { var ( // IsOpenshift is true if environment is Openshift, it is false if environment is Kubernetes IsOpenshift = isOpenshift() + // IsCSIEnabled is true if environment has CSI provider installed, otherwise false + IsCSIInstalled = isCSIInstalled() ) // GetClients returns a `Clients` object containing both openshift and kubernetes clients with an openshift identifier @@ -52,9 +54,11 @@ func GetClients() Clients { var csiClient *csiclient.Clientset - csiClient, err = GetCSIClient() - if err != nil { - logrus.Warnf("Unable to create CSI client error = %v", err) + if IsCSIInstalled { + csiClient, err = GetCSIClient() + if err != nil { + logrus.Warnf("Unable to create CSI client error = %v", err) + } } return Clients{ @@ -73,6 +77,20 @@ func GetArgoRolloutClient() (*argorollout.Clientset, error) { return argorollout.NewForConfig(config) } +func isCSIInstalled() bool { + client, err := GetKubernetesClient() + if err != nil { + logrus.Fatalf("Unable to create Kubernetes client error = %v", err) + } + _, err = client.RESTClient().Get().AbsPath("/apis/secrets-store.csi.x-k8s.io/v1").Do(context.TODO()).Raw() + if err == nil { + logrus.Info("CSI provider is installed") + return true + } + logrus.Info("CSI provider is not installed") + return false +} + func GetCSIClient() (*csiclient.Clientset, error) { config, err := getConfig() if err != nil { From 69b0d93f31e5fed80c1c65a1cf699f14c7e4c556 Mon Sep 17 00:00:00 2001 From: Zanis <22601571+ZanisO@users.noreply.github.com> Date: Mon, 20 Jan 2025 23:26:07 +0000 Subject: [PATCH 06/45] Added controller tests --- internal/pkg/controller/controller_test.go | 441 ++++++++++++++++++++- internal/pkg/testutil/kube.go | 39 +- 2 files changed, 462 insertions(+), 18 deletions(-) diff --git a/internal/pkg/controller/controller_test.go b/internal/pkg/controller/controller_test.go index ccef5df..ae42c59 100644 --- a/internal/pkg/controller/controller_test.go +++ b/internal/pkg/controller/controller_test.go @@ -25,14 +25,15 @@ import ( ) var ( - clients = kube.GetClients() - namespace = "test-reloader-" + testutil.RandSeq(5) - configmapNamePrefix = "testconfigmap-reloader" - secretNamePrefix = "testsecret-reloader" - data = "dGVzdFNlY3JldEVuY29kaW5nRm9yUmVsb2FkZXI=" - newData = "dGVzdE5ld1NlY3JldEVuY29kaW5nRm9yUmVsb2FkZXI=" - updatedData = "dGVzdFVwZGF0ZWRTZWNyZXRFbmNvZGluZ0ZvclJlbG9hZGVy" - collectors = metrics.NewCollectors() + clients = kube.GetClients() + namespace = "test-reloader-" + testutil.RandSeq(5) + configmapNamePrefix = "testconfigmap-reloader" + secretNamePrefix = "testsecret-reloader" + secretProviderClassPodStatusPrefix = "testsecretproviderclasspodstatus-reloader" + data = "dGVzdFNlY3JldEVuY29kaW5nRm9yUmVsb2FkZXI=" + newData = "dGVzdE5ld1NlY3JldEVuY29kaW5nRm9yUmVsb2FkZXI=" + updatedData = "dGVzdFVwZGF0ZWRTZWNyZXRFbmNvZGluZ0ZvclJlbG9hZGVy" + collectors = metrics.NewCollectors() ) const ( @@ -45,6 +46,10 @@ func TestMain(m *testing.M) { logrus.Infof("Creating controller") for k := range kube.ResourceMap { + // Don't create controller if CSI provider is not installed + if k == "secretproviderclasspodstatuses" && !kube.IsCSIInstalled { + continue + } if k == "namespaces" { continue } @@ -636,6 +641,217 @@ func TestControllerUpdatingSecretLabelsShouldNotCreateOrUpdatePodAnnotationInDep time.Sleep(sleepDuration) } +// Perform rolling upgrade on deployment and create pod annotation var upon updating the secretclassproviderpodstatus +func TestControllerUpdatingSecretProviderClassPodStatusShouldCreatePodAnnotationInDeployment(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + + if !kube.IsCSIInstalled { + return + } + + // Creating secretclassprovider + secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) + _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclassprovider %v", err) + } + + // Creating secretproviderclasspodstatus + spcpsClient, err := testutil.CreateSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclasssproviderpodstatus %v", err) + } + + // Creating deployment + _, err = testutil.CreateDeployment(clients.KubernetesClient, secretproviderclasspodstatusName, namespace, true) + if err != nil { + t.Errorf("Error in deployment creation: %v", err) + } + + // Updating secretproviderclasspodstatus for first time + updateErr := testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "", newData) + if updateErr != nil { + t.Errorf("Secretproviderclasspodstatus was not updated") + } + + // Verifying deployment update + logrus.Infof("Verifying pod annotation has been created") + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, namespace, secretproviderclasspodstatusName, newData) + config := util.Config{ + Namespace: namespace, + ResourceName: secretproviderclasspodstatusName, + SHAValue: shaData, + Annotation: options.SecretProviderClassUpdateOnChangeAnnotation, + } + deploymentFuncs := handler.GetDeploymentRollingUpgradeFuncs() + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + time.Sleep(sleepDuration) + + // Deleting deployment + err = testutil.DeleteDeployment(clients.KubernetesClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the deployment %v", err) + } + + // Deleting secretproviderclass + err = testutil.DeleteSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass %v", err) + } + + // Deleting secretproviderclasspodstatus + err = testutil.DeleteSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclasspodstatus %v", err) + } + time.Sleep(sleepDuration) +} + +// Perform rolling upgrade on deployment and update pod annotation var upon updating the secretproviderclasspodstatus +func TestControllerUpdatingSecretProviderClassPodStatusShouldUpdatePodAnnotationInDeployment(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + + if !kube.IsCSIInstalled { + return + } + + // Creating secretclassprovider + secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) + _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclassprovider %v", err) + } + + // Creating secretproviderclasspodstatus + spcpsClient, err := testutil.CreateSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclasssproviderpodstatus %v", err) + } + + // Creating deployment + _, err = testutil.CreateDeployment(clients.KubernetesClient, secretproviderclasspodstatusName, namespace, true) + if err != nil { + t.Errorf("Error in deployment creation: %v", err) + } + + // Updating Secret + err = testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "", newData) + if err != nil { + t.Errorf("Error while updating secretproviderclasspodstatus %v", err) + } + + // Updating Secret + err = testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "", updatedData) + if err != nil { + t.Errorf("Error while updating secretproviderclasspodstatus %v", err) + } + + // Verifying Upgrade + logrus.Infof("Verifying pod annotation has been updated") + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, namespace, secretproviderclasspodstatusName, updatedData) + config := util.Config{ + Namespace: namespace, + ResourceName: secretproviderclasspodstatusName, + SHAValue: shaData, + Annotation: options.SecretProviderClassUpdateOnChangeAnnotation, + } + deploymentFuncs := handler.GetDeploymentRollingUpgradeFuncs() + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + // Deleting Deployment + err = testutil.DeleteDeployment(clients.KubernetesClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the deployment %v", err) + } + + // Deleting secretproviderclass + err = testutil.DeleteSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass %v", err) + } + + // Deleting secretproviderclasspodstatus + err = testutil.DeleteSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclasspodstatus %v", err) + } + time.Sleep(sleepDuration) + +} + +// Do not Perform rolling upgrade on pod and create or update a pod annotation upon updating the label in secretproviderclasspodstatus +func TestControllerUpdatingSecretProviderClassPodStatusWithSameDataShouldNotCreateOrUpdatePodAnnotationInDeployment(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + + if !kube.IsCSIInstalled { + return + } + + // Creating secretclassprovider + secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) + _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclassprovider %v", err) + } + + // Creating secretproviderclasspodstatus + spcpsClient, err := testutil.CreateSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclasssproviderpodstatus %v", err) + } + + // Creating deployment + _, err = testutil.CreateDeployment(clients.KubernetesClient, secretproviderclasspodstatusName, namespace, true) + if err != nil { + t.Errorf("Error in deployment creation: %v", err) + } + + err = testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "", data) + if err != nil { + t.Errorf("Error while updating secretproviderclasspodstatus %v", err) + } + + // Verifying Upgrade + logrus.Infof("Verifying pod annotation has been created") + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, namespace, secretproviderclasspodstatusName, data) + config := util.Config{ + Namespace: namespace, + ResourceName: secretproviderclasspodstatusName, + SHAValue: shaData, + Annotation: options.SecretProviderClassUpdateOnChangeAnnotation, + } + deploymentFuncs := handler.GetDeploymentRollingUpgradeFuncs() + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if updated { + t.Errorf("Deployment should not be updated by changing in secret") + } + + // Deleting Deployment + err = testutil.DeleteDeployment(clients.KubernetesClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the deployment %v", err) + } + + // Deleting secretproviderclass + err = testutil.DeleteSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass %v", err) + } + + // Deleting secretproviderclasspodstatus + err = testutil.DeleteSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclasspodstatus %v", err) + } + time.Sleep(sleepDuration) +} + // Perform rolling upgrade on DaemonSet and create pod annotation var upon updating the configmap func TestControllerUpdatingConfigmapShouldCreatePodAnnotationInDaemonSet(t *testing.T) { options.ReloadStrategy = constants.AnnotationsReloadStrategy @@ -1646,6 +1862,215 @@ func TestControllerUpdatingSecretLabelsShouldNotCreateOrUpdateEnvInDeployment(t time.Sleep(sleepDuration) } +// Perform rolling upgrade on pod and create a env var upon updating the secretproviderclasspodstatus +func TestControllerUpdatingSecretProviderClassPodStatusShouldCreateEnvInDeployment(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + + if !kube.IsCSIInstalled { + return + } + + // Creating secretclassprovider + secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) + _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclassprovider %v", err) + } + + // Creating secretproviderclasspodstatus + spcpsClient, err := testutil.CreateSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclasssproviderpodstatus %v", err) + } + + // Creating deployment + _, err = testutil.CreateDeployment(clients.KubernetesClient, secretproviderclasspodstatusName, namespace, true) + if err != nil { + t.Errorf("Error in deployment creation: %v", err) + } + + // Updating Secret + err = testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "", newData) + if err != nil { + t.Errorf("Error while updating secretproviderclasspodstatus %v", err) + } + + // Verifying Upgrade + logrus.Infof("Verifying env var has been created") + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, namespace, secretproviderclasspodstatusName, newData) + config := util.Config{ + Namespace: namespace, + ResourceName: secretproviderclasspodstatusName, + SHAValue: shaData, + Annotation: options.SecretProviderClassUpdateOnChangeAnnotation, + } + deploymentFuncs := handler.GetDeploymentRollingUpgradeFuncs() + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, constants.SecretProviderClassEnvVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + // Deleting Deployment + err = testutil.DeleteDeployment(clients.KubernetesClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the deployment %v", err) + } + + // Deleting secretproviderclass + err = testutil.DeleteSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass %v", err) + } + + // Deleting secretproviderclasspodstatus + err = testutil.DeleteSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclasspodstatus %v", err) + } + time.Sleep(sleepDuration) +} + +// Perform rolling upgrade on deployment and update env var upon updating the secretproviderclasspodstatus +func TestControllerUpdatingSecretProviderClassPodStatusShouldUpdateEnvInDeployment(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + + if !kube.IsCSIInstalled { + return + } + + // Creating secretclassprovider + secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) + _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclassprovider %v", err) + } + + // Creating secretproviderclasspodstatus + spcpsClient, err := testutil.CreateSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclasssproviderpodstatus %v", err) + } + + // Creating deployment + _, err = testutil.CreateDeployment(clients.KubernetesClient, secretproviderclasspodstatusName, namespace, true) + if err != nil { + t.Errorf("Error in deployment creation: %v", err) + } + + // Updating secretproviderclasspodstatus + err = testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "", newData) + if err != nil { + t.Errorf("Error while updating secretproviderclasspodstatus %v", err) + } + + // Updating secretproviderclasspodstatus + err = testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "", updatedData) + if err != nil { + t.Errorf("Error while updating secretproviderclasspodstatus %v", err) + } + + // Verifying Upgrade + logrus.Infof("Verifying env var has been updated") + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, namespace, secretproviderclasspodstatusName, updatedData) + config := util.Config{ + Namespace: namespace, + ResourceName: secretproviderclasspodstatusName, + SHAValue: shaData, + Annotation: options.SecretProviderClassUpdateOnChangeAnnotation, + } + deploymentFuncs := handler.GetDeploymentRollingUpgradeFuncs() + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, constants.SecretProviderClassEnvVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + // Deleting Deployment + err = testutil.DeleteDeployment(clients.KubernetesClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the deployment %v", err) + } + + // Deleting secretproviderclass + err = testutil.DeleteSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass %v", err) + } + + // Deleting secretproviderclasspodstatus + err = testutil.DeleteSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclasspodstatus %v", err) + } + time.Sleep(sleepDuration) +} + +// Do not Perform rolling upgrade on pod and create or update a env var upon updating the label in secretclasssproviderpodstatus +func TestControllerUpdatingSecretProviderClassPodStatusLabelsShouldNotCreateOrUpdateEnvInDeployment(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + + if !kube.IsCSIInstalled { + return + } + + // Creating secretclassprovider + secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) + _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclassprovider %v", err) + } + + // Creating secretproviderclasspodstatus + spcpsClient, err := testutil.CreateSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclasssproviderpodstatus %v", err) + } + + // Creating deployment + _, err = testutil.CreateDeployment(clients.KubernetesClient, secretproviderclasspodstatusName, namespace, true) + if err != nil { + t.Errorf("Error in deployment creation: %v", err) + } + + err = testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "test", data) + if err != nil { + t.Errorf("Error while updating secret %v", err) + } + + // Verifying Upgrade + logrus.Infof("Verifying env var has been created") + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, namespace, secretproviderclasspodstatusName, data) + config := util.Config{ + Namespace: namespace, + ResourceName: secretproviderclasspodstatusName, + SHAValue: shaData, + Annotation: options.SecretProviderClassUpdateOnChangeAnnotation, + } + deploymentFuncs := handler.GetDeploymentRollingUpgradeFuncs() + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, constants.SecretProviderClassEnvVarPostfix, deploymentFuncs) + if updated { + t.Errorf("Deployment should not be updated by changing label in secret") + } + + // Deleting Deployment + err = testutil.DeleteDeployment(clients.KubernetesClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the deployment %v", err) + } + + // Deleting secretproviderclass + err = testutil.DeleteSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass %v", err) + } + + // Deleting secretproviderclasspodstatus + err = testutil.DeleteSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclasspodstatus %v", err) + } + time.Sleep(sleepDuration) +} + // Perform rolling upgrade on DaemonSet and create env var upon updating the configmap func TestControllerUpdatingConfigmapShouldCreateEnvInDaemonSet(t *testing.T) { options.ReloadStrategy = constants.EnvVarsReloadStrategy diff --git a/internal/pkg/testutil/kube.go b/internal/pkg/testutil/kube.go index 29e4fd7..3d8b502 100644 --- a/internal/pkg/testutil/kube.go +++ b/internal/pkg/testutil/kube.go @@ -855,16 +855,6 @@ func CreateSecretProviderClassPodStatus(client csiclient.Interface, namespace st return secretProviderClassPodStatusClient, err } -// CreateSecretProviderClassAndPodStatus creates a SecretProviderClass and SecretProviderClassPodStatus in given namespace -func CreateSecretProviderClassAndPodStatus(client csiclient.Interface, namespace string, name string, data string) error { - _, err := CreateSecretProviderClass(client, namespace, name, data) - if err != nil { - return err - } - _, err = CreateSecretProviderClassPodStatus(client, namespace, name, data) - return err -} - // CreateSecret creates a secret in given namespace and returns the SecretInterface func CreateSecret(client kubernetes.Interface, namespace string, secretName string, data string) (core_v1.SecretInterface, error) { logrus.Infof("Creating secret") @@ -1091,6 +1081,27 @@ func UpdateSecret(secretClient core_v1.SecretInterface, namespace string, secret return updateErr } +// UpdateSecretProviderClassPodStatus updates a secretproviderclasspodstatus in given namespace and returns the error if any +func UpdateSecretProviderClassPodStatus(spcpsClient csiclient_v1.SecretProviderClassPodStatusInterface, namespace string, spcpsName string, label string, data string) error { + logrus.Infof("Updating secretproviderclasspodstatus %q.\n", spcpsName) + updatedStatus := GetSecretProviderClassPodStatus(namespace, spcpsName, data).Status + secretproviderclasspodstatus, err := spcpsClient.Get(context.TODO(), spcpsName, metav1.GetOptions{}) + if err != nil { + return err + } + secretproviderclasspodstatus.Status = updatedStatus + if label != "" { + labels := secretproviderclasspodstatus.Labels + if labels == nil { + labels = make(map[string]string) + } + labels["firstLabel"] = label + } + _, updateErr := spcpsClient.Update(context.TODO(), secretproviderclasspodstatus, metav1.UpdateOptions{}) + time.Sleep(3 * time.Second) + return updateErr +} + // DeleteConfigMap deletes a configmap in given namespace and returns the error if any func DeleteConfigMap(client kubernetes.Interface, namespace string, configmapName string) error { logrus.Infof("Deleting configmap %q.\n", configmapName) @@ -1115,6 +1126,14 @@ func DeleteSecretProviderClass(client csiclient.Interface, namespace string, sec return err } +// DeleteSecretProviderClassPodStatus deletes a secretproviderclasspodstatus in given namespace and returns the error if any +func DeleteSecretProviderClassPodStatus(client csiclient.Interface, namespace string, secretProviderClassPodStatusName string) error { + logrus.Infof("Deleting secretproviderclasspodstatus %q.\n", secretProviderClassPodStatusName) + err := client.SecretsstoreV1().SecretProviderClassPodStatuses(namespace).Delete(context.TODO(), secretProviderClassPodStatusName, metav1.DeleteOptions{}) + time.Sleep(3 * time.Second) + return err +} + // RandSeq generates a random sequence func RandSeq(n int) string { b := make([]rune, n) From 570649e56bd58c1cda9a9b2538d01b7e9707a241 Mon Sep 17 00:00:00 2001 From: Zanis <22601571+ZanisO@users.noreply.github.com> Date: Wed, 5 Feb 2025 21:43:56 +0000 Subject: [PATCH 07/45] Minor improvements to tests and handlers --- internal/pkg/cmd/reloader.go | 6 ++--- internal/pkg/constants/constants.go | 2 +- internal/pkg/controller/controller.go | 22 +++++++++++----- internal/pkg/controller/controller_test.go | 30 +++++++++++----------- internal/pkg/handler/upgrade_test.go | 6 ++--- internal/pkg/options/flags.go | 2 +- internal/pkg/testutil/kube.go | 2 +- 7 files changed, 40 insertions(+), 30 deletions(-) diff --git a/internal/pkg/cmd/reloader.go b/internal/pkg/cmd/reloader.go index f17b2a2..03b6262 100644 --- a/internal/pkg/cmd/reloader.go +++ b/internal/pkg/cmd/reloader.go @@ -36,11 +36,11 @@ func NewReloaderCommand() *cobra.Command { cmd.PersistentFlags().BoolVar(&options.AutoReloadAll, "auto-reload-all", false, "Auto reload all resources") cmd.PersistentFlags().StringVar(&options.ConfigmapUpdateOnChangeAnnotation, "configmap-annotation", "configmap.reloader.stakater.com/reload", "annotation to detect changes in configmaps, specified by name") cmd.PersistentFlags().StringVar(&options.SecretUpdateOnChangeAnnotation, "secret-annotation", "secret.reloader.stakater.com/reload", "annotation to detect changes in secrets, specified by name") - cmd.PersistentFlags().StringVar(&options.SecretProviderClassUpdateOnChangeAnnotation, "spc-annotation", "secretproviderclass.reloader.stakater.com/reload", "annotation to detect changes in secretproviderclasses, specified by name") + cmd.PersistentFlags().StringVar(&options.SecretProviderClassUpdateOnChangeAnnotation, "secretproviderclass-annotation", "secretproviderclass.reloader.stakater.com/reload", "annotation to detect changes in secretproviderclasses, specified by name") cmd.PersistentFlags().StringVar(&options.ReloaderAutoAnnotation, "auto-annotation", "reloader.stakater.com/auto", "annotation to detect changes in secrets/configmaps") cmd.PersistentFlags().StringVar(&options.ConfigmapReloaderAutoAnnotation, "configmap-auto-annotation", "configmap.reloader.stakater.com/auto", "annotation to detect changes in configmaps") cmd.PersistentFlags().StringVar(&options.SecretReloaderAutoAnnotation, "secret-auto-annotation", "secret.reloader.stakater.com/auto", "annotation to detect changes in secrets") - cmd.PersistentFlags().StringVar(&options.SecretProviderClassReloaderAutoAnnotation, "spc-auto-annotation", "secretproviderclass.reloader.stakater.com/auto", "annotation to detect changes in secretproviderclasses") + cmd.PersistentFlags().StringVar(&options.SecretProviderClassReloaderAutoAnnotation, "secretproviderclass-auto-annotation", "secretproviderclass.reloader.stakater.com/auto", "annotation to detect changes in secretproviderclasses") cmd.PersistentFlags().StringVar(&options.AutoSearchAnnotation, "auto-search-annotation", "reloader.stakater.com/search", "annotation to detect changes in configmaps or secrets tagged with special match annotation") cmd.PersistentFlags().StringVar(&options.SearchMatchAnnotation, "search-match-annotation", "reloader.stakater.com/match", "annotation to mark secrets or configmaps to match the search") cmd.PersistentFlags().StringVar(&options.LogFormat, "log-format", "", "Log format to use (empty string for text, or JSON)") @@ -184,7 +184,7 @@ func startReloader(cmd *cobra.Command, args []string) { continue } if !kube.IsCSIInstalled { - logrus.Infof("Can't run CSI controller as CSI CRDs are not installed") + logrus.Infof("Can't run secretproviderclasspodstatuses controller as CSI CRDs are not installed") continue } } diff --git a/internal/pkg/constants/constants.go b/internal/pkg/constants/constants.go index 6ad3bd5..0d1f1c7 100644 --- a/internal/pkg/constants/constants.go +++ b/internal/pkg/constants/constants.go @@ -8,7 +8,7 @@ const ( ConfigmapEnvVarPostfix = "CONFIGMAP" // SecretEnvVarPostfix is a postfix for secret envVar SecretEnvVarPostfix = "SECRET" - // SecretEnvVarSecretProviderClassPodStatus is a postfix for secretproviderclasspodstatus envVar + // SecretProviderClassEnvVarPostfix is a postfix for secretproviderclasspodstatus envVar SecretProviderClassEnvVarPostfix = "SECRETPROVIDERCLASS" // EnvVarPrefix is a Prefix for environment variable EnvVarPrefix = "STAKATER_" diff --git a/internal/pkg/controller/controller.go b/internal/pkg/controller/controller.go index bf8ea4b..dca6625 100644 --- a/internal/pkg/controller/controller.go +++ b/internal/pkg/controller/controller.go @@ -22,6 +22,7 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/kubectl/pkg/scheme" "k8s.io/utils/strings/slices" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" ) // Controller for checking events @@ -117,6 +118,8 @@ func (c *Controller) Add(obj interface{}) { case *v1.Namespace: c.addSelectedNamespaceToCache(*object) return + case *csiv1.SecretProviderClassPodStatus: + return } if options.ReloadOnCreate == "true" { @@ -136,6 +139,8 @@ func (c *Controller) resourceInIgnoredNamespace(raw interface{}) bool { return c.ignoredNamespaces.Contains(object.ObjectMeta.Namespace) case *v1.Secret: return c.ignoredNamespaces.Contains(object.ObjectMeta.Namespace) + case *csiv1.SecretProviderClassPodStatus: + return c.ignoredNamespaces.Contains(object.ObjectMeta.Namespace) } return false } @@ -154,6 +159,10 @@ func (c *Controller) resourceInSelectedNamespaces(raw interface{}) bool { if slices.Contains(selectedNamespacesCache, object.GetNamespace()) { return true } + case *csiv1.SecretProviderClassPodStatus: + if slices.Contains(selectedNamespacesCache, object.GetNamespace()) { + return true + } } return false } @@ -192,6 +201,13 @@ func (c *Controller) Update(old interface{}, new interface{}) { // Delete function to add an object to the queue in case of deleting a resource func (c *Controller) Delete(old interface{}) { + switch object := old.(type) { + case *v1.Namespace: + c.removeSelectedNamespaceFromCache(*object) + return + case *csiv1.SecretProviderClassPodStatus: + return + } if options.ReloadOnDelete == "true" { if !c.resourceInIgnoredNamespace(old) && c.resourceInSelectedNamespaces(old) && secretControllerInitialized && configmapControllerInitialized { @@ -202,12 +218,6 @@ func (c *Controller) Delete(old interface{}) { }) } } - - switch object := old.(type) { - case *v1.Namespace: - c.removeSelectedNamespaceFromCache(*object) - return - } } // Run function for controller which handles the queue diff --git a/internal/pkg/controller/controller_test.go b/internal/pkg/controller/controller_test.go index ae42c59..f599923 100644 --- a/internal/pkg/controller/controller_test.go +++ b/internal/pkg/controller/controller_test.go @@ -649,11 +649,11 @@ func TestControllerUpdatingSecretProviderClassPodStatusShouldCreatePodAnnotation return } - // Creating secretclassprovider + // Creating secretproviderclass secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) if err != nil { - t.Errorf("Error while creating the secretclassprovider %v", err) + t.Errorf("Error while creating the secretproviderclass %v", err) } // Creating secretproviderclasspodstatus @@ -718,11 +718,11 @@ func TestControllerUpdatingSecretProviderClassPodStatusShouldUpdatePodAnnotation return } - // Creating secretclassprovider + // Creating secretproviderclass secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) if err != nil { - t.Errorf("Error while creating the secretclassprovider %v", err) + t.Errorf("Error while creating the secretproviderclass %v", err) } // Creating secretproviderclasspodstatus @@ -793,11 +793,11 @@ func TestControllerUpdatingSecretProviderClassPodStatusWithSameDataShouldNotCrea return } - // Creating secretclassprovider + // Creating secretproviderclass secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) if err != nil { - t.Errorf("Error while creating the secretclassprovider %v", err) + t.Errorf("Error while creating the secretproviderclass %v", err) } // Creating secretproviderclasspodstatus @@ -829,7 +829,7 @@ func TestControllerUpdatingSecretProviderClassPodStatusWithSameDataShouldNotCrea deploymentFuncs := handler.GetDeploymentRollingUpgradeFuncs() updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) if updated { - t.Errorf("Deployment should not be updated by changing in secret") + t.Errorf("Deployment should not be updated by changing in secretproviderclasspodstatus") } // Deleting Deployment @@ -1870,11 +1870,11 @@ func TestControllerUpdatingSecretProviderClassPodStatusShouldCreateEnvInDeployme return } - // Creating secretclassprovider + // Creating secretproviderclass secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) if err != nil { - t.Errorf("Error while creating the secretclassprovider %v", err) + t.Errorf("Error while creating the secretproviderclass %v", err) } // Creating secretproviderclasspodstatus @@ -1938,11 +1938,11 @@ func TestControllerUpdatingSecretProviderClassPodStatusShouldUpdateEnvInDeployme return } - // Creating secretclassprovider + // Creating secretproviderclass secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) if err != nil { - t.Errorf("Error while creating the secretclassprovider %v", err) + t.Errorf("Error while creating the secretproviderclass %v", err) } // Creating secretproviderclasspodstatus @@ -2012,11 +2012,11 @@ func TestControllerUpdatingSecretProviderClassPodStatusLabelsShouldNotCreateOrUp return } - // Creating secretclassprovider + // Creating secretproviderclass secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) if err != nil { - t.Errorf("Error while creating the secretclassprovider %v", err) + t.Errorf("Error while creating the secretproviderclass %v", err) } // Creating secretproviderclasspodstatus @@ -2033,7 +2033,7 @@ func TestControllerUpdatingSecretProviderClassPodStatusLabelsShouldNotCreateOrUp err = testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "test", data) if err != nil { - t.Errorf("Error while updating secret %v", err) + t.Errorf("Error while updating secretproviderclasspodstatus %v", err) } // Verifying Upgrade @@ -2048,7 +2048,7 @@ func TestControllerUpdatingSecretProviderClassPodStatusLabelsShouldNotCreateOrUp deploymentFuncs := handler.GetDeploymentRollingUpgradeFuncs() updated := testutil.VerifyResourceEnvVarUpdate(clients, config, constants.SecretProviderClassEnvVarPostfix, deploymentFuncs) if updated { - t.Errorf("Deployment should not be updated by changing label in secret") + t.Errorf("Deployment should not be updated by changing label in secretproviderclasspodstatus") } // Deleting Deployment diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index 35acddb..a0fb657 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -2809,7 +2809,7 @@ func TestRollingUpgradeForDaemonSetWithSecretProviderClassUsingArs(t *testing.T) envVarPostfix := constants.SecretProviderClassEnvVarPostfix shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassName, "testing1") - config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassName, shaData, options.SecretUpdateOnChangeAnnotation, options.SecretReloaderAutoAnnotation) + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassName, shaData, options.SecretProviderClassUpdateOnChangeAnnotation, options.SecretProviderClassReloaderAutoAnnotation) daemonSetFuncs := GetDaemonSetRollingUpgradeFuncs() collectors := getCollectors() @@ -2969,7 +2969,7 @@ func TestRollingUpgradeForStatefulSetWithSecretProviderClassUsingArs(t *testing. envVarPostfix := constants.SecretProviderClassEnvVarPostfix shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassName, "testing1") - config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassName, shaData, options.SecretUpdateOnChangeAnnotation, options.SecretReloaderAutoAnnotation) + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassName, shaData, options.SecretProviderClassUpdateOnChangeAnnotation, options.SecretProviderClassReloaderAutoAnnotation) statefulSetFuncs := GetStatefulSetRollingUpgradeFuncs() collectors := getCollectors() @@ -3776,7 +3776,7 @@ func TestRollingUpgradeForDeploymentWithSecretProviderClassExcludeAnnotationUsin err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) time.Sleep(5 * time.Second) if err != nil { - t.Errorf("Rolling upgrade failed for Deployment with exclude Secret") + t.Errorf("Rolling upgrade failed for Deployment with exclude SecretProviderClass") } logrus.Infof("Verifying deployment did not update") diff --git a/internal/pkg/options/flags.go b/internal/pkg/options/flags.go index 8267bed..dcefade 100644 --- a/internal/pkg/options/flags.go +++ b/internal/pkg/options/flags.go @@ -62,7 +62,7 @@ var ( EnableHA = false // Url to send a request to instead of triggering a reload WebhookUrl = "" - // EnableCsiIntegration Adds support to watch SecretProviderClassPodStatus and restart deployment based on it + // EnableCSIIntegration Adds support to watch SecretProviderClassPodStatus and restart deployment based on it EnableCSIIntegration = false ) diff --git a/internal/pkg/testutil/kube.go b/internal/pkg/testutil/kube.go index 3d8b502..8c843d7 100644 --- a/internal/pkg/testutil/kube.go +++ b/internal/pkg/testutil/kube.go @@ -845,7 +845,7 @@ func CreateSecretProviderClass(client csiclient.Interface, namespace string, sec return secretProviderClassClient, err } -// CreateSecretProviderClass creates a SecretProviderClassPodStatus in given namespace and returns the SecretProviderClassInterface +// CreateSecretProviderClassPodStatus creates a SecretProviderClassPodStatus in given namespace and returns the SecretProviderClassPodStatusInterface func CreateSecretProviderClassPodStatus(client csiclient.Interface, namespace string, secretProviderClassPodStatusName string, data string) (csiclient_v1.SecretProviderClassPodStatusInterface, error) { logrus.Infof("Creating SecretProviderClassPodStatus") secretProviderClassPodStatusClient := client.SecretsstoreV1().SecretProviderClassPodStatuses(namespace) From e7e095cb4bb24ebfe5ab16907f1f878ef11650e4 Mon Sep 17 00:00:00 2001 From: Zanis <22601571+ZanisO@users.noreply.github.com> Date: Wed, 5 Feb 2025 23:36:08 +0000 Subject: [PATCH 08/45] Fixed tests --- internal/pkg/testutil/kube.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/internal/pkg/testutil/kube.go b/internal/pkg/testutil/kube.go index 8c843d7..a61b63d 100644 --- a/internal/pkg/testutil/kube.go +++ b/internal/pkg/testutil/kube.go @@ -97,11 +97,14 @@ func getAnnotations(name string, autoReload bool, secretAutoReload bool, configm if configmapAutoReload { annotations[options.ConfigmapReloaderAutoAnnotation] = "true" } + if secretproviderclass { + annotations[options.SecretProviderClassReloaderAutoAnnotation] = "true" + } if !(len(annotations) > 0) { annotations = map[string]string{ - options.ConfigmapUpdateOnChangeAnnotation: name, - options.SecretUpdateOnChangeAnnotation: name, + options.ConfigmapUpdateOnChangeAnnotation: name, + options.SecretUpdateOnChangeAnnotation: name, options.SecretProviderClassUpdateOnChangeAnnotation: name, } } @@ -700,7 +703,7 @@ func GetSecret(namespace string, secretName string, data string) *v1.Secret { func GetCronJob(namespace string, cronJobName string) *batchv1.CronJob { return &batchv1.CronJob{ - ObjectMeta: getObjectMeta(namespace, cronJobName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, cronJobName, false, false, false, false, map[string]string{}), Spec: batchv1.CronJobSpec{ Schedule: "*/5 * * * *", // Run every 5 minutes JobTemplate: batchv1.JobTemplateSpec{ @@ -717,7 +720,7 @@ func GetCronJob(namespace string, cronJobName string) *batchv1.CronJob { func GetJob(namespace string, jobName string) *batchv1.Job { return &batchv1.Job{ - ObjectMeta: getObjectMeta(namespace, jobName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, jobName, false, false, false, false, map[string]string{}), Spec: batchv1.JobSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -729,7 +732,7 @@ func GetJob(namespace string, jobName string) *batchv1.Job { func GetCronJobWithEnvVar(namespace string, cronJobName string) *batchv1.CronJob { return &batchv1.CronJob{ - ObjectMeta: getObjectMeta(namespace, cronJobName, true, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, cronJobName, true, false, false, false, map[string]string{}), Spec: batchv1.CronJobSpec{ Schedule: "*/5 * * * *", // Run every 5 minutes JobTemplate: batchv1.JobTemplateSpec{ @@ -746,7 +749,7 @@ func GetCronJobWithEnvVar(namespace string, cronJobName string) *batchv1.CronJob func GetJobWithEnvVar(namespace string, jobName string) *batchv1.Job { return &batchv1.Job{ - ObjectMeta: getObjectMeta(namespace, jobName, true, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, jobName, true, false, false, false, map[string]string{}), Spec: batchv1.JobSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -1291,7 +1294,7 @@ func GetSHAfromEmptyData() string { func GetRollout(namespace string, rolloutName string, annotations map[string]string) *argorolloutv1alpha1.Rollout { replicaset := int32(1) return &argorolloutv1alpha1.Rollout{ - ObjectMeta: getObjectMeta(namespace, rolloutName, false, false, false, annotations), + ObjectMeta: getObjectMeta(namespace, rolloutName, false, false, false, false, annotations), Spec: argorolloutv1alpha1.RolloutSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, From 4f8b22e9546bc065dc07d67d17baa8a660b450cd Mon Sep 17 00:00:00 2001 From: Safwan Date: Tue, 25 Nov 2025 01:48:54 +0500 Subject: [PATCH 09/45] resolved comments --- internal/pkg/handler/update.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/internal/pkg/handler/update.go b/internal/pkg/handler/update.go index 262399d..cc1e16b 100644 --- a/internal/pkg/handler/update.go +++ b/internal/pkg/handler/update.go @@ -41,17 +41,18 @@ func (r ResourceUpdatedHandler) Handle() error { func (r ResourceUpdatedHandler) GetConfig() (common.Config, string) { var oldSHAData string var config common.Config - if _, ok := r.Resource.(*v1.ConfigMap); ok { + switch res := r.Resource.(type) { + case *v1.ConfigMap: oldSHAData = util.GetSHAfromConfigmap(r.OldResource.(*v1.ConfigMap)) - config = common.GetConfigmapConfig(r.Resource.(*v1.ConfigMap)) - } else if _, ok := r.Resource.(*v1.Secret); ok { + config = common.GetConfigmapConfig(res) + case *v1.Secret: oldSHAData = util.GetSHAfromSecret(r.OldResource.(*v1.Secret).Data) - config = common.GetSecretConfig(r.Resource.(*v1.Secret)) - } else if _, ok := r.Resource.(*csiv1.SecretProviderClassPodStatus); ok { + config = common.GetSecretConfig(res) + case *csiv1.SecretProviderClassPodStatus: oldSHAData = util.GetSHAfromSecretProviderClassPodStatus(r.OldResource.(*csiv1.SecretProviderClassPodStatus).Status) - config = common.GetSecretProviderClassPodStatusConfig(r.Resource.(*csiv1.SecretProviderClassPodStatus)) - } else { - logrus.Warnf("Invalid resource: Resource should be 'Secret' or 'Configmap' but found, %v", r.Resource) + config = common.GetSecretProviderClassPodStatusConfig(res) + default: + logrus.Warnf("Invalid resource: Resource should be 'Secret', 'Configmap' or 'SecretProviderClassPodStatus' but found, %v", r.Resource) } return config, oldSHAData } From 1725f17b0bc5aff13990792dbdc3087247bf7ef5 Mon Sep 17 00:00:00 2001 From: Safwan Date: Tue, 25 Nov 2025 02:05:41 +0500 Subject: [PATCH 10/45] fixed namespace behavior issue --- internal/pkg/controller/controller.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/pkg/controller/controller.go b/internal/pkg/controller/controller.go index f2a0143..a670d81 100644 --- a/internal/pkg/controller/controller.go +++ b/internal/pkg/controller/controller.go @@ -201,11 +201,7 @@ func (c *Controller) Update(old interface{}, new interface{}) { // Delete function to add an object to the queue in case of deleting a resource func (c *Controller) Delete(old interface{}) { - switch object := old.(type) { - case *v1.Namespace: - c.removeSelectedNamespaceFromCache(*object) - return - case *csiv1.SecretProviderClassPodStatus: + if _, ok := old.(*csiv1.SecretProviderClassPodStatus); ok { return } @@ -218,6 +214,12 @@ func (c *Controller) Delete(old interface{}) { }) } } + + switch object := old.(type) { + case *v1.Namespace: + c.removeSelectedNamespaceFromCache(*object) + return + } } // Run function for controller which handles the queue From c9cab4f6e0131de1676786887523f132fe42ac6f Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Sat, 3 Jan 2026 19:32:43 +0100 Subject: [PATCH 11/45] Update chart for CSI driver Signed-off-by: faizanahmad055 --- .../chart/reloader/templates/clusterrole.yaml | 11 +++++++++++ .../chart/reloader/templates/deployment.yaml | 5 ++++- deployments/kubernetes/chart/reloader/values.yaml | 1 + internal/pkg/cmd/reloader.go | 1 + internal/pkg/util/util.go | 1 + 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/deployments/kubernetes/chart/reloader/templates/clusterrole.yaml b/deployments/kubernetes/chart/reloader/templates/clusterrole.yaml index 9f655aa..bd14dfe 100644 --- a/deployments/kubernetes/chart/reloader/templates/clusterrole.yaml +++ b/deployments/kubernetes/chart/reloader/templates/clusterrole.yaml @@ -105,6 +105,17 @@ rules: - create - get - update +{{- end}} +{{- if .Values.reloader.enableCSIIntegration }} + - apiGroups: + - "secrets-store.csi.x-k8s.io" + resources: + - secretproviderclasspodstatuses + - secretproviderclasses + verbs: + - list + - get + - watch {{- end}} - apiGroups: - "" diff --git a/deployments/kubernetes/chart/reloader/templates/deployment.yaml b/deployments/kubernetes/chart/reloader/templates/deployment.yaml index 16564b2..e568f9f 100644 --- a/deployments/kubernetes/chart/reloader/templates/deployment.yaml +++ b/deployments/kubernetes/chart/reloader/templates/deployment.yaml @@ -210,7 +210,7 @@ spec: {{- . | toYaml | nindent 10 }} {{- end }} {{- end }} - {{- if or (.Values.reloader.logFormat) (.Values.reloader.logLevel) (.Values.reloader.ignoreSecrets) (.Values.reloader.ignoreNamespaces) (include "reloader-namespaceSelector" .) (.Values.reloader.resourceLabelSelector) (.Values.reloader.ignoreConfigMaps) (.Values.reloader.custom_annotations) (eq .Values.reloader.isArgoRollouts true) (eq .Values.reloader.reloadOnCreate true) (eq .Values.reloader.reloadOnDelete true) (ne .Values.reloader.reloadStrategy "default") (.Values.reloader.enableHA) (.Values.reloader.autoReloadAll) (.Values.reloader.ignoreJobs) (.Values.reloader.ignoreCronJobs)}} + {{- if or (.Values.reloader.logFormat) (.Values.reloader.logLevel) (.Values.reloader.ignoreSecrets) (.Values.reloader.ignoreNamespaces) (include "reloader-namespaceSelector" .) (.Values.reloader.resourceLabelSelector) (.Values.reloader.ignoreConfigMaps) (.Values.reloader.custom_annotations) (eq .Values.reloader.isArgoRollouts true) (eq .Values.reloader.reloadOnCreate true) (eq .Values.reloader.reloadOnDelete true) (ne .Values.reloader.reloadStrategy "default") (.Values.reloader.enableHA) (.Values.reloader.autoReloadAll) (.Values.reloader.ignoreJobs) (.Values.reloader.ignoreCronJobs) (.Values.reloader.enableCSIIntegration)}} args: {{- if .Values.reloader.logFormat }} - "--log-format={{ .Values.reloader.logFormat }}" @@ -246,6 +246,9 @@ spec: - "--pprof-addr={{ .Values.reloader.pprofAddr }}" {{- end }} {{- end }} + {{- if .Values.reloader.enableCSIIntegration }} + - "--enable-csi-integration=true" + {{- end }} {{- if .Values.reloader.custom_annotations }} {{- if .Values.reloader.custom_annotations.configmap }} - "--configmap-annotation" diff --git a/deployments/kubernetes/chart/reloader/values.yaml b/deployments/kubernetes/chart/reloader/values.yaml index c9a46a0..a607491 100644 --- a/deployments/kubernetes/chart/reloader/values.yaml +++ b/deployments/kubernetes/chart/reloader/values.yaml @@ -49,6 +49,7 @@ reloader: enableHA: false # Set to true to enable pprof for profiling enablePProf: false + enableCSIIntegration: false # Address to start pprof server on. Default is ":6060" pprofAddr: ":6060" # Set to true if you have a pod security policy that enforces readOnlyRootFilesystem diff --git a/internal/pkg/cmd/reloader.go b/internal/pkg/cmd/reloader.go index f20e0b8..6bdb339 100644 --- a/internal/pkg/cmd/reloader.go +++ b/internal/pkg/cmd/reloader.go @@ -162,6 +162,7 @@ func startReloader(cmd *cobra.Command, args []string) { for k := range kube.ResourceMap { if k == "secretproviderclasspodstatuses" { if !options.EnableCSIIntegration { + logrus.Infof("EnableCSIIntegration is set to false, won't run secretproviderclasspodstatuses controller") continue } if !kube.IsCSIInstalled { diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index 53846f3..047d068 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -106,6 +106,7 @@ func ConfigureReloaderFlags(cmd *cobra.Command) { cmd.PersistentFlags().BoolVar(&options.SyncAfterRestart, "sync-after-restart", false, "Sync add events after reloader restarts") cmd.PersistentFlags().BoolVar(&options.EnablePProf, "enable-pprof", false, "Enable pprof for profiling") cmd.PersistentFlags().StringVar(&options.PProfAddr, "pprof-addr", ":6060", "Address to start pprof server on. Default is :6060") + cmd.PersistentFlags().BoolVar(&options.EnableCSIIntegration, "enable-csi-integration", false, "Enables CSI integration. Default is :true") } func GetIgnoredResourcesList() (List, error) { From 109971d8b70bf7072093fd2e9d3c7b67d514f602 Mon Sep 17 00:00:00 2001 From: Safwan Date: Sat, 27 Dec 2025 21:57:04 +0500 Subject: [PATCH 12/45] prioritize named resource --- pkg/common/common.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/common/common.go b/pkg/common/common.go index b6fe3b5..7c9d61e 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -266,15 +266,6 @@ func ShouldReload(config Config, resourceType string, annotations Map, podAnnota } } - reloaderEnabled, _ := strconv.ParseBool(reloaderEnabledValue) - typedAutoAnnotationEnabled, _ := strconv.ParseBool(typedAutoAnnotationEnabledValue) - if reloaderEnabled || typedAutoAnnotationEnabled || reloaderEnabledValue == "" && typedAutoAnnotationEnabledValue == "" && options.AutoReloadAll { - return ReloadCheckResult{ - ShouldReload: true, - AutoReload: true, - } - } - values := strings.Split(annotationValue, ",") for _, value := range values { value = strings.TrimSpace(value) @@ -297,6 +288,15 @@ func ShouldReload(config Config, resourceType string, annotations Map, podAnnota } } + reloaderEnabled, _ := strconv.ParseBool(reloaderEnabledValue) + typedAutoAnnotationEnabled, _ := strconv.ParseBool(typedAutoAnnotationEnabledValue) + if reloaderEnabled || typedAutoAnnotationEnabled || reloaderEnabledValue == "" && typedAutoAnnotationEnabledValue == "" && options.AutoReloadAll { + return ReloadCheckResult{ + ShouldReload: true, + AutoReload: true, + } + } + return ReloadCheckResult{ ShouldReload: false, } From 9c8c511ae5e38820e2bbcc33fc7f9c4b8f57d254 Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Sun, 4 Jan 2026 00:45:15 +0100 Subject: [PATCH 13/45] Update dependencies and fix shouldReload issue Signed-off-by: faizanahmad055 --- go.mod | 68 +++++++++++++------------- go.sum | 149 ++++++++++++++++++++++++++++++--------------------------- 2 files changed, 112 insertions(+), 105 deletions(-) diff --git a/go.mod b/go.mod index 1b51b65..48f13d8 100644 --- a/go.mod +++ b/go.mod @@ -4,19 +4,19 @@ go 1.25.5 require ( github.com/argoproj/argo-rollouts v1.8.3 - github.com/openshift/api v0.0.0-20250411135543-10a8fa583797 - github.com/openshift/client-go v0.0.0-20250402181141-b3bad3b645f2 + github.com/openshift/api v0.0.0-20260102143802-d2ec16864f86 + github.com/openshift/client-go v0.0.0-20251223102348-558b0eef16bc github.com/parnurzeal/gorequest v0.3.0 - github.com/prometheus/client_golang v1.22.0 + github.com/prometheus/client_golang v1.23.2 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/cobra v1.10.1 - github.com/stretchr/testify v1.10.0 - k8s.io/api v0.32.3 - k8s.io/apimachinery v0.32.3 - k8s.io/client-go v0.32.3 - k8s.io/kubectl v0.32.3 - k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 - sigs.k8s.io/secrets-store-csi-driver v1.5.4 + github.com/spf13/cobra v1.10.2 + github.com/stretchr/testify v1.11.1 + k8s.io/api v0.35.0 + k8s.io/apimachinery v0.35.0 + k8s.io/client-go v0.35.0 + k8s.io/kubectl v0.35.0 + k8s.io/utils v0.0.0-20251222233032-718f0e51e6d2 + sigs.k8s.io/secrets-store-csi-driver v1.5.5 ) require ( @@ -25,16 +25,14 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380 // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect - github.com/fxamacker/cbor/v2 v2.8.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-openapi/jsonpointer v0.21.1 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.9 // indirect + github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -42,41 +40,43 @@ require ( github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/moul/http2curl v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.63.0 // indirect - github.com/prometheus/procfs v0.16.0 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/smartystreets/goconvey v1.7.2 // indirect github.com/spf13/pflag v1.0.9 // indirect github.com/x448/float16 v0.8.4 // indirect - golang.org/x/net v0.39.0 // indirect - golang.org/x/oauth2 v0.29.0 // indirect - golang.org/x/sys v0.32.0 // indirect - golang.org/x/term v0.31.0 // indirect - golang.org/x/text v0.24.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 // indirect golang.org/x/time v0.11.0 // indirect - google.golang.org/protobuf v1.36.6 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) // Replacements for argo-rollouts replace ( github.com/go-check/check => github.com/go-check/check v0.0.0-20201130134442-10cb98267c6c - k8s.io/api v0.0.0 => k8s.io/api v0.32.3 - k8s.io/apimachinery v0.0.0 => k8s.io/apimachinery v0.32.3 - k8s.io/client-go v0.0.0 => k8s.io/client-go v0.32.3 + k8s.io/api v0.0.0 => k8s.io/api v0.35.0 + k8s.io/apimachinery v0.0.0 => k8s.io/apimachinery v0.35.0 + k8s.io/client-go v0.0.0 => k8s.io/client-go v0.35.0 k8s.io/cloud-provider v0.0.0 => k8s.io/cloud-provider v0.24.2 k8s.io/controller-manager v0.0.0 => k8s.io/controller-manager v0.24.2 k8s.io/cri-api v0.0.0 => k8s.io/cri-api v0.20.5-rc.0 @@ -85,7 +85,7 @@ replace ( k8s.io/kube-controller-manager v0.0.0 => k8s.io/kube-controller-manager v0.24.2 k8s.io/kube-proxy v0.0.0 => k8s.io/kube-proxy v0.24.2 k8s.io/kube-scheduler v0.0.0 => k8s.io/kube-scheduler v0.24.2 - k8s.io/kubectl v0.0.0 => k8s.io/kubectl v0.32.3 + k8s.io/kubectl v0.0.0 => k8s.io/kubectl v0.35.0 k8s.io/kubelet v0.0.0 => k8s.io/kubelet v0.24.2 k8s.io/legacy-cloud-providers v0.0.0 => k8s.io/legacy-cloud-providers v0.24.2 k8s.io/mount-utils v0.0.0 => k8s.io/mount-utils v0.20.5-rc.0 diff --git a/go.sum b/go.sum index 05738d7..a1b7e7d 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/argoproj/argo-rollouts v1.8.3 h1:blbtQva4IK9r6gFh+dWkCrLnFdPOWiv9ubQYu36qeaA= github.com/argoproj/argo-rollouts v1.8.3/go.mod h1:kCAUvIfMGfOyVf3lvQbBt0nqQn4Pd+zB5/YwKv+UBa8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -13,10 +15,10 @@ github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380 h1:1NyRx2f4W4WBRyg github.com/elazarl/goproxy v0.0.0-20240726154733-8b0c20506380/go.mod h1:thX175TtLTzLj3p7N/Q9IiKZ7NF+p72cvL91emV0hzo= github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/fxamacker/cbor/v2 v2.8.0 h1:fFtUGXUzXPHTIUdne5+zzMPTfffl3RD5qYnkY40vtxU= -github.com/fxamacker/cbor/v2 v2.8.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic= github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= @@ -27,18 +29,13 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= -github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= -github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -66,21 +63,22 @@ github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUt github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= -github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= -github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= -github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= -github.com/openshift/api v0.0.0-20250411135543-10a8fa583797 h1:8x3G8QOZqo2bRAL8JFlPz/odqQECI/XmlZeRwnFxJ8I= -github.com/openshift/api v0.0.0-20250411135543-10a8fa583797/go.mod h1:yk60tHAmHhtVpJQo3TwVYq2zpuP70iJIFDCmeKMIzPw= -github.com/openshift/client-go v0.0.0-20250402181141-b3bad3b645f2 h1:bPXR0R8zp1o12nSUphN26hSM+OKYq5pMorbDCpApzDQ= -github.com/openshift/client-go v0.0.0-20250402181141-b3bad3b645f2/go.mod h1:dT1cJyVTperQ53GvVRa+GZ27r02fDZy2k5j+9QoQsCo= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= +github.com/openshift/api v0.0.0-20260102143802-d2ec16864f86 h1:Vsqg+WqSA91LjrwK5lzkSCjztK/B+T8MPKI3MIALx3w= +github.com/openshift/api v0.0.0-20260102143802-d2ec16864f86/go.mod h1:d5uzF0YN2nQQFA0jIEWzzOZ+edmo6wzlGLvx5Fhz4uY= +github.com/openshift/client-go v0.0.0-20251223102348-558b0eef16bc h1:nIlRaJfr/yGjPV15MNF5eVHLAGyXFjcUzO+hXeWDDk8= +github.com/openshift/client-go v0.0.0-20251223102348-558b0eef16bc/go.mod h1:cs9BwTu96sm2vQvy7r9rOiltgu90M6ju2qIHFG9WU+o= github.com/parnurzeal/gorequest v0.3.0 h1:SoFyqCDC9COr1xuS6VA8fC8RU7XyrJZN2ona1kEX7FI= github.com/parnurzeal/gorequest v0.3.0/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -88,16 +86,16 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= -github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= -github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= -github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM= -github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -105,50 +103,60 @@ github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= -github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= -github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= -golang.org/x/oauth2 v0.29.0 h1:WdYw2tdTK1S8olAzWHdgeqfy+Mtm9XNhv/xJsY65d98= -golang.org/x/oauth2 v0.29.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= -golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -156,46 +164,45 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.32.3 h1:Hw7KqxRusq+6QSplE3NYG4MBxZw1BZnq4aP4cJVINls= -k8s.io/api v0.32.3/go.mod h1:2wEDTXADtm/HA7CCMD8D8bK4yuBUptzaRhYcYEEYA3k= -k8s.io/apimachinery v0.32.3 h1:JmDuDarhDmA/Li7j3aPrwhpNBA94Nvk5zLeOge9HH1U= -k8s.io/apimachinery v0.32.3/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= -k8s.io/client-go v0.32.3 h1:RKPVltzopkSgHS7aS98QdscAgtgah/+zmpAogooIqVU= -k8s.io/client-go v0.32.3/go.mod h1:3v0+3k4IcT9bXTc4V2rt+d2ZPPG700Xy6Oi0Gdl2PaY= +k8s.io/api v0.35.0 h1:iBAU5LTyBI9vw3L5glmat1njFK34srdLmktWwLTprlY= +k8s.io/api v0.35.0/go.mod h1:AQ0SNTzm4ZAczM03QH42c7l3bih1TbAXYo0DkF8ktnA= +k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= +k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= +k8s.io/client-go v0.35.0 h1:IAW0ifFbfQQwQmga0UdoH0yvdqrbwMdq9vIFEhRpxBE= +k8s.io/client-go v0.35.0/go.mod h1:q2E5AAyqcbeLGPdoRB+Nxe3KYTfPce1Dnu1myQdqz9o= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= -k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= -k8s.io/kubectl v0.32.3 h1:VMi584rbboso+yjfv0d8uBHwwxbC438LKq+dXd5tOAI= -k8s.io/kubectl v0.32.3/go.mod h1:6Euv2aso5GKzo/UVMacV6C7miuyevpfI91SvBvV9Zdg= -k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= -k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= -sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= +k8s.io/kubectl v0.35.0 h1:cL/wJKHDe8E8+rP3G7avnymcMg6bH6JEcR5w5uo06wc= +k8s.io/kubectl v0.35.0/go.mod h1:VR5/TSkYyxZwrRwY5I5dDq6l5KXmiCb+9w8IKplk3Qo= +k8s.io/utils v0.0.0-20251222233032-718f0e51e6d2 h1:OfgiEo21hGiwx1oJUU5MpEaeOEg6coWndBkZF/lkFuE= +k8s.io/utils v0.0.0-20251222233032-718f0e51e6d2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/secrets-store-csi-driver v1.5.4 h1:enl+v1+JbKDyVjdfT/7CillZsc4rLAM9tTHyf7GeLxc= -sigs.k8s.io/secrets-store-csi-driver v1.5.4/go.mod h1:Ct85xqsKLk/dxkj8inRjWA3RJsXXkPLjNSAJ0db5vKs= -sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= -sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +sigs.k8s.io/secrets-store-csi-driver v1.5.5 h1:LJDpDL5TILhlP68nGvtGSlJFxSDgAD2m148NT0Ts7os= +sigs.k8s.io/secrets-store-csi-driver v1.5.5/go.mod h1:i2WqLicYH00hrTG3JAzICPMF4HL4KMEORlDt9UQoZLk= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= From 85f1c13de9e5ac72857a6890bb0814564dfd21de Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Sun, 4 Jan 2026 00:47:13 +0100 Subject: [PATCH 14/45] Add CSI integration in rbac Signed-off-by: faizanahmad055 --- .../kubernetes/chart/reloader/templates/role.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/deployments/kubernetes/chart/reloader/templates/role.yaml b/deployments/kubernetes/chart/reloader/templates/role.yaml index 70a6815..7355d87 100644 --- a/deployments/kubernetes/chart/reloader/templates/role.yaml +++ b/deployments/kubernetes/chart/reloader/templates/role.yaml @@ -92,6 +92,17 @@ rules: - create - get - update +{{- end}} +{{- if .Values.reloader.enableCSIIntegration }} + - apiGroups: + - "secrets-store.csi.x-k8s.io" + resources: + - secretproviderclasspodstatuses + - secretproviderclasses + verbs: + - list + - get + - watch {{- end}} - apiGroups: - "" From 0f1d02e97557ccbb0508e62d4c86527036961907 Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Sun, 4 Jan 2026 02:40:42 +0100 Subject: [PATCH 15/45] Readme update and code refactor Signed-off-by: faizanahmad055 --- README.md | 46 +++++++++++++++++-- .../pkg/callbacks/rolling_upgrade_test.go | 2 +- internal/pkg/cmd/reloader.go | 23 ++++++---- internal/pkg/constants/constants.go | 2 + internal/pkg/controller/controller.go | 24 ++++++---- internal/pkg/controller/controller_test.go | 2 +- internal/pkg/handler/pause_deployment_test.go | 4 +- internal/pkg/handler/update.go | 23 +++++++--- internal/pkg/handler/upgrade.go | 15 +++--- internal/pkg/handler/upgrade_test.go | 2 +- 10 files changed, 104 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index ae0a00a..4fe53f3 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ ## šŸ” What is Reloader? -Reloader is a Kubernetes controller that automatically triggers rollouts of workloads (like Deployments, StatefulSets, and more) whenever referenced `Secrets` or `ConfigMaps` are updated. +Reloader is a Kubernetes controller that automatically triggers rollouts of workloads (like Deployments, StatefulSets, and more) whenever referenced `Secrets`, `ConfigMaps` or **optionally CSI-mounted secrets** are updated. In a traditional Kubernetes setup, updating a `Secret` or `ConfigMap` does not automatically restart or redeploy your workloads. This can lead to stale configurations running in production, especially when dealing with dynamic values like credentials, feature flags, or environment configs. @@ -169,9 +169,11 @@ metadata: This instructs Reloader to skip all reload logic for that resource across all workloads. -### 4. āš™ļø Workload-Specific Rollout Strategy +### 4. āš™ļø Workload-Specific Rollout Strategy (Argo Rollouts Only) -By default, Reloader uses the **rollout** strategy ā€” it updates the pod template to trigger a new rollout. This works well in most cases, but it can cause problems if you're using GitOps tools like ArgoCD, which detect this as configuration drift. +Note: This is only applicable when using [Argo Rollouts](https://argoproj.github.io/argo-rollouts/). It is ignored for standard Kubernetes Deployments, StatefulSets, or DaemonSets. To use this feature, Argo Rollouts support must be enabled in Reloader (for example via --is-argo-rollouts=true). + +By default, Reloader triggers the Argo Rollout controller to perform a standard rollout by updating the pod template. This works well in most cases, however, because this modifies the workload spec, GitOps tools like ArgoCD will detect this as "Configuration Drift" and mark your application as OutOfSync. To avoid that, you can switch to the **restart** strategy, which simply restarts the pod without changing the pod template. @@ -189,8 +191,10 @@ metadata: āœ… Use `restart` if: 1. You're using GitOps and want to avoid drift -1. You want a quick restart without changing the workload spec -1. Your platform restricts metadata changes +2. You want a quick restart without changing the workload spec +3. Your platform restricts metadata changes + +This setting affects Argo Rollouts behavior, not Argo CD sync settings. ### 5. ā— Annotation Behavior Rules & Compatibility @@ -239,6 +243,38 @@ This feature allows you to pause rollouts for a deployment for a specified durat 1. āœ… Your deployment references multiple ConfigMaps or Secrets that may be updated at the same time. 1. āœ… You want to minimize unnecessary rollouts and reduce downtime caused by back-to-back configuration changes. +### 8. šŸ” CSI Secret Provider Support + +Reloader supports the [Secrets Store CSI Driver](https://secrets-store-csi-driver.sigs.k8s.io/), which allows mounting secrets from external secret stores (like AWS Secrets Manager, Azure Key Vault, HashiCorp Vault) directly into pods. +Unlike Kubernetes Secret objects, CSI-mounted secrets do not always trigger native Kubernetes update events. Reloader solves this by watching CSI status resources and restarting affected workloads when mounted secret versions change. + +#### How it works + +When secret rotation is enabled, the Secrets Store CSI Driver updates a Kubernetes resource called: `SecretProviderClassPodStatus` + +This resource reflects the currently mounted secret versions for a pod. +Reloader watches these updates and triggers a rollout when a change is detected. + +#### Prerequisites + +- Secrets Store CSI Driver must be installed in your cluster +- Secret rotation enabled in the CSI driver. +- Enable CSI integration in Reloader: `--enable-csi-integration=true` + +#### Annotations for CSI-mounted Secrets + +| Annotation | Description | +|--------------------------------------------|----------------------------------------------------------------------| +| `reloader.stakater.com/auto: "true"` | Reloads workload when CSI-mounted secrets change | +| `secretproviderclass.reloader.stakater.com/reload: "my-spc"` | Reloads when specific SecretProviderClass changes | + +#### Notes & Limitations + +Reloader reacts to CSI status changes, not direct updates to external secret stores +Secret rotation must be enabled in the CSI driver for updates to be detected +CSI limitations (such as subPath mounts) still apply and may require pod restarts +If secrets are synced to Kubernetes Secret objects, standard Reloader behavior applies and CSI support may not be required + ## šŸš€ Installation ### 1. šŸ“¦ Helm diff --git a/internal/pkg/callbacks/rolling_upgrade_test.go b/internal/pkg/callbacks/rolling_upgrade_test.go index 452867f..75583de 100644 --- a/internal/pkg/callbacks/rolling_upgrade_test.go +++ b/internal/pkg/callbacks/rolling_upgrade_test.go @@ -49,7 +49,7 @@ func newTestFixtures() testFixtures { func setupTestClients() kube.Clients { return kube.Clients{ - KubernetesClient: fake.NewSimpleClientset(), + KubernetesClient: fake.NewClientset(), ArgoRolloutClient: fakeargoclientset.NewSimpleClientset(), } } diff --git a/internal/pkg/cmd/reloader.go b/internal/pkg/cmd/reloader.go index 6bdb339..771e2df 100644 --- a/internal/pkg/cmd/reloader.go +++ b/internal/pkg/cmd/reloader.go @@ -160,15 +160,8 @@ func startReloader(cmd *cobra.Command, args []string) { var controllers []*controller.Controller for k := range kube.ResourceMap { - if k == "secretproviderclasspodstatuses" { - if !options.EnableCSIIntegration { - logrus.Infof("EnableCSIIntegration is set to false, won't run secretproviderclasspodstatuses controller") - continue - } - if !kube.IsCSIInstalled { - logrus.Infof("Can't run secretproviderclasspodstatuses controller as CSI CRDs are not installed") - continue - } + if k == constants.SecretProviderClassController && !shouldRunCSIController() { + continue } if ignoredResourcesList.Contains(k) || (len(namespaceLabelSelector) == 0 && k == "namespaces") { @@ -218,3 +211,15 @@ func startPProfServer() { logrus.Errorf("Failed to start pprof server: %v", err) } } + +func shouldRunCSIController() bool { + if !options.EnableCSIIntegration { + logrus.Info("Skipping secretproviderclasspodstatuses controller: EnableCSIIntegration is disabled") + return false + } + if !kube.IsCSIInstalled { + logrus.Info("Skipping secretproviderclasspodstatuses controller: CSI CRDs not installed") + return false + } + return true +} diff --git a/internal/pkg/constants/constants.go b/internal/pkg/constants/constants.go index 0d1f1c7..8025a29 100644 --- a/internal/pkg/constants/constants.go +++ b/internal/pkg/constants/constants.go @@ -24,6 +24,8 @@ const ( EnvVarsReloadStrategy = "env-vars" // AnnotationsReloadStrategy instructs Reloader to add pod template annotations to facilitate a restart AnnotationsReloadStrategy = "annotations" + // SecretProviderClassController enables support for SecretProviderClassPodStatus resources + SecretProviderClassController = "secretproviderclasspodstatuses" ) // Leadership election related consts diff --git a/internal/pkg/controller/controller.go b/internal/pkg/controller/controller.go index a670d81..519923e 100644 --- a/internal/pkg/controller/controller.go +++ b/internal/pkg/controller/controller.go @@ -2,9 +2,11 @@ package controller import ( "fmt" + "slices" "time" "github.com/sirupsen/logrus" + "github.com/stakater/Reloader/internal/pkg/constants" "github.com/stakater/Reloader/internal/pkg/handler" "github.com/stakater/Reloader/internal/pkg/metrics" "github.com/stakater/Reloader/internal/pkg/options" @@ -21,7 +23,6 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" "k8s.io/kubectl/pkg/scheme" - "k8s.io/utils/strings/slices" csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" ) @@ -80,13 +81,9 @@ func NewController( } } - getterRESTClient := client.CoreV1().RESTClient() - if resource == "secretproviderclasspodstatuses" { - csiClient, err := kube.GetCSIClient() - if err != nil { - logrus.Fatal(err) - } - getterRESTClient = csiClient.SecretsstoreV1().RESTClient() + getterRESTClient, err := getClientForResource(resource, client) + if err != nil { + return nil, fmt.Errorf("failed to initialize REST client for %s: %w", resource, err) } listWatcher := cache.NewFilteredListWatchFromClient(getterRESTClient, resource, namespace, optionsModifier) @@ -301,3 +298,14 @@ func (c *Controller) handleErr(err error, key interface{}) { logrus.Errorf("Dropping key out of the queue: %v", err) logrus.Debugf("Dropping the key %q out of the queue: %v", key, err) } + +func getClientForResource(resource string, coreClient kubernetes.Interface) (cache.Getter, error) { + if resource == constants.SecretProviderClassController { + csiClient, err := kube.GetCSIClient() + if err != nil { + return nil, fmt.Errorf("failed to get CSI client: %w", err) + } + return csiClient.SecretsstoreV1().RESTClient(), nil + } + return coreClient.CoreV1().RESTClient(), nil +} diff --git a/internal/pkg/controller/controller_test.go b/internal/pkg/controller/controller_test.go index 0399933..778b38d 100644 --- a/internal/pkg/controller/controller_test.go +++ b/internal/pkg/controller/controller_test.go @@ -2757,7 +2757,7 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fakeClient := fake.NewSimpleClientset() + fakeClient := fake.NewClientset() namespace, _ := fakeClient.CoreV1().Namespaces().Create(context.Background(), &tt.fields.namespace, metav1.CreateOptions{}) logrus.Infof("created fakeClient namespace for testing = %s", namespace.Name) diff --git a/internal/pkg/handler/pause_deployment_test.go b/internal/pkg/handler/pause_deployment_test.go index c14cbfc..19e7ac6 100644 --- a/internal/pkg/handler/pause_deployment_test.go +++ b/internal/pkg/handler/pause_deployment_test.go @@ -244,7 +244,7 @@ func TestHandleMissingTimerSimple(t *testing.T) { }() t.Run(test.name, func(t *testing.T) { - fakeClient := testclient.NewSimpleClientset() + fakeClient := testclient.NewClientset() clients := kube.Clients{ KubernetesClient: fakeClient, } @@ -337,7 +337,7 @@ func TestPauseDeployment(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - fakeClient := testclient.NewSimpleClientset() + fakeClient := testclient.NewClientset() clients := kube.Clients{ KubernetesClient: fakeClient, } diff --git a/internal/pkg/handler/update.go b/internal/pkg/handler/update.go index cc1e16b..25a4380 100644 --- a/internal/pkg/handler/update.go +++ b/internal/pkg/handler/update.go @@ -39,20 +39,31 @@ func (r ResourceUpdatedHandler) Handle() error { // GetConfig gets configurations containing SHA, annotations, namespace and resource name func (r ResourceUpdatedHandler) GetConfig() (common.Config, string) { - var oldSHAData string - var config common.Config + var ( + oldSHAData string + config common.Config + ) + switch res := r.Resource.(type) { case *v1.ConfigMap: - oldSHAData = util.GetSHAfromConfigmap(r.OldResource.(*v1.ConfigMap)) + if old, ok := r.OldResource.(*v1.ConfigMap); ok && old != nil { + oldSHAData = util.GetSHAfromConfigmap(old) + } config = common.GetConfigmapConfig(res) + case *v1.Secret: - oldSHAData = util.GetSHAfromSecret(r.OldResource.(*v1.Secret).Data) + if old, ok := r.OldResource.(*v1.Secret); ok && old != nil { + oldSHAData = util.GetSHAfromSecret(old.Data) + } config = common.GetSecretConfig(res) + case *csiv1.SecretProviderClassPodStatus: - oldSHAData = util.GetSHAfromSecretProviderClassPodStatus(r.OldResource.(*csiv1.SecretProviderClassPodStatus).Status) + if old, ok := r.OldResource.(*csiv1.SecretProviderClassPodStatus); ok && old != nil && old.Status.Objects != nil { + oldSHAData = util.GetSHAfromSecretProviderClassPodStatus(old.Status) + } config = common.GetSecretProviderClassPodStatusConfig(res) default: - logrus.Warnf("Invalid resource: Resource should be 'Secret', 'Configmap' or 'SecretProviderClassPodStatus' but found, %v", r.Resource) + logrus.Warnf("Invalid resource: Resource should be 'Secret', 'Configmap' or 'SecretProviderClassPodStatus' but found, %T", r.Resource) } return config, oldSHAData } diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index f5b7ead..b10bfbc 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -634,11 +634,10 @@ func updateEnvVar(container *v1.Container, envVar string, shaData string) consta } func secretProviderClassEnvReloaded(containers []v1.Container, envVar string, shaData string) bool { - for i := range containers { - envs := containers[i].Env - for j := range envs { - if envs[j].Name == envVar { - return envs[j].Value == shaData + for _, container := range containers { + for _, env := range container.Env { + if env.Name == envVar { + return env.Value == shaData } } } @@ -649,7 +648,11 @@ func populateAnnotationsFromSecretProviderClass(clients kube.Clients, config *co obj, err := clients.CSIClient.SecretsstoreV1().SecretProviderClasses(config.Namespace).Get(context.TODO(), config.ResourceName, metav1.GetOptions{}) annotations := make(map[string]string) if err != nil { - logrus.Infof("Couldn't find secretproviderclass '%s' in '%s' namespace for typed annotation", config.ResourceName, config.Namespace) + if apierrors.IsNotFound(err) { + logrus.Warnf("SecretProviderClass '%s' not found in namespace '%s'", config.ResourceName, config.Namespace) + } else { + logrus.Errorf("Failed to get SecretProviderClass '%s' in namespace '%s': %v", config.ResourceName, config.Namespace, err) + } } else if obj.Annotations != nil { annotations = obj.Annotations } diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index 5bf490f..c1897f6 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -32,7 +32,7 @@ import ( var ( clients = kube.Clients{ - KubernetesClient: testclient.NewSimpleClientset(), + KubernetesClient: testclient.NewClientset(), CSIClient: csitestclient.NewSimpleClientset(), } From 7e9d571e1e6b562082b2e4c2ede04d17e309ee53 Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Sun, 4 Jan 2026 20:10:52 +0100 Subject: [PATCH 16/45] Readme update and change SHA1 to SHA512 Signed-off-by: faizanahmad055 --- README.md | 31 +++++++++++++++++++--- docs/How-it-works.md | 4 +-- docs/Reloader-vs-ConfigmapController.md | 14 +++++----- docs/Reloader-vs-k8s-trigger-controller.md | 2 +- internal/pkg/crypto/sha.go | 18 +++++-------- 5 files changed, 44 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 4fe53f3..e7f1d4b 100644 --- a/README.md +++ b/README.md @@ -263,10 +263,33 @@ Reloader watches these updates and triggers a rollout when a change is detected. #### Annotations for CSI-mounted Secrets -| Annotation | Description | -|--------------------------------------------|----------------------------------------------------------------------| -| `reloader.stakater.com/auto: "true"` | Reloads workload when CSI-mounted secrets change | -| `secretproviderclass.reloader.stakater.com/reload: "my-spc"` | Reloads when specific SecretProviderClass changes | +| Annotation | Description | +|------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------| +| `reloader.stakater.com/auto: "true"` | Global Discovery: Automatically discovers and reloads the workload when any mounted ConfigMap or Secret is updated. | +| `secretproviderclass.reloader.stakater.com/auto: 'true'` | CSI Discovery: Specifically watches for updates to all SecretProviderClasses used by the workload (CSI driver integration). | +| `secretproviderclass.reloader.stakater.com/reload: "my-secretproviderclass"` | Targeted Reload: Only reloads the workload when the specifically named SecretProviderClass(es) are updated. | + +Reloader monitors changes at the **per-secret level** by watching the `SecretProviderClassPodStatus`. Make sure each secret you want to monitor is properly defined with a `secretKey` in your `SecretProviderClass`: + +```yaml +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: vault-reloader-demo + namespace: test +spec: + provider: vault + parameters: + vaultAddress: "http://vault.vault.svc:8200" + vaultSkipTLSVerify: "true" + roleName: "demo-role" + objects: | + - objectName: "password" + secretPath: "secret/data/reloader-demo" + secretKey: "password" +``` +***Important***: Reloader tracks changes to individual secrets (identified by secretKey). If your SecretProviderClass doesn't specify secretKey for each object, Reloader may not detect updates correctly. + #### Notes & Limitations diff --git a/docs/How-it-works.md b/docs/How-it-works.md index c0ae964..6a946f9 100644 --- a/docs/How-it-works.md +++ b/docs/How-it-works.md @@ -76,7 +76,7 @@ Note: Rolling upgrade also works in the same way for secrets. ### Hash Value Computation -Reloader uses SHA1 to compute hash value. SHA1 is used because it is efficient and less prone to collision. +Reloader uses SHA512 to compute hash value. SHA1 is used because it is efficient and less prone to collision. ## Monitor All Namespaces @@ -90,4 +90,4 @@ The output file can then be used to deploy Reloader in specific namespace. ## Compatibility With Helm Install and Upgrade -Reloader has no impact on helm deployment cycle. Reloader only injects an environment variable in `deployment`, `daemonset` or `statefulset`. The environment variable contains the SHA1 value of `ConfigMaps` or `Secrets` data. So if a deployment is created using Helm and Reloader updates the deployment, then next time you upgrade the helm release, Reloader will do nothing except changing that environment variable value in `deployment` , `daemonset` or `statefulset`. +Reloader has no impact on helm deployment cycle. Reloader only injects an environment variable in `deployment`, `daemonset` or `statefulset`. The environment variable contains the SHA512 value of `ConfigMaps` or `Secrets` data. So if a deployment is created using Helm and Reloader updates the deployment, then next time you upgrade the helm release, Reloader will do nothing except changing that environment variable value in `deployment` , `daemonset` or `statefulset`. diff --git a/docs/Reloader-vs-ConfigmapController.md b/docs/Reloader-vs-ConfigmapController.md index f866f89..c8bcfc8 100644 --- a/docs/Reloader-vs-ConfigmapController.md +++ b/docs/Reloader-vs-ConfigmapController.md @@ -2,10 +2,10 @@ Reloader is inspired from [`configmapcontroller`](https://github.com/fabric8io/configmapcontroller) but there are many ways in which it differs from `configmapcontroller`. Below is the small comparison between these two controllers. -| Reloader | ConfigMap | -|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Reloader can watch both `Secrets` and `ConfigMaps`. | `configmapcontroller` can only watch changes in `ConfigMaps`. It cannot detect changes in other resources like `Secrets`. | -| Reloader can perform rolling upgrades on `deployments` as well as on `statefulsets` and `daemonsets` | `configmapcontroller` can only perform rolling upgrades on `deployments`. It currently does not support rolling upgrades on `statefulsets` and `daemonsets` | -| Reloader provides both unit test cases and end to end integration test cases for future updates. So one can make sure that new changes do not break any old functionality. | Currently there are not any unit test cases or end to end integration test cases in `configmap-controller`. It add difficulties for any additional updates in `configmap-controller` and one can not know for sure whether new changes breaks any old functionality or not. | -| Reloader uses SHA1 to encode the change in `ConfigMap` or `Secret`. It then saves the SHA1 value in `STAKATER_FOO_CONFIGMAP` or `STAKATER_FOO_SECRET` environment variable depending upon where the change has happened. The use of SHA1 provides a concise 40 characters encoded value that is very less prone to collision. | `configmap-controller` uses `FABRICB_FOO_REVISION` environment variable to store any change in `ConfigMap` controller. It does not encode it or convert it in suitable hash value to avoid data pollution in deployment. | -| Reloader allows you to customize your own annotation (for both `Secrets` and `ConfigMaps`) using command line flags | `configmap-controller` restricts you to only their provided annotation | +| Reloader | ConfigMap | +|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Reloader can watch both `Secrets` and `ConfigMaps`. | `configmapcontroller` can only watch changes in `ConfigMaps`. It cannot detect changes in other resources like `Secrets`. | +| Reloader can perform rolling upgrades on `deployments` as well as on `statefulsets` and `daemonsets` | `configmapcontroller` can only perform rolling upgrades on `deployments`. It currently does not support rolling upgrades on `statefulsets` and `daemonsets` | +| Reloader provides both unit test cases and end to end integration test cases for future updates. So one can make sure that new changes do not break any old functionality. | Currently there are not any unit test cases or end to end integration test cases in `configmap-controller`. It add difficulties for any additional updates in `configmap-controller` and one can not know for sure whether new changes breaks any old functionality or not. | +| Reloader uses SHA512 to encode the change in `ConfigMap` or `Secret`. It then saves the SHA1 value in `STAKATER_FOO_CONFIGMAP` or `STAKATER_FOO_SECRET` environment variable depending upon where the change has happened. The use of SHA1 provides a concise 40 characters encoded value that is very less prone to collision. | `configmap-controller` uses `FABRICB_FOO_REVISION` environment variable to store any change in `ConfigMap` controller. It does not encode it or convert it in suitable hash value to avoid data pollution in deployment. | +| Reloader allows you to customize your own annotation (for both `Secrets` and `ConfigMaps`) using command line flags | `configmap-controller` restricts you to only their provided annotation | diff --git a/docs/Reloader-vs-k8s-trigger-controller.md b/docs/Reloader-vs-k8s-trigger-controller.md index 811987a..fe0f6d9 100644 --- a/docs/Reloader-vs-k8s-trigger-controller.md +++ b/docs/Reloader-vs-k8s-trigger-controller.md @@ -6,7 +6,7 @@ Reloader and k8s-trigger-controller are both built for same purpose. So there ar - Both controllers support change detection in `ConfigMaps` and `Secrets` - Both controllers support deployment `rollout` -- Both controllers use SHA1 for hashing +- Reloader controller use SHA512 for hashing - Both controllers have end to end as well as unit test cases. ## Differences diff --git a/internal/pkg/crypto/sha.go b/internal/pkg/crypto/sha.go index 043fc22..9235425 100644 --- a/internal/pkg/crypto/sha.go +++ b/internal/pkg/crypto/sha.go @@ -1,20 +1,16 @@ package crypto import ( - "crypto/sha1" - "fmt" - "io" - - "github.com/sirupsen/logrus" + "crypto/sha512" + "encoding/hex" ) // GenerateSHA generates SHA from string func GenerateSHA(data string) string { - hasher := sha1.New() - _, err := io.WriteString(hasher, data) - if err != nil { - logrus.Errorf("Unable to write data in hash writer %v", err) + if data == "" { + return "" } - sha := hasher.Sum(nil) - return fmt.Sprintf("%x", sha) + + hash := sha512.Sum512_256([]byte(data)) + return hex.EncodeToString(hash[:]) } From 4b90335362096de60340bde801b47d4006908f3b Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Sun, 4 Jan 2026 21:52:07 +0100 Subject: [PATCH 17/45] Readme update and fix tests Signed-off-by: faizanahmad055 --- README.md | 2 +- docs/Reloader-vs-ConfigmapController.md | 14 +++++++------- internal/pkg/handler/upgrade_test.go | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index e7f1d4b..5c22154 100644 --- a/README.md +++ b/README.md @@ -288,8 +288,8 @@ spec: secretPath: "secret/data/reloader-demo" secretKey: "password" ``` -***Important***: Reloader tracks changes to individual secrets (identified by secretKey). If your SecretProviderClass doesn't specify secretKey for each object, Reloader may not detect updates correctly. +***Important***: Reloader tracks changes to individual secrets (identified by secretKey). If your SecretProviderClass doesn't specify secretKey for each object, Reloader may not detect updates correctly. #### Notes & Limitations diff --git a/docs/Reloader-vs-ConfigmapController.md b/docs/Reloader-vs-ConfigmapController.md index c8bcfc8..3ddab08 100644 --- a/docs/Reloader-vs-ConfigmapController.md +++ b/docs/Reloader-vs-ConfigmapController.md @@ -2,10 +2,10 @@ Reloader is inspired from [`configmapcontroller`](https://github.com/fabric8io/configmapcontroller) but there are many ways in which it differs from `configmapcontroller`. Below is the small comparison between these two controllers. -| Reloader | ConfigMap | -|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Reloader can watch both `Secrets` and `ConfigMaps`. | `configmapcontroller` can only watch changes in `ConfigMaps`. It cannot detect changes in other resources like `Secrets`. | -| Reloader can perform rolling upgrades on `deployments` as well as on `statefulsets` and `daemonsets` | `configmapcontroller` can only perform rolling upgrades on `deployments`. It currently does not support rolling upgrades on `statefulsets` and `daemonsets` | -| Reloader provides both unit test cases and end to end integration test cases for future updates. So one can make sure that new changes do not break any old functionality. | Currently there are not any unit test cases or end to end integration test cases in `configmap-controller`. It add difficulties for any additional updates in `configmap-controller` and one can not know for sure whether new changes breaks any old functionality or not. | -| Reloader uses SHA512 to encode the change in `ConfigMap` or `Secret`. It then saves the SHA1 value in `STAKATER_FOO_CONFIGMAP` or `STAKATER_FOO_SECRET` environment variable depending upon where the change has happened. The use of SHA1 provides a concise 40 characters encoded value that is very less prone to collision. | `configmap-controller` uses `FABRICB_FOO_REVISION` environment variable to store any change in `ConfigMap` controller. It does not encode it or convert it in suitable hash value to avoid data pollution in deployment. | -| Reloader allows you to customize your own annotation (for both `Secrets` and `ConfigMaps`) using command line flags | `configmap-controller` restricts you to only their provided annotation | +| Reloader | ConfigMap | +|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Reloader can watch both `Secrets` and `ConfigMaps`. | `configmapcontroller` can only watch changes in `ConfigMaps`. It cannot detect changes in other resources like `Secrets`. | +| Reloader can perform rolling upgrades on `deployments` as well as on `statefulsets` and `daemonsets` | `configmapcontroller` can only perform rolling upgrades on `deployments`. It currently does not support rolling upgrades on `statefulsets` and `daemonsets` | +| Reloader provides both unit test cases and end to end integration test cases for future updates. So one can make sure that new changes do not break any old functionality. | Currently there are not any unit test cases or end to end integration test cases in `configmap-controller`. It adds difficulties for any additional updates in `configmap-controller` and one can not know for sure whether new changes breaks any old functionality or not. | +| Reloader uses SHA512 to encode the change in `ConfigMap` or `Secret`. It then saves the SHA1 value in `STAKATER_FOO_CONFIGMAP` or `STAKATER_FOO_SECRET` environment variable depending upon where the change has happened. The use of SHA1 provides a concise 40 characters encoded value that is very less prone to collision. | `configmap-controller` uses `FABRICB_FOO_REVISION` environment variable to store any change in `ConfigMap` controller. It does not encode it or convert it in suitable hash value to avoid data pollution in deployment. | +| Reloader allows you to customize your own annotation (for both `Secrets` and `ConfigMaps`) using command line flags | `configmap-controller` restricts you to only their provided annotation | diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index c1897f6..a334db0 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -1981,7 +1981,7 @@ func TestRollingUpgradeForDeploymentWithPatchAndRetryUsingArs(t *testing.T) { assert.Equal(t, patchtypes.StrategicMergePatchType, patchType) assert.NotEmpty(t, bytes) assert.Contains(t, string(bytes), `{"spec":{"template":{"metadata":{"annotations":{"reloader.stakater.com/last-reloaded-from":`) - assert.Contains(t, string(bytes), `\"hash\":\"3c9a892aeaedc759abc3df9884a37b8be5680382\"`) + assert.Contains(t, string(bytes), `\"hash\":\"fd9e71a362056bfa864d9859e12978f893d330ce8cbf09218b25d015770ad91f\"`) return nil } @@ -2964,7 +2964,7 @@ func TestRollingUpgradeForDaemonSetWithPatchAndRetryUsingArs(t *testing.T) { assert.Equal(t, patchtypes.StrategicMergePatchType, patchType) assert.NotEmpty(t, bytes) assert.Contains(t, string(bytes), `{"spec":{"template":{"metadata":{"annotations":{"reloader.stakater.com/last-reloaded-from":`) - assert.Contains(t, string(bytes), `\"hash\":\"314a2269170750a974d79f02b5b9ee517de7f280\"`) + assert.Contains(t, string(bytes), `\"hash\":\"43bf9e30e7c4e32a8f8673c462b86d0b1ac626cf498afdc0d0108e79ebe7ee0c\"`) return nil } @@ -3227,7 +3227,7 @@ func TestRollingUpgradeForStatefulSetWithPatchAndRetryUsingArs(t *testing.T) { assert.Equal(t, patchtypes.StrategicMergePatchType, patchType) assert.NotEmpty(t, bytes) assert.Contains(t, string(bytes), `{"spec":{"template":{"metadata":{"annotations":{"reloader.stakater.com/last-reloaded-from":`) - assert.Contains(t, string(bytes), `\"hash\":\"f821414d40d8815fb330763f74a4ff7ab651d4fa\"`) + assert.Contains(t, string(bytes), `\"hash\":\"6aa837180bdf6a93306c71a0cf62b4a45c2d5b021578247b3b64d5baea2b84d9\"`) return nil } @@ -3607,7 +3607,7 @@ func TestRollingUpgradeForDeploymentWithPatchAndRetryUsingErs(t *testing.T) { assert.Equal(t, patchtypes.StrategicMergePatchType, patchType) assert.NotEmpty(t, bytes) assert.Contains(t, string(bytes), `{"spec":{"template":{"spec":{"containers":[{"name":`) - assert.Contains(t, string(bytes), `"value":"3c9a892aeaedc759abc3df9884a37b8be5680382"`) + assert.Contains(t, string(bytes), `"value":"fd9e71a362056bfa864d9859e12978f893d330ce8cbf09218b25d015770ad91f"`) return nil } @@ -4502,7 +4502,7 @@ func TestRollingUpgradeForDaemonSetWithPatchAndRetryUsingErs(t *testing.T) { assert.Equal(t, patchtypes.StrategicMergePatchType, patchType) assert.NotEmpty(t, bytes) assert.Contains(t, string(bytes), `{"spec":{"template":{"spec":{"containers":[{"name":`) - assert.Contains(t, string(bytes), `"value":"314a2269170750a974d79f02b5b9ee517de7f280"`) + assert.Contains(t, string(bytes), `"value":"43bf9e30e7c4e32a8f8673c462b86d0b1ac626cf498afdc0d0108e79ebe7ee0c"`) return nil } @@ -4737,7 +4737,7 @@ func TestRollingUpgradeForStatefulSetWithPatchAndRetryUsingErs(t *testing.T) { assert.Equal(t, patchtypes.StrategicMergePatchType, patchType) assert.NotEmpty(t, bytes) assert.Contains(t, string(bytes), `{"spec":{"template":{"spec":{"containers":[{"name":`) - assert.Contains(t, string(bytes), `"value":"f821414d40d8815fb330763f74a4ff7ab651d4fa"`) + assert.Contains(t, string(bytes), `"value":"6aa837180bdf6a93306c71a0cf62b4a45c2d5b021578247b3b64d5baea2b84d9"`) return nil } From eb38bf7470dac4764fe355104b34c690f7ec747f Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Sun, 4 Jan 2026 22:01:44 +0100 Subject: [PATCH 18/45] Fix linting errors Signed-off-by: faizanahmad055 --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 5c22154..57a1e62 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ This instructs Reloader to skip all reload logic for that resource across all wo ### 4. āš™ļø Workload-Specific Rollout Strategy (Argo Rollouts Only) -Note: This is only applicable when using [Argo Rollouts](https://argoproj.github.io/argo-rollouts/). It is ignored for standard Kubernetes Deployments, StatefulSets, or DaemonSets. To use this feature, Argo Rollouts support must be enabled in Reloader (for example via --is-argo-rollouts=true). +Note: This is only applicable when using [Argo Rollouts](https://argoproj.github.io/argo-rollouts/). It is ignored for standard Kubernetes `Deployments`, `StatefulSets`, or `DaemonSets`. To use this feature, Argo Rollouts support must be enabled in Reloader (for example via --is-argo-rollouts=true). By default, Reloader triggers the Argo Rollout controller to perform a standard rollout by updating the pod template. This works well in most cases, however, because this modifies the workload spec, GitOps tools like ArgoCD will detect this as "Configuration Drift" and mark your application as OutOfSync. @@ -191,8 +191,8 @@ metadata: āœ… Use `restart` if: 1. You're using GitOps and want to avoid drift -2. You want a quick restart without changing the workload spec -3. Your platform restricts metadata changes +1. You want a quick restart without changing the workload spec +1. Your platform restricts metadata changes This setting affects Argo Rollouts behavior, not Argo CD sync settings. @@ -289,13 +289,13 @@ spec: secretKey: "password" ``` -***Important***: Reloader tracks changes to individual secrets (identified by secretKey). If your SecretProviderClass doesn't specify secretKey for each object, Reloader may not detect updates correctly. +***Important***: Reloader tracks changes to individual secrets (identified by `secretKey`). If your SecretProviderClass doesn't specify `secretKey` for each object, Reloader may not detect updates correctly. #### Notes & Limitations Reloader reacts to CSI status changes, not direct updates to external secret stores Secret rotation must be enabled in the CSI driver for updates to be detected -CSI limitations (such as subPath mounts) still apply and may require pod restarts +CSI limitations (such as `subPath` mounts) still apply and may require pod restarts If secrets are synced to Kubernetes Secret objects, standard Reloader behavior applies and CSI support may not be required ## šŸš€ Installation @@ -489,7 +489,7 @@ PRs are welcome. In general, we follow the "fork-and-pull" Git workflow: ## Release Processes -_Repository GitHub releases_: As requested by the community in [issue 685](https://github.com/stakater/Reloader/issues/685), Reloader is now based on a manual release process. Releases are no longer done on every merged PR to the main branch, but manually on request. +*Repository GitHub releases*: As requested by the community in [issue 685](https://github.com/stakater/Reloader/issues/685), Reloader is now based on a manual release process. Releases are no longer done on every merged PR to the main branch, but manually on request. To make a GitHub release: @@ -502,7 +502,7 @@ To make a GitHub release: 1. Code owners create another branch from `master` and bump the helm chart version as well as Reloader image version. - Code owners create a PR with `release/helm-chart` label, example: [PR-846](https://github.com/stakater/Reloader/pull/846) -_Repository git tagging_: Push to the main branch will create a merge-image and merge-tag named `merge-${{ github.event.number }}`, for example `merge-800` when pull request number 800 is merged. +*Repository git tagging*: Push to the main branch will create a merge-image and merge-tag named `merge-${{ github.event.number }}`, for example `merge-800` when pull request number 800 is merged. ## Changelog From 9a3edf13d2a1b87dfd258d26fec532037cdc4301 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Tue, 6 Jan 2026 11:03:26 +0100 Subject: [PATCH 19/45] feat: Load tests --- .github/workflows/loadtest.yml | 222 ++ internal/pkg/controller/controller.go | 77 +- internal/pkg/controller/controller_test.go | 80 +- internal/pkg/handler/create.go | 34 +- internal/pkg/handler/delete.go | 33 +- internal/pkg/handler/handler.go | 11 +- internal/pkg/handler/update.go | 32 +- internal/pkg/handler/upgrade.go | 43 +- internal/pkg/metrics/prometheus.go | 373 ++- test/loadtest/README.md | 544 +++++ test/loadtest/cmd/loadtest/main.go | 1582 +++++++++++++ test/loadtest/go.mod | 50 + test/loadtest/go.sum | 154 ++ test/loadtest/internal/cluster/kind.go | 313 +++ .../internal/prometheus/prometheus.go | 452 ++++ test/loadtest/internal/scenarios/scenarios.go | 2092 +++++++++++++++++ test/loadtest/manifests/prometheus.yaml | 181 ++ 17 files changed, 6191 insertions(+), 82 deletions(-) create mode 100644 .github/workflows/loadtest.yml create mode 100644 test/loadtest/README.md create mode 100644 test/loadtest/cmd/loadtest/main.go create mode 100644 test/loadtest/go.mod create mode 100644 test/loadtest/go.sum create mode 100644 test/loadtest/internal/cluster/kind.go create mode 100644 test/loadtest/internal/prometheus/prometheus.go create mode 100644 test/loadtest/internal/scenarios/scenarios.go create mode 100644 test/loadtest/manifests/prometheus.yaml diff --git a/.github/workflows/loadtest.yml b/.github/workflows/loadtest.yml new file mode 100644 index 0000000..a62e3e3 --- /dev/null +++ b/.github/workflows/loadtest.yml @@ -0,0 +1,222 @@ +name: Load Test + +on: + issue_comment: + types: [created] + +jobs: + loadtest: + # Only run on PR comments with /loadtest command + if: | + github.event.issue.pull_request && + contains(github.event.comment.body, '/loadtest') + runs-on: ubuntu-latest + + steps: + - name: Add reaction to comment + uses: actions/github-script@v7 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: 'rocket' + }); + + - name: Get PR details + id: pr + uses: actions/github-script@v7 + with: + script: | + const pr = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number + }); + core.setOutput('head_ref', pr.data.head.ref); + core.setOutput('head_sha', pr.data.head.sha); + core.setOutput('base_ref', pr.data.base.ref); + core.setOutput('base_sha', pr.data.base.sha); + console.log(`PR #${context.issue.number}: ${pr.data.head.ref} -> ${pr.data.base.ref}`); + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: '1.23' + cache: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Install kind + run: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.20.0/kind-linux-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind + + # Build OLD image from base branch (e.g., main) + - name: Checkout base branch (old) + uses: actions/checkout@v4 + with: + ref: ${{ steps.pr.outputs.base_ref }} + path: old + + - name: Build old image + run: | + cd old + docker build -t localhost/reloader:old -f Dockerfile . + echo "Built old image from ${{ steps.pr.outputs.base_ref }} (${{ steps.pr.outputs.base_sha }})" + + # Build NEW image from PR branch + - name: Checkout PR branch (new) + uses: actions/checkout@v4 + with: + ref: ${{ steps.pr.outputs.head_ref }} + path: new + + - name: Build new image + run: | + cd new + docker build -t localhost/reloader:new -f Dockerfile . + echo "Built new image from ${{ steps.pr.outputs.head_ref }} (${{ steps.pr.outputs.head_sha }})" + + # Build and run loadtest from PR branch + - name: Build loadtest tool + run: | + cd new/test/loadtest + go build -o loadtest ./cmd/loadtest + + - name: Run A/B comparison load test + id: loadtest + run: | + cd new/test/loadtest + ./loadtest run \ + --old-image=localhost/reloader:old \ + --new-image=localhost/reloader:new \ + --scenario=all \ + --duration=60 2>&1 | tee loadtest-output.txt + echo "exitcode=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT + + - name: Upload results + uses: actions/upload-artifact@v4 + if: always() + with: + name: loadtest-results + path: | + new/test/loadtest/results/ + new/test/loadtest/loadtest-output.txt + retention-days: 30 + + - name: Post results comment + uses: actions/github-script@v7 + if: always() + with: + script: | + const fs = require('fs'); + + let results = ''; + const resultsDir = 'new/test/loadtest/results'; + + // Collect summary of all scenarios + let passCount = 0; + let failCount = 0; + const summaries = []; + + if (fs.existsSync(resultsDir)) { + const scenarios = fs.readdirSync(resultsDir).sort(); + for (const scenario of scenarios) { + const reportPath = `${resultsDir}/${scenario}/report.txt`; + if (fs.existsSync(reportPath)) { + const report = fs.readFileSync(reportPath, 'utf8'); + + // Extract status from report + const statusMatch = report.match(/Status:\s+(PASS|FAIL)/); + const status = statusMatch ? statusMatch[1] : 'UNKNOWN'; + + if (status === 'PASS') passCount++; + else failCount++; + + // Extract key metrics for summary + const actionMatch = report.match(/action_total\s+[\d.]+\s+[\d.]+\s+[\d.]+/); + const errorsMatch = report.match(/errors_total\s+[\d.]+\s+[\d.]+/); + + summaries.push(`| ${scenario} | ${status === 'PASS' ? 'āœ…' : 'āŒ'} ${status} |`); + + results += `\n
\n${status === 'PASS' ? 'āœ…' : 'āŒ'} ${scenario}\n\n\`\`\`\n${report}\n\`\`\`\n
\n`; + } + } + } + + if (!results) { + // Read raw output if no reports + if (fs.existsSync('new/test/loadtest/loadtest-output.txt')) { + const output = fs.readFileSync('new/test/loadtest/loadtest-output.txt', 'utf8'); + const maxLen = 60000; + results = output.length > maxLen + ? output.substring(output.length - maxLen) + : output; + results = `\`\`\`\n${results}\n\`\`\``; + } else { + results = 'No results available'; + } + } + + const overallStatus = failCount === 0 ? 'āœ… ALL PASSED' : `āŒ ${failCount} FAILED`; + + const body = `## Load Test Results ${overallStatus} + + **Comparing:** \`${{ steps.pr.outputs.base_ref }}\` (old) vs \`${{ steps.pr.outputs.head_ref }}\` (new) + **Old commit:** ${{ steps.pr.outputs.base_sha }} + **New commit:** ${{ steps.pr.outputs.head_sha }} + **Triggered by:** @${{ github.event.comment.user.login }} + + ### Summary + + | Scenario | Status | + |----------|--------| + ${summaries.join('\n')} + + **Total:** ${passCount} passed, ${failCount} failed + + ### Detailed Results + + ${results} + +
+ šŸ“¦ Download full results + + Artifacts are available in the [workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}). +
+ `; + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body + }); + + - name: Add success reaction + if: success() + uses: actions/github-script@v7 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: '+1' + }); + + - name: Add failure reaction + if: failure() + uses: actions/github-script@v7 + with: + script: | + await github.rest.reactions.createForIssueComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: context.payload.comment.id, + content: '-1' + }); diff --git a/internal/pkg/controller/controller.go b/internal/pkg/controller/controller.go index 15b2e0f..7dd3f5e 100644 --- a/internal/pkg/controller/controller.go +++ b/internal/pkg/controller/controller.go @@ -103,6 +103,8 @@ func NewController( // Add function to add a new object to the queue in case of creating a resource func (c *Controller) Add(obj interface{}) { + // Record event received + c.collectors.RecordEventReceived("add", c.resource) switch object := obj.(type) { case *v1.Namespace: @@ -112,11 +114,14 @@ func (c *Controller) Add(obj interface{}) { if options.ReloadOnCreate == "true" { if !c.resourceInIgnoredNamespace(obj) && c.resourceInSelectedNamespaces(obj) && secretControllerInitialized && configmapControllerInitialized { - c.queue.Add(handler.ResourceCreatedHandler{ - Resource: obj, - Collectors: c.collectors, - Recorder: c.recorder, + c.enqueue(handler.ResourceCreatedHandler{ + Resource: obj, + Collectors: c.collectors, + Recorder: c.recorder, + EnqueueTime: time.Now(), // Track when item was enqueued }) + } else { + c.collectors.RecordSkipped("ignored_or_not_selected") } } } @@ -166,31 +171,42 @@ func (c *Controller) removeSelectedNamespaceFromCache(namespace v1.Namespace) { // Update function to add an old object and a new object to the queue in case of updating a resource func (c *Controller) Update(old interface{}, new interface{}) { + // Record event received + c.collectors.RecordEventReceived("update", c.resource) + switch new.(type) { case *v1.Namespace: return } if !c.resourceInIgnoredNamespace(new) && c.resourceInSelectedNamespaces(new) { - c.queue.Add(handler.ResourceUpdatedHandler{ + c.enqueue(handler.ResourceUpdatedHandler{ Resource: new, OldResource: old, Collectors: c.collectors, Recorder: c.recorder, + EnqueueTime: time.Now(), // Track when item was enqueued }) + } else { + c.collectors.RecordSkipped("ignored_or_not_selected") } } // Delete function to add an object to the queue in case of deleting a resource func (c *Controller) Delete(old interface{}) { + // Record event received + c.collectors.RecordEventReceived("delete", c.resource) if options.ReloadOnDelete == "true" { if !c.resourceInIgnoredNamespace(old) && c.resourceInSelectedNamespaces(old) && secretControllerInitialized && configmapControllerInitialized { - c.queue.Add(handler.ResourceDeleteHandler{ - Resource: old, - Collectors: c.collectors, - Recorder: c.recorder, + c.enqueue(handler.ResourceDeleteHandler{ + Resource: old, + Collectors: c.collectors, + Recorder: c.recorder, + EnqueueTime: time.Now(), // Track when item was enqueued }) + } else { + c.collectors.RecordSkipped("ignored_or_not_selected") } } @@ -201,6 +217,13 @@ func (c *Controller) Delete(old interface{}) { } } +// enqueue adds an item to the queue and records metrics +func (c *Controller) enqueue(item interface{}) { + c.queue.Add(item) + c.collectors.RecordQueueAdd() + c.collectors.SetQueueDepth(c.queue.Len()) +} + // Run function for controller which handles the queue func (c *Controller) Run(threadiness int, stopCh chan struct{}) { defer runtime.HandleCrash() @@ -242,13 +265,36 @@ func (c *Controller) processNextItem() bool { if quit { return false } + + // Update queue depth after getting item + c.collectors.SetQueueDepth(c.queue.Len()) + // Tell the queue that we are done with processing this key. This unblocks the key for other workers // This allows safe parallel processing because two events with the same key are never processed in // parallel. defer c.queue.Done(resourceHandler) + // Record queue latency if the handler supports it + if h, ok := resourceHandler.(handler.TimedHandler); ok { + queueLatency := time.Since(h.GetEnqueueTime()) + c.collectors.RecordQueueLatency(queueLatency) + } + + // Track reconcile/handler duration + startTime := time.Now() + // Invoke the method containing the business logic err := resourceHandler.(handler.ResourceHandler).Handle() + + duration := time.Since(startTime) + + // Record reconcile metrics + if err != nil { + c.collectors.RecordReconcile("error", duration) + } else { + c.collectors.RecordReconcile("success", duration) + } + // Handle the error if something went wrong during the execution of the business logic c.handleErr(err, resourceHandler) return true @@ -261,16 +307,26 @@ func (c *Controller) handleErr(err error, key interface{}) { // This ensures that future processing of updates for this key is not delayed because of // an outdated error history. c.queue.Forget(key) + + // Record successful event processing + c.collectors.RecordEventProcessed("unknown", c.resource, "success") return } + // Record error + c.collectors.RecordError("handler_error") + // This controller retries 5 times if something goes wrong. After that, it stops trying. if c.queue.NumRequeues(key) < 5 { logrus.Errorf("Error syncing events: %v", err) + // Record retry + c.collectors.RecordRetry() + // Re-enqueue the key rate limited. Based on the rate limiter on the // queue and the re-enqueue history, the key will be processed later again. c.queue.AddRateLimited(key) + c.collectors.SetQueueDepth(c.queue.Len()) return } @@ -279,4 +335,7 @@ func (c *Controller) handleErr(err error, key interface{}) { runtime.HandleError(err) logrus.Errorf("Dropping key out of the queue: %v", err) logrus.Debugf("Dropping the key %q out of the queue: %v", key, err) + + // Record failed event processing + c.collectors.RecordEventProcessed("unknown", c.resource, "dropped") } diff --git a/internal/pkg/controller/controller_test.go b/internal/pkg/controller/controller_test.go index 63e6be3..7b4c728 100644 --- a/internal/pkg/controller/controller_test.go +++ b/internal/pkg/controller/controller_test.go @@ -2157,19 +2157,21 @@ func TestController_resourceInIgnoredNamespace(t *testing.T) { }, } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := &Controller{ - client: tt.fields.client, - indexer: tt.fields.indexer, - queue: tt.fields.queue, - informer: tt.fields.informer, - namespace: tt.fields.namespace, - ignoredNamespaces: tt.fields.ignoredNamespaces, - } - if got := c.resourceInIgnoredNamespace(tt.args.raw); got != tt.want { - t.Errorf("Controller.resourceInIgnoredNamespace() = %v, want %v", got, tt.want) - } - }) + t.Run( + tt.name, func(t *testing.T) { + c := &Controller{ + client: tt.fields.client, + indexer: tt.fields.indexer, + queue: tt.fields.queue, + informer: tt.fields.informer, + namespace: tt.fields.namespace, + ignoredNamespaces: tt.fields.ignoredNamespaces, + } + if got := c.resourceInIgnoredNamespace(tt.args.raw); got != tt.want { + t.Errorf("Controller.resourceInIgnoredNamespace() = %v, want %v", got, tt.want) + } + }, + ) } } @@ -2331,35 +2333,37 @@ func TestController_resourceInNamespaceSelector(t *testing.T) { } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - fakeClient := fake.NewSimpleClientset() - namespace, _ := fakeClient.CoreV1().Namespaces().Create(context.Background(), &tt.fields.namespace, metav1.CreateOptions{}) - logrus.Infof("created fakeClient namespace for testing = %s", namespace.Name) + t.Run( + tt.name, func(t *testing.T) { + fakeClient := fake.NewSimpleClientset() + namespace, _ := fakeClient.CoreV1().Namespaces().Create(context.Background(), &tt.fields.namespace, metav1.CreateOptions{}) + logrus.Infof("created fakeClient namespace for testing = %s", namespace.Name) - c := &Controller{ - client: fakeClient, - indexer: tt.fields.indexer, - queue: tt.fields.queue, - informer: tt.fields.informer, - namespace: tt.fields.namespace.Name, - namespaceSelector: tt.fields.namespaceSelector, - } + c := &Controller{ + client: fakeClient, + indexer: tt.fields.indexer, + queue: tt.fields.queue, + informer: tt.fields.informer, + namespace: tt.fields.namespace.Name, + namespaceSelector: tt.fields.namespaceSelector, + } - listOptions := metav1.ListOptions{} - listOptions.LabelSelector = tt.fields.namespaceSelector - namespaces, _ := fakeClient.CoreV1().Namespaces().List(context.Background(), listOptions) + listOptions := metav1.ListOptions{} + listOptions.LabelSelector = tt.fields.namespaceSelector + namespaces, _ := fakeClient.CoreV1().Namespaces().List(context.Background(), listOptions) - for _, ns := range namespaces.Items { - c.addSelectedNamespaceToCache(ns) - } + for _, ns := range namespaces.Items { + c.addSelectedNamespaceToCache(ns) + } - if got := c.resourceInSelectedNamespaces(tt.args.raw); got != tt.want { - t.Errorf("Controller.resourceInNamespaceSelector() = %v, want %v", got, tt.want) - } + if got := c.resourceInSelectedNamespaces(tt.args.raw); got != tt.want { + t.Errorf("Controller.resourceInNamespaceSelector() = %v, want %v", got, tt.want) + } - for _, ns := range namespaces.Items { - c.removeSelectedNamespaceFromCache(ns) - } - }) + for _, ns := range namespaces.Items { + c.removeSelectedNamespaceFromCache(ns) + } + }, + ) } } diff --git a/internal/pkg/handler/create.go b/internal/pkg/handler/create.go index fab7378..5fd3014 100644 --- a/internal/pkg/handler/create.go +++ b/internal/pkg/handler/create.go @@ -1,6 +1,8 @@ package handler import ( + "time" + "github.com/sirupsen/logrus" "github.com/stakater/Reloader/internal/pkg/metrics" "github.com/stakater/Reloader/internal/pkg/options" @@ -11,23 +13,45 @@ import ( // ResourceCreatedHandler contains new objects type ResourceCreatedHandler struct { - Resource interface{} - Collectors metrics.Collectors - Recorder record.EventRecorder + Resource interface{} + Collectors metrics.Collectors + Recorder record.EventRecorder + EnqueueTime time.Time // Time when this handler was added to the queue +} + +// GetEnqueueTime returns when this handler was enqueued +func (r ResourceCreatedHandler) GetEnqueueTime() time.Time { + return r.EnqueueTime } // Handle processes the newly created resource func (r ResourceCreatedHandler) Handle() error { + startTime := time.Now() + result := "success" + + defer func() { + r.Collectors.RecordReconcile(result, time.Since(startTime)) + }() + if r.Resource == nil { logrus.Errorf("Resource creation handler received nil resource") + result = "error" } else { config, _ := r.GetConfig() // Send webhook if options.WebhookUrl != "" { - return sendUpgradeWebhook(config, options.WebhookUrl) + err := sendUpgradeWebhook(config, options.WebhookUrl) + if err != nil { + result = "error" + } + return err } // process resource based on its type - return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy) + err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy) + if err != nil { + result = "error" + } + return err } return nil } diff --git a/internal/pkg/handler/delete.go b/internal/pkg/handler/delete.go index 65c671e..243602c 100644 --- a/internal/pkg/handler/delete.go +++ b/internal/pkg/handler/delete.go @@ -3,6 +3,7 @@ package handler import ( "fmt" "slices" + "time" "github.com/sirupsen/logrus" "github.com/stakater/Reloader/internal/pkg/callbacks" @@ -20,23 +21,45 @@ import ( // ResourceDeleteHandler contains new objects type ResourceDeleteHandler struct { - Resource interface{} - Collectors metrics.Collectors - Recorder record.EventRecorder + Resource interface{} + Collectors metrics.Collectors + Recorder record.EventRecorder + EnqueueTime time.Time // Time when this handler was added to the queue +} + +// GetEnqueueTime returns when this handler was enqueued +func (r ResourceDeleteHandler) GetEnqueueTime() time.Time { + return r.EnqueueTime } // Handle processes resources being deleted func (r ResourceDeleteHandler) Handle() error { + startTime := time.Now() + result := "success" + + defer func() { + r.Collectors.RecordReconcile(result, time.Since(startTime)) + }() + if r.Resource == nil { logrus.Errorf("Resource delete handler received nil resource") + result = "error" } else { config, _ := r.GetConfig() // Send webhook if options.WebhookUrl != "" { - return sendUpgradeWebhook(config, options.WebhookUrl) + err := sendUpgradeWebhook(config, options.WebhookUrl) + if err != nil { + result = "error" + } + return err } // process resource based on its type - return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeDeleteStrategy) + err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeDeleteStrategy) + if err != nil { + result = "error" + } + return err } return nil } diff --git a/internal/pkg/handler/handler.go b/internal/pkg/handler/handler.go index 1f5858e..9018f80 100644 --- a/internal/pkg/handler/handler.go +++ b/internal/pkg/handler/handler.go @@ -1,9 +1,18 @@ package handler -import "github.com/stakater/Reloader/pkg/common" +import ( + "time" + + "github.com/stakater/Reloader/pkg/common" +) // ResourceHandler handles the creation and update of resources type ResourceHandler interface { Handle() error GetConfig() (common.Config, string) } + +// TimedHandler is a handler that tracks when it was enqueued +type TimedHandler interface { + GetEnqueueTime() time.Time +} diff --git a/internal/pkg/handler/update.go b/internal/pkg/handler/update.go index ae0bb1e..3ae1080 100644 --- a/internal/pkg/handler/update.go +++ b/internal/pkg/handler/update.go @@ -1,6 +1,8 @@ package handler import ( + "time" + "github.com/sirupsen/logrus" "github.com/stakater/Reloader/internal/pkg/metrics" "github.com/stakater/Reloader/internal/pkg/options" @@ -16,21 +18,47 @@ type ResourceUpdatedHandler struct { OldResource interface{} Collectors metrics.Collectors Recorder record.EventRecorder + EnqueueTime time.Time // Time when this handler was added to the queue +} + +// GetEnqueueTime returns when this handler was enqueued +func (r ResourceUpdatedHandler) GetEnqueueTime() time.Time { + return r.EnqueueTime } // Handle processes the updated resource func (r ResourceUpdatedHandler) Handle() error { + startTime := time.Now() + result := "success" + + defer func() { + r.Collectors.RecordReconcile(result, time.Since(startTime)) + }() + if r.Resource == nil || r.OldResource == nil { logrus.Errorf("Resource update handler received nil resource") + result = "error" } else { config, oldSHAData := r.GetConfig() if config.SHAValue != oldSHAData { // Send a webhook if update if options.WebhookUrl != "" { - return sendUpgradeWebhook(config, options.WebhookUrl) + err := sendUpgradeWebhook(config, options.WebhookUrl) + if err != nil { + result = "error" + } + return err } // process resource based on its type - return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy) + err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy) + if err != nil { + result = "error" + } + return err + } else { + // No data change - skip + result = "skipped" + r.Collectors.RecordSkipped("no_data_change") } } return nil diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index 6f185f1..e355d5f 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "time" "github.com/parnurzeal/gorequest" "github.com/prometheus/client_golang/prometheus" @@ -236,23 +237,34 @@ func rollingUpgrade(clients kube.Clients, config common.Config, upgradeFuncs cal func PerformAction(clients kube.Clients, config common.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy) error { items := upgradeFuncs.ItemsFunc(clients, config.Namespace) + // Record workloads scanned + collectors.RecordWorkloadsScanned(upgradeFuncs.ResourceType, len(items)) + + matchedCount := 0 for _, item := range items { - err := retryOnConflict(retry.DefaultRetry, func(fetchResource bool) error { - return upgradeResource(clients, config, upgradeFuncs, collectors, recorder, strategy, item, fetchResource) + err := retryOnConflict(retry.DefaultRetry, func(fetchResource bool) (bool, error) { + matched, err := upgradeResource(clients, config, upgradeFuncs, collectors, recorder, strategy, item, fetchResource) + if matched { + matchedCount++ + } + return matched, err }) if err != nil { return err } } + // Record workloads matched + collectors.RecordWorkloadsMatched(upgradeFuncs.ResourceType, matchedCount) + return nil } -func retryOnConflict(backoff wait.Backoff, fn func(_ bool) error) error { +func retryOnConflict(backoff wait.Backoff, fn func(_ bool) (bool, error)) error { var lastError error fetchResource := false // do not fetch resource on first attempt, already done by ItemsFunc err := wait.ExponentialBackoff(backoff, func() (bool, error) { - err := fn(fetchResource) + _, err := fn(fetchResource) fetchResource = true switch { case err == nil: @@ -270,17 +282,19 @@ func retryOnConflict(backoff wait.Backoff, fn func(_ bool) error) error { return err } -func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy, resource runtime.Object, fetchResource bool) error { +func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy, resource runtime.Object, fetchResource bool) (bool, error) { + actionStartTime := time.Now() + accessor, err := meta.Accessor(resource) if err != nil { - return err + return false, err } resourceName := accessor.GetName() if fetchResource { resource, err = upgradeFuncs.ItemFunc(clients, resourceName, config.Namespace) if err != nil { - return err + return false, err } } annotations := upgradeFuncs.AnnotationsFunc(resource) @@ -289,13 +303,14 @@ func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs ca if !result.ShouldReload { logrus.Debugf("No changes detected in '%s' of type '%s' in namespace '%s'", config.ResourceName, config.Type, config.Namespace) - return nil + return false, nil } strategyResult := strategy(upgradeFuncs, resource, config, result.AutoReload) if strategyResult.Result != constants.Updated { - return nil + collectors.RecordSkipped("strategy_not_updated") + return false, nil } // find correct annotation and update the resource @@ -309,7 +324,7 @@ func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs ca _, err = PauseDeployment(deployment, clients, config.Namespace, pauseInterval) if err != nil { logrus.Errorf("Failed to pause deployment '%s' in namespace '%s': %v", resourceName, config.Namespace, err) - return err + return true, err } } } @@ -320,16 +335,19 @@ func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs ca err = upgradeFuncs.UpdateFunc(clients, config.Namespace, resource) } + actionLatency := time.Since(actionStartTime) + if err != nil { message := fmt.Sprintf("Update for '%s' of type '%s' in namespace '%s' failed with error %v", resourceName, upgradeFuncs.ResourceType, config.Namespace, err) logrus.Errorf("Update for '%s' of type '%s' in namespace '%s' failed with error %v", resourceName, upgradeFuncs.ResourceType, config.Namespace, err) collectors.Reloaded.With(prometheus.Labels{"success": "false"}).Inc() collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "false", "namespace": config.Namespace}).Inc() + collectors.RecordAction(upgradeFuncs.ResourceType, "error", actionLatency) if recorder != nil { recorder.Event(resource, v1.EventTypeWarning, "ReloadFail", message) } - return err + return true, err } else { message := fmt.Sprintf("Changes detected in '%s' of type '%s' in namespace '%s'", config.ResourceName, config.Type, config.Namespace) message += fmt.Sprintf(", Updated '%s' of type '%s' in namespace '%s'", resourceName, upgradeFuncs.ResourceType, config.Namespace) @@ -338,6 +356,7 @@ func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs ca collectors.Reloaded.With(prometheus.Labels{"success": "true"}).Inc() collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": config.Namespace}).Inc() + collectors.RecordAction(upgradeFuncs.ResourceType, "success", actionLatency) alert_on_reload, ok := os.LookupEnv("ALERT_ON_RELOAD") if recorder != nil { recorder.Event(resource, v1.EventTypeNormal, "Reloaded", message) @@ -350,7 +369,7 @@ func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs ca } } - return nil + return true, nil } func getVolumeMountName(volumes []v1.Volume, mountType string, volumeName string) string { diff --git a/internal/pkg/metrics/prometheus.go b/internal/pkg/metrics/prometheus.go index 94153ea..e6f2f35 100644 --- a/internal/pkg/metrics/prometheus.go +++ b/internal/pkg/metrics/prometheus.go @@ -1,54 +1,407 @@ package metrics import ( + "context" "net/http" + "net/url" "os" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "k8s.io/client-go/tools/metrics" ) +// clientGoRequestMetrics implements metrics.LatencyMetric and metrics.ResultMetric +// to expose client-go's rest_client_requests_total metric +type clientGoRequestMetrics struct { + requestCounter *prometheus.CounterVec + requestLatency *prometheus.HistogramVec +} + +func (m *clientGoRequestMetrics) Increment(ctx context.Context, code string, method string, host string) { + m.requestCounter.WithLabelValues(code, method, host).Inc() +} + +func (m *clientGoRequestMetrics) Observe(ctx context.Context, verb string, u url.URL, latency time.Duration) { + m.requestLatency.WithLabelValues(verb, u.Host).Observe(latency.Seconds()) +} + +var clientGoMetrics = &clientGoRequestMetrics{ + requestCounter: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "rest_client_requests_total", + Help: "Number of HTTP requests, partitioned by status code, method, and host.", + }, + []string{"code", "method", "host"}, + ), + requestLatency: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "rest_client_request_duration_seconds", + Help: "Request latency in seconds. Broken down by verb and host.", + Buckets: []float64{0.001, 0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30}, + }, + []string{"verb", "host"}, + ), +} + +func init() { + // Register the metrics collectors + prometheus.MustRegister(clientGoMetrics.requestCounter) + prometheus.MustRegister(clientGoMetrics.requestLatency) + + // Register our metrics implementation with client-go + metrics.RequestResult = clientGoMetrics + metrics.RequestLatency = clientGoMetrics +} + +// Collectors holds all Prometheus metrics collectors for Reloader. type Collectors struct { + // Existing metrics (preserved for backward compatibility) Reloaded *prometheus.CounterVec ReloadedByNamespace *prometheus.CounterVec + countByNamespace bool + + // Reconcile/Handler metrics + ReconcileTotal *prometheus.CounterVec // Total reconcile calls by result + ReconcileDuration *prometheus.HistogramVec // Time spent in reconcile/handler + + // Action metrics + ActionTotal *prometheus.CounterVec // Total actions by workload kind and result + ActionLatency *prometheus.HistogramVec // Time from event to action applied + + // Skip metrics + SkippedTotal *prometheus.CounterVec // Skipped operations by reason + + // Queue metrics + QueueDepth prometheus.Gauge // Current queue depth + QueueAdds prometheus.Counter // Total items added to queue + QueueLatency *prometheus.HistogramVec // Time spent in queue + + // Error and retry metrics + ErrorsTotal *prometheus.CounterVec // Errors by type + RetriesTotal prometheus.Counter // Total retries + + // Event processing metrics + EventsReceived *prometheus.CounterVec // Events received by type (add/update/delete) + EventsProcessed *prometheus.CounterVec // Events processed by type and result + + // Resource discovery metrics + WorkloadsScanned *prometheus.CounterVec // Workloads scanned by kind + WorkloadsMatched *prometheus.CounterVec // Workloads matched for reload by kind +} + +// RecordReload records a reload event with the given success status and namespace. +// Preserved for backward compatibility. +func (c *Collectors) RecordReload(success bool, namespace string) { + if c == nil { + return + } + + successLabel := "false" + if success { + successLabel = "true" + } + + c.Reloaded.With(prometheus.Labels{"success": successLabel}).Inc() + + if c.countByNamespace { + c.ReloadedByNamespace.With(prometheus.Labels{ + "success": successLabel, + "namespace": namespace, + }).Inc() + } +} + +// RecordReconcile records a reconcile/handler invocation. +func (c *Collectors) RecordReconcile(result string, duration time.Duration) { + if c == nil { + return + } + c.ReconcileTotal.With(prometheus.Labels{"result": result}).Inc() + c.ReconcileDuration.With(prometheus.Labels{"result": result}).Observe(duration.Seconds()) +} + +// RecordAction records a reload action on a workload. +func (c *Collectors) RecordAction(workloadKind string, result string, latency time.Duration) { + if c == nil { + return + } + c.ActionTotal.With(prometheus.Labels{"workload_kind": workloadKind, "result": result}).Inc() + c.ActionLatency.With(prometheus.Labels{"workload_kind": workloadKind}).Observe(latency.Seconds()) +} + +// RecordSkipped records a skipped operation with reason. +func (c *Collectors) RecordSkipped(reason string) { + if c == nil { + return + } + c.SkippedTotal.With(prometheus.Labels{"reason": reason}).Inc() +} + +// RecordQueueAdd records an item being added to the queue. +func (c *Collectors) RecordQueueAdd() { + if c == nil { + return + } + c.QueueAdds.Inc() +} + +// SetQueueDepth sets the current queue depth. +func (c *Collectors) SetQueueDepth(depth int) { + if c == nil { + return + } + c.QueueDepth.Set(float64(depth)) +} + +// RecordQueueLatency records how long an item spent in the queue. +func (c *Collectors) RecordQueueLatency(latency time.Duration) { + if c == nil { + return + } + c.QueueLatency.With(prometheus.Labels{}).Observe(latency.Seconds()) +} + +// RecordError records an error by type. +func (c *Collectors) RecordError(errorType string) { + if c == nil { + return + } + c.ErrorsTotal.With(prometheus.Labels{"type": errorType}).Inc() +} + +// RecordRetry records a retry attempt. +func (c *Collectors) RecordRetry() { + if c == nil { + return + } + c.RetriesTotal.Inc() +} + +// RecordEventReceived records an event being received. +func (c *Collectors) RecordEventReceived(eventType string, resourceType string) { + if c == nil { + return + } + c.EventsReceived.With(prometheus.Labels{"event_type": eventType, "resource_type": resourceType}).Inc() +} + +// RecordEventProcessed records an event being processed. +func (c *Collectors) RecordEventProcessed(eventType string, resourceType string, result string) { + if c == nil { + return + } + c.EventsProcessed.With(prometheus.Labels{"event_type": eventType, "resource_type": resourceType, "result": result}).Inc() +} + +// RecordWorkloadsScanned records workloads scanned during a reconcile. +func (c *Collectors) RecordWorkloadsScanned(kind string, count int) { + if c == nil { + return + } + c.WorkloadsScanned.With(prometheus.Labels{"kind": kind}).Add(float64(count)) +} + +// RecordWorkloadsMatched records workloads matched for reload. +func (c *Collectors) RecordWorkloadsMatched(kind string, count int) { + if c == nil { + return + } + c.WorkloadsMatched.With(prometheus.Labels{"kind": kind}).Add(float64(count)) } func NewCollectors() Collectors { + // Existing metrics (preserved) reloaded := prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "reloader", Name: "reload_executed_total", Help: "Counter of reloads executed by Reloader.", }, - []string{ - "success", - }, + []string{"success"}, ) - - //set 0 as default value reloaded.With(prometheus.Labels{"success": "true"}).Add(0) reloaded.With(prometheus.Labels{"success": "false"}).Add(0) - reloaded_by_namespace := prometheus.NewCounterVec( + reloadedByNamespace := prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "reloader", Name: "reload_executed_total_by_namespace", Help: "Counter of reloads executed by Reloader by namespace.", }, - []string{ - "success", - "namespace", + []string{"success", "namespace"}, + ) + + // === NEW: Comprehensive metrics === + + reconcileTotal := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "reconcile_total", + Help: "Total number of reconcile/handler invocations by result.", + }, + []string{"result"}, + ) + + reconcileDuration := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "reloader", + Name: "reconcile_duration_seconds", + Help: "Time spent in reconcile/handler in seconds.", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10}, + }, + []string{"result"}, + ) + + actionTotal := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "action_total", + Help: "Total number of reload actions by workload kind and result.", + }, + []string{"workload_kind", "result"}, + ) + + actionLatency := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "reloader", + Name: "action_latency_seconds", + Help: "Time from event received to action applied in seconds.", + Buckets: []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60}, + }, + []string{"workload_kind"}, + ) + + skippedTotal := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "skipped_total", + Help: "Total number of skipped operations by reason.", + }, + []string{"reason"}, + ) + + queueDepth := prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "reloader", + Name: "workqueue_depth", + Help: "Current depth of the work queue.", }, ) + + queueAdds := prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "workqueue_adds_total", + Help: "Total number of items added to the work queue.", + }, + ) + + queueLatency := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "reloader", + Name: "workqueue_latency_seconds", + Help: "Time spent in the work queue in seconds.", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5}, + }, + []string{}, + ) + + errorsTotal := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "errors_total", + Help: "Total number of errors by type.", + }, + []string{"type"}, + ) + + retriesTotal := prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "retries_total", + Help: "Total number of retry attempts.", + }, + ) + + eventsReceived := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "events_received_total", + Help: "Total number of events received by type and resource.", + }, + []string{"event_type", "resource_type"}, + ) + + eventsProcessed := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "events_processed_total", + Help: "Total number of events processed by type, resource, and result.", + }, + []string{"event_type", "resource_type", "result"}, + ) + + workloadsScanned := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "workloads_scanned_total", + Help: "Total number of workloads scanned by kind.", + }, + []string{"kind"}, + ) + + workloadsMatched := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "reloader", + Name: "workloads_matched_total", + Help: "Total number of workloads matched for reload by kind.", + }, + []string{"kind"}, + ) + return Collectors{ Reloaded: reloaded, - ReloadedByNamespace: reloaded_by_namespace, + ReloadedByNamespace: reloadedByNamespace, + countByNamespace: os.Getenv("METRICS_COUNT_BY_NAMESPACE") == "enabled", + + ReconcileTotal: reconcileTotal, + ReconcileDuration: reconcileDuration, + ActionTotal: actionTotal, + ActionLatency: actionLatency, + SkippedTotal: skippedTotal, + QueueDepth: queueDepth, + QueueAdds: queueAdds, + QueueLatency: queueLatency, + ErrorsTotal: errorsTotal, + RetriesTotal: retriesTotal, + EventsReceived: eventsReceived, + EventsProcessed: eventsProcessed, + WorkloadsScanned: workloadsScanned, + WorkloadsMatched: workloadsMatched, } } func SetupPrometheusEndpoint() Collectors { collectors := NewCollectors() + + // Register all metrics prometheus.MustRegister(collectors.Reloaded) + prometheus.MustRegister(collectors.ReconcileTotal) + prometheus.MustRegister(collectors.ReconcileDuration) + prometheus.MustRegister(collectors.ActionTotal) + prometheus.MustRegister(collectors.ActionLatency) + prometheus.MustRegister(collectors.SkippedTotal) + prometheus.MustRegister(collectors.QueueDepth) + prometheus.MustRegister(collectors.QueueAdds) + prometheus.MustRegister(collectors.QueueLatency) + prometheus.MustRegister(collectors.ErrorsTotal) + prometheus.MustRegister(collectors.RetriesTotal) + prometheus.MustRegister(collectors.EventsReceived) + prometheus.MustRegister(collectors.EventsProcessed) + prometheus.MustRegister(collectors.WorkloadsScanned) + prometheus.MustRegister(collectors.WorkloadsMatched) if os.Getenv("METRICS_COUNT_BY_NAMESPACE") == "enabled" { prometheus.MustRegister(collectors.ReloadedByNamespace) diff --git a/test/loadtest/README.md b/test/loadtest/README.md new file mode 100644 index 0000000..7182bb3 --- /dev/null +++ b/test/loadtest/README.md @@ -0,0 +1,544 @@ +# Reloader Load Test Framework + +This framework provides A/B comparison testing between two Reloader container images. + +## Overview + +The load test framework: +1. Creates a local kind cluster (1 control-plane + 6 worker nodes) +2. Deploys Prometheus for metrics collection +3. Loads the provided Reloader container images into the cluster +4. Runs standardized test scenarios (S1-S13) +5. Collects metrics via Prometheus scraping +6. Generates comparison reports with pass/fail criteria + +## Prerequisites + +- Docker or Podman +- kind (Kubernetes in Docker) +- kubectl +- Go 1.22+ + +## Building + +```bash +cd test/loadtest +go build -o loadtest ./cmd/loadtest +``` + +## Quick Start + +```bash +# Compare two published images (e.g., different versions) +./loadtest run \ + --old-image=stakater/reloader:v1.0.0 \ + --new-image=stakater/reloader:v1.1.0 + +# Run a specific scenario +./loadtest run \ + --old-image=stakater/reloader:v1.0.0 \ + --new-image=stakater/reloader:v1.1.0 \ + --scenario=S2 \ + --duration=120 + +# Test only a single image (no comparison) +./loadtest run --new-image=myregistry/reloader:dev + +# Use local images built with docker/podman +./loadtest run \ + --old-image=localhost/reloader:baseline \ + --new-image=localhost/reloader:feature-branch + +# Skip cluster creation (use existing kind cluster) +./loadtest run \ + --old-image=stakater/reloader:v1.0.0 \ + --new-image=stakater/reloader:v1.1.0 \ + --skip-cluster + +# Run all scenarios in parallel on 4 clusters (faster execution) +./loadtest run \ + --new-image=localhost/reloader:dev \ + --parallelism=4 + +# Run all 13 scenarios in parallel (one cluster per scenario) +./loadtest run \ + --new-image=localhost/reloader:dev \ + --parallelism=13 + +# Generate report from existing results +./loadtest report --scenario=S2 --results-dir=./results +``` + +## Command Line Options + +### Run Command + +| Option | Description | Default | +|--------|-------------|---------| +| `--old-image=IMAGE` | Container image for "old" version | - | +| `--new-image=IMAGE` | Container image for "new" version | - | +| `--scenario=ID` | Test scenario: S1-S13 or "all" | all | +| `--duration=SECONDS` | Test duration in seconds | 60 | +| `--parallelism=N` | Run N scenarios in parallel on N kind clusters | 1 | +| `--skip-cluster` | Skip kind cluster creation (use existing, only for parallelism=1) | false | +| `--results-dir=DIR` | Directory for results | ./results | + +**Note:** At least one of `--old-image` or `--new-image` is required. Provide both for A/B comparison. + +### Report Command + +| Option | Description | Default | +|--------|-------------|---------| +| `--scenario=ID` | Scenario to report on (required) | - | +| `--results-dir=DIR` | Directory containing results | ./results | +| `--output=FILE` | Output file (default: stdout) | - | + +## Test Scenarios + +| ID | Name | Description | +|-----|-----------------------|-------------------------------------------------| +| S1 | Burst Updates | Many ConfigMap/Secret updates in quick succession | +| S2 | Fan-Out | One ConfigMap used by many (50) workloads | +| S3 | High Cardinality | Many CMs/Secrets across many namespaces | +| S4 | No-Op Updates | Updates that don't change data (annotation only)| +| S5 | Workload Churn | Deployments created/deleted rapidly | +| S6 | Controller Restart | Restart controller pod under load | +| S7 | API Pressure | Many concurrent update requests | +| S8 | Large Objects | ConfigMaps > 100KB | +| S9 | Multi-Workload Types | Tests all workload types (Deploy, STS, DS) | +| S10 | Secrets + Mixed | Secrets and mixed ConfigMap+Secret workloads | +| S11 | Annotation Strategy | Tests `--reload-strategy=annotations` | +| S12 | Pause & Resume | Tests pause-period during rapid updates | +| S13 | Complex References | Init containers, valueFrom, projected volumes | + +## Metrics Reference + +This section explains each metric collected during load tests, what it measures, and what different values might indicate. + +### Counter Metrics (Totals) + +#### `reconcile_total` +**What it measures:** The total number of reconciliation loops executed by the controller. + +**What it indicates:** +- **Higher in new vs old:** The new controller-runtime implementation may batch events differently. This is often expected behavior, not a problem. +- **Lower in new vs old:** Better event batching/deduplication. Controller-runtime's work queue naturally deduplicates events. +- **Expected behavior:** The new implementation typically has *fewer* reconciles due to intelligent event batching. + +#### `action_total` +**What it measures:** The total number of reload actions triggered (rolling restarts of Deployments/StatefulSets/DaemonSets). + +**What it indicates:** +- **Should match expected value:** Both implementations should trigger the same number of reloads for the same workload. +- **Lower than expected:** Some updates were missed - potential bug or race condition. +- **Higher than expected:** Duplicate reloads triggered - inefficiency but not data loss. + +#### `reload_executed_total` +**What it measures:** Successful reload operations executed, labeled by `success=true/false`. + +**What it indicates:** +- **`success=true` count:** Number of workloads successfully restarted. +- **`success=false` count:** Failed restart attempts (API errors, permission issues). +- **Should match `action_total`:** If significantly lower, reloads are failing. + +#### `workloads_scanned_total` +**What it measures:** Number of workloads (Deployments, etc.) scanned when checking for ConfigMap/Secret references. + +**What it indicates:** +- **High count:** Controller is scanning many workloads per reconcile. +- **Expected behavior:** Should roughly match the number of workloads Ɨ number of reconciles. +- **Optimization signal:** If very high, namespace filtering or label selectors could help. + +#### `workloads_matched_total` +**What it measures:** Number of workloads that matched (reference the changed ConfigMap/Secret). + +**What it indicates:** +- **Should match `reload_executed_total`:** Every matched workload should be reloaded. +- **Higher than reloads:** Some matched workloads weren't reloaded (potential issue). + +#### `errors_total` +**What it measures:** Total errors encountered, labeled by error type. + +**What it indicates:** +- **Should be 0:** Any errors indicate problems. +- **Common causes:** API server timeouts, RBAC issues, resource conflicts. +- **Critical metric:** Non-zero errors in production should be investigated. + +### API Efficiency Metrics (REST Client) + +These metrics track Kubernetes API server calls made by Reloader. Lower values indicate more efficient operation with less API server load. + +#### `rest_client_requests_total` +**What it measures:** Total number of HTTP requests made to the Kubernetes API server. + +**What it indicates:** +- **Lower is better:** Fewer API calls means less load on the API server. +- **High count:** May indicate inefficient caching or excessive reconciles. +- **Comparison use:** Shows overall API efficiency between implementations. + +#### `rest_client_requests_get` +**What it measures:** Number of GET requests (fetching individual resources or listings). + +**What it indicates:** +- **Includes:** Fetching ConfigMaps, Secrets, Deployments, etc. +- **Higher count:** More frequent resource fetching, possibly due to cache misses. +- **Expected behavior:** Controller-runtime's caching should reduce GET requests compared to direct API calls. + +#### `rest_client_requests_patch` +**What it measures:** Number of PATCH requests (partial updates to resources). + +**What it indicates:** +- **Used for:** Rolling restart annotations on workloads. +- **Should correlate with:** `reload_executed_total` - each reload typically requires one PATCH. +- **Lower is better:** Fewer patches means more efficient batching or deduplication. + +#### `rest_client_requests_put` +**What it measures:** Number of PUT requests (full resource updates). + +**What it indicates:** +- **Used for:** Full object replacements (less common than PATCH). +- **Should be low:** Most updates use PATCH for efficiency. +- **High count:** May indicate suboptimal update strategy. + +#### `rest_client_requests_errors` +**What it measures:** Number of failed API requests (4xx/5xx responses). + +**What it indicates:** +- **Should be 0:** Errors indicate API server issues or permission problems. +- **Common causes:** Rate limiting, RBAC issues, resource conflicts, network issues. +- **Non-zero:** Investigate API server logs and Reloader permissions. + +### Latency Metrics (Percentiles) + +All latency metrics are reported in **seconds**. The report shows p50 (median), p95, and p99 percentiles. + +#### `reconcile_duration (s)` +**What it measures:** Time spent inside each reconcile loop, from start to finish. + +**What it indicates:** +- **p50 (median):** Typical reconcile time. Should be < 100ms for good performance. +- **p95:** 95th percentile - only 5% of reconciles take longer than this. +- **p99:** 99th percentile - indicates worst-case performance. + +**Interpreting differences:** +- **New higher than old:** Controller-runtime reconciles may do more work per loop but run fewer times. Check `reconcile_total` - if it's lower, this is expected. +- **Minor differences (< 0.5s absolute):** Not significant for sub-second values. + +#### `action_latency (s)` +**What it measures:** End-to-end time from ConfigMap/Secret change detection to workload restart triggered. + +**What it indicates:** +- **This is the user-facing latency:** How long users wait for their config changes to take effect. +- **p50 < 1s:** Excellent - most changes apply within a second. +- **p95 < 5s:** Good - even under load, changes apply quickly. +- **p99 > 10s:** May need investigation - some changes take too long. + +**What affects this:** +- API server responsiveness +- Number of workloads to scan +- Concurrent updates competing for resources + +### Understanding the Report + +#### Report Columns + +``` +Metric Old New Expected Oldāœ“ Newāœ“ Status +------ --- --- -------- ---- ---- ------ +action_total 100.00 100.00 100 āœ“ āœ“ pass +action_latency_p95 (s) 0.15 0.04 - - - pass +``` + +- **Old/New:** Measured values from each implementation +- **Expected:** Known expected value (for throughput metrics) +- **Oldāœ“/Newāœ“:** Whether the value is within 15% of expected (āœ“ = yes, āœ— = no, - = no expected value) +- **Status:** pass/fail based on comparison thresholds + +#### Pass/Fail Logic + +| Metric Type | Pass Condition | +|-------------|----------------| +| Throughput (action_total, reload_executed_total) | New value within 15% of expected | +| Latency (p50, p95, p99) | New not more than threshold% worse than old, OR absolute difference < minimum threshold | +| Errors | New ā‰¤ Old (ideally both 0) | +| API Efficiency (rest_client_requests_*) | New ā‰¤ Old (lower is better), or New not more than 50% higher | + +#### Latency Thresholds + +Latency comparisons use both percentage AND absolute thresholds to avoid false failures: + +| Metric | Max % Worse | Min Absolute Diff | +|--------|-------------|-------------------| +| p50 | 100% | 0.5s | +| p95 | 100% | 1.0s | +| p99 | 100% | 1.0s | + +**Example:** If old p50 = 0.01s and new p50 = 0.08s: +- Percentage difference: +700% (would fail % check) +- Absolute difference: 0.07s (< 0.5s threshold) +- **Result: PASS** (both values are fast enough that the difference doesn't matter) + +### Resource Consumption Metrics + +These metrics track CPU, memory, and Go runtime resource usage. Lower values generally indicate more efficient operation. + +#### Memory Metrics + +| Metric | Description | Unit | +|--------|-------------|------| +| `memory_rss_mb_avg` | Average RSS (resident set size) memory | MB | +| `memory_rss_mb_max` | Peak RSS memory during test | MB | +| `memory_heap_mb_avg` | Average Go heap allocation | MB | +| `memory_heap_mb_max` | Peak Go heap allocation | MB | + +**What to watch for:** +- **High RSS:** May indicate memory leaks or inefficient caching +- **High heap:** Many objects being created (check GC metrics) +- **Growing over time:** Potential memory leak + +#### CPU Metrics + +| Metric | Description | Unit | +|--------|-------------|------| +| `cpu_cores_avg` | Average CPU usage rate | cores | +| `cpu_cores_max` | Peak CPU usage rate | cores | + +**What to watch for:** +- **High CPU:** Inefficient algorithms or excessive reconciles +- **Spiky max:** May indicate burst handling issues + +#### Go Runtime Metrics + +| Metric | Description | Unit | +|--------|-------------|------| +| `goroutines_avg` | Average goroutine count | count | +| `goroutines_max` | Peak goroutine count | count | +| `gc_pause_p99_ms` | 99th percentile GC pause time | ms | + +**What to watch for:** +- **High goroutines:** Potential goroutine leak or unbounded concurrency +- **High GC pause:** Large heap or allocation pressure + +### Scenario-Specific Expectations + +| Scenario | Key Metrics to Watch | Expected Behavior | +|----------|---------------------|-------------------| +| S1 (Burst) | action_latency_p99, cpu_cores_max, goroutines_max | Should handle bursts without queue backup | +| S2 (Fan-Out) | reconcile_total, workloads_matched, memory_rss_mb_max | One CM change ā†’ 50 workload reloads | +| S3 (High Cardinality) | reconcile_duration, memory_heap_mb_avg | Many namespaces shouldn't increase memory | +| S4 (No-Op) | action_total = 0, cpu_cores_avg should be low | Minimal resource usage for no-op | +| S5 (Churn) | errors_total, goroutines_avg | Graceful handling, no goroutine leak | +| S6 (Restart) | All metrics captured | Metrics survive controller restart | +| S7 (API Pressure) | errors_total, cpu_cores_max, goroutines_max | No errors under concurrent load | +| S8 (Large Objects) | memory_rss_mb_max, gc_pause_p99_ms | Large ConfigMaps don't cause OOM or GC issues | +| S9 (Multi-Workload) | reload_executed_total per type | All workload types (Deploy, STS, DS) reload | +| S10 (Secrets) | reload_executed_total, workloads_matched | Both Secrets and ConfigMaps trigger reloads | +| S11 (Annotation) | workload annotations present | Deployments get `last-reloaded-from` annotation | +| S12 (Pause) | reload_executed_total << updates | Pause-period reduces reload frequency | +| S13 (Complex) | reload_executed_total | All reference types trigger reloads | + +### Troubleshooting + +#### New implementation shows 0 for all metrics +- Check if Prometheus is scraping the new Reloader pod +- Verify pod annotations: `prometheus.io/scrape: "true"` +- Check Prometheus targets: `http://localhost:9091/targets` + +#### Metrics don't match expected values +- Verify test ran to completion (check logs) +- Ensure Prometheus scraped final metrics (18s wait after test) +- Check for pod restarts during test (metrics reset on restart - handled by `increase()`) + +#### High latency in new implementation +- Check Reloader pod resource limits +- Look for API server throttling in logs +- Compare `reconcile_total` - fewer reconciles with higher duration may be normal + +#### REST client errors are non-zero +- **Common causes:** + - Optional CRD schemes registered but CRDs not installed (e.g., Argo Rollouts, OpenShift DeploymentConfig) + - API server rate limiting under high load + - RBAC permissions missing for certain resource types +- **Argo Rollouts errors:** If you see ~4 errors per test, ensure `--enable-argo-rollouts=false` if not using Argo Rollouts +- **OpenShift errors:** Similarly, ensure DeploymentConfig support is disabled on non-OpenShift clusters + +#### REST client requests much higher in new implementation +- Check if caching is working correctly +- Look for excessive re-queuing in controller logs +- Compare `reconcile_total` - more reconciles naturally means more API calls + +## Report Format + +The report generator produces a comparison table with units and expected value indicators: + +``` +================================================================================ + RELOADER A/B COMPARISON REPORT +================================================================================ + +Scenario: S2 +Generated: 2026-01-03 14:30:00 +Status: PASS +Summary: All metrics within acceptable thresholds + +Test: S2: Fan-out test - 1 CM update triggers 50 deployment reloads + +-------------------------------------------------------------------------------- + METRIC COMPARISONS +-------------------------------------------------------------------------------- +(Oldāœ“/Newāœ“ = meets expected value within 15%) + +Metric Old New Expected Oldāœ“ Newāœ“ Status +------ --- --- -------- ---- ---- ------ +reconcile_total 50.00 25.00 - - - pass +reconcile_duration_p50 (s) 0.01 0.05 - - - pass +reconcile_duration_p95 (s) 0.02 0.15 - - - pass +action_total 50.00 50.00 50 āœ“ āœ“ pass +action_latency_p50 (s) 0.05 0.03 - - - pass +action_latency_p95 (s) 0.12 0.08 - - - pass +errors_total 0.00 0.00 - - - pass +reload_executed_total 50.00 50.00 50 āœ“ āœ“ pass +workloads_scanned_total 50.00 50.00 50 āœ“ āœ“ pass +workloads_matched_total 50.00 50.00 50 āœ“ āœ“ pass +rest_client_requests_total 850 720 - - - pass +rest_client_requests_get 500 420 - - - pass +rest_client_requests_patch 300 250 - - - pass +rest_client_requests_errors 0 0 - - - pass +``` + +Reports are saved to `results//report.txt` after each test. + +## Directory Structure + +``` +test/loadtest/ +ā”œā”€ā”€ cmd/ +ā”‚ ā””ā”€ā”€ loadtest/ # Unified CLI (run + report) +ā”‚ ā””ā”€ā”€ main.go +ā”œā”€ā”€ internal/ +ā”‚ ā”œā”€ā”€ cluster/ # Kind cluster management +ā”‚ ā”‚ ā””ā”€ā”€ kind.go +ā”‚ ā”œā”€ā”€ prometheus/ # Prometheus deployment & querying +ā”‚ ā”‚ ā””ā”€ā”€ prometheus.go +ā”‚ ā”œā”€ā”€ reloader/ # Reloader deployment +ā”‚ ā”‚ ā””ā”€ā”€ deploy.go +ā”‚ ā””ā”€ā”€ scenarios/ # Test scenario implementations +ā”‚ ā””ā”€ā”€ scenarios.go +ā”œā”€ā”€ manifests/ +ā”‚ ā””ā”€ā”€ prometheus.yaml # Prometheus deployment manifest +ā”œā”€ā”€ results/ # Generated after tests +ā”‚ ā””ā”€ā”€ / +ā”‚ ā”œā”€ā”€ old/ # Old version data +ā”‚ ā”‚ ā”œā”€ā”€ *.json # Prometheus metric snapshots +ā”‚ ā”‚ ā””ā”€ā”€ reloader.log # Reloader pod logs +ā”‚ ā”œā”€ā”€ new/ # New version data +ā”‚ ā”‚ ā”œā”€ā”€ *.json # Prometheus metric snapshots +ā”‚ ā”‚ ā””ā”€ā”€ reloader.log # Reloader pod logs +ā”‚ ā”œā”€ā”€ expected.json # Expected values from test +ā”‚ ā””ā”€ā”€ report.txt # Comparison report +ā”œā”€ā”€ go.mod +ā”œā”€ā”€ go.sum +ā””ā”€ā”€ README.md +``` + +## Building Local Images for Testing + +If you want to test local code changes: + +```bash +# Build the new Reloader image from current source +docker build -t localhost/reloader:dev -f Dockerfile . + +# Build from a different branch/commit +git checkout feature-branch +docker build -t localhost/reloader:feature -f Dockerfile . + +# Then run comparison +./loadtest run \ + --old-image=stakater/reloader:v1.0.0 \ + --new-image=localhost/reloader:feature +``` + +## Interpreting Results + +### PASS +All metrics are within acceptable thresholds. The new implementation is comparable or better than the old one. + +### FAIL +One or more metrics exceeded thresholds. Review the specific metrics: +- **Latency degradation**: p95/p99 latencies are significantly higher +- **Missed reloads**: `reload_executed_total` differs significantly +- **Errors increased**: `errors_total` is higher in new version + +### Investigation + +If tests fail, check: +1. Pod logs: `kubectl logs -n reloader-new deployment/reloader` (or check `results//new/reloader.log`) +2. Resource usage: `kubectl top pods -n reloader-new` +3. Events: `kubectl get events -n reloader-test` + +## Parallel Execution + +The `--parallelism` option enables running scenarios on multiple kind clusters simultaneously, significantly reducing total test time. + +### How It Works + +1. **Multiple Clusters**: Creates N kind clusters named `reloader-loadtest-0`, `reloader-loadtest-1`, etc. +2. **Separate Prometheus**: Each cluster gets its own Prometheus instance with a unique port (9091, 9092, etc.) +3. **Worker Pool**: Scenarios are distributed to workers via a channel, with each worker running on its own cluster +4. **Independent Execution**: Each scenario runs in complete isolation with no resource contention + +### Usage + +```bash +# Run 4 scenarios at a time (creates 4 clusters) +./loadtest run --new-image=my-image:tag --parallelism=4 + +# Run all 13 scenarios in parallel (creates 13 clusters) +./loadtest run --new-image=my-image:tag --parallelism=13 --scenario=all +``` + +### Resource Requirements + +Parallel execution requires significant system resources: + +| Parallelism | Clusters | Est. Memory | Est. CPU | +|-------------|----------|-------------|----------| +| 1 (default) | 1 | ~4GB | 2-4 cores | +| 4 | 4 | ~16GB | 8-16 cores | +| 13 | 13 | ~52GB | 26-52 cores | + +### Notes + +- The `--skip-cluster` option is not supported with parallelism > 1 +- Each worker loads images independently, so initial setup takes longer +- All results are written to the same `--results-dir` with per-scenario subdirectories +- If a cluster setup fails, remaining workers continue with available clusters +- Parallelism automatically reduces to match scenario count if set higher + +## CI Integration + +### GitHub Actions + +Load tests can be triggered on pull requests by commenting `/loadtest`: + +``` +/loadtest +``` + +This will: +1. Build a container image from the PR branch +2. Run all load test scenarios against it +3. Post results as a PR comment +4. Upload detailed results as artifacts + +### Make Target + +Run load tests locally or in CI: + +```bash +# From repository root +make loadtest +``` + +This builds the container image and runs all scenarios with a 60-second duration. diff --git a/test/loadtest/cmd/loadtest/main.go b/test/loadtest/cmd/loadtest/main.go new file mode 100644 index 0000000..19f7b1d --- /dev/null +++ b/test/loadtest/cmd/loadtest/main.go @@ -0,0 +1,1582 @@ +// Package main implements the unified load test CLI for Reloader A/B comparison. +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "math" + "os" + "os/exec" + "os/signal" + "path/filepath" + "sort" + "strings" + "sync" + "syscall" + "time" + + "github.com/stakater/Reloader/test/loadtest/internal/cluster" + "github.com/stakater/Reloader/test/loadtest/internal/prometheus" + "github.com/stakater/Reloader/test/loadtest/internal/reloader" + "github.com/stakater/Reloader/test/loadtest/internal/scenarios" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + clusterName = "reloader-loadtest" + testNamespace = "reloader-test" +) + +// workerContext holds all resources for a single worker (cluster + prometheus). +type workerContext struct { + id int + clusterMgr *cluster.Manager + promMgr *prometheus.Manager + kubeClient kubernetes.Interface + kubeContext string + runtime string +} + +// Config holds CLI configuration. +type Config struct { + OldImage string + NewImage string + Scenario string + Duration int + SkipCluster bool + ResultsDir string + ManifestsDir string + Parallelism int // Number of parallel clusters (1 = sequential) +} + +func main() { + if len(os.Args) < 2 { + printUsage() + os.Exit(1) + } + + cmd := os.Args[1] + switch cmd { + case "run": + runCommand(os.Args[2:]) + case "report": + reportCommand(os.Args[2:]) + case "help", "--help", "-h": + printUsage() + default: + fmt.Printf("Unknown command: %s\n", cmd) + printUsage() + os.Exit(1) + } +} + +func printUsage() { + fmt.Print(`Reloader Load Test CLI + +Usage: + loadtest run [options] Run A/B comparison tests + loadtest report [options] Generate comparison report + loadtest help Show this help + +Run Options: + --old-image=IMAGE Container image for "old" version (required for comparison) + --new-image=IMAGE Container image for "new" version (required for comparison) + --scenario=ID Test scenario: S1-S13 or "all" (default: all) + --duration=SECONDS Test duration in seconds (default: 60) + --parallelism=N Run N scenarios in parallel on N clusters (default: 1) + --skip-cluster Skip kind cluster creation (use existing, only for parallelism=1) + --results-dir=DIR Directory for results (default: ./results) + +Report Options: + --scenario=ID Scenario to report on (required) + --results-dir=DIR Directory containing results (default: ./results) + --output=FILE Output file (default: stdout) + +Examples: + # Compare two images + loadtest run --old-image=stakater/reloader:v1.0.0 --new-image=stakater/reloader:v1.1.0 + + # Run specific scenario + loadtest run --old-image=stakater/reloader:v1.0.0 --new-image=localhost/reloader:dev --scenario=S2 + + # Test single image (no comparison) + loadtest run --new-image=localhost/reloader:test + + # Run all scenarios in parallel on 4 clusters + loadtest run --new-image=localhost/reloader:test --parallelism=4 + + # Run all 13 scenarios in parallel (one cluster per scenario) + loadtest run --new-image=localhost/reloader:test --parallelism=13 + + # Generate report + loadtest report --scenario=S2 --results-dir=./results +`) +} + +func parseArgs(args []string) Config { + cfg := Config{ + Scenario: "all", + Duration: 60, + ResultsDir: "./results", + Parallelism: 1, + } + + // Find manifests dir relative to executable or current dir + execPath, _ := os.Executable() + execDir := filepath.Dir(execPath) + cfg.ManifestsDir = filepath.Join(execDir, "..", "..", "manifests") + if _, err := os.Stat(cfg.ManifestsDir); os.IsNotExist(err) { + // Try relative to current dir + cfg.ManifestsDir = "./manifests" + } + + for _, arg := range args { + switch { + case strings.HasPrefix(arg, "--old-image="): + cfg.OldImage = strings.TrimPrefix(arg, "--old-image=") + case strings.HasPrefix(arg, "--new-image="): + cfg.NewImage = strings.TrimPrefix(arg, "--new-image=") + case strings.HasPrefix(arg, "--scenario="): + cfg.Scenario = strings.TrimPrefix(arg, "--scenario=") + case strings.HasPrefix(arg, "--duration="): + fmt.Sscanf(strings.TrimPrefix(arg, "--duration="), "%d", &cfg.Duration) + case strings.HasPrefix(arg, "--parallelism="): + fmt.Sscanf(strings.TrimPrefix(arg, "--parallelism="), "%d", &cfg.Parallelism) + case arg == "--skip-cluster": + cfg.SkipCluster = true + case strings.HasPrefix(arg, "--results-dir="): + cfg.ResultsDir = strings.TrimPrefix(arg, "--results-dir=") + case strings.HasPrefix(arg, "--manifests-dir="): + cfg.ManifestsDir = strings.TrimPrefix(arg, "--manifests-dir=") + } + } + + // Validate parallelism + if cfg.Parallelism < 1 { + cfg.Parallelism = 1 + } + + return cfg +} + +func runCommand(args []string) { + cfg := parseArgs(args) + + // Validate required args + if cfg.OldImage == "" && cfg.NewImage == "" { + log.Fatal("At least one of --old-image or --new-image is required") + } + + // Determine mode + runOld := cfg.OldImage != "" + runNew := cfg.NewImage != "" + runBoth := runOld && runNew + + log.Printf("Configuration:") + log.Printf(" Scenario: %s", cfg.Scenario) + log.Printf(" Duration: %ds", cfg.Duration) + log.Printf(" Parallelism: %d", cfg.Parallelism) + if cfg.OldImage != "" { + log.Printf(" Old image: %s", cfg.OldImage) + } + if cfg.NewImage != "" { + log.Printf(" New image: %s", cfg.NewImage) + } + + // Detect container runtime + runtime, err := cluster.DetectContainerRuntime() + if err != nil { + log.Fatalf("Failed to detect container runtime: %v", err) + } + log.Printf(" Container runtime: %s", runtime) + + // Setup context with signal handling + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigCh + log.Println("Received shutdown signal...") + cancel() + }() + + // Determine scenarios to run + scenariosToRun := []string{cfg.Scenario} + if cfg.Scenario == "all" { + scenariosToRun = []string{"S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8", "S9", "S10", "S11", "S12", "S13"} + } + + // Skip-cluster only works for parallelism=1 + if cfg.SkipCluster && cfg.Parallelism > 1 { + log.Fatal("--skip-cluster is not supported with --parallelism > 1") + } + + // If parallelism > 1, use parallel execution + if cfg.Parallelism > 1 { + runParallel(ctx, cfg, scenariosToRun, runtime, runOld, runNew, runBoth) + return + } + + // Sequential execution (parallelism == 1) + runSequential(ctx, cfg, scenariosToRun, runtime, runOld, runNew, runBoth) +} + +// runSequential runs scenarios one by one on a single cluster. +func runSequential(ctx context.Context, cfg Config, scenariosToRun []string, runtime string, runOld, runNew, runBoth bool) { + // Create cluster manager + clusterMgr := cluster.NewManager(cluster.Config{ + Name: clusterName, + ContainerRuntime: runtime, + }) + + // Create/verify cluster + if cfg.SkipCluster { + log.Println("Skipping cluster creation (using existing)") + if !clusterMgr.Exists() { + log.Fatalf("Cluster %s does not exist. Remove --skip-cluster to create it.", clusterName) + } + } else { + log.Println("Creating kind cluster...") + if err := clusterMgr.Create(ctx); err != nil { + log.Fatalf("Failed to create cluster: %v", err) + } + } + + // Deploy Prometheus + promManifest := filepath.Join(cfg.ManifestsDir, "prometheus.yaml") + promMgr := prometheus.NewManager(promManifest) + + log.Println("Installing Prometheus...") + if err := promMgr.Deploy(ctx); err != nil { + log.Fatalf("Failed to deploy Prometheus: %v", err) + } + + if err := promMgr.StartPortForward(ctx); err != nil { + log.Fatalf("Failed to start Prometheus port-forward: %v", err) + } + defer promMgr.StopPortForward() + + // Load images into kind + log.Println("Loading images into kind cluster...") + if runOld { + log.Printf("Loading old image: %s", cfg.OldImage) + if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { + log.Fatalf("Failed to load old image: %v", err) + } + } + if runNew { + log.Printf("Loading new image: %s", cfg.NewImage) + if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { + log.Fatalf("Failed to load new image: %v", err) + } + } + + // Pre-pull test images + log.Println("Pre-loading test images...") + testImage := "gcr.io/google-containers/busybox:1.27" + clusterMgr.LoadImage(ctx, testImage) // Ignore errors + + // Get kubernetes client + kubeClient, err := getKubeClient("") + if err != nil { + log.Fatalf("Failed to create kubernetes client: %v", err) + } + + for _, scenarioID := range scenariosToRun { + log.Printf("========================================") + log.Printf("=== Starting scenario %s ===", scenarioID) + log.Printf("========================================") + + // Clean up from previous scenario + cleanupTestNamespaces(ctx, "") + cleanupReloader(ctx, "old", "") + cleanupReloader(ctx, "new", "") + + // Reset Prometheus + if err := promMgr.Reset(ctx); err != nil { + log.Printf("Warning: failed to reset Prometheus: %v", err) + } + + // Create test namespace + createTestNamespace(ctx, "") + + if runOld { + // Test old version + oldMgr := reloader.NewManager(reloader.Config{ + Version: "old", + Image: cfg.OldImage, + }) + + if err := oldMgr.Deploy(ctx); err != nil { + log.Printf("Failed to deploy old Reloader: %v", err) + continue + } + + // Wait for Prometheus to discover and scrape the Reloader + if err := promMgr.WaitForTarget(ctx, oldMgr.Job(), 60*time.Second); err != nil { + log.Printf("Warning: %v", err) + log.Println("Proceeding anyway, but metrics may be incomplete") + } + + runScenario(ctx, kubeClient, scenarioID, "old", cfg.OldImage, cfg.Duration, cfg.ResultsDir) + collectMetrics(ctx, promMgr, oldMgr.Job(), scenarioID, "old", cfg.ResultsDir) + collectLogs(ctx, oldMgr, scenarioID, "old", cfg.ResultsDir) + + if runBoth { + // Clean up for new version + cleanupTestNamespaces(ctx, "") + oldMgr.Cleanup(ctx) + promMgr.Reset(ctx) + createTestNamespace(ctx, "") + } + } + + if runNew { + // Test new version + newMgr := reloader.NewManager(reloader.Config{ + Version: "new", + Image: cfg.NewImage, + }) + + if err := newMgr.Deploy(ctx); err != nil { + log.Printf("Failed to deploy new Reloader: %v", err) + continue + } + + // Wait for Prometheus to discover and scrape the Reloader + if err := promMgr.WaitForTarget(ctx, newMgr.Job(), 60*time.Second); err != nil { + log.Printf("Warning: %v", err) + log.Println("Proceeding anyway, but metrics may be incomplete") + } + + runScenario(ctx, kubeClient, scenarioID, "new", cfg.NewImage, cfg.Duration, cfg.ResultsDir) + collectMetrics(ctx, promMgr, newMgr.Job(), scenarioID, "new", cfg.ResultsDir) + collectLogs(ctx, newMgr, scenarioID, "new", cfg.ResultsDir) + } + + // Generate report + generateReport(scenarioID, cfg.ResultsDir, runBoth) + + log.Printf("=== Scenario %s complete ===", scenarioID) + } + + log.Println("Load test complete!") + log.Printf("Results available in: %s", cfg.ResultsDir) +} + +// runParallel runs scenarios in parallel on N separate kind clusters. +func runParallel(ctx context.Context, cfg Config, scenariosToRun []string, runtime string, runOld, runNew, runBoth bool) { + numWorkers := cfg.Parallelism + if numWorkers > len(scenariosToRun) { + numWorkers = len(scenariosToRun) + log.Printf("Reducing parallelism to %d (number of scenarios)", numWorkers) + } + + log.Printf("Starting parallel execution with %d workers", numWorkers) + + // Create workers + workers := make([]*workerContext, numWorkers) + var setupWg sync.WaitGroup + setupErrors := make(chan error, numWorkers) + + log.Println("Setting up worker clusters...") + for i := 0; i < numWorkers; i++ { + setupWg.Add(1) + go func(workerID int) { + defer setupWg.Done() + worker, err := setupWorker(ctx, cfg, workerID, runtime, runOld, runNew) + if err != nil { + setupErrors <- fmt.Errorf("worker %d setup failed: %w", workerID, err) + return + } + workers[workerID] = worker + }(i) + } + + setupWg.Wait() + close(setupErrors) + + // Check for setup errors + for err := range setupErrors { + log.Printf("Error: %v", err) + } + + // Verify all workers are ready + readyWorkers := 0 + for _, w := range workers { + if w != nil { + readyWorkers++ + } + } + if readyWorkers == 0 { + log.Fatal("No workers ready, aborting") + } + if readyWorkers < numWorkers { + log.Printf("Warning: only %d/%d workers ready", readyWorkers, numWorkers) + } + + // Cleanup workers on exit + defer func() { + log.Println("Cleaning up worker clusters...") + for _, w := range workers { + if w != nil { + w.promMgr.StopPortForward() + } + } + }() + + // Create scenario channel + scenarioCh := make(chan string, len(scenariosToRun)) + for _, s := range scenariosToRun { + scenarioCh <- s + } + close(scenarioCh) + + // Results tracking + var resultsMu sync.Mutex + completedScenarios := make([]string, 0, len(scenariosToRun)) + + // Start workers + var wg sync.WaitGroup + for _, worker := range workers { + if worker == nil { + continue + } + wg.Add(1) + go func(w *workerContext) { + defer wg.Done() + for scenarioID := range scenarioCh { + select { + case <-ctx.Done(): + return + default: + } + + log.Printf("[Worker %d] Starting scenario %s", w.id, scenarioID) + + // Clean up from previous scenario + cleanupTestNamespaces(ctx, w.kubeContext) + cleanupReloader(ctx, "old", w.kubeContext) + cleanupReloader(ctx, "new", w.kubeContext) + + // Reset Prometheus + if err := w.promMgr.Reset(ctx); err != nil { + log.Printf("[Worker %d] Warning: failed to reset Prometheus: %v", w.id, err) + } + + // Create test namespace + createTestNamespace(ctx, w.kubeContext) + + if runOld { + runVersionOnWorker(ctx, w, cfg, scenarioID, "old", cfg.OldImage, runBoth) + } + + if runNew { + runVersionOnWorker(ctx, w, cfg, scenarioID, "new", cfg.NewImage, false) + } + + // Generate report + generateReport(scenarioID, cfg.ResultsDir, runBoth) + + resultsMu.Lock() + completedScenarios = append(completedScenarios, scenarioID) + resultsMu.Unlock() + + log.Printf("[Worker %d] Scenario %s complete", w.id, scenarioID) + } + }(worker) + } + + wg.Wait() + + log.Println("Load test complete!") + log.Printf("Completed %d/%d scenarios", len(completedScenarios), len(scenariosToRun)) + log.Printf("Results available in: %s", cfg.ResultsDir) +} + +// setupWorker creates a cluster and deploys prometheus for a single worker. +func setupWorker(ctx context.Context, cfg Config, workerID int, runtime string, runOld, runNew bool) (*workerContext, error) { + workerName := fmt.Sprintf("%s-%d", clusterName, workerID) + promPort := 9091 + workerID + + log.Printf("[Worker %d] Creating cluster %s (ports %d/%d)...", workerID, workerName, 8080+workerID, 8443+workerID) + + clusterMgr := cluster.NewManager(cluster.Config{ + Name: workerName, + ContainerRuntime: runtime, + PortOffset: workerID, // Each cluster gets unique ports + }) + + if err := clusterMgr.Create(ctx); err != nil { + return nil, fmt.Errorf("creating cluster: %w", err) + } + + kubeContext := clusterMgr.Context() + + // Deploy Prometheus + promManifest := filepath.Join(cfg.ManifestsDir, "prometheus.yaml") + promMgr := prometheus.NewManagerWithPort(promManifest, promPort, kubeContext) + + log.Printf("[Worker %d] Installing Prometheus (port %d)...", workerID, promPort) + if err := promMgr.Deploy(ctx); err != nil { + return nil, fmt.Errorf("deploying prometheus: %w", err) + } + + if err := promMgr.StartPortForward(ctx); err != nil { + return nil, fmt.Errorf("starting prometheus port-forward: %w", err) + } + + // Load images + log.Printf("[Worker %d] Loading images...", workerID) + if runOld { + if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { + log.Printf("[Worker %d] Warning: failed to load old image: %v", workerID, err) + } + } + if runNew { + if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { + log.Printf("[Worker %d] Warning: failed to load new image: %v", workerID, err) + } + } + + // Pre-pull test images + testImage := "gcr.io/google-containers/busybox:1.27" + clusterMgr.LoadImage(ctx, testImage) + + // Get kubernetes client for this context + kubeClient, err := getKubeClient(kubeContext) + if err != nil { + return nil, fmt.Errorf("creating kubernetes client: %w", err) + } + + log.Printf("[Worker %d] Ready", workerID) + return &workerContext{ + id: workerID, + clusterMgr: clusterMgr, + promMgr: promMgr, + kubeClient: kubeClient, + kubeContext: kubeContext, + runtime: runtime, + }, nil +} + +// runVersionOnWorker runs a single version test on a worker. +func runVersionOnWorker(ctx context.Context, w *workerContext, cfg Config, scenarioID, version, image string, cleanupAfter bool) { + mgr := reloader.NewManager(reloader.Config{ + Version: version, + Image: image, + }) + mgr.SetKubeContext(w.kubeContext) + + if err := mgr.Deploy(ctx); err != nil { + log.Printf("[Worker %d] Failed to deploy %s Reloader: %v", w.id, version, err) + return + } + + // Wait for Prometheus to discover and scrape the Reloader + if err := w.promMgr.WaitForTarget(ctx, mgr.Job(), 60*time.Second); err != nil { + log.Printf("[Worker %d] Warning: %v", w.id, err) + log.Printf("[Worker %d] Proceeding anyway, but metrics may be incomplete", w.id) + } + + runScenario(ctx, w.kubeClient, scenarioID, version, image, cfg.Duration, cfg.ResultsDir) + collectMetrics(ctx, w.promMgr, mgr.Job(), scenarioID, version, cfg.ResultsDir) + collectLogs(ctx, mgr, scenarioID, version, cfg.ResultsDir) + + if cleanupAfter { + cleanupTestNamespaces(ctx, w.kubeContext) + mgr.Cleanup(ctx) + w.promMgr.Reset(ctx) + createTestNamespace(ctx, w.kubeContext) + } +} + +func runScenario(ctx context.Context, client kubernetes.Interface, scenarioID, version, image string, duration int, resultsDir string) { + runner, ok := scenarios.Registry[scenarioID] + if !ok { + log.Printf("Unknown scenario: %s", scenarioID) + return + } + + // For S6, set the reloader version + if s6, ok := runner.(*scenarios.ControllerRestartScenario); ok { + s6.ReloaderVersion = version + } + + // For S11, set the image to deploy its own Reloader + if s11, ok := runner.(*scenarios.AnnotationStrategyScenario); ok { + s11.Image = image + } + + log.Printf("Running scenario %s (%s): %s", scenarioID, version, runner.Description()) + + // Debug: check parent context state + if ctx.Err() != nil { + log.Printf("WARNING: Parent context already done: %v", ctx.Err()) + } + + // Add extra time for scenario setup (creating deployments, waiting for ready state) + // Some scenarios like S2 create 50 deployments which can take 2-3 minutes + timeout := time.Duration(duration)*time.Second + 5*time.Minute + log.Printf("Creating scenario context with timeout: %v (duration=%ds)", timeout, duration) + + scenarioCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + expected, err := runner.Run(scenarioCtx, client, testNamespace, time.Duration(duration)*time.Second) + if err != nil { + log.Printf("Scenario %s failed: %v", scenarioID, err) + } + + scenarios.WriteExpectedMetrics(scenarioID, resultsDir, expected) +} + +func collectMetrics(ctx context.Context, promMgr *prometheus.Manager, job, scenarioID, version, resultsDir string) { + log.Printf("Waiting 5s for Reloader to finish processing events...") + time.Sleep(5 * time.Second) + + log.Printf("Waiting 8s for Prometheus to scrape final metrics...") + time.Sleep(8 * time.Second) + + log.Printf("Collecting metrics for %s...", version) + outputDir := filepath.Join(resultsDir, scenarioID, version) + if err := promMgr.CollectMetrics(ctx, job, outputDir, scenarioID); err != nil { + log.Printf("Failed to collect metrics: %v", err) + } +} + +func collectLogs(ctx context.Context, mgr *reloader.Manager, scenarioID, version, resultsDir string) { + log.Printf("Collecting logs for %s...", version) + logPath := filepath.Join(resultsDir, scenarioID, version, "reloader.log") + if err := mgr.CollectLogs(ctx, logPath); err != nil { + log.Printf("Failed to collect logs: %v", err) + } +} + +func generateReport(scenarioID, resultsDir string, isComparison bool) { + if isComparison { + log.Println("Generating comparison report...") + } else { + log.Println("Generating single-version report...") + } + + reportPath := filepath.Join(resultsDir, scenarioID, "report.txt") + + // Use the report command + cmd := exec.Command(os.Args[0], "report", + fmt.Sprintf("--scenario=%s", scenarioID), + fmt.Sprintf("--results-dir=%s", resultsDir), + fmt.Sprintf("--output=%s", reportPath)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Run() + + // Also print to stdout + if data, err := os.ReadFile(reportPath); err == nil { + fmt.Println(string(data)) + } + + log.Printf("Report saved to: %s", reportPath) +} + +func getKubeClient(kubeContext string) (kubernetes.Interface, error) { + kubeconfig := os.Getenv("KUBECONFIG") + if kubeconfig == "" { + home, _ := os.UserHomeDir() + kubeconfig = filepath.Join(home, ".kube", "config") + } + + loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig} + configOverrides := &clientcmd.ConfigOverrides{} + if kubeContext != "" { + configOverrides.CurrentContext = kubeContext + } + + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + config, err := kubeConfig.ClientConfig() + if err != nil { + return nil, err + } + + return kubernetes.NewForConfig(config) +} + +func createTestNamespace(ctx context.Context, kubeContext string) { + args := []string{"create", "namespace", testNamespace, "--dry-run=client", "-o", "yaml"} + if kubeContext != "" { + args = append([]string{"--context", kubeContext}, args...) + } + cmd := exec.CommandContext(ctx, "kubectl", args...) + out, _ := cmd.Output() + + applyArgs := []string{"apply", "-f", "-"} + if kubeContext != "" { + applyArgs = append([]string{"--context", kubeContext}, applyArgs...) + } + applyCmd := exec.CommandContext(ctx, "kubectl", applyArgs...) + applyCmd.Stdin = strings.NewReader(string(out)) + applyCmd.Run() +} + +func cleanupTestNamespaces(ctx context.Context, kubeContext string) { + log.Println("Cleaning up test resources...") + + // Main namespace + S3 extra namespaces + namespaces := []string{testNamespace} + for i := 0; i < 10; i++ { + namespaces = append(namespaces, fmt.Sprintf("%s-%d", testNamespace, i)) + } + + for _, ns := range namespaces { + args := []string{"delete", "namespace", ns, "--wait=false", "--ignore-not-found"} + if kubeContext != "" { + args = append([]string{"--context", kubeContext}, args...) + } + exec.CommandContext(ctx, "kubectl", args...).Run() + } + + // Wait a bit for cleanup + time.Sleep(2 * time.Second) + + // Force delete remaining pods + for _, ns := range namespaces { + args := []string{"delete", "pods", "--all", "-n", ns, "--grace-period=0", "--force"} + if kubeContext != "" { + args = append([]string{"--context", kubeContext}, args...) + } + exec.CommandContext(ctx, "kubectl", args...).Run() + } +} + +func cleanupReloader(ctx context.Context, version string, kubeContext string) { + ns := fmt.Sprintf("reloader-%s", version) + + nsArgs := []string{"delete", "namespace", ns, "--wait=false", "--ignore-not-found"} + crArgs := []string{"delete", "clusterrole", fmt.Sprintf("reloader-%s", version), "--ignore-not-found"} + crbArgs := []string{"delete", "clusterrolebinding", fmt.Sprintf("reloader-%s", version), "--ignore-not-found"} + + if kubeContext != "" { + nsArgs = append([]string{"--context", kubeContext}, nsArgs...) + crArgs = append([]string{"--context", kubeContext}, crArgs...) + crbArgs = append([]string{"--context", kubeContext}, crbArgs...) + } + + exec.CommandContext(ctx, "kubectl", nsArgs...).Run() + exec.CommandContext(ctx, "kubectl", crArgs...).Run() + exec.CommandContext(ctx, "kubectl", crbArgs...).Run() +} + +// ============================================================================ +// REPORT COMMAND +// ============================================================================ + +func reportCommand(args []string) { + var scenarioID, resultsDir, outputFile string + resultsDir = "./results" + + for _, arg := range args { + switch { + case strings.HasPrefix(arg, "--scenario="): + scenarioID = strings.TrimPrefix(arg, "--scenario=") + case strings.HasPrefix(arg, "--results-dir="): + resultsDir = strings.TrimPrefix(arg, "--results-dir=") + case strings.HasPrefix(arg, "--output="): + outputFile = strings.TrimPrefix(arg, "--output=") + } + } + + if scenarioID == "" { + log.Fatal("--scenario is required for report command") + } + + report, err := generateScenarioReport(scenarioID, resultsDir) + if err != nil { + log.Fatalf("Failed to generate report: %v", err) + } + + output := renderScenarioReport(report) + + if outputFile != "" { + if err := os.WriteFile(outputFile, []byte(output), 0644); err != nil { + log.Fatalf("Failed to write output file: %v", err) + } + log.Printf("Report written to %s", outputFile) + } else { + fmt.Println(output) + } +} + +// PrometheusResponse represents a Prometheus API response for report parsing. +type PrometheusResponse struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + Result []struct { + Metric map[string]string `json:"metric"` + Value []interface{} `json:"value"` + } `json:"result"` + } `json:"data"` +} + +// MetricComparison represents the comparison of a single metric. +type MetricComparison struct { + Name string + DisplayName string + Unit string + IsCounter bool + OldValue float64 + NewValue float64 + Expected float64 + Difference float64 + DiffPct float64 + Status string + Threshold float64 + OldMeetsExpected string + NewMeetsExpected string +} + +type metricInfo struct { + unit string + isCounter bool +} + +var metricInfoMap = map[string]metricInfo{ + "reconcile_total": {unit: "count", isCounter: true}, + "reconcile_duration_p50": {unit: "s", isCounter: false}, + "reconcile_duration_p95": {unit: "s", isCounter: false}, + "reconcile_duration_p99": {unit: "s", isCounter: false}, + "action_total": {unit: "count", isCounter: true}, + "action_latency_p50": {unit: "s", isCounter: false}, + "action_latency_p95": {unit: "s", isCounter: false}, + "action_latency_p99": {unit: "s", isCounter: false}, + "errors_total": {unit: "count", isCounter: true}, + "reload_executed_total": {unit: "count", isCounter: true}, + "workloads_scanned_total": {unit: "count", isCounter: true}, + "workloads_matched_total": {unit: "count", isCounter: true}, + "skipped_total_no_data_change": {unit: "count", isCounter: true}, + "rest_client_requests_total": {unit: "count", isCounter: true}, + "rest_client_requests_get": {unit: "count", isCounter: true}, + "rest_client_requests_patch": {unit: "count", isCounter: true}, + "rest_client_requests_put": {unit: "count", isCounter: true}, + "rest_client_requests_errors": {unit: "count", isCounter: true}, + + // Resource consumption metrics (gauges, not counters) + "memory_rss_mb_avg": {unit: "MB", isCounter: false}, + "memory_rss_mb_max": {unit: "MB", isCounter: false}, + "memory_heap_mb_avg": {unit: "MB", isCounter: false}, + "memory_heap_mb_max": {unit: "MB", isCounter: false}, + "cpu_cores_avg": {unit: "cores", isCounter: false}, + "cpu_cores_max": {unit: "cores", isCounter: false}, + "goroutines_avg": {unit: "count", isCounter: false}, + "goroutines_max": {unit: "count", isCounter: false}, + "gc_pause_p99_ms": {unit: "ms", isCounter: false}, +} + +// ReportExpectedMetrics matches the expected metrics from test scenarios. +type ReportExpectedMetrics struct { + ActionTotal int `json:"action_total"` + ReloadExecutedTotal int `json:"reload_executed_total"` + ReconcileTotal int `json:"reconcile_total"` + WorkloadsScannedTotal int `json:"workloads_scanned_total"` + WorkloadsMatchedTotal int `json:"workloads_matched_total"` + SkippedTotal int `json:"skipped_total"` + Description string `json:"description"` +} + +// ScenarioReport represents the full report for a scenario. +type ScenarioReport struct { + Scenario string + Timestamp time.Time + Comparisons []MetricComparison + OverallStatus string + Summary string + PassCriteria []string + FailedCriteria []string + Expected ReportExpectedMetrics + TestDescription string +} + +// MetricType defines how to evaluate a metric. +type MetricType int + +const ( + LowerIsBetter MetricType = iota + ShouldMatch + HigherIsBetter + Informational // Reports values but doesn't affect pass/fail +) + +type ThresholdConfig struct { + maxDiff float64 + metricType MetricType + minAbsDiff float64 +} + +var thresholds = map[string]ThresholdConfig{ + "reconcile_total": {maxDiff: 60.0, metricType: LowerIsBetter}, + "reconcile_duration_p50": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.5}, + "reconcile_duration_p95": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, + "reconcile_duration_p99": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, + "action_latency_p50": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.5}, + "action_latency_p95": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, + "action_latency_p99": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, + "errors_total": {maxDiff: 0.0, metricType: LowerIsBetter}, + "action_total": {maxDiff: 15.0, metricType: ShouldMatch}, + "reload_executed_total": {maxDiff: 15.0, metricType: ShouldMatch}, + "workloads_scanned_total": {maxDiff: 15.0, metricType: ShouldMatch}, + "workloads_matched_total": {maxDiff: 15.0, metricType: ShouldMatch}, + "skipped_total_no_data_change": {maxDiff: 20.0, metricType: ShouldMatch}, + // API metrics - use minAbsDiff to allow small differences + "rest_client_requests_total": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, + "rest_client_requests_get": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, + "rest_client_requests_patch": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, + "rest_client_requests_put": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 20}, + "rest_client_requests_errors": {maxDiff: 0.0, metricType: LowerIsBetter, minAbsDiff: 100}, // Pass if both < 100 + + // Resource consumption metrics + "memory_rss_mb_avg": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 20}, // 50% or 20MB + "memory_rss_mb_max": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 30}, // 50% or 30MB + "memory_heap_mb_avg": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 15}, // 50% or 15MB + "memory_heap_mb_max": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 20}, // 50% or 20MB + "cpu_cores_avg": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.1}, // 100% or 0.1 cores + "cpu_cores_max": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.2}, // 100% or 0.2 cores + "goroutines_avg": {metricType: Informational}, // Info only - different architectures may use more goroutines + "goroutines_max": {metricType: Informational}, // Info only - different architectures may use more goroutines + "gc_pause_p99_ms": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 5}, // 100% or 5ms +} + +func generateScenarioReport(scenario, resultsDir string) (*ScenarioReport, error) { + oldDir := filepath.Join(resultsDir, scenario, "old") + newDir := filepath.Join(resultsDir, scenario, "new") + scenarioDir := filepath.Join(resultsDir, scenario) + + // Check which directories exist to determine mode + _, oldErr := os.Stat(oldDir) + _, newErr := os.Stat(newDir) + hasOld := oldErr == nil + hasNew := newErr == nil + isComparison := hasOld && hasNew + + // For single-version mode, determine which version we have + singleVersion := "" + singleDir := "" + if !isComparison { + if hasNew { + singleVersion = "new" + singleDir = newDir + } else if hasOld { + singleVersion = "old" + singleDir = oldDir + } else { + return nil, fmt.Errorf("no results found in %s", scenarioDir) + } + } + + report := &ScenarioReport{ + Scenario: scenario, + Timestamp: time.Now(), + } + + // Load expected metrics + expectedPath := filepath.Join(scenarioDir, "expected.json") + if data, err := os.ReadFile(expectedPath); err == nil { + if err := json.Unmarshal(data, &report.Expected); err != nil { + log.Printf("Warning: Could not parse expected metrics: %v", err) + } else { + report.TestDescription = report.Expected.Description + } + } + + // Handle single-version mode + if !isComparison { + return generateSingleVersionReport(report, singleDir, singleVersion, scenario) + } + + // Define metrics to compare + metricsToCompare := []struct { + name string + file string + selector func(data PrometheusResponse) float64 + }{ + {"reconcile_total", "reloader_reconcile_total.json", sumAllValues}, + {"reconcile_duration_p50", "reconcile_p50.json", getFirstValue}, + {"reconcile_duration_p95", "reconcile_p95.json", getFirstValue}, + {"reconcile_duration_p99", "reconcile_p99.json", getFirstValue}, + {"action_total", "reloader_action_total.json", sumAllValues}, + {"action_latency_p50", "action_p50.json", getFirstValue}, + {"action_latency_p95", "action_p95.json", getFirstValue}, + {"action_latency_p99", "action_p99.json", getFirstValue}, + {"errors_total", "reloader_errors_total.json", sumAllValues}, + {"reload_executed_total", "reloader_reload_executed_total.json", sumSuccessValues}, + {"workloads_scanned_total", "reloader_workloads_scanned_total.json", sumAllValues}, + {"workloads_matched_total", "reloader_workloads_matched_total.json", sumAllValues}, + {"rest_client_requests_total", "rest_client_requests_total.json", getFirstValue}, + {"rest_client_requests_get", "rest_client_requests_get.json", getFirstValue}, + {"rest_client_requests_patch", "rest_client_requests_patch.json", getFirstValue}, + {"rest_client_requests_put", "rest_client_requests_put.json", getFirstValue}, + {"rest_client_requests_errors", "rest_client_requests_errors.json", getFirstValue}, + + // Resource consumption metrics + {"memory_rss_mb_avg", "memory_rss_bytes_avg.json", bytesToMB}, + {"memory_rss_mb_max", "memory_rss_bytes_max.json", bytesToMB}, + {"memory_heap_mb_avg", "memory_heap_bytes_avg.json", bytesToMB}, + {"memory_heap_mb_max", "memory_heap_bytes_max.json", bytesToMB}, + {"cpu_cores_avg", "cpu_usage_cores_avg.json", getFirstValue}, + {"cpu_cores_max", "cpu_usage_cores_max.json", getFirstValue}, + {"goroutines_avg", "goroutines_avg.json", getFirstValue}, + {"goroutines_max", "goroutines_max.json", getFirstValue}, + {"gc_pause_p99_ms", "gc_duration_seconds_p99.json", secondsToMs}, + } + + // Build expected values map + expectedValues := map[string]float64{ + "action_total": float64(report.Expected.ActionTotal), + "reload_executed_total": float64(report.Expected.ReloadExecutedTotal), + "reconcile_total": float64(report.Expected.ReconcileTotal), + "workloads_scanned_total": float64(report.Expected.WorkloadsScannedTotal), + "workloads_matched_total": float64(report.Expected.WorkloadsMatchedTotal), + "skipped_total": float64(report.Expected.SkippedTotal), + } + + // First pass: collect all metric values + metricValues := make(map[string]struct{ old, new, expected float64 }) + + for _, m := range metricsToCompare { + oldData, err := loadMetricFile(filepath.Join(oldDir, m.file)) + if err != nil { + log.Printf("Warning: Could not load old metric %s: %v", m.name, err) + continue + } + + newData, err := loadMetricFile(filepath.Join(newDir, m.file)) + if err != nil { + log.Printf("Warning: Could not load new metric %s: %v", m.name, err) + continue + } + + oldValue := m.selector(oldData) + newValue := m.selector(newData) + expected := expectedValues[m.name] + + metricValues[m.name] = struct{ old, new, expected float64 }{oldValue, newValue, expected} + } + + // Check context for smart pass/fail decisions + newMeetsActionExpected := false + newReconcileIsZero := false + isChurnScenario := scenario == "S5" // Workload churn has special pass/fail rules + if v, ok := metricValues["action_total"]; ok && v.expected > 0 { + tolerance := v.expected * 0.15 + newMeetsActionExpected = math.Abs(v.new-v.expected) <= tolerance + } + if v, ok := metricValues["reconcile_total"]; ok { + newReconcileIsZero = v.new == 0 + } + + // Second pass: generate comparisons with context awareness + for _, m := range metricsToCompare { + v, ok := metricValues[m.name] + if !ok { + continue + } + + comparison := compareMetricWithExpected(m.name, v.old, v.new, v.expected) + + // Context-aware adjustments for API metrics + if strings.HasPrefix(m.name, "rest_client_requests") { + // If new correctly processed all expected reloads but old didn't, + // higher API calls in new is expected (it's doing the work correctly) + if newMeetsActionExpected && comparison.Status != "pass" { + if oldMeets, ok := metricValues["action_total"]; ok { + oldTolerance := oldMeets.expected * 0.15 + oldMissed := math.Abs(oldMeets.old-oldMeets.expected) > oldTolerance + if oldMissed { + comparison.Status = "pass" + } + } + } + // If new has 0 reconciles (no-op scenario), API differences are fine + if newReconcileIsZero && comparison.Status != "pass" { + comparison.Status = "pass" + } + } + + // S5 (Workload Churn) specific adjustments: + // - "Not found" errors are expected when deployments are deleted during processing + // - No expected values for throughput, so compare old vs new (should be similar) + if isChurnScenario { + if m.name == "errors_total" { + // In churn scenarios, "not found" errors are expected when workloads + // are deleted while Reloader is processing them. Allow up to 50 errors. + if v.new < 50 && v.old < 50 { + comparison.Status = "pass" + } else if v.new <= v.old*1.5 { + // Also pass if new has similar or fewer errors than old + comparison.Status = "pass" + } + } + if m.name == "action_total" || m.name == "reload_executed_total" { + // No expected value for churn - compare old vs new + // Both should be similar (within 20% of each other) + if v.old > 0 { + diff := math.Abs(v.new-v.old) / v.old * 100 + if diff <= 20 { + comparison.Status = "pass" + } + } else if v.new > 0 { + // Old is 0, new has value - that's fine + comparison.Status = "pass" + } + } + } + + report.Comparisons = append(report.Comparisons, comparison) + + if comparison.Status == "pass" { + report.PassCriteria = append(report.PassCriteria, m.name) + } else if comparison.Status == "fail" { + report.FailedCriteria = append(report.FailedCriteria, m.name) + } + } + + // Determine overall status + if len(report.FailedCriteria) == 0 { + report.OverallStatus = "PASS" + report.Summary = "All metrics within acceptable thresholds" + } else { + report.OverallStatus = "FAIL" + report.Summary = fmt.Sprintf("%d metrics failed: %s", + len(report.FailedCriteria), + strings.Join(report.FailedCriteria, ", ")) + } + + return report, nil +} + +// generateSingleVersionReport creates a report for a single version (no comparison). +func generateSingleVersionReport(report *ScenarioReport, dataDir, version, scenario string) (*ScenarioReport, error) { + // Define metrics to collect + metricsToCollect := []struct { + name string + file string + selector func(data PrometheusResponse) float64 + }{ + {"reconcile_total", "reloader_reconcile_total.json", sumAllValues}, + {"reconcile_duration_p50", "reconcile_p50.json", getFirstValue}, + {"reconcile_duration_p95", "reconcile_p95.json", getFirstValue}, + {"reconcile_duration_p99", "reconcile_p99.json", getFirstValue}, + {"action_total", "reloader_action_total.json", sumAllValues}, + {"action_latency_p50", "action_p50.json", getFirstValue}, + {"action_latency_p95", "action_p95.json", getFirstValue}, + {"action_latency_p99", "action_p99.json", getFirstValue}, + {"errors_total", "reloader_errors_total.json", sumAllValues}, + {"reload_executed_total", "reloader_reload_executed_total.json", sumSuccessValues}, + {"workloads_scanned_total", "reloader_workloads_scanned_total.json", sumAllValues}, + {"workloads_matched_total", "reloader_workloads_matched_total.json", sumAllValues}, + {"rest_client_requests_total", "rest_client_requests_total.json", getFirstValue}, + {"rest_client_requests_get", "rest_client_requests_get.json", getFirstValue}, + {"rest_client_requests_patch", "rest_client_requests_patch.json", getFirstValue}, + {"rest_client_requests_put", "rest_client_requests_put.json", getFirstValue}, + {"rest_client_requests_errors", "rest_client_requests_errors.json", getFirstValue}, + {"memory_rss_mb_avg", "memory_rss_bytes_avg.json", bytesToMB}, + {"memory_rss_mb_max", "memory_rss_bytes_max.json", bytesToMB}, + {"memory_heap_mb_avg", "memory_heap_bytes_avg.json", bytesToMB}, + {"memory_heap_mb_max", "memory_heap_bytes_max.json", bytesToMB}, + {"cpu_cores_avg", "cpu_usage_cores_avg.json", getFirstValue}, + {"cpu_cores_max", "cpu_usage_cores_max.json", getFirstValue}, + {"goroutines_avg", "goroutines_avg.json", getFirstValue}, + {"goroutines_max", "goroutines_max.json", getFirstValue}, + {"gc_pause_p99_ms", "gc_duration_seconds_p99.json", secondsToMs}, + } + + // Build expected values map + expectedValues := map[string]float64{ + "action_total": float64(report.Expected.ActionTotal), + "reload_executed_total": float64(report.Expected.ReloadExecutedTotal), + "reconcile_total": float64(report.Expected.ReconcileTotal), + "workloads_scanned_total": float64(report.Expected.WorkloadsScannedTotal), + "workloads_matched_total": float64(report.Expected.WorkloadsMatchedTotal), + "skipped_total": float64(report.Expected.SkippedTotal), + } + + for _, m := range metricsToCollect { + data, err := loadMetricFile(filepath.Join(dataDir, m.file)) + if err != nil { + log.Printf("Warning: Could not load metric %s: %v", m.name, err) + continue + } + + value := m.selector(data) + expected := expectedValues[m.name] + + info := metricInfoMap[m.name] + if info.unit == "" { + info = metricInfo{unit: "count", isCounter: true} + } + + displayName := m.name + if info.unit != "count" { + displayName = fmt.Sprintf("%s (%s)", m.name, info.unit) + } + + // For single-version, put the value in NewValue column + status := "info" + meetsExp := "-" + + // Check against expected if available + if expected > 0 { + meetsExp = meetsExpected(value, expected) + threshold, ok := thresholds[m.name] + if ok && threshold.metricType == ShouldMatch { + if meetsExp == "āœ“" { + status = "pass" + report.PassCriteria = append(report.PassCriteria, m.name) + } else { + status = "fail" + report.FailedCriteria = append(report.FailedCriteria, m.name) + } + } + } + + if info.isCounter { + value = math.Round(value) + } + + report.Comparisons = append(report.Comparisons, MetricComparison{ + Name: m.name, + DisplayName: displayName, + Unit: info.unit, + IsCounter: info.isCounter, + OldValue: 0, // No old value in single-version mode + NewValue: value, + Expected: expected, + OldMeetsExpected: "-", + NewMeetsExpected: meetsExp, + Status: status, + }) + } + + if len(report.FailedCriteria) == 0 { + report.OverallStatus = "PASS" + report.Summary = fmt.Sprintf("Single-version test (%s) completed successfully", version) + } else { + report.OverallStatus = "FAIL" + report.Summary = fmt.Sprintf("%d metrics failed: %s", + len(report.FailedCriteria), + strings.Join(report.FailedCriteria, ", ")) + } + + return report, nil +} + +func loadMetricFile(path string) (PrometheusResponse, error) { + var resp PrometheusResponse + data, err := os.ReadFile(path) + if err != nil { + return resp, err + } + err = json.Unmarshal(data, &resp) + return resp, err +} + +func sumAllValues(data PrometheusResponse) float64 { + var sum float64 + for _, result := range data.Data.Result { + if len(result.Value) >= 2 { + if v, ok := result.Value[1].(string); ok { + var f float64 + fmt.Sscanf(v, "%f", &f) + sum += f + } + } + } + return sum +} + +func sumSuccessValues(data PrometheusResponse) float64 { + var sum float64 + for _, result := range data.Data.Result { + if result.Metric["success"] == "true" { + if len(result.Value) >= 2 { + if v, ok := result.Value[1].(string); ok { + var f float64 + fmt.Sscanf(v, "%f", &f) + sum += f + } + } + } + } + return sum +} + +func getFirstValue(data PrometheusResponse) float64 { + if len(data.Data.Result) > 0 && len(data.Data.Result[0].Value) >= 2 { + if v, ok := data.Data.Result[0].Value[1].(string); ok { + var f float64 + fmt.Sscanf(v, "%f", &f) + return f + } + } + return 0 +} + +// bytesToMB converts bytes to megabytes. +func bytesToMB(data PrometheusResponse) float64 { + bytes := getFirstValue(data) + return bytes / (1024 * 1024) +} + +// secondsToMs converts seconds to milliseconds. +func secondsToMs(data PrometheusResponse) float64 { + seconds := getFirstValue(data) + return seconds * 1000 +} + +func meetsExpected(value, expected float64) string { + if expected == 0 { + return "-" + } + tolerance := expected * 0.15 + if math.Abs(value-expected) <= tolerance { + return "āœ“" + } + return "āœ—" +} + +func compareMetricWithExpected(name string, oldValue, newValue, expected float64) MetricComparison { + diff := newValue - oldValue + absDiff := math.Abs(diff) + var diffPct float64 + if oldValue != 0 { + diffPct = (diff / oldValue) * 100 + } else if newValue != 0 { + diffPct = 100 + } + + threshold, ok := thresholds[name] + if !ok { + threshold = ThresholdConfig{maxDiff: 10.0, metricType: ShouldMatch} + } + + info := metricInfoMap[name] + if info.unit == "" { + info = metricInfo{unit: "count", isCounter: true} + } + displayName := name + if info.unit != "count" { + displayName = fmt.Sprintf("%s (%s)", name, info.unit) + } + + if info.isCounter { + oldValue = math.Round(oldValue) + newValue = math.Round(newValue) + } + + status := "pass" + oldMeetsExp := meetsExpected(oldValue, expected) + newMeetsExp := meetsExpected(newValue, expected) + + if expected > 0 && threshold.metricType == ShouldMatch { + if newMeetsExp == "āœ—" { + status = "fail" + } + } else { + switch threshold.metricType { + case LowerIsBetter: + if threshold.minAbsDiff > 0 && absDiff < threshold.minAbsDiff { + status = "pass" + } else if diffPct > threshold.maxDiff { + status = "fail" + } + case HigherIsBetter: + if diffPct < -threshold.maxDiff { + status = "fail" + } + case ShouldMatch: + if math.Abs(diffPct) > threshold.maxDiff { + status = "fail" + } + case Informational: + status = "info" + } + } + + return MetricComparison{ + Name: name, + DisplayName: displayName, + Unit: info.unit, + IsCounter: info.isCounter, + Expected: expected, + OldMeetsExpected: oldMeetsExp, + NewMeetsExpected: newMeetsExp, + OldValue: oldValue, + NewValue: newValue, + Difference: diff, + DiffPct: diffPct, + Status: status, + Threshold: threshold.maxDiff, + } +} + +func renderScenarioReport(report *ScenarioReport) string { + var sb strings.Builder + + // Detect single-version mode by checking if all OldValues are 0 + isSingleVersion := true + for _, c := range report.Comparisons { + if c.OldValue != 0 { + isSingleVersion = false + break + } + } + + sb.WriteString("\n") + sb.WriteString("================================================================================\n") + if isSingleVersion { + sb.WriteString(" RELOADER TEST REPORT\n") + } else { + sb.WriteString(" RELOADER A/B COMPARISON REPORT\n") + } + sb.WriteString("================================================================================\n\n") + + fmt.Fprintf(&sb, "Scenario: %s\n", report.Scenario) + fmt.Fprintf(&sb, "Generated: %s\n", report.Timestamp.Format("2006-01-02 15:04:05")) + fmt.Fprintf(&sb, "Status: %s\n", report.OverallStatus) + fmt.Fprintf(&sb, "Summary: %s\n", report.Summary) + + if report.TestDescription != "" { + fmt.Fprintf(&sb, "Test: %s\n", report.TestDescription) + } + + if report.Expected.ActionTotal > 0 { + sb.WriteString("\n--------------------------------------------------------------------------------\n") + sb.WriteString(" EXPECTED VALUES\n") + sb.WriteString("--------------------------------------------------------------------------------\n") + fmt.Fprintf(&sb, "Expected Action Total: %d\n", report.Expected.ActionTotal) + fmt.Fprintf(&sb, "Expected Reload Executed Total: %d\n", report.Expected.ReloadExecutedTotal) + if report.Expected.SkippedTotal > 0 { + fmt.Fprintf(&sb, "Expected Skipped Total: %d\n", report.Expected.SkippedTotal) + } + } + + sb.WriteString("\n--------------------------------------------------------------------------------\n") + if isSingleVersion { + sb.WriteString(" METRICS\n") + } else { + sb.WriteString(" METRIC COMPARISONS\n") + } + sb.WriteString("--------------------------------------------------------------------------------\n") + + if isSingleVersion { + sb.WriteString("(āœ“ = meets expected value within 15%)\n\n") + fmt.Fprintf(&sb, "%-32s %12s %10s %5s %8s\n", + "Metric", "Value", "Expected", "Met?", "Status") + fmt.Fprintf(&sb, "%-32s %12s %10s %5s %8s\n", + "------", "-----", "--------", "----", "------") + + for _, c := range report.Comparisons { + if c.IsCounter { + if c.Expected > 0 { + fmt.Fprintf(&sb, "%-32s %12.0f %10.0f %5s %8s\n", + c.DisplayName, c.NewValue, c.Expected, + c.NewMeetsExpected, c.Status) + } else { + fmt.Fprintf(&sb, "%-32s %12.0f %10s %5s %8s\n", + c.DisplayName, c.NewValue, "-", + c.NewMeetsExpected, c.Status) + } + } else { + fmt.Fprintf(&sb, "%-32s %12.4f %10s %5s %8s\n", + c.DisplayName, c.NewValue, "-", + c.NewMeetsExpected, c.Status) + } + } + } else { + sb.WriteString("(Oldāœ“/Newāœ“ = meets expected value within 15%)\n\n") + + fmt.Fprintf(&sb, "%-32s %12s %12s %10s %5s %5s %8s\n", + "Metric", "Old", "New", "Expected", "Oldāœ“", "Newāœ“", "Status") + fmt.Fprintf(&sb, "%-32s %12s %12s %10s %5s %5s %8s\n", + "------", "---", "---", "--------", "----", "----", "------") + + for _, c := range report.Comparisons { + if c.IsCounter { + if c.Expected > 0 { + fmt.Fprintf(&sb, "%-32s %12.0f %12.0f %10.0f %5s %5s %8s\n", + c.DisplayName, c.OldValue, c.NewValue, c.Expected, + c.OldMeetsExpected, c.NewMeetsExpected, c.Status) + } else { + fmt.Fprintf(&sb, "%-32s %12.0f %12.0f %10s %5s %5s %8s\n", + c.DisplayName, c.OldValue, c.NewValue, "-", + c.OldMeetsExpected, c.NewMeetsExpected, c.Status) + } + } else { + fmt.Fprintf(&sb, "%-32s %12.4f %12.4f %10s %5s %5s %8s\n", + c.DisplayName, c.OldValue, c.NewValue, "-", + c.OldMeetsExpected, c.NewMeetsExpected, c.Status) + } + } + } + + sb.WriteString("\n--------------------------------------------------------------------------------\n") + sb.WriteString(" PASS/FAIL CRITERIA\n") + sb.WriteString("--------------------------------------------------------------------------------\n\n") + + fmt.Fprintf(&sb, "Passed (%d):\n", len(report.PassCriteria)) + for _, p := range report.PassCriteria { + fmt.Fprintf(&sb, " āœ“ %s\n", p) + } + + if len(report.FailedCriteria) > 0 { + fmt.Fprintf(&sb, "\nFailed (%d):\n", len(report.FailedCriteria)) + for _, f := range report.FailedCriteria { + fmt.Fprintf(&sb, " āœ— %s\n", f) + } + } + + sb.WriteString("\n--------------------------------------------------------------------------------\n") + sb.WriteString(" THRESHOLDS USED\n") + sb.WriteString("--------------------------------------------------------------------------------\n\n") + + fmt.Fprintf(&sb, "%-35s %10s %15s %18s\n", + "Metric", "Max Diff%", "Min Abs Diff", "Direction") + fmt.Fprintf(&sb, "%-35s %10s %15s %18s\n", + "------", "---------", "------------", "---------") + + // Sort threshold names + var names []string + for name := range thresholds { + names = append(names, name) + } + sort.Strings(names) + + for _, name := range names { + t := thresholds[name] + var direction string + switch t.metricType { + case LowerIsBetter: + direction = "lower is better" + case HigherIsBetter: + direction = "higher is better" + case ShouldMatch: + direction = "should match" + case Informational: + direction = "info only" + } + minAbsDiff := "-" + if t.minAbsDiff > 0 { + minAbsDiff = fmt.Sprintf("%.1fs", t.minAbsDiff) + } + fmt.Fprintf(&sb, "%-35s %9.1f%% %15s %18s\n", + name, t.maxDiff, minAbsDiff, direction) + } + + sb.WriteString("\n================================================================================\n") + + return sb.String() +} diff --git a/test/loadtest/go.mod b/test/loadtest/go.mod new file mode 100644 index 0000000..ed52882 --- /dev/null +++ b/test/loadtest/go.mod @@ -0,0 +1,50 @@ +module github.com/stakater/Reloader/test/loadtest + +go 1.22.0 + +require ( + k8s.io/api v0.31.0 + k8s.io/apimachinery v0.31.0 + k8s.io/client-go v0.31.0 +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/time v0.3.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/test/loadtest/go.sum b/test/loadtest/go.sum new file mode 100644 index 0000000..a8edbda --- /dev/null +++ b/test/loadtest/go.sum @@ -0,0 +1,154 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= +k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= +k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= +k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= +k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/test/loadtest/internal/cluster/kind.go b/test/loadtest/internal/cluster/kind.go new file mode 100644 index 0000000..bcd5e19 --- /dev/null +++ b/test/loadtest/internal/cluster/kind.go @@ -0,0 +1,313 @@ +// Package cluster provides kind cluster management functionality. +package cluster + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "strings" + "time" +) + +// Config holds configuration for kind cluster operations. +type Config struct { + Name string + ContainerRuntime string // "docker" or "podman" + PortOffset int // Offset for host port mappings (for parallel clusters) +} + +// Manager handles kind cluster operations. +type Manager struct { + cfg Config +} + +// NewManager creates a new cluster manager. +func NewManager(cfg Config) *Manager { + return &Manager{cfg: cfg} +} + +// DetectContainerRuntime finds available container runtime. +func DetectContainerRuntime() (string, error) { + if _, err := exec.LookPath("podman"); err == nil { + return "podman", nil + } + if _, err := exec.LookPath("docker"); err == nil { + return "docker", nil + } + return "", fmt.Errorf("neither docker nor podman found in PATH") +} + +// Exists checks if the cluster already exists. +func (m *Manager) Exists() bool { + cmd := exec.Command("kind", "get", "clusters") + out, err := cmd.Output() + if err != nil { + return false + } + for _, line := range strings.Split(string(out), "\n") { + if strings.TrimSpace(line) == m.cfg.Name { + return true + } + } + return false +} + +// Delete deletes the kind cluster. +func (m *Manager) Delete(ctx context.Context) error { + cmd := exec.CommandContext(ctx, "kind", "delete", "cluster", "--name", m.cfg.Name) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +// Create creates a new kind cluster with optimized settings. +func (m *Manager) Create(ctx context.Context) error { + if m.cfg.ContainerRuntime == "podman" { + os.Setenv("KIND_EXPERIMENTAL_PROVIDER", "podman") + } + + if m.Exists() { + fmt.Printf("Cluster %s already exists, deleting...\n", m.cfg.Name) + if err := m.Delete(ctx); err != nil { + return fmt.Errorf("deleting existing cluster: %w", err) + } + } + + // Calculate unique ports based on offset (for parallel clusters) + httpPort := 8080 + m.cfg.PortOffset + httpsPort := 8443 + m.cfg.PortOffset + + config := fmt.Sprintf(`kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + podSubnet: "10.244.0.0/16" + serviceSubnet: "10.96.0.0/16" +nodes: +- role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + kube-api-qps: "50" + kube-api-burst: "100" + serialize-image-pulls: "false" + event-qps: "50" + event-burst: "100" + - | + kind: ClusterConfiguration + apiServer: + extraArgs: + max-requests-inflight: "800" + max-mutating-requests-inflight: "400" + watch-cache-sizes: "configmaps#1000,secrets#1000,pods#1000" + controllerManager: + extraArgs: + kube-api-qps: "200" + kube-api-burst: "200" + scheduler: + extraArgs: + kube-api-qps: "200" + kube-api-burst: "200" + extraPortMappings: + - containerPort: 80 + hostPort: %d + protocol: TCP + - containerPort: 443 + hostPort: %d + protocol: TCP +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "250" + kube-api-qps: "50" + kube-api-burst: "100" + serialize-image-pulls: "false" + event-qps: "50" + event-burst: "100" +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "250" + kube-api-qps: "50" + kube-api-burst: "100" + serialize-image-pulls: "false" + event-qps: "50" + event-burst: "100" +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "250" + kube-api-qps: "50" + kube-api-burst: "100" + serialize-image-pulls: "false" + event-qps: "50" + event-burst: "100" +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "250" + kube-api-qps: "50" + kube-api-burst: "100" + serialize-image-pulls: "false" + event-qps: "50" + event-burst: "100" +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "250" + kube-api-qps: "50" + kube-api-burst: "100" + serialize-image-pulls: "false" + event-qps: "50" + event-burst: "100" +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + max-pods: "250" + kube-api-qps: "50" + kube-api-burst: "100" + serialize-image-pulls: "false" + event-qps: "50" + event-burst: "100" +`, httpPort, httpsPort) + cmd := exec.CommandContext(ctx, "kind", "create", "cluster", "--name", m.cfg.Name, "--config=-") + cmd.Stdin = strings.NewReader(config) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +// GetKubeconfig returns the kubeconfig for the cluster. +func (m *Manager) GetKubeconfig() (string, error) { + cmd := exec.Command("kind", "get", "kubeconfig", "--name", m.cfg.Name) + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("getting kubeconfig: %w", err) + } + return string(out), nil +} + +// Context returns the kubectl context name for this cluster. +func (m *Manager) Context() string { + return "kind-" + m.cfg.Name +} + +// Name returns the cluster name. +func (m *Manager) Name() string { + return m.cfg.Name +} + +// LoadImage loads a container image into the kind cluster. +func (m *Manager) LoadImage(ctx context.Context, image string) error { + // First check if image exists locally + if !m.imageExistsLocally(image) { + fmt.Printf(" Image not found locally, pulling: %s\n", image) + pullCmd := exec.CommandContext(ctx, m.cfg.ContainerRuntime, "pull", image) + pullCmd.Stdout = os.Stdout + pullCmd.Stderr = os.Stderr + if err := pullCmd.Run(); err != nil { + return fmt.Errorf("pulling image %s: %w", image, err) + } + } else { + fmt.Printf(" Image found locally: %s\n", image) + } + + fmt.Printf(" Copying image to kind cluster...\n") + + if m.cfg.ContainerRuntime == "podman" { + // For podman, save to archive and load + tmpFile := fmt.Sprintf("/tmp/kind-image-%d.tar", time.Now().UnixNano()) + defer os.Remove(tmpFile) + + saveCmd := exec.CommandContext(ctx, m.cfg.ContainerRuntime, "save", image, "-o", tmpFile) + if err := saveCmd.Run(); err != nil { + return fmt.Errorf("saving image %s: %w", image, err) + } + + loadCmd := exec.CommandContext(ctx, "kind", "load", "image-archive", tmpFile, "--name", m.cfg.Name) + loadCmd.Stdout = os.Stdout + loadCmd.Stderr = os.Stderr + if err := loadCmd.Run(); err != nil { + return fmt.Errorf("loading image archive: %w", err) + } + } else { + loadCmd := exec.CommandContext(ctx, "kind", "load", "docker-image", image, "--name", m.cfg.Name) + loadCmd.Stdout = os.Stdout + loadCmd.Stderr = os.Stderr + if err := loadCmd.Run(); err != nil { + return fmt.Errorf("loading image %s: %w", image, err) + } + } + + return nil +} + +// imageExistsLocally checks if an image exists in the local container runtime. +func (m *Manager) imageExistsLocally(image string) bool { + // Try "image exists" command (works for podman) + cmd := exec.Command(m.cfg.ContainerRuntime, "image", "exists", image) + if err := cmd.Run(); err == nil { + return true + } + + // Try "image inspect" (works for both docker and podman) + cmd = exec.Command(m.cfg.ContainerRuntime, "image", "inspect", image) + if err := cmd.Run(); err == nil { + return true + } + + // Try listing images and grep + cmd = exec.Command(m.cfg.ContainerRuntime, "images", "--format", "{{.Repository}}:{{.Tag}}") + out, err := cmd.Output() + if err == nil { + for _, line := range strings.Split(string(out), "\n") { + if strings.TrimSpace(line) == image { + return true + } + } + } + + return false +} + +// PullImage pulls an image using the container runtime. +func (m *Manager) PullImage(ctx context.Context, image string) error { + cmd := exec.CommandContext(ctx, m.cfg.ContainerRuntime, "pull", image) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +// ExecKubectl runs a kubectl command against the cluster. +func (m *Manager) ExecKubectl(ctx context.Context, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, "kubectl", args...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("%w: %s", err, stderr.String()) + } + return stdout.Bytes(), nil +} diff --git a/test/loadtest/internal/prometheus/prometheus.go b/test/loadtest/internal/prometheus/prometheus.go new file mode 100644 index 0000000..f16df78 --- /dev/null +++ b/test/loadtest/internal/prometheus/prometheus.go @@ -0,0 +1,452 @@ +// Package prometheus provides Prometheus deployment and querying functionality. +package prometheus + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// Manager handles Prometheus operations. +type Manager struct { + manifestPath string + portForward *exec.Cmd + localPort int + kubeContext string // Optional: use specific kubeconfig context +} + +// NewManager creates a new Prometheus manager. +func NewManager(manifestPath string) *Manager { + return &Manager{ + manifestPath: manifestPath, + localPort: 9091, // Use 9091 to avoid conflicts + } +} + +// NewManagerWithPort creates a Prometheus manager with a custom port. +func NewManagerWithPort(manifestPath string, port int, kubeContext string) *Manager { + return &Manager{ + manifestPath: manifestPath, + localPort: port, + kubeContext: kubeContext, + } +} + +// kubectl returns kubectl args with optional context +func (m *Manager) kubectl(args ...string) []string { + if m.kubeContext != "" { + return append([]string{"--context", m.kubeContext}, args...) + } + return args +} + +// Deploy deploys Prometheus to the cluster. +func (m *Manager) Deploy(ctx context.Context) error { + // Create namespace + cmd := exec.CommandContext(ctx, "kubectl", m.kubectl("create", "namespace", "monitoring", "--dry-run=client", "-o", "yaml")...) + out, err := cmd.Output() + if err != nil { + return fmt.Errorf("generating namespace yaml: %w", err) + } + + applyCmd := exec.CommandContext(ctx, "kubectl", m.kubectl("apply", "-f", "-")...) + applyCmd.Stdin = strings.NewReader(string(out)) + if err := applyCmd.Run(); err != nil { + return fmt.Errorf("applying namespace: %w", err) + } + + // Apply Prometheus manifest + applyCmd = exec.CommandContext(ctx, "kubectl", m.kubectl("apply", "-f", m.manifestPath)...) + applyCmd.Stdout = os.Stdout + applyCmd.Stderr = os.Stderr + if err := applyCmd.Run(); err != nil { + return fmt.Errorf("applying prometheus manifest: %w", err) + } + + // Wait for Prometheus to be ready + fmt.Println("Waiting for Prometheus to be ready...") + waitCmd := exec.CommandContext(ctx, "kubectl", m.kubectl("wait", "--for=condition=ready", "pod", + "-l", "app=prometheus", "-n", "monitoring", "--timeout=120s")...) + waitCmd.Stdout = os.Stdout + waitCmd.Stderr = os.Stderr + if err := waitCmd.Run(); err != nil { + return fmt.Errorf("waiting for prometheus: %w", err) + } + + return nil +} + +// StartPortForward starts port-forwarding to Prometheus. +func (m *Manager) StartPortForward(ctx context.Context) error { + m.StopPortForward() + + // Start port-forward + m.portForward = exec.CommandContext(ctx, "kubectl", m.kubectl("port-forward", + "-n", "monitoring", "svc/prometheus", fmt.Sprintf("%d:9090", m.localPort))...) + + if err := m.portForward.Start(); err != nil { + return fmt.Errorf("starting port-forward: %w", err) + } + + // Wait for port-forward to be ready + for i := 0; i < 30; i++ { + time.Sleep(time.Second) + if m.isAccessible() { + fmt.Printf("Prometheus accessible at http://localhost:%d\n", m.localPort) + return nil + } + } + + return fmt.Errorf("prometheus port-forward not ready after 30s") +} + +// StopPortForward stops the port-forward process. +func (m *Manager) StopPortForward() { + if m.portForward != nil && m.portForward.Process != nil { + m.portForward.Process.Kill() + m.portForward = nil + } + // Also kill any lingering port-forwards + exec.Command("pkill", "-f", fmt.Sprintf("kubectl port-forward.*prometheus.*%d", m.localPort)).Run() +} + +// Reset restarts Prometheus to clear all metrics. +func (m *Manager) Reset(ctx context.Context) error { + m.StopPortForward() + + // Delete Prometheus pod to reset metrics + cmd := exec.CommandContext(ctx, "kubectl", m.kubectl("delete", "pod", "-n", "monitoring", + "-l", "app=prometheus", "--grace-period=0", "--force")...) + cmd.Run() // Ignore errors + + // Wait for new pod + fmt.Println("Waiting for Prometheus to restart...") + waitCmd := exec.CommandContext(ctx, "kubectl", m.kubectl("wait", "--for=condition=ready", "pod", + "-l", "app=prometheus", "-n", "monitoring", "--timeout=120s")...) + if err := waitCmd.Run(); err != nil { + return fmt.Errorf("waiting for prometheus restart: %w", err) + } + + // Restart port-forward + if err := m.StartPortForward(ctx); err != nil { + return err + } + + // Wait for scraping to initialize + fmt.Println("Waiting 5s for Prometheus to initialize scraping...") + time.Sleep(5 * time.Second) + + return nil +} + +func (m *Manager) isAccessible() bool { + conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", m.localPort), 2*time.Second) + if err != nil { + return false + } + conn.Close() + + // Also try HTTP + resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/status/config", m.localPort)) + if err != nil { + return false + } + resp.Body.Close() + return resp.StatusCode == 200 +} + +// URL returns the local Prometheus URL. +func (m *Manager) URL() string { + return fmt.Sprintf("http://localhost:%d", m.localPort) +} + +// WaitForTarget waits for a specific job to be scraped by Prometheus. +func (m *Manager) WaitForTarget(ctx context.Context, job string, timeout time.Duration) error { + fmt.Printf("Waiting for Prometheus to discover and scrape job '%s'...\n", job) + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if m.isTargetHealthy(job) { + fmt.Printf("Prometheus is scraping job '%s'\n", job) + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(2 * time.Second): + } + } + + // Print debug info on timeout + m.printTargetStatus(job) + return fmt.Errorf("timeout waiting for Prometheus to scrape job '%s'", job) +} + +// isTargetHealthy checks if a job has at least one healthy target. +func (m *Manager) isTargetHealthy(job string) bool { + resp, err := http.Get(fmt.Sprintf("%s/api/v1/targets", m.URL())) + if err != nil { + return false + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return false + } + + var result struct { + Status string `json:"status"` + Data struct { + ActiveTargets []struct { + Labels map[string]string `json:"labels"` + Health string `json:"health"` + } `json:"activeTargets"` + } `json:"data"` + } + + if err := json.Unmarshal(body, &result); err != nil { + return false + } + + for _, target := range result.Data.ActiveTargets { + if target.Labels["job"] == job && target.Health == "up" { + return true + } + } + return false +} + +// printTargetStatus prints debug info about targets. +func (m *Manager) printTargetStatus(job string) { + resp, err := http.Get(fmt.Sprintf("%s/api/v1/targets", m.URL())) + if err != nil { + fmt.Printf("Failed to get targets: %v\n", err) + return + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + + var result struct { + Data struct { + ActiveTargets []struct { + Labels map[string]string `json:"labels"` + Health string `json:"health"` + LastError string `json:"lastError"` + ScrapeURL string `json:"scrapeUrl"` + } `json:"activeTargets"` + } `json:"data"` + } + + if err := json.Unmarshal(body, &result); err != nil { + fmt.Printf("Failed to parse targets: %v\n", err) + return + } + + fmt.Printf("Prometheus targets for job '%s':\n", job) + found := false + for _, target := range result.Data.ActiveTargets { + if target.Labels["job"] == job { + found = true + fmt.Printf(" - %s: health=%s, lastError=%s\n", + target.ScrapeURL, target.Health, target.LastError) + } + } + if !found { + fmt.Printf(" No targets found for job '%s'\n", job) + fmt.Printf(" Available jobs: ") + jobs := make(map[string]bool) + for _, target := range result.Data.ActiveTargets { + jobs[target.Labels["job"]] = true + } + for j := range jobs { + fmt.Printf("%s ", j) + } + fmt.Println() + } +} + +// HasMetrics checks if the specified job has any metrics available. +func (m *Manager) HasMetrics(ctx context.Context, job string) bool { + query := fmt.Sprintf(`up{job="%s"}`, job) + result, err := m.Query(ctx, query) + if err != nil { + return false + } + return len(result.Data.Result) > 0 && result.Data.Result[0].Value[1] == "1" +} + +// QueryResponse represents a Prometheus query response. +type QueryResponse struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + Result []struct { + Metric map[string]string `json:"metric"` + Value []interface{} `json:"value"` + } `json:"result"` + } `json:"data"` +} + +// Query executes a PromQL query and returns the response. +func (m *Manager) Query(ctx context.Context, query string) (*QueryResponse, error) { + u := fmt.Sprintf("%s/api/v1/query?query=%s", m.URL(), url.QueryEscape(query)) + + req, err := http.NewRequestWithContext(ctx, "GET", u, nil) + if err != nil { + return nil, err + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("querying prometheus: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response: %w", err) + } + + var result QueryResponse + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + + return &result, nil +} + +// CollectMetrics collects all metrics for a scenario and writes to output directory. +func (m *Manager) CollectMetrics(ctx context.Context, job, outputDir, scenario string) error { + if err := os.MkdirAll(outputDir, 0755); err != nil { + return fmt.Errorf("creating output directory: %w", err) + } + + timeRange := "10m" + + // For S6 (restart scenario), use increase() to handle counter resets + useIncrease := scenario == "S6" + + // Counter metrics + counterMetrics := []string{ + "reloader_reconcile_total", + "reloader_action_total", + "reloader_skipped_total", + "reloader_errors_total", + "reloader_events_received_total", + "reloader_workloads_scanned_total", + "reloader_workloads_matched_total", + "reloader_reload_executed_total", + } + + for _, metric := range counterMetrics { + var query string + if useIncrease { + query = fmt.Sprintf(`sum(increase(%s{job="%s"}[%s])) by (success, reason)`, metric, job, timeRange) + } else { + query = fmt.Sprintf(`sum(%s{job="%s"}) by (success, reason)`, metric, job) + } + + if err := m.queryAndSave(ctx, query, filepath.Join(outputDir, metric+".json")); err != nil { + fmt.Printf("Warning: failed to collect %s: %v\n", metric, err) + } + } + + // Histogram percentiles + histogramMetrics := []struct { + name string + prefix string + }{ + {"reloader_reconcile_duration_seconds", "reconcile"}, + {"reloader_action_latency_seconds", "action"}, + } + + for _, hm := range histogramMetrics { + for _, pct := range []int{50, 95, 99} { + quantile := float64(pct) / 100 + query := fmt.Sprintf(`histogram_quantile(%v, sum(rate(%s_bucket{job="%s"}[%s])) by (le))`, + quantile, hm.name, job, timeRange) + outFile := filepath.Join(outputDir, fmt.Sprintf("%s_p%d.json", hm.prefix, pct)) + if err := m.queryAndSave(ctx, query, outFile); err != nil { + fmt.Printf("Warning: failed to collect %s p%d: %v\n", hm.name, pct, err) + } + } + } + + // REST client metrics + restQueries := map[string]string{ + "rest_client_requests_total.json": fmt.Sprintf(`sum(rest_client_requests_total{job="%s"})`, job), + "rest_client_requests_get.json": fmt.Sprintf(`sum(rest_client_requests_total{job="%s",method="GET"})`, job), + "rest_client_requests_patch.json": fmt.Sprintf(`sum(rest_client_requests_total{job="%s",method="PATCH"})`, job), + "rest_client_requests_put.json": fmt.Sprintf(`sum(rest_client_requests_total{job="%s",method="PUT"})`, job), + "rest_client_requests_errors.json": fmt.Sprintf(`sum(rest_client_requests_total{job="%s",code=~"[45].."}) or vector(0)`, job), + } + + for filename, query := range restQueries { + if err := m.queryAndSave(ctx, query, filepath.Join(outputDir, filename)); err != nil { + fmt.Printf("Warning: failed to collect %s: %v\n", filename, err) + } + } + + // Resource consumption metrics (memory, CPU, goroutines) + resourceQueries := map[string]string{ + // Memory metrics (in bytes) + "memory_rss_bytes_avg.json": fmt.Sprintf(`avg_over_time(process_resident_memory_bytes{job="%s"}[%s])`, job, timeRange), + "memory_rss_bytes_max.json": fmt.Sprintf(`max_over_time(process_resident_memory_bytes{job="%s"}[%s])`, job, timeRange), + "memory_rss_bytes_cur.json": fmt.Sprintf(`process_resident_memory_bytes{job="%s"}`, job), + + // Heap memory (Go runtime) + "memory_heap_bytes_avg.json": fmt.Sprintf(`avg_over_time(go_memstats_heap_alloc_bytes{job="%s"}[%s])`, job, timeRange), + "memory_heap_bytes_max.json": fmt.Sprintf(`max_over_time(go_memstats_heap_alloc_bytes{job="%s"}[%s])`, job, timeRange), + + // CPU metrics (rate of CPU seconds used) + "cpu_usage_cores_avg.json": fmt.Sprintf(`rate(process_cpu_seconds_total{job="%s"}[%s])`, job, timeRange), + "cpu_usage_cores_max.json": fmt.Sprintf(`max_over_time(rate(process_cpu_seconds_total{job="%s"}[1m])[%s:1m])`, job, timeRange), + + // Goroutines (concurrency indicator) + "goroutines_avg.json": fmt.Sprintf(`avg_over_time(go_goroutines{job="%s"}[%s])`, job, timeRange), + "goroutines_max.json": fmt.Sprintf(`max_over_time(go_goroutines{job="%s"}[%s])`, job, timeRange), + "goroutines_cur.json": fmt.Sprintf(`go_goroutines{job="%s"}`, job), + + // GC metrics + "gc_duration_seconds_p99.json": fmt.Sprintf(`histogram_quantile(0.99, sum(rate(go_gc_duration_seconds_bucket{job="%s"}[%s])) by (le))`, job, timeRange), + + // Threads + "threads_cur.json": fmt.Sprintf(`go_threads{job="%s"}`, job), + } + + for filename, query := range resourceQueries { + if err := m.queryAndSave(ctx, query, filepath.Join(outputDir, filename)); err != nil { + fmt.Printf("Warning: failed to collect %s: %v\n", filename, err) + } + } + + return nil +} + +func (m *Manager) queryAndSave(ctx context.Context, query, outputPath string) error { + result, err := m.Query(ctx, query) + if err != nil { + // Write empty result on error + emptyResult := `{"status":"success","data":{"resultType":"vector","result":[]}}` + return os.WriteFile(outputPath, []byte(emptyResult), 0644) + } + + data, err := json.MarshalIndent(result, "", " ") + if err != nil { + return err + } + + return os.WriteFile(outputPath, data, 0644) +} diff --git a/test/loadtest/internal/scenarios/scenarios.go b/test/loadtest/internal/scenarios/scenarios.go new file mode 100644 index 0000000..ed48a3b --- /dev/null +++ b/test/loadtest/internal/scenarios/scenarios.go @@ -0,0 +1,2092 @@ +// Package scenarios contains all load test scenario implementations. +package scenarios + +import ( + "context" + "encoding/json" + "fmt" + "log" + "math/rand" + "os" + "path/filepath" + "sync" + "time" + + "github.com/stakater/Reloader/test/loadtest/internal/reloader" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" +) + +// ExpectedMetrics holds the expected values for metrics verification. +type ExpectedMetrics struct { + ActionTotal int `json:"action_total"` + ReloadExecutedTotal int `json:"reload_executed_total"` + ReconcileTotal int `json:"reconcile_total"` + WorkloadsScannedTotal int `json:"workloads_scanned_total"` + WorkloadsMatchedTotal int `json:"workloads_matched_total"` + SkippedTotal int `json:"skipped_total"` + Description string `json:"description"` +} + +// Runner defines the interface for test scenarios. +type Runner interface { + Name() string + Description() string + Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) +} + +// Registry holds all available test scenarios. +var Registry = map[string]Runner{ + "S1": &BurstUpdateScenario{}, + "S2": &FanOutScenario{}, + "S3": &HighCardinalityScenario{}, + "S4": &NoOpUpdateScenario{}, + "S5": &WorkloadChurnScenario{}, + "S6": &ControllerRestartScenario{}, + "S7": &APIPressureScenario{}, + "S8": &LargeObjectScenario{}, + "S9": &MultiWorkloadTypeScenario{}, + "S10": &SecretsAndMixedScenario{}, + "S11": &AnnotationStrategyScenario{}, + "S12": &PauseResumeScenario{}, + "S13": &ComplexReferencesScenario{}, +} + +// WriteExpectedMetrics writes expected metrics to a JSON file. +func WriteExpectedMetrics(scenario, resultsDir string, expected ExpectedMetrics) error { + if resultsDir == "" { + return nil + } + + dir := filepath.Join(resultsDir, scenario) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("creating results directory: %w", err) + } + + data, err := json.MarshalIndent(expected, "", " ") + if err != nil { + return fmt.Errorf("marshaling expected metrics: %w", err) + } + + path := filepath.Join(dir, "expected.json") + if err := os.WriteFile(path, data, 0644); err != nil { + return fmt.Errorf("writing expected metrics: %w", err) + } + + log.Printf("Expected metrics written to %s", path) + return nil +} + +// BurstUpdateScenario - Many ConfigMap/Secret updates in quick succession. +type BurstUpdateScenario struct{} + +func (s *BurstUpdateScenario) Name() string { return "S1" } +func (s *BurstUpdateScenario) Description() string { return "Burst ConfigMap/Secret updates" } + +func (s *BurstUpdateScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S1: Creating base ConfigMaps and Deployments...") + + const numConfigMaps = 10 + const numDeployments = 10 + + setupCtx := context.Background() + + for i := 0; i < numConfigMaps; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("burst-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{"key": "initial-value"}, + } + if _, err := client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create ConfigMap %s: %v", cm.Name, err) + } + } + + for i := 0; i < numDeployments; i++ { + deploy := createDeployment(fmt.Sprintf("burst-deploy-%d", i), namespace, fmt.Sprintf("burst-cm-%d", i)) + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create Deployment: %v", err) + } + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S1: Starting burst updates...") + + updateCount := 0 + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + log.Printf("S1: Context cancelled, completed %d burst updates", updateCount) + return ExpectedMetrics{ + ActionTotal: updateCount, + ReloadExecutedTotal: updateCount, + WorkloadsMatchedTotal: updateCount, + Description: fmt.Sprintf("S1: %d burst updates, each triggers 1 deployment reload", updateCount), + }, nil + case <-ticker.C: + cmIndex := rand.Intn(numConfigMaps) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("burst-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["key"] = fmt.Sprintf("value-%d-%d", updateCount, time.Now().UnixNano()) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err != nil { + log.Printf("Failed to update ConfigMap: %v", err) + } else { + updateCount++ + } + } + } + + log.Printf("S1: Completed %d burst updates", updateCount) + return ExpectedMetrics{ + ActionTotal: updateCount, + ReloadExecutedTotal: updateCount, + WorkloadsMatchedTotal: updateCount, + Description: fmt.Sprintf("S1: %d burst updates, each triggers 1 deployment reload", updateCount), + }, nil +} + +// FanOutScenario - One ConfigMap used by many workloads. +type FanOutScenario struct{} + +func (s *FanOutScenario) Name() string { return "S2" } +func (s *FanOutScenario) Description() string { return "Fan-out (one CM -> many workloads)" } + +func (s *FanOutScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S2: Creating shared ConfigMap and multiple Deployments...") + + const numDeployments = 50 + setupCtx := context.Background() + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "shared-cm", + Namespace: namespace, + }, + Data: map[string]string{"config": "initial"}, + } + if _, err := client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}); err != nil { + return ExpectedMetrics{}, fmt.Errorf("failed to create shared ConfigMap: %w", err) + } + + for i := 0; i < numDeployments; i++ { + deploy := createDeployment(fmt.Sprintf("fanout-deploy-%d", i), namespace, "shared-cm") + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create Deployment %d: %v", i, err) + } + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 5*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S2: Updating shared ConfigMap...") + + // Check context state before starting update loop + if ctx.Err() != nil { + log.Printf("S2: WARNING - Context already done before update loop: %v", ctx.Err()) + } + if deadline, ok := ctx.Deadline(); ok { + remaining := time.Until(deadline) + log.Printf("S2: Context deadline in %v", remaining) + if remaining < 10*time.Second { + log.Printf("S2: WARNING - Very little time remaining on context!") + } + } else { + log.Println("S2: Context has no deadline") + } + + updateCount := 0 + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + log.Printf("S2: Will run updates for %v (duration=%v)", duration-5*time.Second, duration) + + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + expectedActions := updateCount * numDeployments + log.Printf("S2: Context done (err=%v), completed %d fan-out updates", ctx.Err(), updateCount) + return ExpectedMetrics{ + ActionTotal: expectedActions, + ReloadExecutedTotal: expectedActions, + WorkloadsScannedTotal: expectedActions, + WorkloadsMatchedTotal: expectedActions, + Description: fmt.Sprintf("S2: %d updates Ɨ %d deployments = %d expected reloads", updateCount, numDeployments, expectedActions), + }, nil + case <-ticker.C: + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, "shared-cm", metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["config"] = fmt.Sprintf("update-%d", updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err != nil { + log.Printf("Failed to update shared ConfigMap: %v", err) + } else { + updateCount++ + log.Printf("S2: Updated shared ConfigMap (should trigger %d reloads)", numDeployments) + } + } + } + + expectedActions := updateCount * numDeployments + log.Printf("S2: Completed %d fan-out updates, expected %d total actions", updateCount, expectedActions) + return ExpectedMetrics{ + ActionTotal: expectedActions, + ReloadExecutedTotal: expectedActions, + WorkloadsScannedTotal: expectedActions, + WorkloadsMatchedTotal: expectedActions, + Description: fmt.Sprintf("S2: %d updates Ɨ %d deployments = %d expected reloads", updateCount, numDeployments, expectedActions), + }, nil +} + +// HighCardinalityScenario - Many ConfigMaps/Secrets across many namespaces. +type HighCardinalityScenario struct{} + +func (s *HighCardinalityScenario) Name() string { return "S3" } +func (s *HighCardinalityScenario) Description() string { + return "High cardinality (many CMs, many namespaces)" +} + +func (s *HighCardinalityScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S3: Creating high cardinality resources...") + + setupCtx := context.Background() + + namespaces := []string{namespace} + for i := 0; i < 10; i++ { + ns := fmt.Sprintf("%s-%d", namespace, i) + if _, err := client.CoreV1().Namespaces().Create(setupCtx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: ns}, + }, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create namespace %s: %v", ns, err) + } else { + namespaces = append(namespaces, ns) + } + } + + for _, ns := range namespaces { + for i := 0; i < 20; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("hc-cm-%d", i), + Namespace: ns, + }, + Data: map[string]string{"key": "value"}, + } + client.CoreV1().ConfigMaps(ns).Create(setupCtx, cm, metav1.CreateOptions{}) + deploy := createDeployment(fmt.Sprintf("hc-deploy-%d", i), ns, fmt.Sprintf("hc-cm-%d", i)) + client.AppsV1().Deployments(ns).Create(setupCtx, deploy, metav1.CreateOptions{}) + } + } + + if err := waitForAllNamespacesReady(setupCtx, client, namespaces, 5*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S3: Starting random updates across namespaces...") + + updateDuration := duration - 5*time.Second + if updateDuration < 30*time.Second { + updateDuration = 30 * time.Second + } + + updateCount := 0 + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + + updateCtx, updateCancel := context.WithTimeout(context.Background(), updateDuration) + defer updateCancel() + + endTime := time.Now().Add(updateDuration) + log.Printf("S3: Will run updates for %v (until %v)", updateDuration, endTime.Format("15:04:05")) + + for time.Now().Before(endTime) { + select { + case <-updateCtx.Done(): + log.Printf("S3: Completed %d high cardinality updates", updateCount) + return ExpectedMetrics{ + ActionTotal: updateCount, + ReloadExecutedTotal: updateCount, + Description: fmt.Sprintf("S3: %d updates across %d namespaces", updateCount, len(namespaces)), + }, nil + case <-ticker.C: + ns := namespaces[rand.Intn(len(namespaces))] + cmIndex := rand.Intn(20) + cm, err := client.CoreV1().ConfigMaps(ns).Get(setupCtx, fmt.Sprintf("hc-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["key"] = fmt.Sprintf("update-%d", updateCount) + if _, err := client.CoreV1().ConfigMaps(ns).Update(setupCtx, cm, metav1.UpdateOptions{}); err == nil { + updateCount++ + } + } + } + + log.Printf("S3: Completed %d high cardinality updates", updateCount) + return ExpectedMetrics{ + ActionTotal: updateCount, + ReloadExecutedTotal: updateCount, + Description: fmt.Sprintf("S3: %d updates across %d namespaces", updateCount, len(namespaces)), + }, nil +} + +// NoOpUpdateScenario - Updates that don't actually change data. +type NoOpUpdateScenario struct{} + +func (s *NoOpUpdateScenario) Name() string { return "S4" } +func (s *NoOpUpdateScenario) Description() string { return "No-op updates (same data)" } + +func (s *NoOpUpdateScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S4: Creating ConfigMaps and Deployments for no-op test...") + + setupCtx := context.Background() + + for i := 0; i < 10; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("noop-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{"key": "static-value"}, + } + client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}) + deploy := createDeployment(fmt.Sprintf("noop-deploy-%d", i), namespace, fmt.Sprintf("noop-cm-%d", i)) + client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}) + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S4: Starting no-op updates (annotation changes only)...") + + updateCount := 0 + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + log.Printf("S4: Completed %d no-op updates", updateCount) + return ExpectedMetrics{ + ActionTotal: 0, + ReloadExecutedTotal: 0, + SkippedTotal: updateCount, + Description: fmt.Sprintf("S4: %d no-op updates, all should be skipped", updateCount), + }, nil + case <-ticker.C: + cmIndex := rand.Intn(10) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("noop-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + if cm.Annotations == nil { + cm.Annotations = make(map[string]string) + } + cm.Annotations["noop-counter"] = fmt.Sprintf("%d", updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err == nil { + updateCount++ + } + } + } + + log.Printf("S4: Completed %d no-op updates (should see 0 actions)", updateCount) + return ExpectedMetrics{ + ActionTotal: 0, + ReloadExecutedTotal: 0, + SkippedTotal: updateCount, + Description: fmt.Sprintf("S4: %d no-op updates, all should be skipped", updateCount), + }, nil +} + +// WorkloadChurnScenario - Deployments created and deleted rapidly. +type WorkloadChurnScenario struct{} + +func (s *WorkloadChurnScenario) Name() string { return "S5" } +func (s *WorkloadChurnScenario) Description() string { return "Workload churn (rapid create/delete)" } + +func (s *WorkloadChurnScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S5: Creating base ConfigMap...") + + setupCtx := context.Background() + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "churn-cm", Namespace: namespace}, + Data: map[string]string{"key": "value"}, + } + client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}) + + log.Println("S5: Starting workload churn...") + + var wg sync.WaitGroup + var mu sync.Mutex + deployCounter := 0 + deleteCounter := 0 + cmUpdateCount := 0 + + wg.Add(1) + go func() { + defer wg.Done() + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + return + case <-ticker.C: + deployName := fmt.Sprintf("churn-deploy-%d", deployCounter) + deploy := createDeployment(deployName, namespace, "churn-cm") + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err == nil { + mu.Lock() + deployCounter++ + mu.Unlock() + } + if deployCounter > 10 { + oldName := fmt.Sprintf("churn-deploy-%d", deployCounter-10) + if err := client.AppsV1().Deployments(namespace).Delete(setupCtx, oldName, metav1.DeleteOptions{}); err == nil { + mu.Lock() + deleteCounter++ + mu.Unlock() + } + } + } + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + return + case <-ticker.C: + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, "churn-cm", metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["key"] = fmt.Sprintf("update-%d", cmUpdateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err == nil { + mu.Lock() + cmUpdateCount++ + mu.Unlock() + } + } + } + }() + + wg.Wait() + log.Printf("S5: Created %d, deleted %d deployments, %d CM updates", deployCounter, deleteCounter, cmUpdateCount) + + // S5 does NOT set expected values for action_total/reload_executed_total because: + // - There are ~10 active deployments at any time (creates new, deletes old) + // - Each CM update triggers reloads on ALL active deployments + // - Exact counts depend on timing of creates/deletes vs CM updates + // - "Not found" errors are expected when a deployment is deleted during processing + // Instead, S5 pass/fail compares old vs new (both should be similar) + return ExpectedMetrics{ + // No expected values - churn makes exact counts unpredictable + Description: fmt.Sprintf("S5: Churn test - %d deploys created, %d deleted, %d CM updates, ~10 active deploys at any time", deployCounter, deleteCounter, cmUpdateCount), + }, nil +} + +// ControllerRestartScenario - Restart controller under load. +type ControllerRestartScenario struct { + ReloaderVersion string +} + +func (s *ControllerRestartScenario) Name() string { return "S6" } +func (s *ControllerRestartScenario) Description() string { + return "Controller restart under load" +} + +func (s *ControllerRestartScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S6: Creating resources and generating load...") + + setupCtx := context.Background() + + for i := 0; i < 20; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("restart-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{"key": "initial"}, + } + client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}) + deploy := createDeployment(fmt.Sprintf("restart-deploy-%d", i), namespace, fmt.Sprintf("restart-cm-%d", i)) + client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}) + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + var wg sync.WaitGroup + var mu sync.Mutex + updateCount := 0 + + wg.Add(1) + go func() { + defer wg.Done() + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + return + case <-ticker.C: + cmIndex := rand.Intn(20) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("restart-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["key"] = fmt.Sprintf("update-%d", updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err == nil { + mu.Lock() + updateCount++ + mu.Unlock() + } + } + } + }() + + reloaderNS := fmt.Sprintf("reloader-%s", s.ReloaderVersion) + if s.ReloaderVersion == "" { + reloaderNS = "reloader-new" + } + + log.Println("S6: Waiting 20 seconds before restarting controller...") + time.Sleep(20 * time.Second) + + log.Println("S6: Restarting Reloader pod...") + pods, err := client.CoreV1().Pods(reloaderNS).List(setupCtx, metav1.ListOptions{ + LabelSelector: "app=reloader", + }) + if err == nil && len(pods.Items) > 0 { + client.CoreV1().Pods(reloaderNS).Delete(setupCtx, pods.Items[0].Name, metav1.DeleteOptions{}) + } + + wg.Wait() + log.Printf("S6: Controller restart scenario completed with %d updates", updateCount) + return ExpectedMetrics{ + Description: fmt.Sprintf("S6: Restart test - %d updates during restart", updateCount), + }, nil +} + +// APIPressureScenario - Simulate API server pressure with many concurrent requests. +type APIPressureScenario struct{} + +func (s *APIPressureScenario) Name() string { return "S7" } +func (s *APIPressureScenario) Description() string { return "API pressure (many concurrent requests)" } + +func (s *APIPressureScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S7: Creating resources for API pressure test...") + + const numConfigMaps = 50 + setupCtx := context.Background() + + for i := 0; i < numConfigMaps; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("api-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{"key": "value"}, + } + client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}) + deploy := createDeployment(fmt.Sprintf("api-deploy-%d", i), namespace, fmt.Sprintf("api-cm-%d", i)) + client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}) + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 5*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S7: Starting concurrent updates from multiple goroutines...") + + updateDuration := duration - 5*time.Second + if updateDuration < 30*time.Second { + updateDuration = 30 * time.Second + } + + updateCtx, updateCancel := context.WithTimeout(context.Background(), updateDuration) + defer updateCancel() + + endTime := time.Now().Add(updateDuration) + log.Printf("S7: Will run updates for %v (until %v)", updateDuration, endTime.Format("15:04:05")) + + var wg sync.WaitGroup + var mu sync.Mutex + totalUpdates := 0 + + for g := 0; g < 10; g++ { + wg.Add(1) + go func(goroutineID int) { + defer wg.Done() + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + updateCount := 0 + for time.Now().Before(endTime) { + select { + case <-updateCtx.Done(): + return + case <-ticker.C: + cmIndex := rand.Intn(numConfigMaps) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("api-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["key"] = fmt.Sprintf("g%d-update-%d", goroutineID, updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err == nil { + updateCount++ + } + } + } + mu.Lock() + totalUpdates += updateCount + mu.Unlock() + log.Printf("S7: Goroutine %d completed %d updates", goroutineID, updateCount) + }(g) + } + + wg.Wait() + log.Printf("S7: API pressure scenario completed with %d total updates", totalUpdates) + return ExpectedMetrics{ + ActionTotal: totalUpdates, + ReloadExecutedTotal: totalUpdates, + Description: fmt.Sprintf("S7: %d concurrent updates from 10 goroutines", totalUpdates), + }, nil +} + +// LargeObjectScenario - Large ConfigMaps/Secrets. +type LargeObjectScenario struct{} + +func (s *LargeObjectScenario) Name() string { return "S8" } +func (s *LargeObjectScenario) Description() string { return "Large ConfigMaps/Secrets (>100KB)" } + +func (s *LargeObjectScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S8: Creating large ConfigMaps...") + + setupCtx := context.Background() + + largeData := make([]byte, 100*1024) + for i := range largeData { + largeData[i] = byte('a' + (i % 26)) + } + largeValue := string(largeData) + + for i := 0; i < 10; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("large-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{ + "large-key-1": largeValue, + "large-key-2": largeValue, + }, + } + if _, err := client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create large ConfigMap %d: %v", i, err) + } + deploy := createDeployment(fmt.Sprintf("large-deploy-%d", i), namespace, fmt.Sprintf("large-cm-%d", i)) + client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}) + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S8: Starting large object updates...") + + updateCount := 0 + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + log.Printf("S8: Completed %d large object updates", updateCount) + return ExpectedMetrics{ + ActionTotal: updateCount, + ReloadExecutedTotal: updateCount, + Description: fmt.Sprintf("S8: %d large object (100KB) updates", updateCount), + }, nil + case <-ticker.C: + cmIndex := rand.Intn(10) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("large-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["large-key-1"] = largeValue[:len(largeValue)-10] + fmt.Sprintf("-%d", updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err != nil { + log.Printf("Failed to update large ConfigMap: %v", err) + } else { + updateCount++ + } + } + } + + log.Printf("S8: Completed %d large object updates", updateCount) + return ExpectedMetrics{ + ActionTotal: updateCount, + ReloadExecutedTotal: updateCount, + Description: fmt.Sprintf("S8: %d large object (100KB) updates", updateCount), + }, nil +} + +// Helper functions + +func waitForDeploymentsReady(ctx context.Context, client kubernetes.Interface, namespace string, timeout time.Duration) error { + log.Printf("Waiting for all deployments in %s to be ready (timeout: %v)...", namespace, timeout) + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + deployments, err := client.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list deployments: %w", err) + } + + allReady := true + notReady := 0 + for _, d := range deployments.Items { + if d.Status.ReadyReplicas < *d.Spec.Replicas { + allReady = false + notReady++ + } + } + + if allReady && len(deployments.Items) > 0 { + log.Printf("All %d deployments in %s are ready", len(deployments.Items), namespace) + return nil + } + + log.Printf("Waiting for deployments: %d/%d not ready yet...", notReady, len(deployments.Items)) + time.Sleep(5 * time.Second) + } + + return fmt.Errorf("timeout waiting for deployments to be ready") +} + +func waitForAllNamespacesReady(ctx context.Context, client kubernetes.Interface, namespaces []string, timeout time.Duration) error { + log.Printf("Waiting for deployments in %d namespaces to be ready...", len(namespaces)) + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + allReady := true + totalDeploys := 0 + notReady := 0 + + for _, ns := range namespaces { + deployments, err := client.AppsV1().Deployments(ns).List(ctx, metav1.ListOptions{}) + if err != nil { + continue + } + for _, d := range deployments.Items { + totalDeploys++ + if d.Status.ReadyReplicas < *d.Spec.Replicas { + allReady = false + notReady++ + } + } + } + + if allReady && totalDeploys > 0 { + log.Printf("All %d deployments across %d namespaces are ready", totalDeploys, len(namespaces)) + return nil + } + + log.Printf("Waiting: %d/%d deployments not ready yet...", notReady, totalDeploys) + time.Sleep(5 * time.Second) + } + + return fmt.Errorf("timeout waiting for deployments to be ready") +} + +func createDeployment(name, namespace, configMapName string) *appsv1.Deployment { + replicas := int32(1) + maxSurge := intstr.FromInt(1) + maxUnavailable := intstr.FromInt(1) + terminationGracePeriod := int64(0) + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "reloader.stakater.com/auto": "true", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": name}, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: &terminationGracePeriod, + Containers: []corev1.Container{ + { + Name: "app", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "sleep 999999999"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMapName, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func createDeploymentWithSecret(name, namespace, secretName string) *appsv1.Deployment { + replicas := int32(1) + maxSurge := intstr.FromInt(1) + maxUnavailable := intstr.FromInt(1) + terminationGracePeriod := int64(0) + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "reloader.stakater.com/auto": "true", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": name}, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: &terminationGracePeriod, + Containers: []corev1.Container{ + { + Name: "app", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "sleep 999999999"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + EnvFrom: []corev1.EnvFromSource{ + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretName, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func createDeploymentWithBoth(name, namespace, configMapName, secretName string) *appsv1.Deployment { + replicas := int32(1) + maxSurge := intstr.FromInt(1) + maxUnavailable := intstr.FromInt(1) + terminationGracePeriod := int64(0) + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "reloader.stakater.com/auto": "true", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": name}, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: &terminationGracePeriod, + Containers: []corev1.Container{ + { + Name: "app", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "sleep 999999999"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMapName, + }, + }, + }, + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretName, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +// SecretsAndMixedScenario - Tests Secrets and mixed ConfigMap+Secret workloads. +type SecretsAndMixedScenario struct{} + +func (s *SecretsAndMixedScenario) Name() string { return "S10" } +func (s *SecretsAndMixedScenario) Description() string { + return "Secrets and mixed ConfigMap+Secret workloads" +} + +func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S10: Creating Secrets, ConfigMaps, and mixed workloads...") + + const numSecrets = 5 + const numConfigMaps = 5 + const numSecretOnlyDeploys = 5 + const numConfigMapOnlyDeploys = 3 + const numMixedDeploys = 2 + + setupCtx := context.Background() + + // Create Secrets + for i := 0; i < numSecrets; i++ { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("mixed-secret-%d", i), + Namespace: namespace, + }, + StringData: map[string]string{ + "password": fmt.Sprintf("initial-secret-%d", i), + }, + } + if _, err := client.CoreV1().Secrets(namespace).Create(setupCtx, secret, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create Secret %s: %v", secret.Name, err) + } + } + + // Create ConfigMaps + for i := 0; i < numConfigMaps; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("mixed-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{ + "config": fmt.Sprintf("initial-config-%d", i), + }, + } + if _, err := client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create ConfigMap %s: %v", cm.Name, err) + } + } + + // Create Secret-only deployments + for i := 0; i < numSecretOnlyDeploys; i++ { + deploy := createDeploymentWithSecret( + fmt.Sprintf("secret-only-deploy-%d", i), + namespace, + fmt.Sprintf("mixed-secret-%d", i%numSecrets), + ) + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create Secret-only Deployment: %v", err) + } + } + + // Create ConfigMap-only deployments + for i := 0; i < numConfigMapOnlyDeploys; i++ { + deploy := createDeployment( + fmt.Sprintf("cm-only-deploy-%d", i), + namespace, + fmt.Sprintf("mixed-cm-%d", i%numConfigMaps), + ) + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create ConfigMap-only Deployment: %v", err) + } + } + + // Create mixed deployments (using both Secret and ConfigMap) + for i := 0; i < numMixedDeploys; i++ { + deploy := createDeploymentWithBoth( + fmt.Sprintf("mixed-deploy-%d", i), + namespace, + fmt.Sprintf("mixed-cm-%d", i%numConfigMaps), + fmt.Sprintf("mixed-secret-%d", i%numSecrets), + ) + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create mixed Deployment: %v", err) + } + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S10: Starting alternating Secret and ConfigMap updates...") + + secretUpdateCount := 0 + cmUpdateCount := 0 + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + updateSecret := true // Alternate between Secret and ConfigMap updates + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + return s.calculateExpected(secretUpdateCount, cmUpdateCount, numSecretOnlyDeploys, numConfigMapOnlyDeploys, numMixedDeploys), nil + case <-ticker.C: + if updateSecret { + // Update a random Secret + secretIndex := rand.Intn(numSecrets) + secret, err := client.CoreV1().Secrets(namespace).Get(setupCtx, fmt.Sprintf("mixed-secret-%d", secretIndex), metav1.GetOptions{}) + if err != nil { + continue + } + secret.StringData = map[string]string{ + "password": fmt.Sprintf("updated-secret-%d-%d", secretIndex, secretUpdateCount), + } + if _, err := client.CoreV1().Secrets(namespace).Update(setupCtx, secret, metav1.UpdateOptions{}); err == nil { + secretUpdateCount++ + } + } else { + // Update a random ConfigMap + cmIndex := rand.Intn(numConfigMaps) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("mixed-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["config"] = fmt.Sprintf("updated-config-%d-%d", cmIndex, cmUpdateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err == nil { + cmUpdateCount++ + } + } + updateSecret = !updateSecret + } + } + + log.Printf("S10: Completed %d Secret updates and %d ConfigMap updates", secretUpdateCount, cmUpdateCount) + return s.calculateExpected(secretUpdateCount, cmUpdateCount, numSecretOnlyDeploys, numConfigMapOnlyDeploys, numMixedDeploys), nil +} + +func (s *SecretsAndMixedScenario) calculateExpected(secretUpdates, cmUpdates, secretOnlyDeploys, cmOnlyDeploys, mixedDeploys int) ExpectedMetrics { + // Secret updates trigger: secret-only deploys + mixed deploys + secretTriggeredReloads := secretUpdates * (secretOnlyDeploys + mixedDeploys) + // ConfigMap updates trigger: cm-only deploys + mixed deploys + cmTriggeredReloads := cmUpdates * (cmOnlyDeploys + mixedDeploys) + totalExpectedReloads := secretTriggeredReloads + cmTriggeredReloads + + return ExpectedMetrics{ + ActionTotal: totalExpectedReloads, + ReloadExecutedTotal: totalExpectedReloads, + Description: fmt.Sprintf("S10: %d Secret updates (ā†’%d reloads) + %d CM updates (ā†’%d reloads) = %d total", + secretUpdates, secretTriggeredReloads, cmUpdates, cmTriggeredReloads, totalExpectedReloads), + } +} + +// MultiWorkloadTypeScenario - Tests all supported workload types with a shared ConfigMap. +type MultiWorkloadTypeScenario struct{} + +func (s *MultiWorkloadTypeScenario) Name() string { return "S9" } +func (s *MultiWorkloadTypeScenario) Description() string { + return "Multi-workload types (Deploy, StatefulSet, DaemonSet, Job, CronJob)" +} + +func (s *MultiWorkloadTypeScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S9: Creating shared ConfigMap and multiple workload types...") + + const numDeployments = 5 + const numStatefulSets = 3 + const numDaemonSets = 2 + + setupCtx := context.Background() + + // Create shared ConfigMap + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "multi-type-cm", + Namespace: namespace, + }, + Data: map[string]string{"config": "initial"}, + } + if _, err := client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}); err != nil { + return ExpectedMetrics{}, fmt.Errorf("failed to create shared ConfigMap: %w", err) + } + + // Create Deployments + for i := 0; i < numDeployments; i++ { + deploy := createDeployment(fmt.Sprintf("multi-deploy-%d", i), namespace, "multi-type-cm") + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create Deployment %d: %v", i, err) + } + } + + // Create StatefulSets + for i := 0; i < numStatefulSets; i++ { + sts := createStatefulSet(fmt.Sprintf("multi-sts-%d", i), namespace, "multi-type-cm") + if _, err := client.AppsV1().StatefulSets(namespace).Create(setupCtx, sts, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create StatefulSet %d: %v", i, err) + } + } + + // Create DaemonSets + for i := 0; i < numDaemonSets; i++ { + ds := createDaemonSet(fmt.Sprintf("multi-ds-%d", i), namespace, "multi-type-cm") + if _, err := client.AppsV1().DaemonSets(namespace).Create(setupCtx, ds, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create DaemonSet %d: %v", i, err) + } + } + + // Wait for workloads to be ready + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + if err := waitForStatefulSetsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + if err := waitForDaemonSetsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S9: Starting ConfigMap updates to trigger reloads on all workload types...") + + updateCount := 0 + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + return s.calculateExpected(updateCount, numDeployments, numStatefulSets, numDaemonSets), nil + case <-ticker.C: + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, "multi-type-cm", metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["config"] = fmt.Sprintf("update-%d", updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err != nil { + log.Printf("Failed to update shared ConfigMap: %v", err) + } else { + updateCount++ + log.Printf("S9: Updated shared ConfigMap (update #%d)", updateCount) + } + } + } + + log.Printf("S9: Completed %d ConfigMap updates", updateCount) + return s.calculateExpected(updateCount, numDeployments, numStatefulSets, numDaemonSets), nil +} + +func (s *MultiWorkloadTypeScenario) calculateExpected(updateCount, numDeployments, numStatefulSets, numDaemonSets int) ExpectedMetrics { + // Each CM update triggers reload on all workloads + totalWorkloads := numDeployments + numStatefulSets + numDaemonSets + expectedReloads := updateCount * totalWorkloads + + return ExpectedMetrics{ + ActionTotal: expectedReloads, + ReloadExecutedTotal: expectedReloads, + WorkloadsMatchedTotal: expectedReloads, + Description: fmt.Sprintf("S9: %d CM updates Ɨ %d workloads (%d Deploys + %d STS + %d DS) = %d reloads", + updateCount, totalWorkloads, numDeployments, numStatefulSets, numDaemonSets, expectedReloads), + } +} + +func createStatefulSet(name, namespace, configMapName string) *appsv1.StatefulSet { + replicas := int32(1) + terminationGracePeriod := int64(0) + + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "reloader.stakater.com/auto": "true", + }, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + ServiceName: name, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": name}, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: &terminationGracePeriod, + Containers: []corev1.Container{ + { + Name: "app", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "sleep 999999999"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMapName, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func createDaemonSet(name, namespace, configMapName string) *appsv1.DaemonSet { + terminationGracePeriod := int64(0) + + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "reloader.stakater.com/auto": "true", + }, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": name}, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: &terminationGracePeriod, + // Use tolerations to run on all nodes including control-plane + Tolerations: []corev1.Toleration{ + { + Key: "node-role.kubernetes.io/control-plane", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "node-role.kubernetes.io/master", + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + }, + Containers: []corev1.Container{ + { + Name: "app", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "sleep 999999999"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMapName, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func waitForStatefulSetsReady(ctx context.Context, client kubernetes.Interface, namespace string, timeout time.Duration) error { + log.Printf("Waiting for all StatefulSets in %s to be ready (timeout: %v)...", namespace, timeout) + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + stsList, err := client.AppsV1().StatefulSets(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list StatefulSets: %w", err) + } + + if len(stsList.Items) == 0 { + log.Printf("No StatefulSets found in %s", namespace) + return nil + } + + allReady := true + notReady := 0 + for _, sts := range stsList.Items { + if sts.Status.ReadyReplicas < *sts.Spec.Replicas { + allReady = false + notReady++ + } + } + + if allReady { + log.Printf("All %d StatefulSets in %s are ready", len(stsList.Items), namespace) + return nil + } + + log.Printf("Waiting for StatefulSets: %d/%d not ready yet...", notReady, len(stsList.Items)) + time.Sleep(5 * time.Second) + } + + return fmt.Errorf("timeout waiting for StatefulSets to be ready") +} + +func waitForDaemonSetsReady(ctx context.Context, client kubernetes.Interface, namespace string, timeout time.Duration) error { + log.Printf("Waiting for all DaemonSets in %s to be ready (timeout: %v)...", namespace, timeout) + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + dsList, err := client.AppsV1().DaemonSets(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list DaemonSets: %w", err) + } + + if len(dsList.Items) == 0 { + log.Printf("No DaemonSets found in %s", namespace) + return nil + } + + allReady := true + notReady := 0 + for _, ds := range dsList.Items { + if ds.Status.NumberReady < ds.Status.DesiredNumberScheduled { + allReady = false + notReady++ + } + } + + if allReady { + log.Printf("All %d DaemonSets in %s are ready", len(dsList.Items), namespace) + return nil + } + + log.Printf("Waiting for DaemonSets: %d/%d not ready yet...", notReady, len(dsList.Items)) + time.Sleep(5 * time.Second) + } + + return fmt.Errorf("timeout waiting for DaemonSets to be ready") +} + +// ComplexReferencesScenario - Tests init containers, valueFrom, and projected volumes. +type ComplexReferencesScenario struct{} + +func (s *ComplexReferencesScenario) Name() string { return "S13" } +func (s *ComplexReferencesScenario) Description() string { + return "Complex references (init containers, valueFrom, projected volumes)" +} + +func (s *ComplexReferencesScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S13: Creating ConfigMaps and complex deployments with various reference types...") + + const numConfigMaps = 5 + const numDeployments = 5 + + setupCtx := context.Background() + + // Create ConfigMaps with multiple keys + for i := 0; i < numConfigMaps; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("complex-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{ + "key1": fmt.Sprintf("value1-%d", i), + "key2": fmt.Sprintf("value2-%d", i), + "config": fmt.Sprintf("config-%d", i), + }, + } + if _, err := client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create ConfigMap %s: %v", cm.Name, err) + } + } + + // Create complex deployments with various reference types + for i := 0; i < numDeployments; i++ { + // Each deployment references multiple ConfigMaps in different ways + primaryCM := fmt.Sprintf("complex-cm-%d", i) + secondaryCM := fmt.Sprintf("complex-cm-%d", (i+1)%numConfigMaps) + + deploy := createComplexDeployment( + fmt.Sprintf("complex-deploy-%d", i), + namespace, + primaryCM, + secondaryCM, + ) + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create complex Deployment: %v", err) + } + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S13: Starting ConfigMap updates to test all reference types...") + + updateCount := 0 + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + return s.calculateExpected(updateCount, numConfigMaps, numDeployments), nil + case <-ticker.C: + // Update a random ConfigMap + cmIndex := rand.Intn(numConfigMaps) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("complex-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["key1"] = fmt.Sprintf("updated-value1-%d-%d", cmIndex, updateCount) + cm.Data["config"] = fmt.Sprintf("updated-config-%d-%d", cmIndex, updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err != nil { + log.Printf("Failed to update ConfigMap: %v", err) + } else { + updateCount++ + log.Printf("S13: Updated complex-cm-%d (update #%d)", cmIndex, updateCount) + } + } + } + + log.Printf("S13: Completed %d ConfigMap updates", updateCount) + return s.calculateExpected(updateCount, numConfigMaps, numDeployments), nil +} + +func (s *ComplexReferencesScenario) calculateExpected(updateCount, numConfigMaps, numDeployments int) ExpectedMetrics { + // Each ConfigMap is referenced by: + // - 1 deployment as primary (envFrom in init + valueFrom in main + volume mount) + // - 1 deployment as secondary (projected volume) + // So each CM update triggers 2 deployments (on average with random updates) + // But since we're randomly updating, each update affects those 2 deployments + expectedReloadsPerUpdate := 2 + expectedReloads := updateCount * expectedReloadsPerUpdate + + return ExpectedMetrics{ + ActionTotal: expectedReloads, + ReloadExecutedTotal: expectedReloads, + Description: fmt.Sprintf("S13: %d CM updates Ɨ ~%d affected deploys = ~%d reloads (init containers, valueFrom, volumes, projected)", + updateCount, expectedReloadsPerUpdate, expectedReloads), + } +} + +// PauseResumeScenario - Tests pause-period functionality under rapid updates. +type PauseResumeScenario struct{} + +func (s *PauseResumeScenario) Name() string { return "S12" } +func (s *PauseResumeScenario) Description() string { + return "Pause & Resume (rapid updates with pause-period)" +} + +func (s *PauseResumeScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + log.Println("S12: Creating ConfigMaps and Deployments with pause-period annotation...") + + const numConfigMaps = 10 + const numDeployments = 10 + const pausePeriod = 15 * time.Second + const updateInterval = 2 * time.Second + + setupCtx := context.Background() + + // Create ConfigMaps + for i := 0; i < numConfigMaps; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("pause-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{"key": "initial-value"}, + } + if _, err := client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create ConfigMap %s: %v", cm.Name, err) + } + } + + // Create Deployments with pause-period annotation + for i := 0; i < numDeployments; i++ { + deploy := createDeploymentWithPause( + fmt.Sprintf("pause-deploy-%d", i), + namespace, + fmt.Sprintf("pause-cm-%d", i), + pausePeriod, + ) + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create Deployment: %v", err) + } + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Printf("S12: Starting rapid ConfigMap updates (every %v) with %v pause-period...", updateInterval, pausePeriod) + + updateCount := 0 + ticker := time.NewTicker(updateInterval) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 5*time.Second) + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + return s.calculateExpected(updateCount, duration, updateInterval, pausePeriod), nil + case <-ticker.C: + // Update a random ConfigMap + cmIndex := rand.Intn(numConfigMaps) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("pause-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["key"] = fmt.Sprintf("update-%d-%d", cmIndex, updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err != nil { + log.Printf("Failed to update ConfigMap: %v", err) + } else { + updateCount++ + } + } + } + + log.Printf("S12: Completed %d rapid updates (pause-period should reduce actual reloads)", updateCount) + return s.calculateExpected(updateCount, duration, updateInterval, pausePeriod), nil +} + +func (s *PauseResumeScenario) calculateExpected(updateCount int, duration, updateInterval, pausePeriod time.Duration) ExpectedMetrics { + // With pause-period, we expect fewer reloads than updates + // Each deployment gets updates at random, and pause-period prevents rapid consecutive reloads + // The exact count depends on the distribution of updates across ConfigMaps + // Rough estimate: each CM gets updated ~(updateCount/10) times + // With 15s pause and 2s interval, we get roughly 1 reload per pause period per CM + // So expected reloads ā‰ˆ duration / pausePeriod per deployment = (duration/pausePeriod) * numDeployments + + // This is an approximation - the actual value depends on random distribution + expectedCycles := int(duration / pausePeriod) + if expectedCycles < 1 { + expectedCycles = 1 + } + + return ExpectedMetrics{ + // Don't set exact expected values since pause-period makes counts unpredictable + // The scenario validates that reloads << updates due to pause behavior + Description: fmt.Sprintf("S12: %d updates with %v pause-period (expect ~%d reload cycles, actual reloads << updates)", + updateCount, pausePeriod, expectedCycles), + } +} + +// AnnotationStrategyScenario - Tests annotation-based reload strategy. +// This scenario deploys its own Reloader instance with --reload-strategy=annotations. +type AnnotationStrategyScenario struct { + // Image is the Reloader image to use. Must be set before running. + Image string +} + +func (s *AnnotationStrategyScenario) Name() string { return "S11" } +func (s *AnnotationStrategyScenario) Description() string { + return "Annotation reload strategy (--reload-strategy=annotations)" +} + +func (s *AnnotationStrategyScenario) Run(ctx context.Context, client kubernetes.Interface, namespace string, duration time.Duration) (ExpectedMetrics, error) { + if s.Image == "" { + return ExpectedMetrics{}, fmt.Errorf("S11 requires Image to be set (use the same image as --new-image)") + } + + log.Println("S11: Deploying Reloader with --reload-strategy=annotations...") + + // Deploy S11's own Reloader instance + reloaderNS := "reloader-s11" + mgr := reloader.NewManager(reloader.Config{ + Version: "s11", + Image: s.Image, + Namespace: reloaderNS, + ReloadStrategy: "annotations", + }) + + if err := mgr.Deploy(ctx); err != nil { + return ExpectedMetrics{}, fmt.Errorf("deploying S11 reloader: %w", err) + } + + // Ensure cleanup on exit + defer func() { + log.Println("S11: Cleaning up S11-specific Reloader...") + cleanupCtx := context.Background() + if err := mgr.Cleanup(cleanupCtx); err != nil { + log.Printf("Warning: failed to cleanup S11 reloader: %v", err) + } + }() + + log.Println("S11: Creating ConfigMaps and Deployments...") + + const numConfigMaps = 10 + const numDeployments = 10 + + setupCtx := context.Background() + + // Create ConfigMaps + for i := 0; i < numConfigMaps; i++ { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("annot-cm-%d", i), + Namespace: namespace, + }, + Data: map[string]string{"key": "initial-value"}, + } + if _, err := client.CoreV1().ConfigMaps(namespace).Create(setupCtx, cm, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create ConfigMap %s: %v", cm.Name, err) + } + } + + // Create Deployments + for i := 0; i < numDeployments; i++ { + deploy := createDeployment(fmt.Sprintf("annot-deploy-%d", i), namespace, fmt.Sprintf("annot-cm-%d", i)) + if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { + log.Printf("Failed to create Deployment: %v", err) + } + } + + if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { + log.Printf("Warning: %v - continuing anyway", err) + } + + log.Println("S11: Starting ConfigMap updates with annotation strategy...") + + updateCount := 0 + annotationUpdatesSeen := 0 + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + endTime := time.Now().Add(duration - 10*time.Second) // Extra time for cleanup + for time.Now().Before(endTime) { + select { + case <-ctx.Done(): + return s.calculateExpected(updateCount, annotationUpdatesSeen), nil + case <-ticker.C: + // Update a random ConfigMap + cmIndex := rand.Intn(numConfigMaps) + cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("annot-cm-%d", cmIndex), metav1.GetOptions{}) + if err != nil { + continue + } + cm.Data["key"] = fmt.Sprintf("update-%d-%d", cmIndex, updateCount) + if _, err := client.CoreV1().ConfigMaps(namespace).Update(setupCtx, cm, metav1.UpdateOptions{}); err != nil { + log.Printf("Failed to update ConfigMap: %v", err) + } else { + updateCount++ + } + + // Periodically check for annotation updates on deployments + if updateCount%10 == 0 { + deploy, err := client.AppsV1().Deployments(namespace).Get(setupCtx, fmt.Sprintf("annot-deploy-%d", cmIndex), metav1.GetOptions{}) + if err == nil { + if _, hasAnnotation := deploy.Spec.Template.Annotations["reloader.stakater.com/last-reloaded-from"]; hasAnnotation { + annotationUpdatesSeen++ + } + } + } + } + } + + // Final check: verify annotation strategy is working + log.Println("S11: Verifying annotation-based reload...") + time.Sleep(5 * time.Second) // Allow time for final updates to propagate + + deploysWithAnnotation := 0 + for i := 0; i < numDeployments; i++ { + deploy, err := client.AppsV1().Deployments(namespace).Get(setupCtx, fmt.Sprintf("annot-deploy-%d", i), metav1.GetOptions{}) + if err != nil { + continue + } + if deploy.Spec.Template.Annotations != nil { + if _, ok := deploy.Spec.Template.Annotations["reloader.stakater.com/last-reloaded-from"]; ok { + deploysWithAnnotation++ + } + } + } + + log.Printf("S11: Completed %d updates, %d deployments have reload annotation", updateCount, deploysWithAnnotation) + return s.calculateExpected(updateCount, deploysWithAnnotation), nil +} + +func (s *AnnotationStrategyScenario) calculateExpected(updateCount, deploysWithAnnotation int) ExpectedMetrics { + return ExpectedMetrics{ + ActionTotal: updateCount, + ReloadExecutedTotal: updateCount, + Description: fmt.Sprintf("S11: %d updates with annotation strategy, %d deployments received annotation", + updateCount, deploysWithAnnotation), + } +} + +func createDeploymentWithPause(name, namespace, configMapName string, pausePeriod time.Duration) *appsv1.Deployment { + replicas := int32(1) + maxSurge := intstr.FromInt(1) + maxUnavailable := intstr.FromInt(1) + terminationGracePeriod := int64(0) + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "reloader.stakater.com/auto": "true", + // Deployment-specific pause-period annotation + "deployment.reloader.stakater.com/pause-period": fmt.Sprintf("%ds", int(pausePeriod.Seconds())), + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": name}, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: &terminationGracePeriod, + Containers: []corev1.Container{ + { + Name: "app", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "sleep 999999999"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMapName, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +// createComplexDeployment creates a deployment with multiple ConfigMap reference types. +// - Init container using envFrom +// - Main container using env.valueFrom.configMapKeyRef +// - Sidecar container using volume mount +// - Projected volume combining multiple ConfigMaps +func createComplexDeployment(name, namespace, primaryCM, secondaryCM string) *appsv1.Deployment { + replicas := int32(1) + maxSurge := intstr.FromInt(1) + maxUnavailable := intstr.FromInt(1) + terminationGracePeriod := int64(0) + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Annotations: map[string]string{ + "reloader.stakater.com/auto": "true", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &maxSurge, + MaxUnavailable: &maxUnavailable, + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": name}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": name}, + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: &terminationGracePeriod, + // Init container using envFrom + InitContainers: []corev1.Container{ + { + Name: "init", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "echo Init done"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: primaryCM, + }, + }, + }, + }, + }, + }, + Containers: []corev1.Container{ + // Main container using valueFrom (individual keys) + { + Name: "main", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "sleep 999999999"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + Env: []corev1.EnvVar{ + { + Name: "CONFIG_KEY1", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: primaryCM, + }, + Key: "key1", + }, + }, + }, + { + Name: "CONFIG_KEY2", + ValueFrom: &corev1.EnvVarSource{ + ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: primaryCM, + }, + Key: "key2", + }, + }, + }, + }, + }, + // Sidecar using volume mount + { + Name: "sidecar", + Image: "gcr.io/google-containers/busybox:1.27", + Command: []string{"sh", "-c", "sleep 999999999"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1m"), + corev1.ResourceMemory: resource.MustParse("4Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("16Mi"), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "config-volume", + MountPath: "/etc/config", + }, + { + Name: "projected-volume", + MountPath: "/etc/projected", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + // Regular ConfigMap volume + { + Name: "config-volume", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: primaryCM, + }, + }, + }, + }, + // Projected volume combining multiple ConfigMaps + { + Name: "projected-volume", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: primaryCM, + }, + Items: []corev1.KeyToPath{ + { + Key: "key1", + Path: "primary-key1", + }, + }, + }, + }, + { + ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secondaryCM, + }, + Items: []corev1.KeyToPath{ + { + Key: "key1", + Path: "secondary-key1", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} diff --git a/test/loadtest/manifests/prometheus.yaml b/test/loadtest/manifests/prometheus.yaml new file mode 100644 index 0000000..f826f52 --- /dev/null +++ b/test/loadtest/manifests/prometheus.yaml @@ -0,0 +1,181 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + namespace: monitoring +data: + prometheus.yml: | + global: + scrape_interval: 2s + evaluation_interval: 2s + + scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'reloader-old' + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - reloader-old + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + + - job_name: 'reloader-new' + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - reloader-new + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: + - apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: + - configmaps + verbs: ["get"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: + - kind: ServiceAccount + name: prometheus + namespace: monitoring +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + serviceAccountName: prometheus + containers: + - name: prometheus + image: quay.io/prometheus/prometheus:v2.47.0 + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --web.console.libraries=/usr/share/prometheus/console_libraries + - --web.console.templates=/usr/share/prometheus/consoles + - --web.enable-lifecycle + ports: + - containerPort: 9090 + volumeMounts: + - name: config + mountPath: /etc/prometheus + - name: data + mountPath: /prometheus + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 200m + memory: 512Mi + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: config + configMap: + name: prometheus-config + - name: data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: monitoring +spec: + selector: + app: prometheus + ports: + - port: 9090 + targetPort: 9090 + type: NodePort From 512278d740f0cadb9a4133b2d08a4d0da2bfd919 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Tue, 6 Jan 2026 11:06:44 +0100 Subject: [PATCH 20/45] ci: Allow manual trigger --- .github/workflows/loadtest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/loadtest.yml b/.github/workflows/loadtest.yml index a62e3e3..a822fe7 100644 --- a/.github/workflows/loadtest.yml +++ b/.github/workflows/loadtest.yml @@ -3,6 +3,7 @@ name: Load Test on: issue_comment: types: [created] + workflow_dispatch: jobs: loadtest: From 5b63610f4f28fe2f5600260377e9d63f077eaf2f Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Tue, 6 Jan 2026 11:47:23 +0100 Subject: [PATCH 21/45] ci: Remove manual_dispatch from loadtests --- .github/workflows/loadtest.yml | 63 ++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/.github/workflows/loadtest.yml b/.github/workflows/loadtest.yml index a822fe7..db01e7e 100644 --- a/.github/workflows/loadtest.yml +++ b/.github/workflows/loadtest.yml @@ -3,7 +3,11 @@ name: Load Test on: issue_comment: types: [created] - workflow_dispatch: + +permissions: + contents: read + pull-requests: write + issues: write jobs: loadtest: @@ -56,6 +60,12 @@ jobs: chmod +x ./kind sudo mv ./kind /usr/local/bin/kind + - name: Install kubectl + run: | + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl + sudo mv kubectl /usr/local/bin/kubectl + # Build OLD image from base branch (e.g., main) - name: Checkout base branch (old) uses: actions/checkout@v4 @@ -165,31 +175,32 @@ jobs: const overallStatus = failCount === 0 ? 'āœ… ALL PASSED' : `āŒ ${failCount} FAILED`; - const body = `## Load Test Results ${overallStatus} - - **Comparing:** \`${{ steps.pr.outputs.base_ref }}\` (old) vs \`${{ steps.pr.outputs.head_ref }}\` (new) - **Old commit:** ${{ steps.pr.outputs.base_sha }} - **New commit:** ${{ steps.pr.outputs.head_sha }} - **Triggered by:** @${{ github.event.comment.user.login }} - - ### Summary - - | Scenario | Status | - |----------|--------| - ${summaries.join('\n')} - - **Total:** ${passCount} passed, ${failCount} failed - - ### Detailed Results - - ${results} - -
- šŸ“¦ Download full results - - Artifacts are available in the [workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}). -
- `; + const body = [ + `## Load Test Results ${overallStatus}`, + '', + `**Comparing:** \`${{ steps.pr.outputs.base_ref }}\` (old) vs \`${{ steps.pr.outputs.head_ref }}\` (new)`, + `**Old commit:** ${{ steps.pr.outputs.base_sha }}`, + `**New commit:** ${{ steps.pr.outputs.head_sha }}`, + `**Triggered by:** @${{ github.event.comment.user.login }}`, + '', + '### Summary', + '', + '| Scenario | Status |', + '|----------|--------|', + summaries.join('\n'), + '', + `**Total:** ${passCount} passed, ${failCount} failed`, + '', + '### Detailed Results', + '', + results, + '', + '
', + 'šŸ“¦ Download full results', + '', + `Artifacts are available in the [workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}).`, + '
', + ].join('\n'); await github.rest.issues.createComment({ owner: context.repo.owner, From e1db875efeba113b068c470b92e24173054f0e6f Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Tue, 6 Jan 2026 22:20:20 +0100 Subject: [PATCH 22/45] Fix failing tests Signed-off-by: faizanahmad055 --- internal/pkg/crypto/sha.go | 6 ++---- internal/pkg/crypto/sha_test.go | 13 +++++++++++++ internal/pkg/handler/upgrade.go | 6 +++--- internal/pkg/handler/upgrade_test.go | 6 +++--- internal/pkg/testutil/kube.go | 6 +++++- internal/pkg/util/util.go | 2 +- 6 files changed, 27 insertions(+), 12 deletions(-) diff --git a/internal/pkg/crypto/sha.go b/internal/pkg/crypto/sha.go index 9235425..f9ae235 100644 --- a/internal/pkg/crypto/sha.go +++ b/internal/pkg/crypto/sha.go @@ -6,11 +6,9 @@ import ( ) // GenerateSHA generates SHA from string +// Always returns a hash value, even for empty strings, to ensure consistent behavior +// and avoid issues with string matching operations (e.g., strings.Contains(str, "") always returns true) func GenerateSHA(data string) string { - if data == "" { - return "" - } - hash := sha512.Sum512_256([]byte(data)) return hex.EncodeToString(hash[:]) } diff --git a/internal/pkg/crypto/sha_test.go b/internal/pkg/crypto/sha_test.go index 60d5af6..761f8d0 100644 --- a/internal/pkg/crypto/sha_test.go +++ b/internal/pkg/crypto/sha_test.go @@ -13,3 +13,16 @@ func TestGenerateSHA(t *testing.T) { t.Errorf("Failed to generate SHA") } } + +// TestGenerateSHAEmptyString verifies that empty string generates a valid hash +// This ensures consistent behavior and avoids issues with string matching operations +func TestGenerateSHAEmptyString(t *testing.T) { + result := GenerateSHA("") + expected := "c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a" + if result != expected { + t.Errorf("Failed to generate SHA for empty string. Expected: %s, Got: %s", expected, result) + } + if len(result) != 64 { + t.Errorf("SHA hash should be 64 characters long, got %d", len(result)) + } +} diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index b10bfbc..6d63d5c 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -539,8 +539,8 @@ func updatePodAnnotations(upgradeFuncs callbacks.RollingUpgradeFuncs, item runti } func secretProviderClassAnnotationReloaded(oldAnnotations map[string]string, newConfig common.Config) bool { - annotaion := oldAnnotations[getReloaderAnnotationKey()] - return strings.Contains(annotaion, newConfig.ResourceName) && strings.Contains(annotaion, newConfig.SHAValue) + annotation := oldAnnotations[getReloaderAnnotationKey()] + return strings.Contains(annotation, newConfig.ResourceName) && strings.Contains(annotation, newConfig.SHAValue) } func getReloaderAnnotationKey() string { @@ -645,7 +645,7 @@ func secretProviderClassEnvReloaded(containers []v1.Container, envVar string, sh } func populateAnnotationsFromSecretProviderClass(clients kube.Clients, config *common.Config) { - obj, err := clients.CSIClient.SecretsstoreV1().SecretProviderClasses(config.Namespace).Get(context.TODO(), config.ResourceName, metav1.GetOptions{}) + obj, err := clients.CSIClient.SecretsstoreV1().SecretProviderClasses(config.Namespace).Get(context.Background(), config.ResourceName, metav1.GetOptions{}) annotations := make(map[string]string) if err != nil { if apierrors.IsNotFound(err) { diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index a334db0..e905ee0 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -3327,7 +3327,7 @@ func TestRollingUpgradeForStatefulSetWithSecretProviderClassUsingArs(t *testing. err := PerformAction(clients, config, statefulSetFuncs, collectors, nil, invokeReloadStrategy) time.Sleep(5 * time.Second) if err != nil { - t.Errorf("Rolling upgrade failed for StatefulSet with SecretProviderClass") + t.Errorf("Rolling upgrade failed for StatefulSet with SecretProviderClass: %v", err) } logrus.Infof("Verifying statefulSet update") @@ -3337,11 +3337,11 @@ func TestRollingUpgradeForStatefulSetWithSecretProviderClassUsingArs(t *testing. } if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { - t.Errorf("Counter was not increased") + t.Errorf("Counter was not increased, expected 1 but got %f", promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded))) } if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": arsNamespace})) != 1 { - t.Errorf("Counter by namespace was not increased") + t.Errorf("Counter by namespace was not increased, expected 1 but got %f", promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": arsNamespace}))) } testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, statefulSetFuncs, collectors, envVarPostfix) diff --git a/internal/pkg/testutil/kube.go b/internal/pkg/testutil/kube.go index 4901d9a..a778eb1 100644 --- a/internal/pkg/testutil/kube.go +++ b/internal/pkg/testutil/kube.go @@ -1327,7 +1327,11 @@ func VerifyResourceAnnotationUpdate(clients kube.Clients, config common.Config, } func GetSHAfromEmptyData() string { - return crypto.GenerateSHA("") + // Use a special marker that represents "deleted" or "empty" state + // This ensures we have a distinct, deterministic hash for the delete strategy + // Note: We could use GenerateSHA("") which now returns a hash, but using a marker + // makes the intent clearer and avoids potential confusion with actual empty data + return crypto.GenerateSHA("__RELOADER_EMPTY_DELETE_MARKER__") } // GetRollout provides rollout for testing diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index 047d068..476cdb9 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -106,7 +106,7 @@ func ConfigureReloaderFlags(cmd *cobra.Command) { cmd.PersistentFlags().BoolVar(&options.SyncAfterRestart, "sync-after-restart", false, "Sync add events after reloader restarts") cmd.PersistentFlags().BoolVar(&options.EnablePProf, "enable-pprof", false, "Enable pprof for profiling") cmd.PersistentFlags().StringVar(&options.PProfAddr, "pprof-addr", ":6060", "Address to start pprof server on. Default is :6060") - cmd.PersistentFlags().BoolVar(&options.EnableCSIIntegration, "enable-csi-integration", false, "Enables CSI integration. Default is :true") + cmd.PersistentFlags().BoolVar(&options.EnableCSIIntegration, "enable-csi-integration", false, "Enables CSI integration. Default is :false") } func GetIgnoredResourcesList() (List, error) { From 8b64c9b9cda9ef1609e3e9dc078dd8ccc5c9bf81 Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Tue, 6 Jan 2026 22:53:24 +0100 Subject: [PATCH 23/45] Fix failing SHA test Signed-off-by: faizanahmad055 --- internal/pkg/crypto/sha_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/crypto/sha_test.go b/internal/pkg/crypto/sha_test.go index 761f8d0..ee530e3 100644 --- a/internal/pkg/crypto/sha_test.go +++ b/internal/pkg/crypto/sha_test.go @@ -7,7 +7,7 @@ import ( // TestGenerateSHA generates the sha from given data and verifies whether it is correct or not func TestGenerateSHA(t *testing.T) { data := "www.stakater.com" - sha := "abd4ed82fb04548388a6cf3c339fd9dc84d275df" + sha := "2e9aa975331b22861b4f62b7fcc69b63e001f938361fee3b4ed888adf26a10e3" result := GenerateSHA(data) if result != sha { t.Errorf("Failed to generate SHA") From b0ca635e4984dec799b4c15a9689eae26dfb630e Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Wed, 7 Jan 2026 09:11:48 +0100 Subject: [PATCH 24/45] Add file filtering in UBI docker image Signed-off-by: faizanahmad055 --- Dockerfile.ubi | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/Dockerfile.ubi b/Dockerfile.ubi index 4359730..20e2b16 100644 --- a/Dockerfile.ubi +++ b/Dockerfile.ubi @@ -20,7 +20,25 @@ RUN mkdir /image && \ COPY ubi-build-files-${TARGETARCH}.txt /tmp # Copy all the required files from the base UBI image into the image directory # As the go binary is not statically compiled this includes everything needed for CGO to work, cacerts, tzdata and RH release files -RUN tar cf /tmp/files.tar -T /tmp/ubi-build-files-${TARGETARCH}.txt && tar xf /tmp/files.tar -C /image/ +# Filter existing files and exclude temporary entitlement files that may be removed during build +RUN set -e && \ + # Filter files that actually exist (files, directories, or symlinks) + while IFS= read -r file; do \ + [ -n "$file" ] && ([ -e "$file" ] || [ -L "$file" ]) && echo "$file"; \ + done < /tmp/ubi-build-files-${TARGETARCH}.txt > /tmp/existing-files.txt && \ + # Create tarball if we have files to archive + if [ -s /tmp/existing-files.txt ]; then \ + tar -chf /tmp/files.tar \ + --exclude='etc/pki/entitlement-host*' \ + -T /tmp/existing-files.txt 2>&1 | grep -vE "(File removed before we read it|Cannot stat)" || true; \ + # Extract only if tarball was created successfully + if [ -f /tmp/files.tar ]; then \ + tar -xf /tmp/files.tar -C /image/ && \ + rm -f /tmp/files.tar; \ + fi; \ + fi && \ + # Clean up temporary file list + rm -f /tmp/existing-files.txt # Generate a rpm database which contains all the packages that you said were needed in ubi-build-files-*.txt RUN rpm --root /image --initdb \ From 703319e732f11ac6314b265ffe59098039ac2bd7 Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Wed, 7 Jan 2026 09:27:29 +0100 Subject: [PATCH 25/45] Improve file filtering in UBI docker image Signed-off-by: faizanahmad055 --- Dockerfile.ubi | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/Dockerfile.ubi b/Dockerfile.ubi index 20e2b16..9d8c88d 100644 --- a/Dockerfile.ubi +++ b/Dockerfile.ubi @@ -23,19 +23,29 @@ COPY ubi-build-files-${TARGETARCH}.txt /tmp # Filter existing files and exclude temporary entitlement files that may be removed during build RUN set -e && \ # Filter files that actually exist (files, directories, or symlinks) + # This ensures we only copy files that are present, avoiding "Cannot stat" errors while IFS= read -r file; do \ [ -n "$file" ] && ([ -e "$file" ] || [ -L "$file" ]) && echo "$file"; \ done < /tmp/ubi-build-files-${TARGETARCH}.txt > /tmp/existing-files.txt && \ - # Create tarball if we have files to archive - if [ -s /tmp/existing-files.txt ]; then \ - tar -chf /tmp/files.tar \ - --exclude='etc/pki/entitlement-host*' \ - -T /tmp/existing-files.txt 2>&1 | grep -vE "(File removed before we read it|Cannot stat)" || true; \ - # Extract only if tarball was created successfully - if [ -f /tmp/files.tar ]; then \ - tar -xf /tmp/files.tar -C /image/ && \ - rm -f /tmp/files.tar; \ - fi; \ + # Verify we have files to copy (fail if list is empty to catch configuration issues) + if [ ! -s /tmp/existing-files.txt ]; then \ + echo "ERROR: No files found to copy from ubi-build-files-${TARGETARCH}.txt" >&2; \ + echo "This indicates the base image may be missing required files or the file list is incorrect." >&2; \ + echo "Expected files from ubi-build-files-${TARGETARCH}.txt:" >&2; \ + cat /tmp/ubi-build-files-${TARGETARCH}.txt >&2; \ + exit 1; \ + fi && \ + # Create tarball, excluding only temporary entitlement files (safe to exclude) + # Note: --exclude only affects files within directories being archived, not the directories themselves + tar -chf /tmp/files.tar \ + --exclude='etc/pki/entitlement-host*' \ + -T /tmp/existing-files.txt 2>&1 | grep -vE "(File removed before we read it|Cannot stat)" || true; \ + # Extract tarball (critical files like libc.so.6, ld-linux, etc/ssl/certs are included) + if [ -f /tmp/files.tar ]; then \ + tar -xf /tmp/files.tar -C /image/ && \ + rm -f /tmp/files.tar; \ + else \ + echo "WARNING: Tarball was not created, but continuing..." >&2; \ fi && \ # Clean up temporary file list rm -f /tmp/existing-files.txt From 6fd7c8254a1644833f3194b101e3a8036191a754 Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Wed, 7 Jan 2026 10:28:38 +0100 Subject: [PATCH 26/45] Update filtering in UBI image Signed-off-by: faizanahmad055 --- Dockerfile.ubi | 36 +++++++++++------------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/Dockerfile.ubi b/Dockerfile.ubi index 9d8c88d..2f92b38 100644 --- a/Dockerfile.ubi +++ b/Dockerfile.ubi @@ -21,33 +21,19 @@ COPY ubi-build-files-${TARGETARCH}.txt /tmp # Copy all the required files from the base UBI image into the image directory # As the go binary is not statically compiled this includes everything needed for CGO to work, cacerts, tzdata and RH release files # Filter existing files and exclude temporary entitlement files that may be removed during build -RUN set -e && \ - # Filter files that actually exist (files, directories, or symlinks) - # This ensures we only copy files that are present, avoiding "Cannot stat" errors - while IFS= read -r file; do \ - [ -n "$file" ] && ([ -e "$file" ] || [ -L "$file" ]) && echo "$file"; \ +RUN while IFS= read -r file; do \ + [ -z "$file" ] && continue; \ + if [ -e "$file" ] || [ -L "$file" ]; then \ + echo "$file"; \ + fi; \ done < /tmp/ubi-build-files-${TARGETARCH}.txt > /tmp/existing-files.txt && \ - # Verify we have files to copy (fail if list is empty to catch configuration issues) - if [ ! -s /tmp/existing-files.txt ]; then \ - echo "ERROR: No files found to copy from ubi-build-files-${TARGETARCH}.txt" >&2; \ - echo "This indicates the base image may be missing required files or the file list is incorrect." >&2; \ - echo "Expected files from ubi-build-files-${TARGETARCH}.txt:" >&2; \ - cat /tmp/ubi-build-files-${TARGETARCH}.txt >&2; \ - exit 1; \ + if [ -s /tmp/existing-files.txt ]; then \ + tar -chf /tmp/files.tar --exclude='etc/pki/entitlement-host*' -T /tmp/existing-files.txt 2>&1 | grep -vE "(File removed before we read it|Cannot stat)" || true; \ + if [ -f /tmp/files.tar ]; then \ + tar xf /tmp/files.tar -C /image/ 2>/dev/null || true; \ + rm -f /tmp/files.tar; \ + fi; \ fi && \ - # Create tarball, excluding only temporary entitlement files (safe to exclude) - # Note: --exclude only affects files within directories being archived, not the directories themselves - tar -chf /tmp/files.tar \ - --exclude='etc/pki/entitlement-host*' \ - -T /tmp/existing-files.txt 2>&1 | grep -vE "(File removed before we read it|Cannot stat)" || true; \ - # Extract tarball (critical files like libc.so.6, ld-linux, etc/ssl/certs are included) - if [ -f /tmp/files.tar ]; then \ - tar -xf /tmp/files.tar -C /image/ && \ - rm -f /tmp/files.tar; \ - else \ - echo "WARNING: Tarball was not created, but continuing..." >&2; \ - fi && \ - # Clean up temporary file list rm -f /tmp/existing-files.txt # Generate a rpm database which contains all the packages that you said were needed in ubi-build-files-*.txt From 157cf0f2e4ae367f4b0f78f578ce40fd6dc3f3ad Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Wed, 7 Jan 2026 12:13:04 +0100 Subject: [PATCH 27/45] Remove SHA1 changes Signed-off-by: faizanahmad055 --- docs/How-it-works.md | 4 ++-- docs/Reloader-vs-ConfigmapController.md | 14 +++++++------- docs/Reloader-vs-k8s-trigger-controller.md | 2 +- internal/pkg/crypto/sha.go | 18 ++++++++++++------ internal/pkg/crypto/sha_test.go | 8 ++++---- internal/pkg/handler/upgrade_test.go | 12 ++++++------ 6 files changed, 32 insertions(+), 26 deletions(-) diff --git a/docs/How-it-works.md b/docs/How-it-works.md index 6a946f9..c0ae964 100644 --- a/docs/How-it-works.md +++ b/docs/How-it-works.md @@ -76,7 +76,7 @@ Note: Rolling upgrade also works in the same way for secrets. ### Hash Value Computation -Reloader uses SHA512 to compute hash value. SHA1 is used because it is efficient and less prone to collision. +Reloader uses SHA1 to compute hash value. SHA1 is used because it is efficient and less prone to collision. ## Monitor All Namespaces @@ -90,4 +90,4 @@ The output file can then be used to deploy Reloader in specific namespace. ## Compatibility With Helm Install and Upgrade -Reloader has no impact on helm deployment cycle. Reloader only injects an environment variable in `deployment`, `daemonset` or `statefulset`. The environment variable contains the SHA512 value of `ConfigMaps` or `Secrets` data. So if a deployment is created using Helm and Reloader updates the deployment, then next time you upgrade the helm release, Reloader will do nothing except changing that environment variable value in `deployment` , `daemonset` or `statefulset`. +Reloader has no impact on helm deployment cycle. Reloader only injects an environment variable in `deployment`, `daemonset` or `statefulset`. The environment variable contains the SHA1 value of `ConfigMaps` or `Secrets` data. So if a deployment is created using Helm and Reloader updates the deployment, then next time you upgrade the helm release, Reloader will do nothing except changing that environment variable value in `deployment` , `daemonset` or `statefulset`. diff --git a/docs/Reloader-vs-ConfigmapController.md b/docs/Reloader-vs-ConfigmapController.md index 3ddab08..1433daa 100644 --- a/docs/Reloader-vs-ConfigmapController.md +++ b/docs/Reloader-vs-ConfigmapController.md @@ -2,10 +2,10 @@ Reloader is inspired from [`configmapcontroller`](https://github.com/fabric8io/configmapcontroller) but there are many ways in which it differs from `configmapcontroller`. Below is the small comparison between these two controllers. -| Reloader | ConfigMap | -|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Reloader can watch both `Secrets` and `ConfigMaps`. | `configmapcontroller` can only watch changes in `ConfigMaps`. It cannot detect changes in other resources like `Secrets`. | -| Reloader can perform rolling upgrades on `deployments` as well as on `statefulsets` and `daemonsets` | `configmapcontroller` can only perform rolling upgrades on `deployments`. It currently does not support rolling upgrades on `statefulsets` and `daemonsets` | -| Reloader provides both unit test cases and end to end integration test cases for future updates. So one can make sure that new changes do not break any old functionality. | Currently there are not any unit test cases or end to end integration test cases in `configmap-controller`. It adds difficulties for any additional updates in `configmap-controller` and one can not know for sure whether new changes breaks any old functionality or not. | -| Reloader uses SHA512 to encode the change in `ConfigMap` or `Secret`. It then saves the SHA1 value in `STAKATER_FOO_CONFIGMAP` or `STAKATER_FOO_SECRET` environment variable depending upon where the change has happened. The use of SHA1 provides a concise 40 characters encoded value that is very less prone to collision. | `configmap-controller` uses `FABRICB_FOO_REVISION` environment variable to store any change in `ConfigMap` controller. It does not encode it or convert it in suitable hash value to avoid data pollution in deployment. | -| Reloader allows you to customize your own annotation (for both `Secrets` and `ConfigMaps`) using command line flags | `configmap-controller` restricts you to only their provided annotation | +| Reloader | ConfigMap | +|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Reloader can watch both `Secrets` and `ConfigMaps`. | `configmapcontroller` can only watch changes in `ConfigMaps`. It cannot detect changes in other resources like `Secrets`. | +| Reloader can perform rolling upgrades on `deployments` as well as on `statefulsets` and `daemonsets` | `configmapcontroller` can only perform rolling upgrades on `deployments`. It currently does not support rolling upgrades on `statefulsets` and `daemonsets` | +| Reloader provides both unit test cases and end to end integration test cases for future updates. So one can make sure that new changes do not break any old functionality. | Currently there are not any unit test cases or end to end integration test cases in `configmap-controller`. It adds difficulties for any additional updates in `configmap-controller` and one can not know for sure whether new changes breaks any old functionality or not. | +| Reloader uses SHA1 to encode the change in `ConfigMap` or `Secret`. It then saves the SHA1 value in `STAKATER_FOO_CONFIGMAP` or `STAKATER_FOO_SECRET` environment variable depending upon where the change has happened. The use of SHA1 provides a concise 40 characters encoded value that is very less prone to collision. | `configmap-controller` uses `FABRICB_FOO_REVISION` environment variable to store any change in `ConfigMap` controller. It does not encode it or convert it in suitable hash value to avoid data pollution in deployment. | +| Reloader allows you to customize your own annotation (for both `Secrets` and `ConfigMaps`) using command line flags | `configmap-controller` restricts you to only their provided annotation | diff --git a/docs/Reloader-vs-k8s-trigger-controller.md b/docs/Reloader-vs-k8s-trigger-controller.md index fe0f6d9..561dca5 100644 --- a/docs/Reloader-vs-k8s-trigger-controller.md +++ b/docs/Reloader-vs-k8s-trigger-controller.md @@ -6,7 +6,7 @@ Reloader and k8s-trigger-controller are both built for same purpose. So there ar - Both controllers support change detection in `ConfigMaps` and `Secrets` - Both controllers support deployment `rollout` -- Reloader controller use SHA512 for hashing +- Reloader controller use SHA1 for hashing - Both controllers have end to end as well as unit test cases. ## Differences diff --git a/internal/pkg/crypto/sha.go b/internal/pkg/crypto/sha.go index f9ae235..043fc22 100644 --- a/internal/pkg/crypto/sha.go +++ b/internal/pkg/crypto/sha.go @@ -1,14 +1,20 @@ package crypto import ( - "crypto/sha512" - "encoding/hex" + "crypto/sha1" + "fmt" + "io" + + "github.com/sirupsen/logrus" ) // GenerateSHA generates SHA from string -// Always returns a hash value, even for empty strings, to ensure consistent behavior -// and avoid issues with string matching operations (e.g., strings.Contains(str, "") always returns true) func GenerateSHA(data string) string { - hash := sha512.Sum512_256([]byte(data)) - return hex.EncodeToString(hash[:]) + hasher := sha1.New() + _, err := io.WriteString(hasher, data) + if err != nil { + logrus.Errorf("Unable to write data in hash writer %v", err) + } + sha := hasher.Sum(nil) + return fmt.Sprintf("%x", sha) } diff --git a/internal/pkg/crypto/sha_test.go b/internal/pkg/crypto/sha_test.go index ee530e3..5cb0afc 100644 --- a/internal/pkg/crypto/sha_test.go +++ b/internal/pkg/crypto/sha_test.go @@ -7,7 +7,7 @@ import ( // TestGenerateSHA generates the sha from given data and verifies whether it is correct or not func TestGenerateSHA(t *testing.T) { data := "www.stakater.com" - sha := "2e9aa975331b22861b4f62b7fcc69b63e001f938361fee3b4ed888adf26a10e3" + sha := "abd4ed82fb04548388a6cf3c339fd9dc84d275df" result := GenerateSHA(data) if result != sha { t.Errorf("Failed to generate SHA") @@ -18,11 +18,11 @@ func TestGenerateSHA(t *testing.T) { // This ensures consistent behavior and avoids issues with string matching operations func TestGenerateSHAEmptyString(t *testing.T) { result := GenerateSHA("") - expected := "c672b8d1ef56ed28ab87c3622c5114069bdd3ad7b8f9737498d0c01ecef0967a" + expected := "da39a3ee5e6b4b0d3255bfef95601890afd80709" if result != expected { t.Errorf("Failed to generate SHA for empty string. Expected: %s, Got: %s", expected, result) } - if len(result) != 64 { - t.Errorf("SHA hash should be 64 characters long, got %d", len(result)) + if len(result) != 40 { + t.Errorf("SHA hash should be 40 characters long, got %d", len(result)) } } diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index e905ee0..68ba94d 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -1981,7 +1981,7 @@ func TestRollingUpgradeForDeploymentWithPatchAndRetryUsingArs(t *testing.T) { assert.Equal(t, patchtypes.StrategicMergePatchType, patchType) assert.NotEmpty(t, bytes) assert.Contains(t, string(bytes), `{"spec":{"template":{"metadata":{"annotations":{"reloader.stakater.com/last-reloaded-from":`) - assert.Contains(t, string(bytes), `\"hash\":\"fd9e71a362056bfa864d9859e12978f893d330ce8cbf09218b25d015770ad91f\"`) + assert.Contains(t, string(bytes), `\"hash\":\"3c9a892aeaedc759abc3df9884a37b8be5680382\"`) return nil } @@ -2964,7 +2964,7 @@ func TestRollingUpgradeForDaemonSetWithPatchAndRetryUsingArs(t *testing.T) { assert.Equal(t, patchtypes.StrategicMergePatchType, patchType) assert.NotEmpty(t, bytes) assert.Contains(t, string(bytes), `{"spec":{"template":{"metadata":{"annotations":{"reloader.stakater.com/last-reloaded-from":`) - assert.Contains(t, string(bytes), `\"hash\":\"43bf9e30e7c4e32a8f8673c462b86d0b1ac626cf498afdc0d0108e79ebe7ee0c\"`) + assert.Contains(t, string(bytes), `\"hash\":\"314a2269170750a974d79f02b5b9ee517de7f280\"`) return nil } @@ -3227,7 +3227,7 @@ func TestRollingUpgradeForStatefulSetWithPatchAndRetryUsingArs(t *testing.T) { assert.Equal(t, patchtypes.StrategicMergePatchType, patchType) assert.NotEmpty(t, bytes) assert.Contains(t, string(bytes), `{"spec":{"template":{"metadata":{"annotations":{"reloader.stakater.com/last-reloaded-from":`) - assert.Contains(t, string(bytes), `\"hash\":\"6aa837180bdf6a93306c71a0cf62b4a45c2d5b021578247b3b64d5baea2b84d9\"`) + assert.Contains(t, string(bytes), `\"hash\":\"f821414d40d8815fb330763f74a4ff7ab651d4fa\"`) return nil } @@ -3607,7 +3607,7 @@ func TestRollingUpgradeForDeploymentWithPatchAndRetryUsingErs(t *testing.T) { assert.Equal(t, patchtypes.StrategicMergePatchType, patchType) assert.NotEmpty(t, bytes) assert.Contains(t, string(bytes), `{"spec":{"template":{"spec":{"containers":[{"name":`) - assert.Contains(t, string(bytes), `"value":"fd9e71a362056bfa864d9859e12978f893d330ce8cbf09218b25d015770ad91f"`) + assert.Contains(t, string(bytes), `"value":"3c9a892aeaedc759abc3df9884a37b8be5680382"`) return nil } @@ -4502,7 +4502,7 @@ func TestRollingUpgradeForDaemonSetWithPatchAndRetryUsingErs(t *testing.T) { assert.Equal(t, patchtypes.StrategicMergePatchType, patchType) assert.NotEmpty(t, bytes) assert.Contains(t, string(bytes), `{"spec":{"template":{"spec":{"containers":[{"name":`) - assert.Contains(t, string(bytes), `"value":"43bf9e30e7c4e32a8f8673c462b86d0b1ac626cf498afdc0d0108e79ebe7ee0c"`) + assert.Contains(t, string(bytes), `"value":"314a2269170750a974d79f02b5b9ee517de7f280"`) return nil } @@ -4737,7 +4737,7 @@ func TestRollingUpgradeForStatefulSetWithPatchAndRetryUsingErs(t *testing.T) { assert.Equal(t, patchtypes.StrategicMergePatchType, patchType) assert.NotEmpty(t, bytes) assert.Contains(t, string(bytes), `{"spec":{"template":{"spec":{"containers":[{"name":`) - assert.Contains(t, string(bytes), `"value":"6aa837180bdf6a93306c71a0cf62b4a45c2d5b021578247b3b64d5baea2b84d9"`) + assert.Contains(t, string(bytes), `"value":"f821414d40d8815fb330763f74a4ff7ab651d4fa"`) return nil } From 27f49ecc0fea8e0cd4a976f1ac22236f2304a429 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Thu, 8 Jan 2026 11:07:38 +0100 Subject: [PATCH 28/45] fix: Use go 1.25 for load tests in CI --- .github/workflows/loadtest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/loadtest.yml b/.github/workflows/loadtest.yml index db01e7e..a677444 100644 --- a/.github/workflows/loadtest.yml +++ b/.github/workflows/loadtest.yml @@ -48,7 +48,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.23' + go-version: '1.25' cache: false - name: Set up Docker Buildx From eb3bc2447e2fefc9add1ef2b7113af88db7ca01b Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Thu, 8 Jan 2026 11:12:10 +0100 Subject: [PATCH 29/45] refactor(upgrade.go): simplify retryOnConflict to return matched status and error --- internal/pkg/handler/upgrade.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index e355d5f..932fade 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -242,16 +242,15 @@ func PerformAction(clients kube.Clients, config common.Config, upgradeFuncs call matchedCount := 0 for _, item := range items { - err := retryOnConflict(retry.DefaultRetry, func(fetchResource bool) (bool, error) { - matched, err := upgradeResource(clients, config, upgradeFuncs, collectors, recorder, strategy, item, fetchResource) - if matched { - matchedCount++ - } - return matched, err + matched, err := retryOnConflict(retry.DefaultRetry, func(fetchResource bool) (bool, error) { + return upgradeResource(clients, config, upgradeFuncs, collectors, recorder, strategy, item, fetchResource) }) if err != nil { return err } + if matched { + matchedCount++ + } } // Record workloads matched @@ -260,11 +259,13 @@ func PerformAction(clients kube.Clients, config common.Config, upgradeFuncs call return nil } -func retryOnConflict(backoff wait.Backoff, fn func(_ bool) (bool, error)) error { +func retryOnConflict(backoff wait.Backoff, fn func(_ bool) (bool, error)) (bool, error) { var lastError error + var matched bool fetchResource := false // do not fetch resource on first attempt, already done by ItemsFunc err := wait.ExponentialBackoff(backoff, func() (bool, error) { - _, err := fn(fetchResource) + var err error + matched, err = fn(fetchResource) fetchResource = true switch { case err == nil: @@ -279,7 +280,7 @@ func retryOnConflict(backoff wait.Backoff, fn func(_ bool) (bool, error)) error if wait.Interrupted(err) { err = lastError } - return err + return matched, err } func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy, resource runtime.Object, fetchResource bool) (bool, error) { From f7210204d40e43b08f37a237997a7063da4ae023 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Thu, 8 Jan 2026 11:41:44 +0100 Subject: [PATCH 30/45] fix: Safe parsing for duration --- test/loadtest/cmd/loadtest/main.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/loadtest/cmd/loadtest/main.go b/test/loadtest/cmd/loadtest/main.go index 19f7b1d..5725090 100644 --- a/test/loadtest/cmd/loadtest/main.go +++ b/test/loadtest/cmd/loadtest/main.go @@ -142,9 +142,13 @@ func parseArgs(args []string) Config { case strings.HasPrefix(arg, "--scenario="): cfg.Scenario = strings.TrimPrefix(arg, "--scenario=") case strings.HasPrefix(arg, "--duration="): - fmt.Sscanf(strings.TrimPrefix(arg, "--duration="), "%d", &cfg.Duration) + if n, _ := fmt.Sscanf(strings.TrimPrefix(arg, "--duration="), "%d", &cfg.Duration); n != 1 { + log.Printf("Warning: invalid --duration value, using default (%d)", cfg.Duration) + } case strings.HasPrefix(arg, "--parallelism="): - fmt.Sscanf(strings.TrimPrefix(arg, "--parallelism="), "%d", &cfg.Parallelism) + if n, _ := fmt.Sscanf(strings.TrimPrefix(arg, "--parallelism="), "%d", &cfg.Parallelism); n != 1 { + log.Printf("Warning: invalid --parallelism value, using default (%d)", cfg.Parallelism) + } case arg == "--skip-cluster": cfg.SkipCluster = true case strings.HasPrefix(arg, "--results-dir="): From 193f64c0ec3fc9c6225a2004b8f01fc2b68e3d58 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Thu, 8 Jan 2026 14:22:43 +0100 Subject: [PATCH 31/45] fix: Missing reloader.go --- .gitignore | 1 + test/loadtest/internal/reloader/reloader.go | 250 ++++++++++++++++++ test/loadtest/internal/scenarios/scenarios.go | 22 +- 3 files changed, 264 insertions(+), 9 deletions(-) create mode 100644 test/loadtest/internal/reloader/reloader.go diff --git a/.gitignore b/.gitignore index 73da63e..a398472 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ vendor dist Reloader !**/chart/reloader +!**/internal/reloader *.tgz styles/ site/ diff --git a/test/loadtest/internal/reloader/reloader.go b/test/loadtest/internal/reloader/reloader.go new file mode 100644 index 0000000..ff3cfdb --- /dev/null +++ b/test/loadtest/internal/reloader/reloader.go @@ -0,0 +1,250 @@ +package reloader + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// Config holds configuration for a Reloader deployment. +type Config struct { + Version string + Image string + Namespace string + ReloadStrategy string +} + +// Manager handles Reloader deployment operations. +type Manager struct { + config Config + kubeContext string +} + +// NewManager creates a new Reloader manager. +func NewManager(config Config) *Manager { + return &Manager{ + config: config, + } +} + +// SetKubeContext sets the kubeconfig context to use. +func (m *Manager) SetKubeContext(kubeContext string) { + m.kubeContext = kubeContext +} + +// kubectl returns kubectl command with optional context. +func (m *Manager) kubectl(ctx context.Context, args ...string) *exec.Cmd { + if m.kubeContext != "" { + args = append([]string{"--context", m.kubeContext}, args...) + } + return exec.CommandContext(ctx, "kubectl", args...) +} + +// namespace returns the namespace for this reloader instance. +func (m *Manager) namespace() string { + if m.config.Namespace != "" { + return m.config.Namespace + } + return fmt.Sprintf("reloader-%s", m.config.Version) +} + +// releaseName returns the release name for this instance. +func (m *Manager) releaseName() string { + return fmt.Sprintf("reloader-%s", m.config.Version) +} + +// Job returns the Prometheus job name for this Reloader instance. +func (m *Manager) Job() string { + return fmt.Sprintf("reloader-%s", m.config.Version) +} + +// Deploy deploys Reloader to the cluster using raw manifests. +func (m *Manager) Deploy(ctx context.Context) error { + ns := m.namespace() + name := m.releaseName() + + fmt.Printf("Deploying Reloader (%s) with image %s...\n", m.config.Version, m.config.Image) + + manifest := m.buildManifest(ns, name) + + applyCmd := m.kubectl(ctx, "apply", "-f", "-") + applyCmd.Stdin = strings.NewReader(manifest) + applyCmd.Stdout = os.Stdout + applyCmd.Stderr = os.Stderr + if err := applyCmd.Run(); err != nil { + return fmt.Errorf("applying manifest: %w", err) + } + + fmt.Printf("Waiting for Reloader deployment to be ready...\n") + waitCmd := m.kubectl(ctx, "rollout", "status", "deployment", name, + "-n", ns, + "--timeout=120s") + waitCmd.Stdout = os.Stdout + waitCmd.Stderr = os.Stderr + if err := waitCmd.Run(); err != nil { + return fmt.Errorf("waiting for deployment: %w", err) + } + + time.Sleep(2 * time.Second) + + fmt.Printf("Reloader (%s) deployed successfully\n", m.config.Version) + return nil +} + +// buildManifest creates the raw Kubernetes manifest for Reloader. +func (m *Manager) buildManifest(ns, name string) string { + var args []string + args = append(args, "--log-format=json") + if m.config.ReloadStrategy != "" && m.config.ReloadStrategy != "default" { + args = append(args, fmt.Sprintf("--reload-strategy=%s", m.config.ReloadStrategy)) + } + + argsYAML := "" + if len(args) > 0 { + argsYAML = " args:\n" + for _, arg := range args { + argsYAML += fmt.Sprintf(" - %q\n", arg) + } + } + + return fmt.Sprintf(`--- +apiVersion: v1 +kind: Namespace +metadata: + name: %[1]s +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: %[2]s + namespace: %[1]s +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: %[2]s +rules: +- apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: %[2]s +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: %[2]s +subjects: +- kind: ServiceAccount + name: %[2]s + namespace: %[1]s +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: %[2]s + namespace: %[1]s + labels: + app: %[2]s + app.kubernetes.io/name: reloader + loadtest-version: %[3]s +spec: + replicas: 1 + selector: + matchLabels: + app: %[2]s + template: + metadata: + labels: + app: %[2]s + app.kubernetes.io/name: reloader + loadtest-version: %[3]s + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + prometheus.io/path: "/metrics" + spec: + serviceAccountName: %[2]s + securityContext: + runAsNonRoot: true + runAsUser: 65534 + containers: + - name: reloader + image: %[4]s + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 9090 +%[5]s resources: + requests: + cpu: 10m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true +`, ns, name, m.config.Version, m.config.Image, argsYAML) +} + +// Cleanup removes all Reloader resources from the cluster. +func (m *Manager) Cleanup(ctx context.Context) error { + ns := m.namespace() + name := m.releaseName() + + delDeploy := m.kubectl(ctx, "delete", "deployment", name, "-n", ns, "--ignore-not-found") + delDeploy.Run() + + delCRB := m.kubectl(ctx, "delete", "clusterrolebinding", name, "--ignore-not-found") + delCRB.Run() + + delCR := m.kubectl(ctx, "delete", "clusterrole", name, "--ignore-not-found") + delCR.Run() + + delNS := m.kubectl(ctx, "delete", "namespace", ns, "--wait=false", "--ignore-not-found") + if err := delNS.Run(); err != nil { + return fmt.Errorf("deleting namespace: %w", err) + } + + return nil +} + +// CollectLogs collects logs from the Reloader pod and writes them to the specified file. +func (m *Manager) CollectLogs(ctx context.Context, logPath string) error { + ns := m.namespace() + name := m.releaseName() + + if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { + return fmt.Errorf("creating log directory: %w", err) + } + + cmd := m.kubectl(ctx, "logs", + "-n", ns, + "-l", fmt.Sprintf("app=%s", name), + "--tail=-1") + + out, err := cmd.Output() + if err != nil { + cmd = m.kubectl(ctx, "logs", + "-n", ns, + "-l", "app.kubernetes.io/name=reloader", + "--tail=-1") + out, err = cmd.Output() + if err != nil { + return fmt.Errorf("collecting logs: %w", err) + } + } + + if err := os.WriteFile(logPath, out, 0644); err != nil { + return fmt.Errorf("writing logs: %w", err) + } + + return nil +} diff --git a/test/loadtest/internal/scenarios/scenarios.go b/test/loadtest/internal/scenarios/scenarios.go index ed48a3b..794f1a6 100644 --- a/test/loadtest/internal/scenarios/scenarios.go +++ b/test/loadtest/internal/scenarios/scenarios.go @@ -1137,7 +1137,7 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int for time.Now().Before(endTime) { select { case <-ctx.Done(): - return s.calculateExpected(secretUpdateCount, cmUpdateCount, numSecretOnlyDeploys, numConfigMapOnlyDeploys, numMixedDeploys), nil + return s.calculateExpected(secretUpdateCount, cmUpdateCount, numSecrets, numConfigMaps, numSecretOnlyDeploys, numConfigMapOnlyDeploys, numMixedDeploys), nil case <-ticker.C: if updateSecret { // Update a random Secret @@ -1169,21 +1169,25 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int } log.Printf("S10: Completed %d Secret updates and %d ConfigMap updates", secretUpdateCount, cmUpdateCount) - return s.calculateExpected(secretUpdateCount, cmUpdateCount, numSecretOnlyDeploys, numConfigMapOnlyDeploys, numMixedDeploys), nil + return s.calculateExpected(secretUpdateCount, cmUpdateCount, numSecrets, numConfigMaps, numSecretOnlyDeploys, numConfigMapOnlyDeploys, numMixedDeploys), nil } -func (s *SecretsAndMixedScenario) calculateExpected(secretUpdates, cmUpdates, secretOnlyDeploys, cmOnlyDeploys, mixedDeploys int) ExpectedMetrics { - // Secret updates trigger: secret-only deploys + mixed deploys - secretTriggeredReloads := secretUpdates * (secretOnlyDeploys + mixedDeploys) - // ConfigMap updates trigger: cm-only deploys + mixed deploys - cmTriggeredReloads := cmUpdates * (cmOnlyDeploys + mixedDeploys) +func (s *SecretsAndMixedScenario) calculateExpected(secretUpdates, cmUpdates, numSecrets, numConfigMaps, secretOnlyDeploys, cmOnlyDeploys, mixedDeploys int) ExpectedMetrics { + // Average deploys triggered per random secret update + avgSecretReloads := float64(secretOnlyDeploys)/float64(numSecrets) + float64(mixedDeploys)/float64(numSecrets) + secretTriggeredReloads := int(float64(secretUpdates) * avgSecretReloads) + + // Average deploys triggered per random CM update + avgCMReloads := float64(cmOnlyDeploys)/float64(numConfigMaps) + float64(mixedDeploys)/float64(numConfigMaps) + cmTriggeredReloads := int(float64(cmUpdates) * avgCMReloads) + totalExpectedReloads := secretTriggeredReloads + cmTriggeredReloads return ExpectedMetrics{ ActionTotal: totalExpectedReloads, ReloadExecutedTotal: totalExpectedReloads, - Description: fmt.Sprintf("S10: %d Secret updates (ā†’%d reloads) + %d CM updates (ā†’%d reloads) = %d total", - secretUpdates, secretTriggeredReloads, cmUpdates, cmTriggeredReloads, totalExpectedReloads), + Description: fmt.Sprintf("S10: %d Secret updates (ā†’%d reloads, avg %.1f/update) + %d CM updates (ā†’%d reloads, avg %.1f/update) = %d total", + secretUpdates, secretTriggeredReloads, avgSecretReloads, cmUpdates, cmTriggeredReloads, avgCMReloads, totalExpectedReloads), } } From 922cac120aa5bf3f5fc9c979d57d5e4e68d38bbd Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Thu, 8 Jan 2026 22:15:21 +0100 Subject: [PATCH 32/45] fix: Update gitignore to include results and nfs files --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a398472..3f28c3f 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,8 @@ styles/ site/ /mkdocs.yml yq -bin \ No newline at end of file +bin +test/loadtest/results +test/loadtest/loadtest +# Temporary NFS files +.nfs* From 958c6c2be77a433d91b638419f030761d09faa5f Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Thu, 8 Jan 2026 22:52:07 +0100 Subject: [PATCH 33/45] feat(ci): Separate action for loadtests --- .github/actions/loadtest/action.yml | 256 ++++++++++++++++++++ .github/workflows/loadtest.yml | 165 ++----------- .github/workflows/pull_request.yaml | 12 + Makefile | 40 ++++ test/loadtest/cmd/loadtest/main.go | 354 +++++++++++++++++++++++++++- 5 files changed, 677 insertions(+), 150 deletions(-) create mode 100644 .github/actions/loadtest/action.yml diff --git a/.github/actions/loadtest/action.yml b/.github/actions/loadtest/action.yml new file mode 100644 index 0000000..3b91957 --- /dev/null +++ b/.github/actions/loadtest/action.yml @@ -0,0 +1,256 @@ +name: 'Reloader Load Test' +description: 'Run Reloader load tests with A/B comparison support' + +inputs: + old-ref: + description: 'Git ref for "old" version (optional, enables A/B comparison)' + required: false + default: '' + new-ref: + description: 'Git ref for "new" version (defaults to current checkout)' + required: false + default: '' + old-image: + description: 'Pre-built container image for "old" version (alternative to old-ref)' + required: false + default: '' + new-image: + description: 'Pre-built container image for "new" version (alternative to new-ref)' + required: false + default: '' + scenarios: + description: 'Scenarios to run: S1,S4,S6 or all' + required: false + default: 'S1,S4,S6' + test-type: + description: 'Test type label for summary: quick or full' + required: false + default: 'quick' + duration: + description: 'Test duration in seconds' + required: false + default: '60' + kind-cluster: + description: 'Name of existing Kind cluster (if empty, creates new one)' + required: false + default: '' + post-comment: + description: 'Post results as PR comment' + required: false + default: 'false' + pr-number: + description: 'PR number for commenting (required if post-comment is true)' + required: false + default: '' + github-token: + description: 'GitHub token for posting comments' + required: false + default: ${{ github.token }} + comment-header: + description: 'Optional header text for the comment' + required: false + default: '' + +outputs: + status: + description: 'Overall test status: pass or fail' + value: ${{ steps.run.outputs.status }} + summary: + description: 'Markdown summary of results' + value: ${{ steps.summary.outputs.summary }} + pass-count: + description: 'Number of passed scenarios' + value: ${{ steps.summary.outputs.pass_count }} + fail-count: + description: 'Number of failed scenarios' + value: ${{ steps.summary.outputs.fail_count }} + +runs: + using: 'composite' + steps: + - name: Determine images to use + id: images + shell: bash + run: | + # Determine old image + if [ -n "${{ inputs.old-image }}" ]; then + echo "old=${{ inputs.old-image }}" >> $GITHUB_OUTPUT + elif [ -n "${{ inputs.old-ref }}" ]; then + echo "old=localhost/reloader:old" >> $GITHUB_OUTPUT + echo "build_old=true" >> $GITHUB_OUTPUT + else + echo "old=" >> $GITHUB_OUTPUT + fi + + # Determine new image + if [ -n "${{ inputs.new-image }}" ]; then + echo "new=${{ inputs.new-image }}" >> $GITHUB_OUTPUT + elif [ -n "${{ inputs.new-ref }}" ]; then + echo "new=localhost/reloader:new" >> $GITHUB_OUTPUT + echo "build_new=true" >> $GITHUB_OUTPUT + else + # Default: build from current checkout + echo "new=localhost/reloader:new" >> $GITHUB_OUTPUT + echo "build_new_current=true" >> $GITHUB_OUTPUT + fi + + - name: Build old image from ref + if: steps.images.outputs.build_old == 'true' + shell: bash + run: | + CURRENT_SHA=$(git rev-parse HEAD) + git checkout ${{ inputs.old-ref }} + docker build -t localhost/reloader:old . + echo "Built old image from ref: ${{ inputs.old-ref }}" + git checkout $CURRENT_SHA + + - name: Build new image from ref + if: steps.images.outputs.build_new == 'true' + shell: bash + run: | + CURRENT_SHA=$(git rev-parse HEAD) + git checkout ${{ inputs.new-ref }} + docker build -t localhost/reloader:new . + echo "Built new image from ref: ${{ inputs.new-ref }}" + git checkout $CURRENT_SHA + + - name: Build new image from current checkout + if: steps.images.outputs.build_new_current == 'true' + shell: bash + run: | + docker build -t localhost/reloader:new . + echo "Built new image from current checkout" + + - name: Build loadtest binary + shell: bash + run: | + cd ${{ github.workspace }}/test/loadtest + go build -o loadtest ./cmd/loadtest + + - name: Determine cluster name + id: cluster + shell: bash + run: | + if [ -n "${{ inputs.kind-cluster }}" ]; then + echo "name=${{ inputs.kind-cluster }}" >> $GITHUB_OUTPUT + echo "skip=true" >> $GITHUB_OUTPUT + else + echo "name=reloader-loadtest" >> $GITHUB_OUTPUT + echo "skip=false" >> $GITHUB_OUTPUT + fi + + - name: Load images into Kind + shell: bash + run: | + CLUSTER="${{ steps.cluster.outputs.name }}" + + if [ -n "${{ steps.images.outputs.old }}" ]; then + echo "Loading old image: ${{ steps.images.outputs.old }}" + kind load docker-image "${{ steps.images.outputs.old }}" --name "$CLUSTER" || true + fi + + echo "Loading new image: ${{ steps.images.outputs.new }}" + kind load docker-image "${{ steps.images.outputs.new }}" --name "$CLUSTER" || true + + - name: Run load tests + id: run + shell: bash + run: | + cd ${{ github.workspace }}/test/loadtest + + ARGS="--new-image=${{ steps.images.outputs.new }}" + ARGS="$ARGS --scenario=${{ inputs.scenarios }}" + ARGS="$ARGS --duration=${{ inputs.duration }}" + ARGS="$ARGS --cluster-name=${{ steps.cluster.outputs.name }}" + + if [ -n "${{ steps.images.outputs.old }}" ]; then + ARGS="$ARGS --old-image=${{ steps.images.outputs.old }}" + fi + + if [ "${{ steps.cluster.outputs.skip }}" = "true" ]; then + ARGS="$ARGS --skip-cluster" + fi + + echo "Running: ./loadtest run $ARGS" + if ./loadtest run $ARGS; then + echo "status=pass" >> $GITHUB_OUTPUT + else + echo "status=fail" >> $GITHUB_OUTPUT + fi + + - name: Generate summary + id: summary + shell: bash + run: | + cd ${{ github.workspace }}/test/loadtest + + # Generate markdown summary + ./loadtest summary \ + --results-dir=./results \ + --test-type=${{ inputs.test-type }} \ + --format=markdown > summary.md 2>/dev/null || true + + # Output to GitHub Step Summary + cat summary.md >> $GITHUB_STEP_SUMMARY + + # Store summary for output (using heredoc for multiline) + { + echo 'summary<> $GITHUB_OUTPUT + + # Get pass/fail counts from JSON + COUNTS=$(./loadtest summary --format=json 2>/dev/null | head -20 || echo '{}') + echo "pass_count=$(echo "$COUNTS" | grep -o '"pass_count": [0-9]*' | grep -o '[0-9]*' || echo 0)" >> $GITHUB_OUTPUT + echo "fail_count=$(echo "$COUNTS" | grep -o '"fail_count": [0-9]*' | grep -o '[0-9]*' || echo 0)" >> $GITHUB_OUTPUT + + - name: Post PR comment + if: inputs.post-comment == 'true' && inputs.pr-number != '' + uses: actions/github-script@v7 + with: + github-token: ${{ inputs.github-token }} + script: | + const fs = require('fs'); + const summaryPath = '${{ github.workspace }}/test/loadtest/summary.md'; + let summary = 'No results available'; + try { + summary = fs.readFileSync(summaryPath, 'utf8'); + } catch (e) { + console.log('Could not read summary file:', e.message); + } + + const header = '${{ inputs.comment-header }}'; + const status = '${{ steps.run.outputs.status }}'; + const statusEmoji = status === 'pass' ? ':white_check_mark:' : ':x:'; + + const body = [ + header ? header : `## ${statusEmoji} Load Test Results (${{ inputs.test-type }})`, + '', + summary, + '', + '---', + `**Artifacts:** [Download](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})`, + ].join('\n'); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ${{ inputs.pr-number }}, + body: body + }); + + - name: Upload results + uses: actions/upload-artifact@v4 + if: always() + with: + name: loadtest-${{ inputs.test-type }}-results + path: | + ${{ github.workspace }}/test/loadtest/results/ + retention-days: 30 + + - name: Cleanup Kind cluster (only if we created it) + if: always() && steps.cluster.outputs.skip == 'false' + shell: bash + run: | + kind delete cluster --name ${{ steps.cluster.outputs.name }} || true diff --git a/.github/workflows/loadtest.yml b/.github/workflows/loadtest.yml index a677444..f4eb322 100644 --- a/.github/workflows/loadtest.yml +++ b/.github/workflows/loadtest.yml @@ -1,4 +1,4 @@ -name: Load Test +name: Load Test (Full) on: issue_comment: @@ -10,6 +10,7 @@ permissions: issues: write jobs: + # Full load test suite triggered by /loadtest command loadtest: # Only run on PR comments with /loadtest command if: | @@ -45,6 +46,12 @@ jobs: core.setOutput('base_sha', pr.data.base.sha); console.log(`PR #${context.issue.number}: ${pr.data.head.ref} -> ${pr.data.base.ref}`); + - name: Checkout PR branch + uses: actions/checkout@v4 + with: + ref: ${{ steps.pr.outputs.head_sha }} + fetch-depth: 0 # Full history for building from base ref + - name: Set up Go uses: actions/setup-go@v5 with: @@ -66,151 +73,23 @@ jobs: chmod +x kubectl sudo mv kubectl /usr/local/bin/kubectl - # Build OLD image from base branch (e.g., main) - - name: Checkout base branch (old) - uses: actions/checkout@v4 - with: - ref: ${{ steps.pr.outputs.base_ref }} - path: old - - - name: Build old image - run: | - cd old - docker build -t localhost/reloader:old -f Dockerfile . - echo "Built old image from ${{ steps.pr.outputs.base_ref }} (${{ steps.pr.outputs.base_sha }})" - - # Build NEW image from PR branch - - name: Checkout PR branch (new) - uses: actions/checkout@v4 - with: - ref: ${{ steps.pr.outputs.head_ref }} - path: new - - - name: Build new image - run: | - cd new - docker build -t localhost/reloader:new -f Dockerfile . - echo "Built new image from ${{ steps.pr.outputs.head_ref }} (${{ steps.pr.outputs.head_sha }})" - - # Build and run loadtest from PR branch - - name: Build loadtest tool - run: | - cd new/test/loadtest - go build -o loadtest ./cmd/loadtest - - - name: Run A/B comparison load test + - name: Run full A/B comparison load test id: loadtest - run: | - cd new/test/loadtest - ./loadtest run \ - --old-image=localhost/reloader:old \ - --new-image=localhost/reloader:new \ - --scenario=all \ - --duration=60 2>&1 | tee loadtest-output.txt - echo "exitcode=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT - - - name: Upload results - uses: actions/upload-artifact@v4 - if: always() + uses: ./.github/actions/loadtest with: - name: loadtest-results - path: | - new/test/loadtest/results/ - new/test/loadtest/loadtest-output.txt - retention-days: 30 - - - name: Post results comment - uses: actions/github-script@v7 - if: always() - with: - script: | - const fs = require('fs'); - - let results = ''; - const resultsDir = 'new/test/loadtest/results'; - - // Collect summary of all scenarios - let passCount = 0; - let failCount = 0; - const summaries = []; - - if (fs.existsSync(resultsDir)) { - const scenarios = fs.readdirSync(resultsDir).sort(); - for (const scenario of scenarios) { - const reportPath = `${resultsDir}/${scenario}/report.txt`; - if (fs.existsSync(reportPath)) { - const report = fs.readFileSync(reportPath, 'utf8'); - - // Extract status from report - const statusMatch = report.match(/Status:\s+(PASS|FAIL)/); - const status = statusMatch ? statusMatch[1] : 'UNKNOWN'; - - if (status === 'PASS') passCount++; - else failCount++; - - // Extract key metrics for summary - const actionMatch = report.match(/action_total\s+[\d.]+\s+[\d.]+\s+[\d.]+/); - const errorsMatch = report.match(/errors_total\s+[\d.]+\s+[\d.]+/); - - summaries.push(`| ${scenario} | ${status === 'PASS' ? 'āœ…' : 'āŒ'} ${status} |`); - - results += `\n
\n${status === 'PASS' ? 'āœ…' : 'āŒ'} ${scenario}\n\n\`\`\`\n${report}\n\`\`\`\n
\n`; - } - } - } - - if (!results) { - // Read raw output if no reports - if (fs.existsSync('new/test/loadtest/loadtest-output.txt')) { - const output = fs.readFileSync('new/test/loadtest/loadtest-output.txt', 'utf8'); - const maxLen = 60000; - results = output.length > maxLen - ? output.substring(output.length - maxLen) - : output; - results = `\`\`\`\n${results}\n\`\`\``; - } else { - results = 'No results available'; - } - } - - const overallStatus = failCount === 0 ? 'āœ… ALL PASSED' : `āŒ ${failCount} FAILED`; - - const body = [ - `## Load Test Results ${overallStatus}`, - '', - `**Comparing:** \`${{ steps.pr.outputs.base_ref }}\` (old) vs \`${{ steps.pr.outputs.head_ref }}\` (new)`, - `**Old commit:** ${{ steps.pr.outputs.base_sha }}`, - `**New commit:** ${{ steps.pr.outputs.head_sha }}`, - `**Triggered by:** @${{ github.event.comment.user.login }}`, - '', - '### Summary', - '', - '| Scenario | Status |', - '|----------|--------|', - summaries.join('\n'), - '', - `**Total:** ${passCount} passed, ${failCount} failed`, - '', - '### Detailed Results', - '', - results, - '', - '
', - 'šŸ“¦ Download full results', - '', - `Artifacts are available in the [workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}).`, - '
', - ].join('\n'); - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: body - }); + old-ref: ${{ steps.pr.outputs.base_sha }} + new-ref: ${{ steps.pr.outputs.head_sha }} + scenarios: 'all' + test-type: 'full' + post-comment: 'true' + pr-number: ${{ github.event.issue.number }} + comment-header: | + ## Load Test Results (Full A/B Comparison) + **Comparing:** `${{ steps.pr.outputs.base_ref }}` ā†’ `${{ steps.pr.outputs.head_ref }}` + **Triggered by:** @${{ github.event.comment.user.login }} - name: Add success reaction - if: success() + if: steps.loadtest.outputs.status == 'pass' uses: actions/github-script@v7 with: script: | @@ -222,7 +101,7 @@ jobs: }); - name: Add failure reaction - if: failure() + if: steps.loadtest.outputs.status == 'fail' uses: actions/github-script@v7 with: script: | diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index e4b1c6f..fbf59ab 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -35,6 +35,7 @@ jobs: permissions: contents: read + pull-requests: write runs-on: ubuntu-latest name: Build @@ -109,6 +110,17 @@ jobs: - name: Test run: make test + - name: Run quick A/B load tests + uses: ./.github/actions/loadtest + with: + old-ref: ${{ github.event.pull_request.base.sha }} + # new-ref defaults to current checkout (PR branch) + scenarios: 'S1,S4,S6' + test-type: 'quick' + kind-cluster: 'kind' # Use the existing cluster created above + post-comment: 'true' + pr-number: ${{ github.event.pull_request.number }} + - name: Generate Tags id: generate_tag run: | diff --git a/Makefile b/Makefile index 8444e1f..8c0aed8 100644 --- a/Makefile +++ b/Makefile @@ -169,3 +169,43 @@ yq-install: @curl -sL $(YQ_DOWNLOAD_URL) -o $(YQ_BIN) @chmod +x $(YQ_BIN) @echo "yq $(YQ_VERSION) installed at $(YQ_BIN)" + +# ============================================================================= +# Load Testing +# ============================================================================= + +LOADTEST_BIN = test/loadtest/loadtest +LOADTEST_OLD_IMAGE ?= localhost/reloader:old +LOADTEST_NEW_IMAGE ?= localhost/reloader:new +LOADTEST_DURATION ?= 60 +LOADTEST_SCENARIOS ?= all + +.PHONY: loadtest-build loadtest-quick loadtest-full loadtest loadtest-clean + +loadtest-build: ## Build loadtest binary + cd test/loadtest && $(GOCMD) build -o loadtest ./cmd/loadtest + +loadtest-quick: loadtest-build ## Run quick load tests (S1, S4, S6) + cd test/loadtest && ./loadtest run \ + --old-image=$(LOADTEST_OLD_IMAGE) \ + --new-image=$(LOADTEST_NEW_IMAGE) \ + --scenario=S1,S4,S6 \ + --duration=$(LOADTEST_DURATION) + +loadtest-full: loadtest-build ## Run full load test suite + cd test/loadtest && ./loadtest run \ + --old-image=$(LOADTEST_OLD_IMAGE) \ + --new-image=$(LOADTEST_NEW_IMAGE) \ + --scenario=all \ + --duration=$(LOADTEST_DURATION) + +loadtest: loadtest-build ## Run load tests with configurable scenarios (default: all) + cd test/loadtest && ./loadtest run \ + --old-image=$(LOADTEST_OLD_IMAGE) \ + --new-image=$(LOADTEST_NEW_IMAGE) \ + --scenario=$(LOADTEST_SCENARIOS) \ + --duration=$(LOADTEST_DURATION) + +loadtest-clean: ## Clean loadtest binary and results + rm -f $(LOADTEST_BIN) + rm -rf test/loadtest/results diff --git a/test/loadtest/cmd/loadtest/main.go b/test/loadtest/cmd/loadtest/main.go index 5725090..fb05db5 100644 --- a/test/loadtest/cmd/loadtest/main.go +++ b/test/loadtest/cmd/loadtest/main.go @@ -30,6 +30,15 @@ const ( testNamespace = "reloader-test" ) +// OutputFormat defines the output format for reports. +type OutputFormat string + +const ( + OutputFormatText OutputFormat = "text" + OutputFormatJSON OutputFormat = "json" + OutputFormatMarkdown OutputFormat = "markdown" +) + // workerContext holds all resources for a single worker (cluster + prometheus). type workerContext struct { id int @@ -47,6 +56,7 @@ type Config struct { Scenario string Duration int SkipCluster bool + ClusterName string // Custom cluster name (default: reloader-loadtest) ResultsDir string ManifestsDir string Parallelism int // Number of parallel clusters (1 = sequential) @@ -64,6 +74,8 @@ func main() { runCommand(os.Args[2:]) case "report": reportCommand(os.Args[2:]) + case "summary": + summaryCommand(os.Args[2:]) case "help", "--help", "-h": printUsage() default: @@ -78,7 +90,8 @@ func printUsage() { Usage: loadtest run [options] Run A/B comparison tests - loadtest report [options] Generate comparison report + loadtest report [options] Generate comparison report for a scenario + loadtest summary [options] Generate summary across all scenarios (for CI) loadtest help Show this help Run Options: @@ -87,13 +100,21 @@ Run Options: --scenario=ID Test scenario: S1-S13 or "all" (default: all) --duration=SECONDS Test duration in seconds (default: 60) --parallelism=N Run N scenarios in parallel on N clusters (default: 1) - --skip-cluster Skip kind cluster creation (use existing, only for parallelism=1) + --skip-cluster Skip kind cluster creation (use existing) + --cluster-name=NAME Kind cluster name (default: reloader-loadtest) --results-dir=DIR Directory for results (default: ./results) Report Options: --scenario=ID Scenario to report on (required) --results-dir=DIR Directory containing results (default: ./results) --output=FILE Output file (default: stdout) + --format=FORMAT Output format: text, json, markdown (default: text) + +Summary Options: + --results-dir=DIR Directory containing results (default: ./results) + --output=FILE Output file (default: stdout) + --format=FORMAT Output format: text, json, markdown (default: markdown) + --test-type=TYPE Test type label: quick, full (default: full) Examples: # Compare two images @@ -111,8 +132,14 @@ Examples: # Run all 13 scenarios in parallel (one cluster per scenario) loadtest run --new-image=localhost/reloader:test --parallelism=13 - # Generate report + # Generate report for a scenario loadtest report --scenario=S2 --results-dir=./results + + # Generate JSON report + loadtest report --scenario=S2 --format=json + + # Generate markdown summary for CI + loadtest summary --results-dir=./results --format=markdown `) } @@ -122,6 +149,7 @@ func parseArgs(args []string) Config { Duration: 60, ResultsDir: "./results", Parallelism: 1, + ClusterName: clusterName, // default } // Find manifests dir relative to executable or current dir @@ -151,6 +179,8 @@ func parseArgs(args []string) Config { } case arg == "--skip-cluster": cfg.SkipCluster = true + case strings.HasPrefix(arg, "--cluster-name="): + cfg.ClusterName = strings.TrimPrefix(arg, "--cluster-name=") case strings.HasPrefix(arg, "--results-dir="): cfg.ResultsDir = strings.TrimPrefix(arg, "--results-dir=") case strings.HasPrefix(arg, "--manifests-dir="): @@ -234,15 +264,15 @@ func runCommand(args []string) { func runSequential(ctx context.Context, cfg Config, scenariosToRun []string, runtime string, runOld, runNew, runBoth bool) { // Create cluster manager clusterMgr := cluster.NewManager(cluster.Config{ - Name: clusterName, + Name: cfg.ClusterName, ContainerRuntime: runtime, }) // Create/verify cluster if cfg.SkipCluster { - log.Println("Skipping cluster creation (using existing)") + log.Printf("Skipping cluster creation (using existing cluster: %s)", cfg.ClusterName) if !clusterMgr.Exists() { - log.Fatalf("Cluster %s does not exist. Remove --skip-cluster to create it.", clusterName) + log.Fatalf("Cluster %s does not exist. Remove --skip-cluster to create it.", cfg.ClusterName) } } else { log.Println("Creating kind cluster...") @@ -781,6 +811,7 @@ func cleanupReloader(ctx context.Context, version string, kubeContext string) { func reportCommand(args []string) { var scenarioID, resultsDir, outputFile string + format := OutputFormatText resultsDir = "./results" for _, arg := range args { @@ -791,6 +822,8 @@ func reportCommand(args []string) { resultsDir = strings.TrimPrefix(arg, "--results-dir=") case strings.HasPrefix(arg, "--output="): outputFile = strings.TrimPrefix(arg, "--output=") + case strings.HasPrefix(arg, "--format="): + format = OutputFormat(strings.TrimPrefix(arg, "--format=")) } } @@ -803,7 +836,15 @@ func reportCommand(args []string) { log.Fatalf("Failed to generate report: %v", err) } - output := renderScenarioReport(report) + var output string + switch format { + case OutputFormatJSON: + output = renderScenarioReportJSON(report) + case OutputFormatMarkdown: + output = renderScenarioReportMarkdown(report) + default: + output = renderScenarioReport(report) + } if outputFile != "" { if err := os.WriteFile(outputFile, []byte(output), 0644); err != nil { @@ -1584,3 +1625,302 @@ func renderScenarioReport(report *ScenarioReport) string { return sb.String() } + +// renderScenarioReportJSON renders a scenario report as JSON. +func renderScenarioReportJSON(report *ScenarioReport) string { + data, err := json.MarshalIndent(report, "", " ") + if err != nil { + return fmt.Sprintf(`{"error": "%s"}`, err.Error()) + } + return string(data) +} + +// renderScenarioReportMarkdown renders a scenario report as concise markdown. +func renderScenarioReportMarkdown(report *ScenarioReport) string { + var sb strings.Builder + + // Status emoji + emoji := "āœ…" + if report.OverallStatus != "PASS" { + emoji = "āŒ" + } + + sb.WriteString(fmt.Sprintf("## %s %s: %s\n\n", emoji, report.Scenario, report.OverallStatus)) + + if report.TestDescription != "" { + sb.WriteString(fmt.Sprintf("> %s\n\n", report.TestDescription)) + } + + // Key metrics table + sb.WriteString("| Metric | Value | Expected | Status |\n") + sb.WriteString("|--------|------:|:--------:|:------:|\n") + + // Show only key metrics + keyMetrics := []string{"action_total", "reload_executed_total", "errors_total", "reconcile_total"} + for _, name := range keyMetrics { + for _, c := range report.Comparisons { + if c.Name == name { + value := fmt.Sprintf("%.0f", c.NewValue) + expected := "-" + if c.Expected > 0 { + expected = fmt.Sprintf("%.0f", c.Expected) + } + status := "āœ…" + if c.Status == "fail" { + status = "āŒ" + } else if c.Status == "info" { + status = "ā„¹ļø" + } + sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s |\n", c.DisplayName, value, expected, status)) + break + } + } + } + + return sb.String() +} + +// ============================================================================ +// SUMMARY COMMAND +// ============================================================================ + +// SummaryReport aggregates results from multiple scenarios. +type SummaryReport struct { + Timestamp time.Time `json:"timestamp"` + TestType string `json:"test_type"` + PassCount int `json:"pass_count"` + FailCount int `json:"fail_count"` + TotalCount int `json:"total_count"` + Scenarios []ScenarioSummary `json:"scenarios"` +} + +// ScenarioSummary provides a brief summary of a single scenario. +type ScenarioSummary struct { + ID string `json:"id"` + Status string `json:"status"` + Description string `json:"description"` + ActionTotal float64 `json:"action_total"` + ActionExp float64 `json:"action_expected"` + ErrorsTotal float64 `json:"errors_total"` +} + +func summaryCommand(args []string) { + var resultsDir, outputFile, testType string + format := OutputFormatMarkdown // Default to markdown for CI + resultsDir = "./results" + testType = "full" + + for _, arg := range args { + switch { + case strings.HasPrefix(arg, "--results-dir="): + resultsDir = strings.TrimPrefix(arg, "--results-dir=") + case strings.HasPrefix(arg, "--output="): + outputFile = strings.TrimPrefix(arg, "--output=") + case strings.HasPrefix(arg, "--format="): + format = OutputFormat(strings.TrimPrefix(arg, "--format=")) + case strings.HasPrefix(arg, "--test-type="): + testType = strings.TrimPrefix(arg, "--test-type=") + } + } + + summary, err := generateSummaryReport(resultsDir, testType) + if err != nil { + log.Fatalf("Failed to generate summary: %v", err) + } + + var output string + switch format { + case OutputFormatJSON: + output = renderSummaryJSON(summary) + case OutputFormatText: + output = renderSummaryText(summary) + default: + output = renderSummaryMarkdown(summary) + } + + if outputFile != "" { + if err := os.WriteFile(outputFile, []byte(output), 0644); err != nil { + log.Fatalf("Failed to write output file: %v", err) + } + log.Printf("Summary written to %s", outputFile) + } else { + fmt.Print(output) + } + + // Exit with non-zero status if any tests failed + if summary.FailCount > 0 { + os.Exit(1) + } +} + +func generateSummaryReport(resultsDir, testType string) (*SummaryReport, error) { + summary := &SummaryReport{ + Timestamp: time.Now(), + TestType: testType, + } + + // Find all scenario directories + entries, err := os.ReadDir(resultsDir) + if err != nil { + return nil, fmt.Errorf("failed to read results directory: %w", err) + } + + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "S") { + continue + } + + scenarioID := entry.Name() + report, err := generateScenarioReport(scenarioID, resultsDir) + if err != nil { + log.Printf("Warning: failed to load scenario %s: %v", scenarioID, err) + continue + } + + scenarioSummary := ScenarioSummary{ + ID: scenarioID, + Status: report.OverallStatus, + Description: report.TestDescription, + } + + // Extract key metrics + for _, c := range report.Comparisons { + switch c.Name { + case "action_total": + scenarioSummary.ActionTotal = c.NewValue + scenarioSummary.ActionExp = c.Expected + case "errors_total": + scenarioSummary.ErrorsTotal = c.NewValue + } + } + + summary.Scenarios = append(summary.Scenarios, scenarioSummary) + summary.TotalCount++ + if report.OverallStatus == "PASS" { + summary.PassCount++ + } else { + summary.FailCount++ + } + } + + // Sort scenarios by ID + sort.Slice(summary.Scenarios, func(i, j int) bool { + return naturalSort(summary.Scenarios[i].ID, summary.Scenarios[j].ID) + }) + + return summary, nil +} + +// naturalSort compares two scenario IDs (S1, S2, ..., S10, S11) +func naturalSort(a, b string) bool { + var aNum, bNum int + fmt.Sscanf(a, "S%d", &aNum) + fmt.Sscanf(b, "S%d", &bNum) + return aNum < bNum +} + +func renderSummaryJSON(summary *SummaryReport) string { + data, err := json.MarshalIndent(summary, "", " ") + if err != nil { + return fmt.Sprintf(`{"error": "%s"}`, err.Error()) + } + return string(data) +} + +func renderSummaryText(summary *SummaryReport) string { + var sb strings.Builder + + sb.WriteString("================================================================================\n") + sb.WriteString(" LOAD TEST SUMMARY\n") + sb.WriteString("================================================================================\n\n") + + passRate := 0 + if summary.TotalCount > 0 { + passRate = summary.PassCount * 100 / summary.TotalCount + } + + fmt.Fprintf(&sb, "Test Type: %s\n", summary.TestType) + fmt.Fprintf(&sb, "Results: %d/%d passed (%d%%)\n\n", summary.PassCount, summary.TotalCount, passRate) + + fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8s\n", "ID", "Status", "Description", "Actions", "Errors") + fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8s\n", "------", "--------", strings.Repeat("-", 45), "----------", "--------") + + for _, s := range summary.Scenarios { + desc := s.Description + if len(desc) > 45 { + desc = desc[:42] + "..." + } + actions := fmt.Sprintf("%.0f", s.ActionTotal) + if s.ActionExp > 0 { + actions = fmt.Sprintf("%.0f/%.0f", s.ActionTotal, s.ActionExp) + } + fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8.0f\n", s.ID, s.Status, desc, actions, s.ErrorsTotal) + } + + sb.WriteString("\n================================================================================\n") + return sb.String() +} + +func renderSummaryMarkdown(summary *SummaryReport) string { + var sb strings.Builder + + // Overall status + emoji := "āœ…" + title := "ALL TESTS PASSED" + if summary.FailCount > 0 { + emoji = "āŒ" + title = fmt.Sprintf("%d TEST(S) FAILED", summary.FailCount) + } else if summary.TotalCount == 0 { + emoji = "āš ļø" + title = "NO RESULTS" + } + + sb.WriteString(fmt.Sprintf("## %s Load Test Results: %s\n\n", emoji, title)) + + // Test type note + if summary.TestType == "quick" { + sb.WriteString("> šŸš€ **Quick Test** (S1, S4, S6) ā€” Use `/loadtest` for full suite\n\n") + } + + // Pass rate + passRate := 0 + if summary.TotalCount > 0 { + passRate = summary.PassCount * 100 / summary.TotalCount + } + sb.WriteString(fmt.Sprintf("**%d/%d passed** (%d%%)\n\n", summary.PassCount, summary.TotalCount, passRate)) + + // Results table + sb.WriteString("| | Scenario | Description | Actions | Errors |\n") + sb.WriteString("|:-:|:--------:|-------------|:-------:|:------:|\n") + + for _, s := range summary.Scenarios { + icon := "āœ…" + if s.Status != "PASS" { + icon = "āŒ" + } + + // Truncate description + desc := s.Description + if len(desc) > 45 { + desc = desc[:42] + "..." + } + + // Format actions + actions := fmt.Sprintf("%.0f", s.ActionTotal) + if s.ActionExp > 0 { + actions = fmt.Sprintf("%.0f/%.0f", s.ActionTotal, s.ActionExp) + } + + // Format errors + errors := fmt.Sprintf("%.0f", s.ErrorsTotal) + if s.ErrorsTotal > 0 { + errors = fmt.Sprintf("āš ļø %.0f", s.ErrorsTotal) + } + + sb.WriteString(fmt.Sprintf("| %s | **%s** | %s | %s | %s |\n", icon, s.ID, desc, actions, errors)) + } + + sb.WriteString("\nšŸ“¦ **[Download detailed results](../artifacts)**\n") + + return sb.String() +} From 322c4bc13027c449fb26ce0ce69f30a33509fa2e Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Thu, 8 Jan 2026 23:26:41 +0100 Subject: [PATCH 34/45] feat: Use cobra for loadtest CLI commands --- test/loadtest/cmd/loadtest/main.go | 1924 +------------------------ test/loadtest/go.mod | 4 +- test/loadtest/go.sum | 6 + test/loadtest/internal/cmd/report.go | 856 +++++++++++ test/loadtest/internal/cmd/root.go | 44 + test/loadtest/internal/cmd/run.go | 648 +++++++++ test/loadtest/internal/cmd/summary.go | 251 ++++ 7 files changed, 1811 insertions(+), 1922 deletions(-) create mode 100644 test/loadtest/internal/cmd/report.go create mode 100644 test/loadtest/internal/cmd/root.go create mode 100644 test/loadtest/internal/cmd/run.go create mode 100644 test/loadtest/internal/cmd/summary.go diff --git a/test/loadtest/cmd/loadtest/main.go b/test/loadtest/cmd/loadtest/main.go index fb05db5..a8925bc 100644 --- a/test/loadtest/cmd/loadtest/main.go +++ b/test/loadtest/cmd/loadtest/main.go @@ -1,1926 +1,8 @@ -// Package main implements the unified load test CLI for Reloader A/B comparison. +// Package main is the entrypoint for the load test CLI. package main -import ( - "context" - "encoding/json" - "fmt" - "log" - "math" - "os" - "os/exec" - "os/signal" - "path/filepath" - "sort" - "strings" - "sync" - "syscall" - "time" - - "github.com/stakater/Reloader/test/loadtest/internal/cluster" - "github.com/stakater/Reloader/test/loadtest/internal/prometheus" - "github.com/stakater/Reloader/test/loadtest/internal/reloader" - "github.com/stakater/Reloader/test/loadtest/internal/scenarios" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" -) - -const ( - clusterName = "reloader-loadtest" - testNamespace = "reloader-test" -) - -// OutputFormat defines the output format for reports. -type OutputFormat string - -const ( - OutputFormatText OutputFormat = "text" - OutputFormatJSON OutputFormat = "json" - OutputFormatMarkdown OutputFormat = "markdown" -) - -// workerContext holds all resources for a single worker (cluster + prometheus). -type workerContext struct { - id int - clusterMgr *cluster.Manager - promMgr *prometheus.Manager - kubeClient kubernetes.Interface - kubeContext string - runtime string -} - -// Config holds CLI configuration. -type Config struct { - OldImage string - NewImage string - Scenario string - Duration int - SkipCluster bool - ClusterName string // Custom cluster name (default: reloader-loadtest) - ResultsDir string - ManifestsDir string - Parallelism int // Number of parallel clusters (1 = sequential) -} +import "github.com/stakater/Reloader/test/loadtest/internal/cmd" func main() { - if len(os.Args) < 2 { - printUsage() - os.Exit(1) - } - - cmd := os.Args[1] - switch cmd { - case "run": - runCommand(os.Args[2:]) - case "report": - reportCommand(os.Args[2:]) - case "summary": - summaryCommand(os.Args[2:]) - case "help", "--help", "-h": - printUsage() - default: - fmt.Printf("Unknown command: %s\n", cmd) - printUsage() - os.Exit(1) - } -} - -func printUsage() { - fmt.Print(`Reloader Load Test CLI - -Usage: - loadtest run [options] Run A/B comparison tests - loadtest report [options] Generate comparison report for a scenario - loadtest summary [options] Generate summary across all scenarios (for CI) - loadtest help Show this help - -Run Options: - --old-image=IMAGE Container image for "old" version (required for comparison) - --new-image=IMAGE Container image for "new" version (required for comparison) - --scenario=ID Test scenario: S1-S13 or "all" (default: all) - --duration=SECONDS Test duration in seconds (default: 60) - --parallelism=N Run N scenarios in parallel on N clusters (default: 1) - --skip-cluster Skip kind cluster creation (use existing) - --cluster-name=NAME Kind cluster name (default: reloader-loadtest) - --results-dir=DIR Directory for results (default: ./results) - -Report Options: - --scenario=ID Scenario to report on (required) - --results-dir=DIR Directory containing results (default: ./results) - --output=FILE Output file (default: stdout) - --format=FORMAT Output format: text, json, markdown (default: text) - -Summary Options: - --results-dir=DIR Directory containing results (default: ./results) - --output=FILE Output file (default: stdout) - --format=FORMAT Output format: text, json, markdown (default: markdown) - --test-type=TYPE Test type label: quick, full (default: full) - -Examples: - # Compare two images - loadtest run --old-image=stakater/reloader:v1.0.0 --new-image=stakater/reloader:v1.1.0 - - # Run specific scenario - loadtest run --old-image=stakater/reloader:v1.0.0 --new-image=localhost/reloader:dev --scenario=S2 - - # Test single image (no comparison) - loadtest run --new-image=localhost/reloader:test - - # Run all scenarios in parallel on 4 clusters - loadtest run --new-image=localhost/reloader:test --parallelism=4 - - # Run all 13 scenarios in parallel (one cluster per scenario) - loadtest run --new-image=localhost/reloader:test --parallelism=13 - - # Generate report for a scenario - loadtest report --scenario=S2 --results-dir=./results - - # Generate JSON report - loadtest report --scenario=S2 --format=json - - # Generate markdown summary for CI - loadtest summary --results-dir=./results --format=markdown -`) -} - -func parseArgs(args []string) Config { - cfg := Config{ - Scenario: "all", - Duration: 60, - ResultsDir: "./results", - Parallelism: 1, - ClusterName: clusterName, // default - } - - // Find manifests dir relative to executable or current dir - execPath, _ := os.Executable() - execDir := filepath.Dir(execPath) - cfg.ManifestsDir = filepath.Join(execDir, "..", "..", "manifests") - if _, err := os.Stat(cfg.ManifestsDir); os.IsNotExist(err) { - // Try relative to current dir - cfg.ManifestsDir = "./manifests" - } - - for _, arg := range args { - switch { - case strings.HasPrefix(arg, "--old-image="): - cfg.OldImage = strings.TrimPrefix(arg, "--old-image=") - case strings.HasPrefix(arg, "--new-image="): - cfg.NewImage = strings.TrimPrefix(arg, "--new-image=") - case strings.HasPrefix(arg, "--scenario="): - cfg.Scenario = strings.TrimPrefix(arg, "--scenario=") - case strings.HasPrefix(arg, "--duration="): - if n, _ := fmt.Sscanf(strings.TrimPrefix(arg, "--duration="), "%d", &cfg.Duration); n != 1 { - log.Printf("Warning: invalid --duration value, using default (%d)", cfg.Duration) - } - case strings.HasPrefix(arg, "--parallelism="): - if n, _ := fmt.Sscanf(strings.TrimPrefix(arg, "--parallelism="), "%d", &cfg.Parallelism); n != 1 { - log.Printf("Warning: invalid --parallelism value, using default (%d)", cfg.Parallelism) - } - case arg == "--skip-cluster": - cfg.SkipCluster = true - case strings.HasPrefix(arg, "--cluster-name="): - cfg.ClusterName = strings.TrimPrefix(arg, "--cluster-name=") - case strings.HasPrefix(arg, "--results-dir="): - cfg.ResultsDir = strings.TrimPrefix(arg, "--results-dir=") - case strings.HasPrefix(arg, "--manifests-dir="): - cfg.ManifestsDir = strings.TrimPrefix(arg, "--manifests-dir=") - } - } - - // Validate parallelism - if cfg.Parallelism < 1 { - cfg.Parallelism = 1 - } - - return cfg -} - -func runCommand(args []string) { - cfg := parseArgs(args) - - // Validate required args - if cfg.OldImage == "" && cfg.NewImage == "" { - log.Fatal("At least one of --old-image or --new-image is required") - } - - // Determine mode - runOld := cfg.OldImage != "" - runNew := cfg.NewImage != "" - runBoth := runOld && runNew - - log.Printf("Configuration:") - log.Printf(" Scenario: %s", cfg.Scenario) - log.Printf(" Duration: %ds", cfg.Duration) - log.Printf(" Parallelism: %d", cfg.Parallelism) - if cfg.OldImage != "" { - log.Printf(" Old image: %s", cfg.OldImage) - } - if cfg.NewImage != "" { - log.Printf(" New image: %s", cfg.NewImage) - } - - // Detect container runtime - runtime, err := cluster.DetectContainerRuntime() - if err != nil { - log.Fatalf("Failed to detect container runtime: %v", err) - } - log.Printf(" Container runtime: %s", runtime) - - // Setup context with signal handling - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) - go func() { - <-sigCh - log.Println("Received shutdown signal...") - cancel() - }() - - // Determine scenarios to run - scenariosToRun := []string{cfg.Scenario} - if cfg.Scenario == "all" { - scenariosToRun = []string{"S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8", "S9", "S10", "S11", "S12", "S13"} - } - - // Skip-cluster only works for parallelism=1 - if cfg.SkipCluster && cfg.Parallelism > 1 { - log.Fatal("--skip-cluster is not supported with --parallelism > 1") - } - - // If parallelism > 1, use parallel execution - if cfg.Parallelism > 1 { - runParallel(ctx, cfg, scenariosToRun, runtime, runOld, runNew, runBoth) - return - } - - // Sequential execution (parallelism == 1) - runSequential(ctx, cfg, scenariosToRun, runtime, runOld, runNew, runBoth) -} - -// runSequential runs scenarios one by one on a single cluster. -func runSequential(ctx context.Context, cfg Config, scenariosToRun []string, runtime string, runOld, runNew, runBoth bool) { - // Create cluster manager - clusterMgr := cluster.NewManager(cluster.Config{ - Name: cfg.ClusterName, - ContainerRuntime: runtime, - }) - - // Create/verify cluster - if cfg.SkipCluster { - log.Printf("Skipping cluster creation (using existing cluster: %s)", cfg.ClusterName) - if !clusterMgr.Exists() { - log.Fatalf("Cluster %s does not exist. Remove --skip-cluster to create it.", cfg.ClusterName) - } - } else { - log.Println("Creating kind cluster...") - if err := clusterMgr.Create(ctx); err != nil { - log.Fatalf("Failed to create cluster: %v", err) - } - } - - // Deploy Prometheus - promManifest := filepath.Join(cfg.ManifestsDir, "prometheus.yaml") - promMgr := prometheus.NewManager(promManifest) - - log.Println("Installing Prometheus...") - if err := promMgr.Deploy(ctx); err != nil { - log.Fatalf("Failed to deploy Prometheus: %v", err) - } - - if err := promMgr.StartPortForward(ctx); err != nil { - log.Fatalf("Failed to start Prometheus port-forward: %v", err) - } - defer promMgr.StopPortForward() - - // Load images into kind - log.Println("Loading images into kind cluster...") - if runOld { - log.Printf("Loading old image: %s", cfg.OldImage) - if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { - log.Fatalf("Failed to load old image: %v", err) - } - } - if runNew { - log.Printf("Loading new image: %s", cfg.NewImage) - if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { - log.Fatalf("Failed to load new image: %v", err) - } - } - - // Pre-pull test images - log.Println("Pre-loading test images...") - testImage := "gcr.io/google-containers/busybox:1.27" - clusterMgr.LoadImage(ctx, testImage) // Ignore errors - - // Get kubernetes client - kubeClient, err := getKubeClient("") - if err != nil { - log.Fatalf("Failed to create kubernetes client: %v", err) - } - - for _, scenarioID := range scenariosToRun { - log.Printf("========================================") - log.Printf("=== Starting scenario %s ===", scenarioID) - log.Printf("========================================") - - // Clean up from previous scenario - cleanupTestNamespaces(ctx, "") - cleanupReloader(ctx, "old", "") - cleanupReloader(ctx, "new", "") - - // Reset Prometheus - if err := promMgr.Reset(ctx); err != nil { - log.Printf("Warning: failed to reset Prometheus: %v", err) - } - - // Create test namespace - createTestNamespace(ctx, "") - - if runOld { - // Test old version - oldMgr := reloader.NewManager(reloader.Config{ - Version: "old", - Image: cfg.OldImage, - }) - - if err := oldMgr.Deploy(ctx); err != nil { - log.Printf("Failed to deploy old Reloader: %v", err) - continue - } - - // Wait for Prometheus to discover and scrape the Reloader - if err := promMgr.WaitForTarget(ctx, oldMgr.Job(), 60*time.Second); err != nil { - log.Printf("Warning: %v", err) - log.Println("Proceeding anyway, but metrics may be incomplete") - } - - runScenario(ctx, kubeClient, scenarioID, "old", cfg.OldImage, cfg.Duration, cfg.ResultsDir) - collectMetrics(ctx, promMgr, oldMgr.Job(), scenarioID, "old", cfg.ResultsDir) - collectLogs(ctx, oldMgr, scenarioID, "old", cfg.ResultsDir) - - if runBoth { - // Clean up for new version - cleanupTestNamespaces(ctx, "") - oldMgr.Cleanup(ctx) - promMgr.Reset(ctx) - createTestNamespace(ctx, "") - } - } - - if runNew { - // Test new version - newMgr := reloader.NewManager(reloader.Config{ - Version: "new", - Image: cfg.NewImage, - }) - - if err := newMgr.Deploy(ctx); err != nil { - log.Printf("Failed to deploy new Reloader: %v", err) - continue - } - - // Wait for Prometheus to discover and scrape the Reloader - if err := promMgr.WaitForTarget(ctx, newMgr.Job(), 60*time.Second); err != nil { - log.Printf("Warning: %v", err) - log.Println("Proceeding anyway, but metrics may be incomplete") - } - - runScenario(ctx, kubeClient, scenarioID, "new", cfg.NewImage, cfg.Duration, cfg.ResultsDir) - collectMetrics(ctx, promMgr, newMgr.Job(), scenarioID, "new", cfg.ResultsDir) - collectLogs(ctx, newMgr, scenarioID, "new", cfg.ResultsDir) - } - - // Generate report - generateReport(scenarioID, cfg.ResultsDir, runBoth) - - log.Printf("=== Scenario %s complete ===", scenarioID) - } - - log.Println("Load test complete!") - log.Printf("Results available in: %s", cfg.ResultsDir) -} - -// runParallel runs scenarios in parallel on N separate kind clusters. -func runParallel(ctx context.Context, cfg Config, scenariosToRun []string, runtime string, runOld, runNew, runBoth bool) { - numWorkers := cfg.Parallelism - if numWorkers > len(scenariosToRun) { - numWorkers = len(scenariosToRun) - log.Printf("Reducing parallelism to %d (number of scenarios)", numWorkers) - } - - log.Printf("Starting parallel execution with %d workers", numWorkers) - - // Create workers - workers := make([]*workerContext, numWorkers) - var setupWg sync.WaitGroup - setupErrors := make(chan error, numWorkers) - - log.Println("Setting up worker clusters...") - for i := 0; i < numWorkers; i++ { - setupWg.Add(1) - go func(workerID int) { - defer setupWg.Done() - worker, err := setupWorker(ctx, cfg, workerID, runtime, runOld, runNew) - if err != nil { - setupErrors <- fmt.Errorf("worker %d setup failed: %w", workerID, err) - return - } - workers[workerID] = worker - }(i) - } - - setupWg.Wait() - close(setupErrors) - - // Check for setup errors - for err := range setupErrors { - log.Printf("Error: %v", err) - } - - // Verify all workers are ready - readyWorkers := 0 - for _, w := range workers { - if w != nil { - readyWorkers++ - } - } - if readyWorkers == 0 { - log.Fatal("No workers ready, aborting") - } - if readyWorkers < numWorkers { - log.Printf("Warning: only %d/%d workers ready", readyWorkers, numWorkers) - } - - // Cleanup workers on exit - defer func() { - log.Println("Cleaning up worker clusters...") - for _, w := range workers { - if w != nil { - w.promMgr.StopPortForward() - } - } - }() - - // Create scenario channel - scenarioCh := make(chan string, len(scenariosToRun)) - for _, s := range scenariosToRun { - scenarioCh <- s - } - close(scenarioCh) - - // Results tracking - var resultsMu sync.Mutex - completedScenarios := make([]string, 0, len(scenariosToRun)) - - // Start workers - var wg sync.WaitGroup - for _, worker := range workers { - if worker == nil { - continue - } - wg.Add(1) - go func(w *workerContext) { - defer wg.Done() - for scenarioID := range scenarioCh { - select { - case <-ctx.Done(): - return - default: - } - - log.Printf("[Worker %d] Starting scenario %s", w.id, scenarioID) - - // Clean up from previous scenario - cleanupTestNamespaces(ctx, w.kubeContext) - cleanupReloader(ctx, "old", w.kubeContext) - cleanupReloader(ctx, "new", w.kubeContext) - - // Reset Prometheus - if err := w.promMgr.Reset(ctx); err != nil { - log.Printf("[Worker %d] Warning: failed to reset Prometheus: %v", w.id, err) - } - - // Create test namespace - createTestNamespace(ctx, w.kubeContext) - - if runOld { - runVersionOnWorker(ctx, w, cfg, scenarioID, "old", cfg.OldImage, runBoth) - } - - if runNew { - runVersionOnWorker(ctx, w, cfg, scenarioID, "new", cfg.NewImage, false) - } - - // Generate report - generateReport(scenarioID, cfg.ResultsDir, runBoth) - - resultsMu.Lock() - completedScenarios = append(completedScenarios, scenarioID) - resultsMu.Unlock() - - log.Printf("[Worker %d] Scenario %s complete", w.id, scenarioID) - } - }(worker) - } - - wg.Wait() - - log.Println("Load test complete!") - log.Printf("Completed %d/%d scenarios", len(completedScenarios), len(scenariosToRun)) - log.Printf("Results available in: %s", cfg.ResultsDir) -} - -// setupWorker creates a cluster and deploys prometheus for a single worker. -func setupWorker(ctx context.Context, cfg Config, workerID int, runtime string, runOld, runNew bool) (*workerContext, error) { - workerName := fmt.Sprintf("%s-%d", clusterName, workerID) - promPort := 9091 + workerID - - log.Printf("[Worker %d] Creating cluster %s (ports %d/%d)...", workerID, workerName, 8080+workerID, 8443+workerID) - - clusterMgr := cluster.NewManager(cluster.Config{ - Name: workerName, - ContainerRuntime: runtime, - PortOffset: workerID, // Each cluster gets unique ports - }) - - if err := clusterMgr.Create(ctx); err != nil { - return nil, fmt.Errorf("creating cluster: %w", err) - } - - kubeContext := clusterMgr.Context() - - // Deploy Prometheus - promManifest := filepath.Join(cfg.ManifestsDir, "prometheus.yaml") - promMgr := prometheus.NewManagerWithPort(promManifest, promPort, kubeContext) - - log.Printf("[Worker %d] Installing Prometheus (port %d)...", workerID, promPort) - if err := promMgr.Deploy(ctx); err != nil { - return nil, fmt.Errorf("deploying prometheus: %w", err) - } - - if err := promMgr.StartPortForward(ctx); err != nil { - return nil, fmt.Errorf("starting prometheus port-forward: %w", err) - } - - // Load images - log.Printf("[Worker %d] Loading images...", workerID) - if runOld { - if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { - log.Printf("[Worker %d] Warning: failed to load old image: %v", workerID, err) - } - } - if runNew { - if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { - log.Printf("[Worker %d] Warning: failed to load new image: %v", workerID, err) - } - } - - // Pre-pull test images - testImage := "gcr.io/google-containers/busybox:1.27" - clusterMgr.LoadImage(ctx, testImage) - - // Get kubernetes client for this context - kubeClient, err := getKubeClient(kubeContext) - if err != nil { - return nil, fmt.Errorf("creating kubernetes client: %w", err) - } - - log.Printf("[Worker %d] Ready", workerID) - return &workerContext{ - id: workerID, - clusterMgr: clusterMgr, - promMgr: promMgr, - kubeClient: kubeClient, - kubeContext: kubeContext, - runtime: runtime, - }, nil -} - -// runVersionOnWorker runs a single version test on a worker. -func runVersionOnWorker(ctx context.Context, w *workerContext, cfg Config, scenarioID, version, image string, cleanupAfter bool) { - mgr := reloader.NewManager(reloader.Config{ - Version: version, - Image: image, - }) - mgr.SetKubeContext(w.kubeContext) - - if err := mgr.Deploy(ctx); err != nil { - log.Printf("[Worker %d] Failed to deploy %s Reloader: %v", w.id, version, err) - return - } - - // Wait for Prometheus to discover and scrape the Reloader - if err := w.promMgr.WaitForTarget(ctx, mgr.Job(), 60*time.Second); err != nil { - log.Printf("[Worker %d] Warning: %v", w.id, err) - log.Printf("[Worker %d] Proceeding anyway, but metrics may be incomplete", w.id) - } - - runScenario(ctx, w.kubeClient, scenarioID, version, image, cfg.Duration, cfg.ResultsDir) - collectMetrics(ctx, w.promMgr, mgr.Job(), scenarioID, version, cfg.ResultsDir) - collectLogs(ctx, mgr, scenarioID, version, cfg.ResultsDir) - - if cleanupAfter { - cleanupTestNamespaces(ctx, w.kubeContext) - mgr.Cleanup(ctx) - w.promMgr.Reset(ctx) - createTestNamespace(ctx, w.kubeContext) - } -} - -func runScenario(ctx context.Context, client kubernetes.Interface, scenarioID, version, image string, duration int, resultsDir string) { - runner, ok := scenarios.Registry[scenarioID] - if !ok { - log.Printf("Unknown scenario: %s", scenarioID) - return - } - - // For S6, set the reloader version - if s6, ok := runner.(*scenarios.ControllerRestartScenario); ok { - s6.ReloaderVersion = version - } - - // For S11, set the image to deploy its own Reloader - if s11, ok := runner.(*scenarios.AnnotationStrategyScenario); ok { - s11.Image = image - } - - log.Printf("Running scenario %s (%s): %s", scenarioID, version, runner.Description()) - - // Debug: check parent context state - if ctx.Err() != nil { - log.Printf("WARNING: Parent context already done: %v", ctx.Err()) - } - - // Add extra time for scenario setup (creating deployments, waiting for ready state) - // Some scenarios like S2 create 50 deployments which can take 2-3 minutes - timeout := time.Duration(duration)*time.Second + 5*time.Minute - log.Printf("Creating scenario context with timeout: %v (duration=%ds)", timeout, duration) - - scenarioCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - expected, err := runner.Run(scenarioCtx, client, testNamespace, time.Duration(duration)*time.Second) - if err != nil { - log.Printf("Scenario %s failed: %v", scenarioID, err) - } - - scenarios.WriteExpectedMetrics(scenarioID, resultsDir, expected) -} - -func collectMetrics(ctx context.Context, promMgr *prometheus.Manager, job, scenarioID, version, resultsDir string) { - log.Printf("Waiting 5s for Reloader to finish processing events...") - time.Sleep(5 * time.Second) - - log.Printf("Waiting 8s for Prometheus to scrape final metrics...") - time.Sleep(8 * time.Second) - - log.Printf("Collecting metrics for %s...", version) - outputDir := filepath.Join(resultsDir, scenarioID, version) - if err := promMgr.CollectMetrics(ctx, job, outputDir, scenarioID); err != nil { - log.Printf("Failed to collect metrics: %v", err) - } -} - -func collectLogs(ctx context.Context, mgr *reloader.Manager, scenarioID, version, resultsDir string) { - log.Printf("Collecting logs for %s...", version) - logPath := filepath.Join(resultsDir, scenarioID, version, "reloader.log") - if err := mgr.CollectLogs(ctx, logPath); err != nil { - log.Printf("Failed to collect logs: %v", err) - } -} - -func generateReport(scenarioID, resultsDir string, isComparison bool) { - if isComparison { - log.Println("Generating comparison report...") - } else { - log.Println("Generating single-version report...") - } - - reportPath := filepath.Join(resultsDir, scenarioID, "report.txt") - - // Use the report command - cmd := exec.Command(os.Args[0], "report", - fmt.Sprintf("--scenario=%s", scenarioID), - fmt.Sprintf("--results-dir=%s", resultsDir), - fmt.Sprintf("--output=%s", reportPath)) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Run() - - // Also print to stdout - if data, err := os.ReadFile(reportPath); err == nil { - fmt.Println(string(data)) - } - - log.Printf("Report saved to: %s", reportPath) -} - -func getKubeClient(kubeContext string) (kubernetes.Interface, error) { - kubeconfig := os.Getenv("KUBECONFIG") - if kubeconfig == "" { - home, _ := os.UserHomeDir() - kubeconfig = filepath.Join(home, ".kube", "config") - } - - loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig} - configOverrides := &clientcmd.ConfigOverrides{} - if kubeContext != "" { - configOverrides.CurrentContext = kubeContext - } - - kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) - config, err := kubeConfig.ClientConfig() - if err != nil { - return nil, err - } - - return kubernetes.NewForConfig(config) -} - -func createTestNamespace(ctx context.Context, kubeContext string) { - args := []string{"create", "namespace", testNamespace, "--dry-run=client", "-o", "yaml"} - if kubeContext != "" { - args = append([]string{"--context", kubeContext}, args...) - } - cmd := exec.CommandContext(ctx, "kubectl", args...) - out, _ := cmd.Output() - - applyArgs := []string{"apply", "-f", "-"} - if kubeContext != "" { - applyArgs = append([]string{"--context", kubeContext}, applyArgs...) - } - applyCmd := exec.CommandContext(ctx, "kubectl", applyArgs...) - applyCmd.Stdin = strings.NewReader(string(out)) - applyCmd.Run() -} - -func cleanupTestNamespaces(ctx context.Context, kubeContext string) { - log.Println("Cleaning up test resources...") - - // Main namespace + S3 extra namespaces - namespaces := []string{testNamespace} - for i := 0; i < 10; i++ { - namespaces = append(namespaces, fmt.Sprintf("%s-%d", testNamespace, i)) - } - - for _, ns := range namespaces { - args := []string{"delete", "namespace", ns, "--wait=false", "--ignore-not-found"} - if kubeContext != "" { - args = append([]string{"--context", kubeContext}, args...) - } - exec.CommandContext(ctx, "kubectl", args...).Run() - } - - // Wait a bit for cleanup - time.Sleep(2 * time.Second) - - // Force delete remaining pods - for _, ns := range namespaces { - args := []string{"delete", "pods", "--all", "-n", ns, "--grace-period=0", "--force"} - if kubeContext != "" { - args = append([]string{"--context", kubeContext}, args...) - } - exec.CommandContext(ctx, "kubectl", args...).Run() - } -} - -func cleanupReloader(ctx context.Context, version string, kubeContext string) { - ns := fmt.Sprintf("reloader-%s", version) - - nsArgs := []string{"delete", "namespace", ns, "--wait=false", "--ignore-not-found"} - crArgs := []string{"delete", "clusterrole", fmt.Sprintf("reloader-%s", version), "--ignore-not-found"} - crbArgs := []string{"delete", "clusterrolebinding", fmt.Sprintf("reloader-%s", version), "--ignore-not-found"} - - if kubeContext != "" { - nsArgs = append([]string{"--context", kubeContext}, nsArgs...) - crArgs = append([]string{"--context", kubeContext}, crArgs...) - crbArgs = append([]string{"--context", kubeContext}, crbArgs...) - } - - exec.CommandContext(ctx, "kubectl", nsArgs...).Run() - exec.CommandContext(ctx, "kubectl", crArgs...).Run() - exec.CommandContext(ctx, "kubectl", crbArgs...).Run() -} - -// ============================================================================ -// REPORT COMMAND -// ============================================================================ - -func reportCommand(args []string) { - var scenarioID, resultsDir, outputFile string - format := OutputFormatText - resultsDir = "./results" - - for _, arg := range args { - switch { - case strings.HasPrefix(arg, "--scenario="): - scenarioID = strings.TrimPrefix(arg, "--scenario=") - case strings.HasPrefix(arg, "--results-dir="): - resultsDir = strings.TrimPrefix(arg, "--results-dir=") - case strings.HasPrefix(arg, "--output="): - outputFile = strings.TrimPrefix(arg, "--output=") - case strings.HasPrefix(arg, "--format="): - format = OutputFormat(strings.TrimPrefix(arg, "--format=")) - } - } - - if scenarioID == "" { - log.Fatal("--scenario is required for report command") - } - - report, err := generateScenarioReport(scenarioID, resultsDir) - if err != nil { - log.Fatalf("Failed to generate report: %v", err) - } - - var output string - switch format { - case OutputFormatJSON: - output = renderScenarioReportJSON(report) - case OutputFormatMarkdown: - output = renderScenarioReportMarkdown(report) - default: - output = renderScenarioReport(report) - } - - if outputFile != "" { - if err := os.WriteFile(outputFile, []byte(output), 0644); err != nil { - log.Fatalf("Failed to write output file: %v", err) - } - log.Printf("Report written to %s", outputFile) - } else { - fmt.Println(output) - } -} - -// PrometheusResponse represents a Prometheus API response for report parsing. -type PrometheusResponse struct { - Status string `json:"status"` - Data struct { - ResultType string `json:"resultType"` - Result []struct { - Metric map[string]string `json:"metric"` - Value []interface{} `json:"value"` - } `json:"result"` - } `json:"data"` -} - -// MetricComparison represents the comparison of a single metric. -type MetricComparison struct { - Name string - DisplayName string - Unit string - IsCounter bool - OldValue float64 - NewValue float64 - Expected float64 - Difference float64 - DiffPct float64 - Status string - Threshold float64 - OldMeetsExpected string - NewMeetsExpected string -} - -type metricInfo struct { - unit string - isCounter bool -} - -var metricInfoMap = map[string]metricInfo{ - "reconcile_total": {unit: "count", isCounter: true}, - "reconcile_duration_p50": {unit: "s", isCounter: false}, - "reconcile_duration_p95": {unit: "s", isCounter: false}, - "reconcile_duration_p99": {unit: "s", isCounter: false}, - "action_total": {unit: "count", isCounter: true}, - "action_latency_p50": {unit: "s", isCounter: false}, - "action_latency_p95": {unit: "s", isCounter: false}, - "action_latency_p99": {unit: "s", isCounter: false}, - "errors_total": {unit: "count", isCounter: true}, - "reload_executed_total": {unit: "count", isCounter: true}, - "workloads_scanned_total": {unit: "count", isCounter: true}, - "workloads_matched_total": {unit: "count", isCounter: true}, - "skipped_total_no_data_change": {unit: "count", isCounter: true}, - "rest_client_requests_total": {unit: "count", isCounter: true}, - "rest_client_requests_get": {unit: "count", isCounter: true}, - "rest_client_requests_patch": {unit: "count", isCounter: true}, - "rest_client_requests_put": {unit: "count", isCounter: true}, - "rest_client_requests_errors": {unit: "count", isCounter: true}, - - // Resource consumption metrics (gauges, not counters) - "memory_rss_mb_avg": {unit: "MB", isCounter: false}, - "memory_rss_mb_max": {unit: "MB", isCounter: false}, - "memory_heap_mb_avg": {unit: "MB", isCounter: false}, - "memory_heap_mb_max": {unit: "MB", isCounter: false}, - "cpu_cores_avg": {unit: "cores", isCounter: false}, - "cpu_cores_max": {unit: "cores", isCounter: false}, - "goroutines_avg": {unit: "count", isCounter: false}, - "goroutines_max": {unit: "count", isCounter: false}, - "gc_pause_p99_ms": {unit: "ms", isCounter: false}, -} - -// ReportExpectedMetrics matches the expected metrics from test scenarios. -type ReportExpectedMetrics struct { - ActionTotal int `json:"action_total"` - ReloadExecutedTotal int `json:"reload_executed_total"` - ReconcileTotal int `json:"reconcile_total"` - WorkloadsScannedTotal int `json:"workloads_scanned_total"` - WorkloadsMatchedTotal int `json:"workloads_matched_total"` - SkippedTotal int `json:"skipped_total"` - Description string `json:"description"` -} - -// ScenarioReport represents the full report for a scenario. -type ScenarioReport struct { - Scenario string - Timestamp time.Time - Comparisons []MetricComparison - OverallStatus string - Summary string - PassCriteria []string - FailedCriteria []string - Expected ReportExpectedMetrics - TestDescription string -} - -// MetricType defines how to evaluate a metric. -type MetricType int - -const ( - LowerIsBetter MetricType = iota - ShouldMatch - HigherIsBetter - Informational // Reports values but doesn't affect pass/fail -) - -type ThresholdConfig struct { - maxDiff float64 - metricType MetricType - minAbsDiff float64 -} - -var thresholds = map[string]ThresholdConfig{ - "reconcile_total": {maxDiff: 60.0, metricType: LowerIsBetter}, - "reconcile_duration_p50": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.5}, - "reconcile_duration_p95": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, - "reconcile_duration_p99": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, - "action_latency_p50": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.5}, - "action_latency_p95": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, - "action_latency_p99": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, - "errors_total": {maxDiff: 0.0, metricType: LowerIsBetter}, - "action_total": {maxDiff: 15.0, metricType: ShouldMatch}, - "reload_executed_total": {maxDiff: 15.0, metricType: ShouldMatch}, - "workloads_scanned_total": {maxDiff: 15.0, metricType: ShouldMatch}, - "workloads_matched_total": {maxDiff: 15.0, metricType: ShouldMatch}, - "skipped_total_no_data_change": {maxDiff: 20.0, metricType: ShouldMatch}, - // API metrics - use minAbsDiff to allow small differences - "rest_client_requests_total": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, - "rest_client_requests_get": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, - "rest_client_requests_patch": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, - "rest_client_requests_put": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 20}, - "rest_client_requests_errors": {maxDiff: 0.0, metricType: LowerIsBetter, minAbsDiff: 100}, // Pass if both < 100 - - // Resource consumption metrics - "memory_rss_mb_avg": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 20}, // 50% or 20MB - "memory_rss_mb_max": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 30}, // 50% or 30MB - "memory_heap_mb_avg": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 15}, // 50% or 15MB - "memory_heap_mb_max": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 20}, // 50% or 20MB - "cpu_cores_avg": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.1}, // 100% or 0.1 cores - "cpu_cores_max": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.2}, // 100% or 0.2 cores - "goroutines_avg": {metricType: Informational}, // Info only - different architectures may use more goroutines - "goroutines_max": {metricType: Informational}, // Info only - different architectures may use more goroutines - "gc_pause_p99_ms": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 5}, // 100% or 5ms -} - -func generateScenarioReport(scenario, resultsDir string) (*ScenarioReport, error) { - oldDir := filepath.Join(resultsDir, scenario, "old") - newDir := filepath.Join(resultsDir, scenario, "new") - scenarioDir := filepath.Join(resultsDir, scenario) - - // Check which directories exist to determine mode - _, oldErr := os.Stat(oldDir) - _, newErr := os.Stat(newDir) - hasOld := oldErr == nil - hasNew := newErr == nil - isComparison := hasOld && hasNew - - // For single-version mode, determine which version we have - singleVersion := "" - singleDir := "" - if !isComparison { - if hasNew { - singleVersion = "new" - singleDir = newDir - } else if hasOld { - singleVersion = "old" - singleDir = oldDir - } else { - return nil, fmt.Errorf("no results found in %s", scenarioDir) - } - } - - report := &ScenarioReport{ - Scenario: scenario, - Timestamp: time.Now(), - } - - // Load expected metrics - expectedPath := filepath.Join(scenarioDir, "expected.json") - if data, err := os.ReadFile(expectedPath); err == nil { - if err := json.Unmarshal(data, &report.Expected); err != nil { - log.Printf("Warning: Could not parse expected metrics: %v", err) - } else { - report.TestDescription = report.Expected.Description - } - } - - // Handle single-version mode - if !isComparison { - return generateSingleVersionReport(report, singleDir, singleVersion, scenario) - } - - // Define metrics to compare - metricsToCompare := []struct { - name string - file string - selector func(data PrometheusResponse) float64 - }{ - {"reconcile_total", "reloader_reconcile_total.json", sumAllValues}, - {"reconcile_duration_p50", "reconcile_p50.json", getFirstValue}, - {"reconcile_duration_p95", "reconcile_p95.json", getFirstValue}, - {"reconcile_duration_p99", "reconcile_p99.json", getFirstValue}, - {"action_total", "reloader_action_total.json", sumAllValues}, - {"action_latency_p50", "action_p50.json", getFirstValue}, - {"action_latency_p95", "action_p95.json", getFirstValue}, - {"action_latency_p99", "action_p99.json", getFirstValue}, - {"errors_total", "reloader_errors_total.json", sumAllValues}, - {"reload_executed_total", "reloader_reload_executed_total.json", sumSuccessValues}, - {"workloads_scanned_total", "reloader_workloads_scanned_total.json", sumAllValues}, - {"workloads_matched_total", "reloader_workloads_matched_total.json", sumAllValues}, - {"rest_client_requests_total", "rest_client_requests_total.json", getFirstValue}, - {"rest_client_requests_get", "rest_client_requests_get.json", getFirstValue}, - {"rest_client_requests_patch", "rest_client_requests_patch.json", getFirstValue}, - {"rest_client_requests_put", "rest_client_requests_put.json", getFirstValue}, - {"rest_client_requests_errors", "rest_client_requests_errors.json", getFirstValue}, - - // Resource consumption metrics - {"memory_rss_mb_avg", "memory_rss_bytes_avg.json", bytesToMB}, - {"memory_rss_mb_max", "memory_rss_bytes_max.json", bytesToMB}, - {"memory_heap_mb_avg", "memory_heap_bytes_avg.json", bytesToMB}, - {"memory_heap_mb_max", "memory_heap_bytes_max.json", bytesToMB}, - {"cpu_cores_avg", "cpu_usage_cores_avg.json", getFirstValue}, - {"cpu_cores_max", "cpu_usage_cores_max.json", getFirstValue}, - {"goroutines_avg", "goroutines_avg.json", getFirstValue}, - {"goroutines_max", "goroutines_max.json", getFirstValue}, - {"gc_pause_p99_ms", "gc_duration_seconds_p99.json", secondsToMs}, - } - - // Build expected values map - expectedValues := map[string]float64{ - "action_total": float64(report.Expected.ActionTotal), - "reload_executed_total": float64(report.Expected.ReloadExecutedTotal), - "reconcile_total": float64(report.Expected.ReconcileTotal), - "workloads_scanned_total": float64(report.Expected.WorkloadsScannedTotal), - "workloads_matched_total": float64(report.Expected.WorkloadsMatchedTotal), - "skipped_total": float64(report.Expected.SkippedTotal), - } - - // First pass: collect all metric values - metricValues := make(map[string]struct{ old, new, expected float64 }) - - for _, m := range metricsToCompare { - oldData, err := loadMetricFile(filepath.Join(oldDir, m.file)) - if err != nil { - log.Printf("Warning: Could not load old metric %s: %v", m.name, err) - continue - } - - newData, err := loadMetricFile(filepath.Join(newDir, m.file)) - if err != nil { - log.Printf("Warning: Could not load new metric %s: %v", m.name, err) - continue - } - - oldValue := m.selector(oldData) - newValue := m.selector(newData) - expected := expectedValues[m.name] - - metricValues[m.name] = struct{ old, new, expected float64 }{oldValue, newValue, expected} - } - - // Check context for smart pass/fail decisions - newMeetsActionExpected := false - newReconcileIsZero := false - isChurnScenario := scenario == "S5" // Workload churn has special pass/fail rules - if v, ok := metricValues["action_total"]; ok && v.expected > 0 { - tolerance := v.expected * 0.15 - newMeetsActionExpected = math.Abs(v.new-v.expected) <= tolerance - } - if v, ok := metricValues["reconcile_total"]; ok { - newReconcileIsZero = v.new == 0 - } - - // Second pass: generate comparisons with context awareness - for _, m := range metricsToCompare { - v, ok := metricValues[m.name] - if !ok { - continue - } - - comparison := compareMetricWithExpected(m.name, v.old, v.new, v.expected) - - // Context-aware adjustments for API metrics - if strings.HasPrefix(m.name, "rest_client_requests") { - // If new correctly processed all expected reloads but old didn't, - // higher API calls in new is expected (it's doing the work correctly) - if newMeetsActionExpected && comparison.Status != "pass" { - if oldMeets, ok := metricValues["action_total"]; ok { - oldTolerance := oldMeets.expected * 0.15 - oldMissed := math.Abs(oldMeets.old-oldMeets.expected) > oldTolerance - if oldMissed { - comparison.Status = "pass" - } - } - } - // If new has 0 reconciles (no-op scenario), API differences are fine - if newReconcileIsZero && comparison.Status != "pass" { - comparison.Status = "pass" - } - } - - // S5 (Workload Churn) specific adjustments: - // - "Not found" errors are expected when deployments are deleted during processing - // - No expected values for throughput, so compare old vs new (should be similar) - if isChurnScenario { - if m.name == "errors_total" { - // In churn scenarios, "not found" errors are expected when workloads - // are deleted while Reloader is processing them. Allow up to 50 errors. - if v.new < 50 && v.old < 50 { - comparison.Status = "pass" - } else if v.new <= v.old*1.5 { - // Also pass if new has similar or fewer errors than old - comparison.Status = "pass" - } - } - if m.name == "action_total" || m.name == "reload_executed_total" { - // No expected value for churn - compare old vs new - // Both should be similar (within 20% of each other) - if v.old > 0 { - diff := math.Abs(v.new-v.old) / v.old * 100 - if diff <= 20 { - comparison.Status = "pass" - } - } else if v.new > 0 { - // Old is 0, new has value - that's fine - comparison.Status = "pass" - } - } - } - - report.Comparisons = append(report.Comparisons, comparison) - - if comparison.Status == "pass" { - report.PassCriteria = append(report.PassCriteria, m.name) - } else if comparison.Status == "fail" { - report.FailedCriteria = append(report.FailedCriteria, m.name) - } - } - - // Determine overall status - if len(report.FailedCriteria) == 0 { - report.OverallStatus = "PASS" - report.Summary = "All metrics within acceptable thresholds" - } else { - report.OverallStatus = "FAIL" - report.Summary = fmt.Sprintf("%d metrics failed: %s", - len(report.FailedCriteria), - strings.Join(report.FailedCriteria, ", ")) - } - - return report, nil -} - -// generateSingleVersionReport creates a report for a single version (no comparison). -func generateSingleVersionReport(report *ScenarioReport, dataDir, version, scenario string) (*ScenarioReport, error) { - // Define metrics to collect - metricsToCollect := []struct { - name string - file string - selector func(data PrometheusResponse) float64 - }{ - {"reconcile_total", "reloader_reconcile_total.json", sumAllValues}, - {"reconcile_duration_p50", "reconcile_p50.json", getFirstValue}, - {"reconcile_duration_p95", "reconcile_p95.json", getFirstValue}, - {"reconcile_duration_p99", "reconcile_p99.json", getFirstValue}, - {"action_total", "reloader_action_total.json", sumAllValues}, - {"action_latency_p50", "action_p50.json", getFirstValue}, - {"action_latency_p95", "action_p95.json", getFirstValue}, - {"action_latency_p99", "action_p99.json", getFirstValue}, - {"errors_total", "reloader_errors_total.json", sumAllValues}, - {"reload_executed_total", "reloader_reload_executed_total.json", sumSuccessValues}, - {"workloads_scanned_total", "reloader_workloads_scanned_total.json", sumAllValues}, - {"workloads_matched_total", "reloader_workloads_matched_total.json", sumAllValues}, - {"rest_client_requests_total", "rest_client_requests_total.json", getFirstValue}, - {"rest_client_requests_get", "rest_client_requests_get.json", getFirstValue}, - {"rest_client_requests_patch", "rest_client_requests_patch.json", getFirstValue}, - {"rest_client_requests_put", "rest_client_requests_put.json", getFirstValue}, - {"rest_client_requests_errors", "rest_client_requests_errors.json", getFirstValue}, - {"memory_rss_mb_avg", "memory_rss_bytes_avg.json", bytesToMB}, - {"memory_rss_mb_max", "memory_rss_bytes_max.json", bytesToMB}, - {"memory_heap_mb_avg", "memory_heap_bytes_avg.json", bytesToMB}, - {"memory_heap_mb_max", "memory_heap_bytes_max.json", bytesToMB}, - {"cpu_cores_avg", "cpu_usage_cores_avg.json", getFirstValue}, - {"cpu_cores_max", "cpu_usage_cores_max.json", getFirstValue}, - {"goroutines_avg", "goroutines_avg.json", getFirstValue}, - {"goroutines_max", "goroutines_max.json", getFirstValue}, - {"gc_pause_p99_ms", "gc_duration_seconds_p99.json", secondsToMs}, - } - - // Build expected values map - expectedValues := map[string]float64{ - "action_total": float64(report.Expected.ActionTotal), - "reload_executed_total": float64(report.Expected.ReloadExecutedTotal), - "reconcile_total": float64(report.Expected.ReconcileTotal), - "workloads_scanned_total": float64(report.Expected.WorkloadsScannedTotal), - "workloads_matched_total": float64(report.Expected.WorkloadsMatchedTotal), - "skipped_total": float64(report.Expected.SkippedTotal), - } - - for _, m := range metricsToCollect { - data, err := loadMetricFile(filepath.Join(dataDir, m.file)) - if err != nil { - log.Printf("Warning: Could not load metric %s: %v", m.name, err) - continue - } - - value := m.selector(data) - expected := expectedValues[m.name] - - info := metricInfoMap[m.name] - if info.unit == "" { - info = metricInfo{unit: "count", isCounter: true} - } - - displayName := m.name - if info.unit != "count" { - displayName = fmt.Sprintf("%s (%s)", m.name, info.unit) - } - - // For single-version, put the value in NewValue column - status := "info" - meetsExp := "-" - - // Check against expected if available - if expected > 0 { - meetsExp = meetsExpected(value, expected) - threshold, ok := thresholds[m.name] - if ok && threshold.metricType == ShouldMatch { - if meetsExp == "āœ“" { - status = "pass" - report.PassCriteria = append(report.PassCriteria, m.name) - } else { - status = "fail" - report.FailedCriteria = append(report.FailedCriteria, m.name) - } - } - } - - if info.isCounter { - value = math.Round(value) - } - - report.Comparisons = append(report.Comparisons, MetricComparison{ - Name: m.name, - DisplayName: displayName, - Unit: info.unit, - IsCounter: info.isCounter, - OldValue: 0, // No old value in single-version mode - NewValue: value, - Expected: expected, - OldMeetsExpected: "-", - NewMeetsExpected: meetsExp, - Status: status, - }) - } - - if len(report.FailedCriteria) == 0 { - report.OverallStatus = "PASS" - report.Summary = fmt.Sprintf("Single-version test (%s) completed successfully", version) - } else { - report.OverallStatus = "FAIL" - report.Summary = fmt.Sprintf("%d metrics failed: %s", - len(report.FailedCriteria), - strings.Join(report.FailedCriteria, ", ")) - } - - return report, nil -} - -func loadMetricFile(path string) (PrometheusResponse, error) { - var resp PrometheusResponse - data, err := os.ReadFile(path) - if err != nil { - return resp, err - } - err = json.Unmarshal(data, &resp) - return resp, err -} - -func sumAllValues(data PrometheusResponse) float64 { - var sum float64 - for _, result := range data.Data.Result { - if len(result.Value) >= 2 { - if v, ok := result.Value[1].(string); ok { - var f float64 - fmt.Sscanf(v, "%f", &f) - sum += f - } - } - } - return sum -} - -func sumSuccessValues(data PrometheusResponse) float64 { - var sum float64 - for _, result := range data.Data.Result { - if result.Metric["success"] == "true" { - if len(result.Value) >= 2 { - if v, ok := result.Value[1].(string); ok { - var f float64 - fmt.Sscanf(v, "%f", &f) - sum += f - } - } - } - } - return sum -} - -func getFirstValue(data PrometheusResponse) float64 { - if len(data.Data.Result) > 0 && len(data.Data.Result[0].Value) >= 2 { - if v, ok := data.Data.Result[0].Value[1].(string); ok { - var f float64 - fmt.Sscanf(v, "%f", &f) - return f - } - } - return 0 -} - -// bytesToMB converts bytes to megabytes. -func bytesToMB(data PrometheusResponse) float64 { - bytes := getFirstValue(data) - return bytes / (1024 * 1024) -} - -// secondsToMs converts seconds to milliseconds. -func secondsToMs(data PrometheusResponse) float64 { - seconds := getFirstValue(data) - return seconds * 1000 -} - -func meetsExpected(value, expected float64) string { - if expected == 0 { - return "-" - } - tolerance := expected * 0.15 - if math.Abs(value-expected) <= tolerance { - return "āœ“" - } - return "āœ—" -} - -func compareMetricWithExpected(name string, oldValue, newValue, expected float64) MetricComparison { - diff := newValue - oldValue - absDiff := math.Abs(diff) - var diffPct float64 - if oldValue != 0 { - diffPct = (diff / oldValue) * 100 - } else if newValue != 0 { - diffPct = 100 - } - - threshold, ok := thresholds[name] - if !ok { - threshold = ThresholdConfig{maxDiff: 10.0, metricType: ShouldMatch} - } - - info := metricInfoMap[name] - if info.unit == "" { - info = metricInfo{unit: "count", isCounter: true} - } - displayName := name - if info.unit != "count" { - displayName = fmt.Sprintf("%s (%s)", name, info.unit) - } - - if info.isCounter { - oldValue = math.Round(oldValue) - newValue = math.Round(newValue) - } - - status := "pass" - oldMeetsExp := meetsExpected(oldValue, expected) - newMeetsExp := meetsExpected(newValue, expected) - - if expected > 0 && threshold.metricType == ShouldMatch { - if newMeetsExp == "āœ—" { - status = "fail" - } - } else { - switch threshold.metricType { - case LowerIsBetter: - if threshold.minAbsDiff > 0 && absDiff < threshold.minAbsDiff { - status = "pass" - } else if diffPct > threshold.maxDiff { - status = "fail" - } - case HigherIsBetter: - if diffPct < -threshold.maxDiff { - status = "fail" - } - case ShouldMatch: - if math.Abs(diffPct) > threshold.maxDiff { - status = "fail" - } - case Informational: - status = "info" - } - } - - return MetricComparison{ - Name: name, - DisplayName: displayName, - Unit: info.unit, - IsCounter: info.isCounter, - Expected: expected, - OldMeetsExpected: oldMeetsExp, - NewMeetsExpected: newMeetsExp, - OldValue: oldValue, - NewValue: newValue, - Difference: diff, - DiffPct: diffPct, - Status: status, - Threshold: threshold.maxDiff, - } -} - -func renderScenarioReport(report *ScenarioReport) string { - var sb strings.Builder - - // Detect single-version mode by checking if all OldValues are 0 - isSingleVersion := true - for _, c := range report.Comparisons { - if c.OldValue != 0 { - isSingleVersion = false - break - } - } - - sb.WriteString("\n") - sb.WriteString("================================================================================\n") - if isSingleVersion { - sb.WriteString(" RELOADER TEST REPORT\n") - } else { - sb.WriteString(" RELOADER A/B COMPARISON REPORT\n") - } - sb.WriteString("================================================================================\n\n") - - fmt.Fprintf(&sb, "Scenario: %s\n", report.Scenario) - fmt.Fprintf(&sb, "Generated: %s\n", report.Timestamp.Format("2006-01-02 15:04:05")) - fmt.Fprintf(&sb, "Status: %s\n", report.OverallStatus) - fmt.Fprintf(&sb, "Summary: %s\n", report.Summary) - - if report.TestDescription != "" { - fmt.Fprintf(&sb, "Test: %s\n", report.TestDescription) - } - - if report.Expected.ActionTotal > 0 { - sb.WriteString("\n--------------------------------------------------------------------------------\n") - sb.WriteString(" EXPECTED VALUES\n") - sb.WriteString("--------------------------------------------------------------------------------\n") - fmt.Fprintf(&sb, "Expected Action Total: %d\n", report.Expected.ActionTotal) - fmt.Fprintf(&sb, "Expected Reload Executed Total: %d\n", report.Expected.ReloadExecutedTotal) - if report.Expected.SkippedTotal > 0 { - fmt.Fprintf(&sb, "Expected Skipped Total: %d\n", report.Expected.SkippedTotal) - } - } - - sb.WriteString("\n--------------------------------------------------------------------------------\n") - if isSingleVersion { - sb.WriteString(" METRICS\n") - } else { - sb.WriteString(" METRIC COMPARISONS\n") - } - sb.WriteString("--------------------------------------------------------------------------------\n") - - if isSingleVersion { - sb.WriteString("(āœ“ = meets expected value within 15%)\n\n") - fmt.Fprintf(&sb, "%-32s %12s %10s %5s %8s\n", - "Metric", "Value", "Expected", "Met?", "Status") - fmt.Fprintf(&sb, "%-32s %12s %10s %5s %8s\n", - "------", "-----", "--------", "----", "------") - - for _, c := range report.Comparisons { - if c.IsCounter { - if c.Expected > 0 { - fmt.Fprintf(&sb, "%-32s %12.0f %10.0f %5s %8s\n", - c.DisplayName, c.NewValue, c.Expected, - c.NewMeetsExpected, c.Status) - } else { - fmt.Fprintf(&sb, "%-32s %12.0f %10s %5s %8s\n", - c.DisplayName, c.NewValue, "-", - c.NewMeetsExpected, c.Status) - } - } else { - fmt.Fprintf(&sb, "%-32s %12.4f %10s %5s %8s\n", - c.DisplayName, c.NewValue, "-", - c.NewMeetsExpected, c.Status) - } - } - } else { - sb.WriteString("(Oldāœ“/Newāœ“ = meets expected value within 15%)\n\n") - - fmt.Fprintf(&sb, "%-32s %12s %12s %10s %5s %5s %8s\n", - "Metric", "Old", "New", "Expected", "Oldāœ“", "Newāœ“", "Status") - fmt.Fprintf(&sb, "%-32s %12s %12s %10s %5s %5s %8s\n", - "------", "---", "---", "--------", "----", "----", "------") - - for _, c := range report.Comparisons { - if c.IsCounter { - if c.Expected > 0 { - fmt.Fprintf(&sb, "%-32s %12.0f %12.0f %10.0f %5s %5s %8s\n", - c.DisplayName, c.OldValue, c.NewValue, c.Expected, - c.OldMeetsExpected, c.NewMeetsExpected, c.Status) - } else { - fmt.Fprintf(&sb, "%-32s %12.0f %12.0f %10s %5s %5s %8s\n", - c.DisplayName, c.OldValue, c.NewValue, "-", - c.OldMeetsExpected, c.NewMeetsExpected, c.Status) - } - } else { - fmt.Fprintf(&sb, "%-32s %12.4f %12.4f %10s %5s %5s %8s\n", - c.DisplayName, c.OldValue, c.NewValue, "-", - c.OldMeetsExpected, c.NewMeetsExpected, c.Status) - } - } - } - - sb.WriteString("\n--------------------------------------------------------------------------------\n") - sb.WriteString(" PASS/FAIL CRITERIA\n") - sb.WriteString("--------------------------------------------------------------------------------\n\n") - - fmt.Fprintf(&sb, "Passed (%d):\n", len(report.PassCriteria)) - for _, p := range report.PassCriteria { - fmt.Fprintf(&sb, " āœ“ %s\n", p) - } - - if len(report.FailedCriteria) > 0 { - fmt.Fprintf(&sb, "\nFailed (%d):\n", len(report.FailedCriteria)) - for _, f := range report.FailedCriteria { - fmt.Fprintf(&sb, " āœ— %s\n", f) - } - } - - sb.WriteString("\n--------------------------------------------------------------------------------\n") - sb.WriteString(" THRESHOLDS USED\n") - sb.WriteString("--------------------------------------------------------------------------------\n\n") - - fmt.Fprintf(&sb, "%-35s %10s %15s %18s\n", - "Metric", "Max Diff%", "Min Abs Diff", "Direction") - fmt.Fprintf(&sb, "%-35s %10s %15s %18s\n", - "------", "---------", "------------", "---------") - - // Sort threshold names - var names []string - for name := range thresholds { - names = append(names, name) - } - sort.Strings(names) - - for _, name := range names { - t := thresholds[name] - var direction string - switch t.metricType { - case LowerIsBetter: - direction = "lower is better" - case HigherIsBetter: - direction = "higher is better" - case ShouldMatch: - direction = "should match" - case Informational: - direction = "info only" - } - minAbsDiff := "-" - if t.minAbsDiff > 0 { - minAbsDiff = fmt.Sprintf("%.1fs", t.minAbsDiff) - } - fmt.Fprintf(&sb, "%-35s %9.1f%% %15s %18s\n", - name, t.maxDiff, minAbsDiff, direction) - } - - sb.WriteString("\n================================================================================\n") - - return sb.String() -} - -// renderScenarioReportJSON renders a scenario report as JSON. -func renderScenarioReportJSON(report *ScenarioReport) string { - data, err := json.MarshalIndent(report, "", " ") - if err != nil { - return fmt.Sprintf(`{"error": "%s"}`, err.Error()) - } - return string(data) -} - -// renderScenarioReportMarkdown renders a scenario report as concise markdown. -func renderScenarioReportMarkdown(report *ScenarioReport) string { - var sb strings.Builder - - // Status emoji - emoji := "āœ…" - if report.OverallStatus != "PASS" { - emoji = "āŒ" - } - - sb.WriteString(fmt.Sprintf("## %s %s: %s\n\n", emoji, report.Scenario, report.OverallStatus)) - - if report.TestDescription != "" { - sb.WriteString(fmt.Sprintf("> %s\n\n", report.TestDescription)) - } - - // Key metrics table - sb.WriteString("| Metric | Value | Expected | Status |\n") - sb.WriteString("|--------|------:|:--------:|:------:|\n") - - // Show only key metrics - keyMetrics := []string{"action_total", "reload_executed_total", "errors_total", "reconcile_total"} - for _, name := range keyMetrics { - for _, c := range report.Comparisons { - if c.Name == name { - value := fmt.Sprintf("%.0f", c.NewValue) - expected := "-" - if c.Expected > 0 { - expected = fmt.Sprintf("%.0f", c.Expected) - } - status := "āœ…" - if c.Status == "fail" { - status = "āŒ" - } else if c.Status == "info" { - status = "ā„¹ļø" - } - sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s |\n", c.DisplayName, value, expected, status)) - break - } - } - } - - return sb.String() -} - -// ============================================================================ -// SUMMARY COMMAND -// ============================================================================ - -// SummaryReport aggregates results from multiple scenarios. -type SummaryReport struct { - Timestamp time.Time `json:"timestamp"` - TestType string `json:"test_type"` - PassCount int `json:"pass_count"` - FailCount int `json:"fail_count"` - TotalCount int `json:"total_count"` - Scenarios []ScenarioSummary `json:"scenarios"` -} - -// ScenarioSummary provides a brief summary of a single scenario. -type ScenarioSummary struct { - ID string `json:"id"` - Status string `json:"status"` - Description string `json:"description"` - ActionTotal float64 `json:"action_total"` - ActionExp float64 `json:"action_expected"` - ErrorsTotal float64 `json:"errors_total"` -} - -func summaryCommand(args []string) { - var resultsDir, outputFile, testType string - format := OutputFormatMarkdown // Default to markdown for CI - resultsDir = "./results" - testType = "full" - - for _, arg := range args { - switch { - case strings.HasPrefix(arg, "--results-dir="): - resultsDir = strings.TrimPrefix(arg, "--results-dir=") - case strings.HasPrefix(arg, "--output="): - outputFile = strings.TrimPrefix(arg, "--output=") - case strings.HasPrefix(arg, "--format="): - format = OutputFormat(strings.TrimPrefix(arg, "--format=")) - case strings.HasPrefix(arg, "--test-type="): - testType = strings.TrimPrefix(arg, "--test-type=") - } - } - - summary, err := generateSummaryReport(resultsDir, testType) - if err != nil { - log.Fatalf("Failed to generate summary: %v", err) - } - - var output string - switch format { - case OutputFormatJSON: - output = renderSummaryJSON(summary) - case OutputFormatText: - output = renderSummaryText(summary) - default: - output = renderSummaryMarkdown(summary) - } - - if outputFile != "" { - if err := os.WriteFile(outputFile, []byte(output), 0644); err != nil { - log.Fatalf("Failed to write output file: %v", err) - } - log.Printf("Summary written to %s", outputFile) - } else { - fmt.Print(output) - } - - // Exit with non-zero status if any tests failed - if summary.FailCount > 0 { - os.Exit(1) - } -} - -func generateSummaryReport(resultsDir, testType string) (*SummaryReport, error) { - summary := &SummaryReport{ - Timestamp: time.Now(), - TestType: testType, - } - - // Find all scenario directories - entries, err := os.ReadDir(resultsDir) - if err != nil { - return nil, fmt.Errorf("failed to read results directory: %w", err) - } - - for _, entry := range entries { - if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "S") { - continue - } - - scenarioID := entry.Name() - report, err := generateScenarioReport(scenarioID, resultsDir) - if err != nil { - log.Printf("Warning: failed to load scenario %s: %v", scenarioID, err) - continue - } - - scenarioSummary := ScenarioSummary{ - ID: scenarioID, - Status: report.OverallStatus, - Description: report.TestDescription, - } - - // Extract key metrics - for _, c := range report.Comparisons { - switch c.Name { - case "action_total": - scenarioSummary.ActionTotal = c.NewValue - scenarioSummary.ActionExp = c.Expected - case "errors_total": - scenarioSummary.ErrorsTotal = c.NewValue - } - } - - summary.Scenarios = append(summary.Scenarios, scenarioSummary) - summary.TotalCount++ - if report.OverallStatus == "PASS" { - summary.PassCount++ - } else { - summary.FailCount++ - } - } - - // Sort scenarios by ID - sort.Slice(summary.Scenarios, func(i, j int) bool { - return naturalSort(summary.Scenarios[i].ID, summary.Scenarios[j].ID) - }) - - return summary, nil -} - -// naturalSort compares two scenario IDs (S1, S2, ..., S10, S11) -func naturalSort(a, b string) bool { - var aNum, bNum int - fmt.Sscanf(a, "S%d", &aNum) - fmt.Sscanf(b, "S%d", &bNum) - return aNum < bNum -} - -func renderSummaryJSON(summary *SummaryReport) string { - data, err := json.MarshalIndent(summary, "", " ") - if err != nil { - return fmt.Sprintf(`{"error": "%s"}`, err.Error()) - } - return string(data) -} - -func renderSummaryText(summary *SummaryReport) string { - var sb strings.Builder - - sb.WriteString("================================================================================\n") - sb.WriteString(" LOAD TEST SUMMARY\n") - sb.WriteString("================================================================================\n\n") - - passRate := 0 - if summary.TotalCount > 0 { - passRate = summary.PassCount * 100 / summary.TotalCount - } - - fmt.Fprintf(&sb, "Test Type: %s\n", summary.TestType) - fmt.Fprintf(&sb, "Results: %d/%d passed (%d%%)\n\n", summary.PassCount, summary.TotalCount, passRate) - - fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8s\n", "ID", "Status", "Description", "Actions", "Errors") - fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8s\n", "------", "--------", strings.Repeat("-", 45), "----------", "--------") - - for _, s := range summary.Scenarios { - desc := s.Description - if len(desc) > 45 { - desc = desc[:42] + "..." - } - actions := fmt.Sprintf("%.0f", s.ActionTotal) - if s.ActionExp > 0 { - actions = fmt.Sprintf("%.0f/%.0f", s.ActionTotal, s.ActionExp) - } - fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8.0f\n", s.ID, s.Status, desc, actions, s.ErrorsTotal) - } - - sb.WriteString("\n================================================================================\n") - return sb.String() -} - -func renderSummaryMarkdown(summary *SummaryReport) string { - var sb strings.Builder - - // Overall status - emoji := "āœ…" - title := "ALL TESTS PASSED" - if summary.FailCount > 0 { - emoji = "āŒ" - title = fmt.Sprintf("%d TEST(S) FAILED", summary.FailCount) - } else if summary.TotalCount == 0 { - emoji = "āš ļø" - title = "NO RESULTS" - } - - sb.WriteString(fmt.Sprintf("## %s Load Test Results: %s\n\n", emoji, title)) - - // Test type note - if summary.TestType == "quick" { - sb.WriteString("> šŸš€ **Quick Test** (S1, S4, S6) ā€” Use `/loadtest` for full suite\n\n") - } - - // Pass rate - passRate := 0 - if summary.TotalCount > 0 { - passRate = summary.PassCount * 100 / summary.TotalCount - } - sb.WriteString(fmt.Sprintf("**%d/%d passed** (%d%%)\n\n", summary.PassCount, summary.TotalCount, passRate)) - - // Results table - sb.WriteString("| | Scenario | Description | Actions | Errors |\n") - sb.WriteString("|:-:|:--------:|-------------|:-------:|:------:|\n") - - for _, s := range summary.Scenarios { - icon := "āœ…" - if s.Status != "PASS" { - icon = "āŒ" - } - - // Truncate description - desc := s.Description - if len(desc) > 45 { - desc = desc[:42] + "..." - } - - // Format actions - actions := fmt.Sprintf("%.0f", s.ActionTotal) - if s.ActionExp > 0 { - actions = fmt.Sprintf("%.0f/%.0f", s.ActionTotal, s.ActionExp) - } - - // Format errors - errors := fmt.Sprintf("%.0f", s.ErrorsTotal) - if s.ErrorsTotal > 0 { - errors = fmt.Sprintf("āš ļø %.0f", s.ErrorsTotal) - } - - sb.WriteString(fmt.Sprintf("| %s | **%s** | %s | %s | %s |\n", icon, s.ID, desc, actions, errors)) - } - - sb.WriteString("\nšŸ“¦ **[Download detailed results](../artifacts)**\n") - - return sb.String() + cmd.Execute() } diff --git a/test/loadtest/go.mod b/test/loadtest/go.mod index ed52882..e96ed76 100644 --- a/test/loadtest/go.mod +++ b/test/loadtest/go.mod @@ -1,8 +1,9 @@ module github.com/stakater/Reloader/test/loadtest -go 1.22.0 +go 1.25 require ( + github.com/spf13/cobra v1.8.1 k8s.io/api v0.31.0 k8s.io/apimachinery v0.31.0 k8s.io/client-go v0.31.0 @@ -23,6 +24,7 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/imdario/mergo v0.3.6 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/test/loadtest/go.sum b/test/loadtest/go.sum index a8edbda..f4f0ad8 100644 --- a/test/loadtest/go.sum +++ b/test/loadtest/go.sum @@ -1,3 +1,4 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -36,6 +37,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -67,6 +70,9 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/test/loadtest/internal/cmd/report.go b/test/loadtest/internal/cmd/report.go new file mode 100644 index 0000000..aa273a5 --- /dev/null +++ b/test/loadtest/internal/cmd/report.go @@ -0,0 +1,856 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "log" + "math" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" +) + +var ( + reportScenario string + reportResultsDir string + reportOutputFile string + reportFormat string +) + +var reportCmd = &cobra.Command{ + Use: "report", + Short: "Generate comparison report for a scenario", + Long: `Generate a detailed report for a specific test scenario. + +Examples: + # Generate report for a scenario + loadtest report --scenario=S2 --results-dir=./results + + # Generate JSON report + loadtest report --scenario=S2 --format=json`, + Run: func(cmd *cobra.Command, args []string) { + reportCommand() + }, +} + +func init() { + reportCmd.Flags().StringVar(&reportScenario, "scenario", "", "Scenario to report on (required)") + reportCmd.Flags().StringVar(&reportResultsDir, "results-dir", "./results", "Directory containing results") + reportCmd.Flags().StringVar(&reportOutputFile, "output", "", "Output file (default: stdout)") + reportCmd.Flags().StringVar(&reportFormat, "format", "text", "Output format: text, json, markdown") + reportCmd.MarkFlagRequired("scenario") +} + +// PrometheusResponse represents a Prometheus API response for report parsing. +type PrometheusResponse struct { + Status string `json:"status"` + Data struct { + ResultType string `json:"resultType"` + Result []struct { + Metric map[string]string `json:"metric"` + Value []interface{} `json:"value"` + } `json:"result"` + } `json:"data"` +} + +// MetricComparison represents the comparison of a single metric. +type MetricComparison struct { + Name string `json:"name"` + DisplayName string `json:"display_name"` + Unit string `json:"unit"` + IsCounter bool `json:"is_counter"` + OldValue float64 `json:"old_value"` + NewValue float64 `json:"new_value"` + Expected float64 `json:"expected"` + Difference float64 `json:"difference"` + DiffPct float64 `json:"diff_pct"` + Status string `json:"status"` + Threshold float64 `json:"threshold"` + OldMeetsExpected string `json:"old_meets_expected"` + NewMeetsExpected string `json:"new_meets_expected"` +} + +type metricInfo struct { + unit string + isCounter bool +} + +var metricInfoMap = map[string]metricInfo{ + "reconcile_total": {unit: "count", isCounter: true}, + "reconcile_duration_p50": {unit: "s", isCounter: false}, + "reconcile_duration_p95": {unit: "s", isCounter: false}, + "reconcile_duration_p99": {unit: "s", isCounter: false}, + "action_total": {unit: "count", isCounter: true}, + "action_latency_p50": {unit: "s", isCounter: false}, + "action_latency_p95": {unit: "s", isCounter: false}, + "action_latency_p99": {unit: "s", isCounter: false}, + "errors_total": {unit: "count", isCounter: true}, + "reload_executed_total": {unit: "count", isCounter: true}, + "workloads_scanned_total": {unit: "count", isCounter: true}, + "workloads_matched_total": {unit: "count", isCounter: true}, + "skipped_total_no_data_change": {unit: "count", isCounter: true}, + "rest_client_requests_total": {unit: "count", isCounter: true}, + "rest_client_requests_get": {unit: "count", isCounter: true}, + "rest_client_requests_patch": {unit: "count", isCounter: true}, + "rest_client_requests_put": {unit: "count", isCounter: true}, + "rest_client_requests_errors": {unit: "count", isCounter: true}, + "memory_rss_mb_avg": {unit: "MB", isCounter: false}, + "memory_rss_mb_max": {unit: "MB", isCounter: false}, + "memory_heap_mb_avg": {unit: "MB", isCounter: false}, + "memory_heap_mb_max": {unit: "MB", isCounter: false}, + "cpu_cores_avg": {unit: "cores", isCounter: false}, + "cpu_cores_max": {unit: "cores", isCounter: false}, + "goroutines_avg": {unit: "count", isCounter: false}, + "goroutines_max": {unit: "count", isCounter: false}, + "gc_pause_p99_ms": {unit: "ms", isCounter: false}, +} + +// ReportExpectedMetrics matches the expected metrics from test scenarios. +type ReportExpectedMetrics struct { + ActionTotal int `json:"action_total"` + ReloadExecutedTotal int `json:"reload_executed_total"` + ReconcileTotal int `json:"reconcile_total"` + WorkloadsScannedTotal int `json:"workloads_scanned_total"` + WorkloadsMatchedTotal int `json:"workloads_matched_total"` + SkippedTotal int `json:"skipped_total"` + Description string `json:"description"` +} + +// ScenarioReport represents the full report for a scenario. +type ScenarioReport struct { + Scenario string `json:"scenario"` + Timestamp time.Time `json:"timestamp"` + Comparisons []MetricComparison `json:"comparisons"` + OverallStatus string `json:"overall_status"` + Summary string `json:"summary"` + PassCriteria []string `json:"pass_criteria"` + FailedCriteria []string `json:"failed_criteria"` + Expected ReportExpectedMetrics `json:"expected"` + TestDescription string `json:"test_description"` +} + +// MetricType defines how to evaluate a metric. +type MetricType int + +const ( + LowerIsBetter MetricType = iota + ShouldMatch + HigherIsBetter + Informational +) + +type thresholdConfig struct { + maxDiff float64 + metricType MetricType + minAbsDiff float64 +} + +var thresholds = map[string]thresholdConfig{ + "reconcile_total": {maxDiff: 60.0, metricType: LowerIsBetter}, + "reconcile_duration_p50": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.5}, + "reconcile_duration_p95": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, + "reconcile_duration_p99": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, + "action_latency_p50": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.5}, + "action_latency_p95": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, + "action_latency_p99": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 1.0}, + "errors_total": {maxDiff: 0.0, metricType: LowerIsBetter}, + "action_total": {maxDiff: 15.0, metricType: ShouldMatch}, + "reload_executed_total": {maxDiff: 15.0, metricType: ShouldMatch}, + "workloads_scanned_total": {maxDiff: 15.0, metricType: ShouldMatch}, + "workloads_matched_total": {maxDiff: 15.0, metricType: ShouldMatch}, + "skipped_total_no_data_change": {maxDiff: 20.0, metricType: ShouldMatch}, + "rest_client_requests_total": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, + "rest_client_requests_get": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, + "rest_client_requests_patch": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 50}, + "rest_client_requests_put": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 20}, + "rest_client_requests_errors": {maxDiff: 0.0, metricType: LowerIsBetter, minAbsDiff: 100}, + "memory_rss_mb_avg": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 20}, + "memory_rss_mb_max": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 30}, + "memory_heap_mb_avg": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 15}, + "memory_heap_mb_max": {maxDiff: 50.0, metricType: LowerIsBetter, minAbsDiff: 20}, + "cpu_cores_avg": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.1}, + "cpu_cores_max": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 0.2}, + "goroutines_avg": {metricType: Informational}, + "goroutines_max": {metricType: Informational}, + "gc_pause_p99_ms": {maxDiff: 100.0, metricType: LowerIsBetter, minAbsDiff: 5}, +} + +func reportCommand() { + if reportScenario == "" { + log.Fatal("--scenario is required for report command") + } + + report, err := generateScenarioReport(reportScenario, reportResultsDir) + if err != nil { + log.Fatalf("Failed to generate report: %v", err) + } + + var output string + switch OutputFormat(reportFormat) { + case OutputFormatJSON: + output = renderScenarioReportJSON(report) + case OutputFormatMarkdown: + output = renderScenarioReportMarkdown(report) + default: + output = renderScenarioReport(report) + } + + if reportOutputFile != "" { + if err := os.WriteFile(reportOutputFile, []byte(output), 0644); err != nil { + log.Fatalf("Failed to write output file: %v", err) + } + log.Printf("Report written to %s", reportOutputFile) + } else { + fmt.Println(output) + } +} + +func generateScenarioReport(scenario, resultsDir string) (*ScenarioReport, error) { + oldDir := filepath.Join(resultsDir, scenario, "old") + newDir := filepath.Join(resultsDir, scenario, "new") + scenarioDir := filepath.Join(resultsDir, scenario) + + _, oldErr := os.Stat(oldDir) + _, newErr := os.Stat(newDir) + hasOld := oldErr == nil + hasNew := newErr == nil + isComparison := hasOld && hasNew + + singleVersion := "" + singleDir := "" + if !isComparison { + if hasNew { + singleVersion = "new" + singleDir = newDir + } else if hasOld { + singleVersion = "old" + singleDir = oldDir + } else { + return nil, fmt.Errorf("no results found in %s", scenarioDir) + } + } + + report := &ScenarioReport{ + Scenario: scenario, + Timestamp: time.Now(), + } + + expectedPath := filepath.Join(scenarioDir, "expected.json") + if data, err := os.ReadFile(expectedPath); err == nil { + if err := json.Unmarshal(data, &report.Expected); err != nil { + log.Printf("Warning: Could not parse expected metrics: %v", err) + } else { + report.TestDescription = report.Expected.Description + } + } + + if !isComparison { + return generateSingleVersionReport(report, singleDir, singleVersion, scenario) + } + + metricsToCompare := []struct { + name string + file string + selector func(data PrometheusResponse) float64 + }{ + {"reconcile_total", "reloader_reconcile_total.json", sumAllValues}, + {"reconcile_duration_p50", "reconcile_p50.json", getFirstValue}, + {"reconcile_duration_p95", "reconcile_p95.json", getFirstValue}, + {"reconcile_duration_p99", "reconcile_p99.json", getFirstValue}, + {"action_total", "reloader_action_total.json", sumAllValues}, + {"action_latency_p50", "action_p50.json", getFirstValue}, + {"action_latency_p95", "action_p95.json", getFirstValue}, + {"action_latency_p99", "action_p99.json", getFirstValue}, + {"errors_total", "reloader_errors_total.json", sumAllValues}, + {"reload_executed_total", "reloader_reload_executed_total.json", sumSuccessValues}, + {"workloads_scanned_total", "reloader_workloads_scanned_total.json", sumAllValues}, + {"workloads_matched_total", "reloader_workloads_matched_total.json", sumAllValues}, + {"rest_client_requests_total", "rest_client_requests_total.json", getFirstValue}, + {"rest_client_requests_get", "rest_client_requests_get.json", getFirstValue}, + {"rest_client_requests_patch", "rest_client_requests_patch.json", getFirstValue}, + {"rest_client_requests_put", "rest_client_requests_put.json", getFirstValue}, + {"rest_client_requests_errors", "rest_client_requests_errors.json", getFirstValue}, + {"memory_rss_mb_avg", "memory_rss_bytes_avg.json", bytesToMB}, + {"memory_rss_mb_max", "memory_rss_bytes_max.json", bytesToMB}, + {"memory_heap_mb_avg", "memory_heap_bytes_avg.json", bytesToMB}, + {"memory_heap_mb_max", "memory_heap_bytes_max.json", bytesToMB}, + {"cpu_cores_avg", "cpu_usage_cores_avg.json", getFirstValue}, + {"cpu_cores_max", "cpu_usage_cores_max.json", getFirstValue}, + {"goroutines_avg", "goroutines_avg.json", getFirstValue}, + {"goroutines_max", "goroutines_max.json", getFirstValue}, + {"gc_pause_p99_ms", "gc_duration_seconds_p99.json", secondsToMs}, + } + + expectedValues := map[string]float64{ + "action_total": float64(report.Expected.ActionTotal), + "reload_executed_total": float64(report.Expected.ReloadExecutedTotal), + "reconcile_total": float64(report.Expected.ReconcileTotal), + "workloads_scanned_total": float64(report.Expected.WorkloadsScannedTotal), + "workloads_matched_total": float64(report.Expected.WorkloadsMatchedTotal), + "skipped_total": float64(report.Expected.SkippedTotal), + } + + metricValues := make(map[string]struct{ old, new, expected float64 }) + + for _, m := range metricsToCompare { + oldData, err := loadMetricFile(filepath.Join(oldDir, m.file)) + if err != nil { + log.Printf("Warning: Could not load old metric %s: %v", m.name, err) + continue + } + + newData, err := loadMetricFile(filepath.Join(newDir, m.file)) + if err != nil { + log.Printf("Warning: Could not load new metric %s: %v", m.name, err) + continue + } + + oldValue := m.selector(oldData) + newValue := m.selector(newData) + expected := expectedValues[m.name] + + metricValues[m.name] = struct{ old, new, expected float64 }{oldValue, newValue, expected} + } + + newMeetsActionExpected := false + newReconcileIsZero := false + isChurnScenario := scenario == "S5" + if v, ok := metricValues["action_total"]; ok && v.expected > 0 { + tolerance := v.expected * 0.15 + newMeetsActionExpected = math.Abs(v.new-v.expected) <= tolerance + } + if v, ok := metricValues["reconcile_total"]; ok { + newReconcileIsZero = v.new == 0 + } + + for _, m := range metricsToCompare { + v, ok := metricValues[m.name] + if !ok { + continue + } + + comparison := compareMetricWithExpected(m.name, v.old, v.new, v.expected) + + if strings.HasPrefix(m.name, "rest_client_requests") { + if newMeetsActionExpected && comparison.Status != "pass" { + if oldMeets, ok := metricValues["action_total"]; ok { + oldTolerance := oldMeets.expected * 0.15 + oldMissed := math.Abs(oldMeets.old-oldMeets.expected) > oldTolerance + if oldMissed { + comparison.Status = "pass" + } + } + } + if newReconcileIsZero && comparison.Status != "pass" { + comparison.Status = "pass" + } + } + + if isChurnScenario { + if m.name == "errors_total" { + if v.new < 50 && v.old < 50 { + comparison.Status = "pass" + } else if v.new <= v.old*1.5 { + comparison.Status = "pass" + } + } + if m.name == "action_total" || m.name == "reload_executed_total" { + if v.old > 0 { + diff := math.Abs(v.new-v.old) / v.old * 100 + if diff <= 20 { + comparison.Status = "pass" + } + } else if v.new > 0 { + comparison.Status = "pass" + } + } + } + + report.Comparisons = append(report.Comparisons, comparison) + + if comparison.Status == "pass" { + report.PassCriteria = append(report.PassCriteria, m.name) + } else if comparison.Status == "fail" { + report.FailedCriteria = append(report.FailedCriteria, m.name) + } + } + + if len(report.FailedCriteria) == 0 { + report.OverallStatus = "PASS" + report.Summary = "All metrics within acceptable thresholds" + } else { + report.OverallStatus = "FAIL" + report.Summary = fmt.Sprintf("%d metrics failed: %s", + len(report.FailedCriteria), + strings.Join(report.FailedCriteria, ", ")) + } + + return report, nil +} + +func generateSingleVersionReport(report *ScenarioReport, dataDir, version, scenario string) (*ScenarioReport, error) { + metricsToCollect := []struct { + name string + file string + selector func(data PrometheusResponse) float64 + }{ + {"reconcile_total", "reloader_reconcile_total.json", sumAllValues}, + {"reconcile_duration_p50", "reconcile_p50.json", getFirstValue}, + {"reconcile_duration_p95", "reconcile_p95.json", getFirstValue}, + {"reconcile_duration_p99", "reconcile_p99.json", getFirstValue}, + {"action_total", "reloader_action_total.json", sumAllValues}, + {"action_latency_p50", "action_p50.json", getFirstValue}, + {"action_latency_p95", "action_p95.json", getFirstValue}, + {"action_latency_p99", "action_p99.json", getFirstValue}, + {"errors_total", "reloader_errors_total.json", sumAllValues}, + {"reload_executed_total", "reloader_reload_executed_total.json", sumSuccessValues}, + {"workloads_scanned_total", "reloader_workloads_scanned_total.json", sumAllValues}, + {"workloads_matched_total", "reloader_workloads_matched_total.json", sumAllValues}, + {"rest_client_requests_total", "rest_client_requests_total.json", getFirstValue}, + {"rest_client_requests_get", "rest_client_requests_get.json", getFirstValue}, + {"rest_client_requests_patch", "rest_client_requests_patch.json", getFirstValue}, + {"rest_client_requests_put", "rest_client_requests_put.json", getFirstValue}, + {"rest_client_requests_errors", "rest_client_requests_errors.json", getFirstValue}, + {"memory_rss_mb_avg", "memory_rss_bytes_avg.json", bytesToMB}, + {"memory_rss_mb_max", "memory_rss_bytes_max.json", bytesToMB}, + {"memory_heap_mb_avg", "memory_heap_bytes_avg.json", bytesToMB}, + {"memory_heap_mb_max", "memory_heap_bytes_max.json", bytesToMB}, + {"cpu_cores_avg", "cpu_usage_cores_avg.json", getFirstValue}, + {"cpu_cores_max", "cpu_usage_cores_max.json", getFirstValue}, + {"goroutines_avg", "goroutines_avg.json", getFirstValue}, + {"goroutines_max", "goroutines_max.json", getFirstValue}, + {"gc_pause_p99_ms", "gc_duration_seconds_p99.json", secondsToMs}, + } + + expectedValues := map[string]float64{ + "action_total": float64(report.Expected.ActionTotal), + "reload_executed_total": float64(report.Expected.ReloadExecutedTotal), + "reconcile_total": float64(report.Expected.ReconcileTotal), + "workloads_scanned_total": float64(report.Expected.WorkloadsScannedTotal), + "workloads_matched_total": float64(report.Expected.WorkloadsMatchedTotal), + "skipped_total": float64(report.Expected.SkippedTotal), + } + + for _, m := range metricsToCollect { + data, err := loadMetricFile(filepath.Join(dataDir, m.file)) + if err != nil { + log.Printf("Warning: Could not load metric %s: %v", m.name, err) + continue + } + + value := m.selector(data) + expected := expectedValues[m.name] + + info := metricInfoMap[m.name] + if info.unit == "" { + info = metricInfo{unit: "count", isCounter: true} + } + + displayName := m.name + if info.unit != "count" { + displayName = fmt.Sprintf("%s (%s)", m.name, info.unit) + } + + status := "info" + meetsExp := "-" + + if expected > 0 { + meetsExp = meetsExpected(value, expected) + threshold, ok := thresholds[m.name] + if ok && threshold.metricType == ShouldMatch { + if meetsExp == "āœ“" { + status = "pass" + report.PassCriteria = append(report.PassCriteria, m.name) + } else { + status = "fail" + report.FailedCriteria = append(report.FailedCriteria, m.name) + } + } + } + + if info.isCounter { + value = math.Round(value) + } + + report.Comparisons = append(report.Comparisons, MetricComparison{ + Name: m.name, + DisplayName: displayName, + Unit: info.unit, + IsCounter: info.isCounter, + OldValue: 0, + NewValue: value, + Expected: expected, + OldMeetsExpected: "-", + NewMeetsExpected: meetsExp, + Status: status, + }) + } + + if len(report.FailedCriteria) == 0 { + report.OverallStatus = "PASS" + report.Summary = fmt.Sprintf("Single-version test (%s) completed successfully", version) + } else { + report.OverallStatus = "FAIL" + report.Summary = fmt.Sprintf("%d metrics failed: %s", + len(report.FailedCriteria), + strings.Join(report.FailedCriteria, ", ")) + } + + return report, nil +} + +func loadMetricFile(path string) (PrometheusResponse, error) { + var resp PrometheusResponse + data, err := os.ReadFile(path) + if err != nil { + return resp, err + } + err = json.Unmarshal(data, &resp) + return resp, err +} + +func sumAllValues(data PrometheusResponse) float64 { + var sum float64 + for _, result := range data.Data.Result { + if len(result.Value) >= 2 { + if v, ok := result.Value[1].(string); ok { + var f float64 + fmt.Sscanf(v, "%f", &f) + sum += f + } + } + } + return sum +} + +func sumSuccessValues(data PrometheusResponse) float64 { + var sum float64 + for _, result := range data.Data.Result { + if result.Metric["success"] == "true" { + if len(result.Value) >= 2 { + if v, ok := result.Value[1].(string); ok { + var f float64 + fmt.Sscanf(v, "%f", &f) + sum += f + } + } + } + } + return sum +} + +func getFirstValue(data PrometheusResponse) float64 { + if len(data.Data.Result) > 0 && len(data.Data.Result[0].Value) >= 2 { + if v, ok := data.Data.Result[0].Value[1].(string); ok { + var f float64 + fmt.Sscanf(v, "%f", &f) + return f + } + } + return 0 +} + +func bytesToMB(data PrometheusResponse) float64 { + bytes := getFirstValue(data) + return bytes / (1024 * 1024) +} + +func secondsToMs(data PrometheusResponse) float64 { + seconds := getFirstValue(data) + return seconds * 1000 +} + +func meetsExpected(value, expected float64) string { + if expected == 0 { + return "-" + } + tolerance := expected * 0.15 + if math.Abs(value-expected) <= tolerance { + return "āœ“" + } + return "āœ—" +} + +func compareMetricWithExpected(name string, oldValue, newValue, expected float64) MetricComparison { + diff := newValue - oldValue + absDiff := math.Abs(diff) + var diffPct float64 + if oldValue != 0 { + diffPct = (diff / oldValue) * 100 + } else if newValue != 0 { + diffPct = 100 + } + + threshold, ok := thresholds[name] + if !ok { + threshold = thresholdConfig{maxDiff: 10.0, metricType: ShouldMatch} + } + + info := metricInfoMap[name] + if info.unit == "" { + info = metricInfo{unit: "count", isCounter: true} + } + displayName := name + if info.unit != "count" { + displayName = fmt.Sprintf("%s (%s)", name, info.unit) + } + + if info.isCounter { + oldValue = math.Round(oldValue) + newValue = math.Round(newValue) + } + + status := "pass" + oldMeetsExp := meetsExpected(oldValue, expected) + newMeetsExp := meetsExpected(newValue, expected) + + if expected > 0 && threshold.metricType == ShouldMatch { + if newMeetsExp == "āœ—" { + status = "fail" + } + } else { + switch threshold.metricType { + case LowerIsBetter: + if threshold.minAbsDiff > 0 && absDiff < threshold.minAbsDiff { + status = "pass" + } else if diffPct > threshold.maxDiff { + status = "fail" + } + case HigherIsBetter: + if diffPct < -threshold.maxDiff { + status = "fail" + } + case ShouldMatch: + if math.Abs(diffPct) > threshold.maxDiff { + status = "fail" + } + case Informational: + status = "info" + } + } + + return MetricComparison{ + Name: name, + DisplayName: displayName, + Unit: info.unit, + IsCounter: info.isCounter, + Expected: expected, + OldMeetsExpected: oldMeetsExp, + NewMeetsExpected: newMeetsExp, + OldValue: oldValue, + NewValue: newValue, + Difference: diff, + DiffPct: diffPct, + Status: status, + Threshold: threshold.maxDiff, + } +} + +func renderScenarioReport(report *ScenarioReport) string { + var sb strings.Builder + + isSingleVersion := true + for _, c := range report.Comparisons { + if c.OldValue != 0 { + isSingleVersion = false + break + } + } + + sb.WriteString("\n") + sb.WriteString("================================================================================\n") + if isSingleVersion { + sb.WriteString(" RELOADER TEST REPORT\n") + } else { + sb.WriteString(" RELOADER A/B COMPARISON REPORT\n") + } + sb.WriteString("================================================================================\n\n") + + fmt.Fprintf(&sb, "Scenario: %s\n", report.Scenario) + fmt.Fprintf(&sb, "Generated: %s\n", report.Timestamp.Format("2006-01-02 15:04:05")) + fmt.Fprintf(&sb, "Status: %s\n", report.OverallStatus) + fmt.Fprintf(&sb, "Summary: %s\n", report.Summary) + + if report.TestDescription != "" { + fmt.Fprintf(&sb, "Test: %s\n", report.TestDescription) + } + + if report.Expected.ActionTotal > 0 { + sb.WriteString("\n--------------------------------------------------------------------------------\n") + sb.WriteString(" EXPECTED VALUES\n") + sb.WriteString("--------------------------------------------------------------------------------\n") + fmt.Fprintf(&sb, "Expected Action Total: %d\n", report.Expected.ActionTotal) + fmt.Fprintf(&sb, "Expected Reload Executed Total: %d\n", report.Expected.ReloadExecutedTotal) + if report.Expected.SkippedTotal > 0 { + fmt.Fprintf(&sb, "Expected Skipped Total: %d\n", report.Expected.SkippedTotal) + } + } + + sb.WriteString("\n--------------------------------------------------------------------------------\n") + if isSingleVersion { + sb.WriteString(" METRICS\n") + } else { + sb.WriteString(" METRIC COMPARISONS\n") + } + sb.WriteString("--------------------------------------------------------------------------------\n") + + if isSingleVersion { + sb.WriteString("(āœ“ = meets expected value within 15%)\n\n") + fmt.Fprintf(&sb, "%-32s %12s %10s %5s %8s\n", + "Metric", "Value", "Expected", "Met?", "Status") + fmt.Fprintf(&sb, "%-32s %12s %10s %5s %8s\n", + "------", "-----", "--------", "----", "------") + + for _, c := range report.Comparisons { + if c.IsCounter { + if c.Expected > 0 { + fmt.Fprintf(&sb, "%-32s %12.0f %10.0f %5s %8s\n", + c.DisplayName, c.NewValue, c.Expected, + c.NewMeetsExpected, c.Status) + } else { + fmt.Fprintf(&sb, "%-32s %12.0f %10s %5s %8s\n", + c.DisplayName, c.NewValue, "-", + c.NewMeetsExpected, c.Status) + } + } else { + fmt.Fprintf(&sb, "%-32s %12.4f %10s %5s %8s\n", + c.DisplayName, c.NewValue, "-", + c.NewMeetsExpected, c.Status) + } + } + } else { + sb.WriteString("(Oldāœ“/Newāœ“ = meets expected value within 15%)\n\n") + + fmt.Fprintf(&sb, "%-32s %12s %12s %10s %5s %5s %8s\n", + "Metric", "Old", "New", "Expected", "Oldāœ“", "Newāœ“", "Status") + fmt.Fprintf(&sb, "%-32s %12s %12s %10s %5s %5s %8s\n", + "------", "---", "---", "--------", "----", "----", "------") + + for _, c := range report.Comparisons { + if c.IsCounter { + if c.Expected > 0 { + fmt.Fprintf(&sb, "%-32s %12.0f %12.0f %10.0f %5s %5s %8s\n", + c.DisplayName, c.OldValue, c.NewValue, c.Expected, + c.OldMeetsExpected, c.NewMeetsExpected, c.Status) + } else { + fmt.Fprintf(&sb, "%-32s %12.0f %12.0f %10s %5s %5s %8s\n", + c.DisplayName, c.OldValue, c.NewValue, "-", + c.OldMeetsExpected, c.NewMeetsExpected, c.Status) + } + } else { + fmt.Fprintf(&sb, "%-32s %12.4f %12.4f %10s %5s %5s %8s\n", + c.DisplayName, c.OldValue, c.NewValue, "-", + c.OldMeetsExpected, c.NewMeetsExpected, c.Status) + } + } + } + + sb.WriteString("\n--------------------------------------------------------------------------------\n") + sb.WriteString(" PASS/FAIL CRITERIA\n") + sb.WriteString("--------------------------------------------------------------------------------\n\n") + + fmt.Fprintf(&sb, "Passed (%d):\n", len(report.PassCriteria)) + for _, p := range report.PassCriteria { + fmt.Fprintf(&sb, " āœ“ %s\n", p) + } + + if len(report.FailedCriteria) > 0 { + fmt.Fprintf(&sb, "\nFailed (%d):\n", len(report.FailedCriteria)) + for _, f := range report.FailedCriteria { + fmt.Fprintf(&sb, " āœ— %s\n", f) + } + } + + sb.WriteString("\n--------------------------------------------------------------------------------\n") + sb.WriteString(" THRESHOLDS USED\n") + sb.WriteString("--------------------------------------------------------------------------------\n\n") + + fmt.Fprintf(&sb, "%-35s %10s %15s %18s\n", + "Metric", "Max Diff%", "Min Abs Diff", "Direction") + fmt.Fprintf(&sb, "%-35s %10s %15s %18s\n", + "------", "---------", "------------", "---------") + + var names []string + for name := range thresholds { + names = append(names, name) + } + sort.Strings(names) + + for _, name := range names { + t := thresholds[name] + var direction string + switch t.metricType { + case LowerIsBetter: + direction = "lower is better" + case HigherIsBetter: + direction = "higher is better" + case ShouldMatch: + direction = "should match" + case Informational: + direction = "info only" + } + minAbsDiff := "-" + if t.minAbsDiff > 0 { + minAbsDiff = fmt.Sprintf("%.1f", t.minAbsDiff) + } + fmt.Fprintf(&sb, "%-35s %9.1f%% %15s %18s\n", + name, t.maxDiff, minAbsDiff, direction) + } + + sb.WriteString("\n================================================================================\n") + + return sb.String() +} + +func renderScenarioReportJSON(report *ScenarioReport) string { + data, err := json.MarshalIndent(report, "", " ") + if err != nil { + return fmt.Sprintf(`{"error": "%s"}`, err.Error()) + } + return string(data) +} + +func renderScenarioReportMarkdown(report *ScenarioReport) string { + var sb strings.Builder + + emoji := "āœ…" + if report.OverallStatus != "PASS" { + emoji = "āŒ" + } + + sb.WriteString(fmt.Sprintf("## %s %s: %s\n\n", emoji, report.Scenario, report.OverallStatus)) + + if report.TestDescription != "" { + sb.WriteString(fmt.Sprintf("> %s\n\n", report.TestDescription)) + } + + sb.WriteString("| Metric | Value | Expected | Status |\n") + sb.WriteString("|--------|------:|:--------:|:------:|\n") + + keyMetrics := []string{"action_total", "reload_executed_total", "errors_total", "reconcile_total"} + for _, name := range keyMetrics { + for _, c := range report.Comparisons { + if c.Name == name { + value := fmt.Sprintf("%.0f", c.NewValue) + expected := "-" + if c.Expected > 0 { + expected = fmt.Sprintf("%.0f", c.Expected) + } + status := "āœ…" + if c.Status == "fail" { + status = "āŒ" + } else if c.Status == "info" { + status = "ā„¹ļø" + } + sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s |\n", c.DisplayName, value, expected, status)) + break + } + } + } + + return sb.String() +} diff --git a/test/loadtest/internal/cmd/root.go b/test/loadtest/internal/cmd/root.go new file mode 100644 index 0000000..406d576 --- /dev/null +++ b/test/loadtest/internal/cmd/root.go @@ -0,0 +1,44 @@ +// Package cmd implements the CLI commands for the load test tool. +package cmd + +import ( + "os" + + "github.com/spf13/cobra" +) + +const ( + // DefaultClusterName is the default kind cluster name. + DefaultClusterName = "reloader-loadtest" + // TestNamespace is the namespace used for test resources. + TestNamespace = "reloader-test" +) + +// OutputFormat defines the output format for reports. +type OutputFormat string + +const ( + OutputFormatText OutputFormat = "text" + OutputFormatJSON OutputFormat = "json" + OutputFormatMarkdown OutputFormat = "markdown" +) + +// rootCmd is the base command. +var rootCmd = &cobra.Command{ + Use: "loadtest", + Short: "Reloader Load Test CLI", + Long: `A CLI tool for running A/B comparison load tests on Reloader.`, +} + +func init() { + rootCmd.AddCommand(runCmd) + rootCmd.AddCommand(reportCmd) + rootCmd.AddCommand(summaryCmd) +} + +// Execute runs the root command. +func Execute() { + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/test/loadtest/internal/cmd/run.go b/test/loadtest/internal/cmd/run.go new file mode 100644 index 0000000..8c88b54 --- /dev/null +++ b/test/loadtest/internal/cmd/run.go @@ -0,0 +1,648 @@ +package cmd + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/spf13/cobra" + "github.com/stakater/Reloader/test/loadtest/internal/cluster" + "github.com/stakater/Reloader/test/loadtest/internal/prometheus" + "github.com/stakater/Reloader/test/loadtest/internal/reloader" + "github.com/stakater/Reloader/test/loadtest/internal/scenarios" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +// RunConfig holds CLI configuration for the run command. +type RunConfig struct { + OldImage string + NewImage string + Scenario string + Duration int + SkipCluster bool + ClusterName string + ResultsDir string + ManifestsDir string + Parallelism int +} + +// workerContext holds all resources for a single worker (cluster + prometheus). +type workerContext struct { + id int + clusterMgr *cluster.Manager + promMgr *prometheus.Manager + kubeClient kubernetes.Interface + kubeContext string + runtime string +} + +var runCfg RunConfig + +var runCmd = &cobra.Command{ + Use: "run", + Short: "Run A/B comparison tests", + Long: `Run load tests comparing old and new versions of Reloader. + +Examples: + # Compare two images + loadtest run --old-image=stakater/reloader:v1.0.0 --new-image=stakater/reloader:v1.1.0 + + # Run specific scenario + loadtest run --old-image=stakater/reloader:v1.0.0 --new-image=localhost/reloader:dev --scenario=S2 + + # Test single image (no comparison) + loadtest run --new-image=localhost/reloader:test + + # Run all scenarios in parallel on 4 clusters + loadtest run --new-image=localhost/reloader:test --parallelism=4`, + Run: func(cmd *cobra.Command, args []string) { + runCommand() + }, +} + +func init() { + runCmd.Flags().StringVar(&runCfg.OldImage, "old-image", "", "Container image for \"old\" version (required for comparison)") + runCmd.Flags().StringVar(&runCfg.NewImage, "new-image", "", "Container image for \"new\" version (required for comparison)") + runCmd.Flags().StringVar(&runCfg.Scenario, "scenario", "all", "Test scenario: S1-S13 or \"all\"") + runCmd.Flags().IntVar(&runCfg.Duration, "duration", 60, "Test duration in seconds") + runCmd.Flags().IntVar(&runCfg.Parallelism, "parallelism", 1, "Run N scenarios in parallel on N clusters") + runCmd.Flags().BoolVar(&runCfg.SkipCluster, "skip-cluster", false, "Skip kind cluster creation (use existing)") + runCmd.Flags().StringVar(&runCfg.ClusterName, "cluster-name", DefaultClusterName, "Kind cluster name") + runCmd.Flags().StringVar(&runCfg.ResultsDir, "results-dir", "./results", "Directory for results") + runCmd.Flags().StringVar(&runCfg.ManifestsDir, "manifests-dir", "", "Directory containing manifests (auto-detected if not set)") +} + +func runCommand() { + if runCfg.ManifestsDir == "" { + execPath, _ := os.Executable() + execDir := filepath.Dir(execPath) + runCfg.ManifestsDir = filepath.Join(execDir, "..", "..", "manifests") + if _, err := os.Stat(runCfg.ManifestsDir); os.IsNotExist(err) { + runCfg.ManifestsDir = "./manifests" + } + } + + if runCfg.Parallelism < 1 { + runCfg.Parallelism = 1 + } + + if runCfg.OldImage == "" && runCfg.NewImage == "" { + log.Fatal("At least one of --old-image or --new-image is required") + } + + runOld := runCfg.OldImage != "" + runNew := runCfg.NewImage != "" + runBoth := runOld && runNew + + log.Printf("Configuration:") + log.Printf(" Scenario: %s", runCfg.Scenario) + log.Printf(" Duration: %ds", runCfg.Duration) + log.Printf(" Parallelism: %d", runCfg.Parallelism) + if runCfg.OldImage != "" { + log.Printf(" Old image: %s", runCfg.OldImage) + } + if runCfg.NewImage != "" { + log.Printf(" New image: %s", runCfg.NewImage) + } + + runtime, err := cluster.DetectContainerRuntime() + if err != nil { + log.Fatalf("Failed to detect container runtime: %v", err) + } + log.Printf(" Container runtime: %s", runtime) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigCh + log.Println("Received shutdown signal...") + cancel() + }() + + scenariosToRun := []string{runCfg.Scenario} + if runCfg.Scenario == "all" { + scenariosToRun = []string{"S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8", "S9", "S10", "S11", "S12", "S13"} + } + + if runCfg.SkipCluster && runCfg.Parallelism > 1 { + log.Fatal("--skip-cluster is not supported with --parallelism > 1") + } + + if runCfg.Parallelism > 1 { + runParallel(ctx, runCfg, scenariosToRun, runtime, runOld, runNew, runBoth) + return + } + + runSequential(ctx, runCfg, scenariosToRun, runtime, runOld, runNew, runBoth) +} + +func runSequential(ctx context.Context, cfg RunConfig, scenariosToRun []string, runtime string, runOld, runNew, runBoth bool) { + clusterMgr := cluster.NewManager(cluster.Config{ + Name: cfg.ClusterName, + ContainerRuntime: runtime, + }) + + if cfg.SkipCluster { + log.Printf("Skipping cluster creation (using existing cluster: %s)", cfg.ClusterName) + if !clusterMgr.Exists() { + log.Fatalf("Cluster %s does not exist. Remove --skip-cluster to create it.", cfg.ClusterName) + } + } else { + log.Println("Creating kind cluster...") + if err := clusterMgr.Create(ctx); err != nil { + log.Fatalf("Failed to create cluster: %v", err) + } + } + + promManifest := filepath.Join(cfg.ManifestsDir, "prometheus.yaml") + promMgr := prometheus.NewManager(promManifest) + + log.Println("Installing Prometheus...") + if err := promMgr.Deploy(ctx); err != nil { + log.Fatalf("Failed to deploy Prometheus: %v", err) + } + + if err := promMgr.StartPortForward(ctx); err != nil { + log.Fatalf("Failed to start Prometheus port-forward: %v", err) + } + defer promMgr.StopPortForward() + + log.Println("Loading images into kind cluster...") + if runOld { + log.Printf("Loading old image: %s", cfg.OldImage) + if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { + log.Fatalf("Failed to load old image: %v", err) + } + } + if runNew { + log.Printf("Loading new image: %s", cfg.NewImage) + if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { + log.Fatalf("Failed to load new image: %v", err) + } + } + + log.Println("Pre-loading test images...") + testImage := "gcr.io/google-containers/busybox:1.27" + clusterMgr.LoadImage(ctx, testImage) + + kubeClient, err := getKubeClient("") + if err != nil { + log.Fatalf("Failed to create kubernetes client: %v", err) + } + + for _, scenarioID := range scenariosToRun { + log.Printf("========================================") + log.Printf("=== Starting scenario %s ===", scenarioID) + log.Printf("========================================") + + cleanupTestNamespaces(ctx, "") + cleanupReloader(ctx, "old", "") + cleanupReloader(ctx, "new", "") + + if err := promMgr.Reset(ctx); err != nil { + log.Printf("Warning: failed to reset Prometheus: %v", err) + } + + createTestNamespace(ctx, "") + + if runOld { + oldMgr := reloader.NewManager(reloader.Config{ + Version: "old", + Image: cfg.OldImage, + }) + + if err := oldMgr.Deploy(ctx); err != nil { + log.Printf("Failed to deploy old Reloader: %v", err) + continue + } + + if err := promMgr.WaitForTarget(ctx, oldMgr.Job(), 60*time.Second); err != nil { + log.Printf("Warning: %v", err) + log.Println("Proceeding anyway, but metrics may be incomplete") + } + + runScenario(ctx, kubeClient, scenarioID, "old", cfg.OldImage, cfg.Duration, cfg.ResultsDir) + collectMetrics(ctx, promMgr, oldMgr.Job(), scenarioID, "old", cfg.ResultsDir) + collectLogs(ctx, oldMgr, scenarioID, "old", cfg.ResultsDir) + + if runBoth { + cleanupTestNamespaces(ctx, "") + oldMgr.Cleanup(ctx) + promMgr.Reset(ctx) + createTestNamespace(ctx, "") + } + } + + if runNew { + newMgr := reloader.NewManager(reloader.Config{ + Version: "new", + Image: cfg.NewImage, + }) + + if err := newMgr.Deploy(ctx); err != nil { + log.Printf("Failed to deploy new Reloader: %v", err) + continue + } + + if err := promMgr.WaitForTarget(ctx, newMgr.Job(), 60*time.Second); err != nil { + log.Printf("Warning: %v", err) + log.Println("Proceeding anyway, but metrics may be incomplete") + } + + runScenario(ctx, kubeClient, scenarioID, "new", cfg.NewImage, cfg.Duration, cfg.ResultsDir) + collectMetrics(ctx, promMgr, newMgr.Job(), scenarioID, "new", cfg.ResultsDir) + collectLogs(ctx, newMgr, scenarioID, "new", cfg.ResultsDir) + } + + generateReport(scenarioID, cfg.ResultsDir, runBoth) + log.Printf("=== Scenario %s complete ===", scenarioID) + } + + log.Println("Load test complete!") + log.Printf("Results available in: %s", cfg.ResultsDir) +} + +func runParallel(ctx context.Context, cfg RunConfig, scenariosToRun []string, runtime string, runOld, runNew, runBoth bool) { + numWorkers := cfg.Parallelism + if numWorkers > len(scenariosToRun) { + numWorkers = len(scenariosToRun) + log.Printf("Reducing parallelism to %d (number of scenarios)", numWorkers) + } + + log.Printf("Starting parallel execution with %d workers", numWorkers) + + workers := make([]*workerContext, numWorkers) + var setupWg sync.WaitGroup + setupErrors := make(chan error, numWorkers) + + log.Println("Setting up worker clusters...") + for i := range numWorkers { + setupWg.Add(1) + go func(workerID int) { + defer setupWg.Done() + worker, err := setupWorker(ctx, cfg, workerID, runtime, runOld, runNew) + if err != nil { + setupErrors <- fmt.Errorf("worker %d setup failed: %w", workerID, err) + return + } + workers[workerID] = worker + }(i) + } + + setupWg.Wait() + close(setupErrors) + + for err := range setupErrors { + log.Printf("Error: %v", err) + } + + readyWorkers := 0 + for _, w := range workers { + if w != nil { + readyWorkers++ + } + } + if readyWorkers == 0 { + log.Fatal("No workers ready, aborting") + } + if readyWorkers < numWorkers { + log.Printf("Warning: only %d/%d workers ready", readyWorkers, numWorkers) + } + + defer func() { + log.Println("Cleaning up worker clusters...") + for _, w := range workers { + if w != nil { + w.promMgr.StopPortForward() + } + } + }() + + scenarioCh := make(chan string, len(scenariosToRun)) + for _, s := range scenariosToRun { + scenarioCh <- s + } + close(scenarioCh) + + var resultsMu sync.Mutex + completedScenarios := make([]string, 0, len(scenariosToRun)) + + var wg sync.WaitGroup + for _, worker := range workers { + if worker == nil { + continue + } + wg.Add(1) + go func(w *workerContext) { + defer wg.Done() + for scenarioID := range scenarioCh { + select { + case <-ctx.Done(): + return + default: + } + + log.Printf("[Worker %d] Starting scenario %s", w.id, scenarioID) + + cleanupTestNamespaces(ctx, w.kubeContext) + cleanupReloader(ctx, "old", w.kubeContext) + cleanupReloader(ctx, "new", w.kubeContext) + + if err := w.promMgr.Reset(ctx); err != nil { + log.Printf("[Worker %d] Warning: failed to reset Prometheus: %v", w.id, err) + } + + createTestNamespace(ctx, w.kubeContext) + + if runOld { + runVersionOnWorker(ctx, w, cfg, scenarioID, "old", cfg.OldImage, runBoth) + } + + if runNew { + runVersionOnWorker(ctx, w, cfg, scenarioID, "new", cfg.NewImage, false) + } + + generateReport(scenarioID, cfg.ResultsDir, runBoth) + + resultsMu.Lock() + completedScenarios = append(completedScenarios, scenarioID) + resultsMu.Unlock() + + log.Printf("[Worker %d] Scenario %s complete", w.id, scenarioID) + } + }(worker) + } + + wg.Wait() + + log.Println("Load test complete!") + log.Printf("Completed %d/%d scenarios", len(completedScenarios), len(scenariosToRun)) + log.Printf("Results available in: %s", cfg.ResultsDir) +} + +func setupWorker(ctx context.Context, cfg RunConfig, workerID int, runtime string, runOld, runNew bool) (*workerContext, error) { + workerName := fmt.Sprintf("%s-%d", DefaultClusterName, workerID) + promPort := 9091 + workerID + + log.Printf("[Worker %d] Creating cluster %s (ports %d/%d)...", workerID, workerName, 8080+workerID, 8443+workerID) + + clusterMgr := cluster.NewManager(cluster.Config{ + Name: workerName, + ContainerRuntime: runtime, + PortOffset: workerID, + }) + + if err := clusterMgr.Create(ctx); err != nil { + return nil, fmt.Errorf("creating cluster: %w", err) + } + + kubeContext := clusterMgr.Context() + + promManifest := filepath.Join(cfg.ManifestsDir, "prometheus.yaml") + promMgr := prometheus.NewManagerWithPort(promManifest, promPort, kubeContext) + + log.Printf("[Worker %d] Installing Prometheus (port %d)...", workerID, promPort) + if err := promMgr.Deploy(ctx); err != nil { + return nil, fmt.Errorf("deploying prometheus: %w", err) + } + + if err := promMgr.StartPortForward(ctx); err != nil { + return nil, fmt.Errorf("starting prometheus port-forward: %w", err) + } + + log.Printf("[Worker %d] Loading images...", workerID) + if runOld { + if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { + log.Printf("[Worker %d] Warning: failed to load old image: %v", workerID, err) + } + } + if runNew { + if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { + log.Printf("[Worker %d] Warning: failed to load new image: %v", workerID, err) + } + } + + testImage := "gcr.io/google-containers/busybox:1.27" + clusterMgr.LoadImage(ctx, testImage) + + kubeClient, err := getKubeClient(kubeContext) + if err != nil { + return nil, fmt.Errorf("creating kubernetes client: %w", err) + } + + log.Printf("[Worker %d] Ready", workerID) + return &workerContext{ + id: workerID, + clusterMgr: clusterMgr, + promMgr: promMgr, + kubeClient: kubeClient, + kubeContext: kubeContext, + runtime: runtime, + }, nil +} + +func runVersionOnWorker(ctx context.Context, w *workerContext, cfg RunConfig, scenarioID, version, image string, cleanupAfter bool) { + mgr := reloader.NewManager(reloader.Config{ + Version: version, + Image: image, + }) + mgr.SetKubeContext(w.kubeContext) + + if err := mgr.Deploy(ctx); err != nil { + log.Printf("[Worker %d] Failed to deploy %s Reloader: %v", w.id, version, err) + return + } + + if err := w.promMgr.WaitForTarget(ctx, mgr.Job(), 60*time.Second); err != nil { + log.Printf("[Worker %d] Warning: %v", w.id, err) + log.Printf("[Worker %d] Proceeding anyway, but metrics may be incomplete", w.id) + } + + runScenario(ctx, w.kubeClient, scenarioID, version, image, cfg.Duration, cfg.ResultsDir) + collectMetrics(ctx, w.promMgr, mgr.Job(), scenarioID, version, cfg.ResultsDir) + collectLogs(ctx, mgr, scenarioID, version, cfg.ResultsDir) + + if cleanupAfter { + cleanupTestNamespaces(ctx, w.kubeContext) + mgr.Cleanup(ctx) + w.promMgr.Reset(ctx) + createTestNamespace(ctx, w.kubeContext) + } +} + +func runScenario(ctx context.Context, client kubernetes.Interface, scenarioID, version, image string, duration int, resultsDir string) { + runner, ok := scenarios.Registry[scenarioID] + if !ok { + log.Printf("Unknown scenario: %s", scenarioID) + return + } + + if s6, ok := runner.(*scenarios.ControllerRestartScenario); ok { + s6.ReloaderVersion = version + } + + if s11, ok := runner.(*scenarios.AnnotationStrategyScenario); ok { + s11.Image = image + } + + log.Printf("Running scenario %s (%s): %s", scenarioID, version, runner.Description()) + + if ctx.Err() != nil { + log.Printf("WARNING: Parent context already done: %v", ctx.Err()) + } + + timeout := time.Duration(duration)*time.Second + 5*time.Minute + log.Printf("Creating scenario context with timeout: %v (duration=%ds)", timeout, duration) + + scenarioCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + expected, err := runner.Run(scenarioCtx, client, TestNamespace, time.Duration(duration)*time.Second) + if err != nil { + log.Printf("Scenario %s failed: %v", scenarioID, err) + } + + scenarios.WriteExpectedMetrics(scenarioID, resultsDir, expected) +} + +func collectMetrics(ctx context.Context, promMgr *prometheus.Manager, job, scenarioID, version, resultsDir string) { + log.Printf("Waiting 5s for Reloader to finish processing events...") + time.Sleep(5 * time.Second) + + log.Printf("Waiting 8s for Prometheus to scrape final metrics...") + time.Sleep(8 * time.Second) + + log.Printf("Collecting metrics for %s...", version) + outputDir := filepath.Join(resultsDir, scenarioID, version) + if err := promMgr.CollectMetrics(ctx, job, outputDir, scenarioID); err != nil { + log.Printf("Failed to collect metrics: %v", err) + } +} + +func collectLogs(ctx context.Context, mgr *reloader.Manager, scenarioID, version, resultsDir string) { + log.Printf("Collecting logs for %s...", version) + logPath := filepath.Join(resultsDir, scenarioID, version, "reloader.log") + if err := mgr.CollectLogs(ctx, logPath); err != nil { + log.Printf("Failed to collect logs: %v", err) + } +} + +func generateReport(scenarioID, resultsDir string, isComparison bool) { + if isComparison { + log.Println("Generating comparison report...") + } else { + log.Println("Generating single-version report...") + } + + reportPath := filepath.Join(resultsDir, scenarioID, "report.txt") + + cmd := exec.Command(os.Args[0], "report", + fmt.Sprintf("--scenario=%s", scenarioID), + fmt.Sprintf("--results-dir=%s", resultsDir), + fmt.Sprintf("--output=%s", reportPath)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Run() + + if data, err := os.ReadFile(reportPath); err == nil { + fmt.Println(string(data)) + } + + log.Printf("Report saved to: %s", reportPath) +} + +func getKubeClient(kubeContext string) (kubernetes.Interface, error) { + kubeconfig := os.Getenv("KUBECONFIG") + if kubeconfig == "" { + home, _ := os.UserHomeDir() + kubeconfig = filepath.Join(home, ".kube", "config") + } + + loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig} + configOverrides := &clientcmd.ConfigOverrides{} + if kubeContext != "" { + configOverrides.CurrentContext = kubeContext + } + + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + config, err := kubeConfig.ClientConfig() + if err != nil { + return nil, err + } + + return kubernetes.NewForConfig(config) +} + +func createTestNamespace(ctx context.Context, kubeContext string) { + args := []string{"create", "namespace", TestNamespace, "--dry-run=client", "-o", "yaml"} + if kubeContext != "" { + args = append([]string{"--context", kubeContext}, args...) + } + cmd := exec.CommandContext(ctx, "kubectl", args...) + out, _ := cmd.Output() + + applyArgs := []string{"apply", "-f", "-"} + if kubeContext != "" { + applyArgs = append([]string{"--context", kubeContext}, applyArgs...) + } + applyCmd := exec.CommandContext(ctx, "kubectl", applyArgs...) + applyCmd.Stdin = strings.NewReader(string(out)) + applyCmd.Run() +} + +func cleanupTestNamespaces(ctx context.Context, kubeContext string) { + log.Println("Cleaning up test resources...") + + namespaces := []string{TestNamespace} + for i := range 10 { + namespaces = append(namespaces, fmt.Sprintf("%s-%d", TestNamespace, i)) + } + + for _, ns := range namespaces { + args := []string{"delete", "namespace", ns, "--wait=false", "--ignore-not-found"} + if kubeContext != "" { + args = append([]string{"--context", kubeContext}, args...) + } + exec.CommandContext(ctx, "kubectl", args...).Run() + } + + time.Sleep(2 * time.Second) + + for _, ns := range namespaces { + args := []string{"delete", "pods", "--all", "-n", ns, "--grace-period=0", "--force"} + if kubeContext != "" { + args = append([]string{"--context", kubeContext}, args...) + } + exec.CommandContext(ctx, "kubectl", args...).Run() + } +} + +func cleanupReloader(ctx context.Context, version string, kubeContext string) { + ns := fmt.Sprintf("reloader-%s", version) + + nsArgs := []string{"delete", "namespace", ns, "--wait=false", "--ignore-not-found"} + crArgs := []string{"delete", "clusterrole", fmt.Sprintf("reloader-%s", version), "--ignore-not-found"} + crbArgs := []string{"delete", "clusterrolebinding", fmt.Sprintf("reloader-%s", version), "--ignore-not-found"} + + if kubeContext != "" { + nsArgs = append([]string{"--context", kubeContext}, nsArgs...) + crArgs = append([]string{"--context", kubeContext}, crArgs...) + crbArgs = append([]string{"--context", kubeContext}, crbArgs...) + } + + exec.CommandContext(ctx, "kubectl", nsArgs...).Run() + exec.CommandContext(ctx, "kubectl", crArgs...).Run() + exec.CommandContext(ctx, "kubectl", crbArgs...).Run() +} diff --git a/test/loadtest/internal/cmd/summary.go b/test/loadtest/internal/cmd/summary.go new file mode 100644 index 0000000..bda40fb --- /dev/null +++ b/test/loadtest/internal/cmd/summary.go @@ -0,0 +1,251 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "log" + "os" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" +) + +var ( + summaryResultsDir string + summaryOutputFile string + summaryFormat string + summaryTestType string +) + +var summaryCmd = &cobra.Command{ + Use: "summary", + Short: "Generate summary across all scenarios (for CI)", + Long: `Generate an aggregated summary report across all test scenarios. + +Examples: + # Generate markdown summary for CI + loadtest summary --results-dir=./results --format=markdown`, + Run: func(cmd *cobra.Command, args []string) { + summaryCommand() + }, +} + +func init() { + summaryCmd.Flags().StringVar(&summaryResultsDir, "results-dir", "./results", "Directory containing results") + summaryCmd.Flags().StringVar(&summaryOutputFile, "output", "", "Output file (default: stdout)") + summaryCmd.Flags().StringVar(&summaryFormat, "format", "markdown", "Output format: text, json, markdown") + summaryCmd.Flags().StringVar(&summaryTestType, "test-type", "full", "Test type label: quick, full") +} + +// SummaryReport aggregates results from multiple scenarios. +type SummaryReport struct { + Timestamp time.Time `json:"timestamp"` + TestType string `json:"test_type"` + PassCount int `json:"pass_count"` + FailCount int `json:"fail_count"` + TotalCount int `json:"total_count"` + Scenarios []ScenarioSummary `json:"scenarios"` +} + +// ScenarioSummary provides a brief summary of a single scenario. +type ScenarioSummary struct { + ID string `json:"id"` + Status string `json:"status"` + Description string `json:"description"` + ActionTotal float64 `json:"action_total"` + ActionExp float64 `json:"action_expected"` + ErrorsTotal float64 `json:"errors_total"` +} + +func summaryCommand() { + summary, err := generateSummaryReport(summaryResultsDir, summaryTestType) + if err != nil { + log.Fatalf("Failed to generate summary: %v", err) + } + + var output string + switch OutputFormat(summaryFormat) { + case OutputFormatJSON: + output = renderSummaryJSON(summary) + case OutputFormatText: + output = renderSummaryText(summary) + default: + output = renderSummaryMarkdown(summary) + } + + if summaryOutputFile != "" { + if err := os.WriteFile(summaryOutputFile, []byte(output), 0644); err != nil { + log.Fatalf("Failed to write output file: %v", err) + } + log.Printf("Summary written to %s", summaryOutputFile) + } else { + fmt.Print(output) + } + + if summary.FailCount > 0 { + os.Exit(1) + } +} + +func generateSummaryReport(resultsDir, testType string) (*SummaryReport, error) { + summary := &SummaryReport{ + Timestamp: time.Now(), + TestType: testType, + } + + entries, err := os.ReadDir(resultsDir) + if err != nil { + return nil, fmt.Errorf("failed to read results directory: %w", err) + } + + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "S") { + continue + } + + scenarioID := entry.Name() + report, err := generateScenarioReport(scenarioID, resultsDir) + if err != nil { + log.Printf("Warning: failed to load scenario %s: %v", scenarioID, err) + continue + } + + scenarioSummary := ScenarioSummary{ + ID: scenarioID, + Status: report.OverallStatus, + Description: report.TestDescription, + } + + for _, c := range report.Comparisons { + switch c.Name { + case "action_total": + scenarioSummary.ActionTotal = c.NewValue + scenarioSummary.ActionExp = c.Expected + case "errors_total": + scenarioSummary.ErrorsTotal = c.NewValue + } + } + + summary.Scenarios = append(summary.Scenarios, scenarioSummary) + summary.TotalCount++ + if report.OverallStatus == "PASS" { + summary.PassCount++ + } else { + summary.FailCount++ + } + } + + sort.Slice(summary.Scenarios, func(i, j int) bool { + return naturalSort(summary.Scenarios[i].ID, summary.Scenarios[j].ID) + }) + + return summary, nil +} + +func naturalSort(a, b string) bool { + var aNum, bNum int + fmt.Sscanf(a, "S%d", &aNum) + fmt.Sscanf(b, "S%d", &bNum) + return aNum < bNum +} + +func renderSummaryJSON(summary *SummaryReport) string { + data, err := json.MarshalIndent(summary, "", " ") + if err != nil { + return fmt.Sprintf(`{"error": "%s"}`, err.Error()) + } + return string(data) +} + +func renderSummaryText(summary *SummaryReport) string { + var sb strings.Builder + + sb.WriteString("================================================================================\n") + sb.WriteString(" LOAD TEST SUMMARY\n") + sb.WriteString("================================================================================\n\n") + + passRate := 0 + if summary.TotalCount > 0 { + passRate = summary.PassCount * 100 / summary.TotalCount + } + + fmt.Fprintf(&sb, "Test Type: %s\n", summary.TestType) + fmt.Fprintf(&sb, "Results: %d/%d passed (%d%%)\n\n", summary.PassCount, summary.TotalCount, passRate) + + fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8s\n", "ID", "Status", "Description", "Actions", "Errors") + fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8s\n", "------", "--------", strings.Repeat("-", 45), "----------", "--------") + + for _, s := range summary.Scenarios { + desc := s.Description + if len(desc) > 45 { + desc = desc[:42] + "..." + } + actions := fmt.Sprintf("%.0f", s.ActionTotal) + if s.ActionExp > 0 { + actions = fmt.Sprintf("%.0f/%.0f", s.ActionTotal, s.ActionExp) + } + fmt.Fprintf(&sb, "%-6s %-8s %-45s %10s %8.0f\n", s.ID, s.Status, desc, actions, s.ErrorsTotal) + } + + sb.WriteString("\n================================================================================\n") + return sb.String() +} + +func renderSummaryMarkdown(summary *SummaryReport) string { + var sb strings.Builder + + emoji := "āœ…" + title := "ALL TESTS PASSED" + if summary.FailCount > 0 { + emoji = "āŒ" + title = fmt.Sprintf("%d TEST(S) FAILED", summary.FailCount) + } else if summary.TotalCount == 0 { + emoji = "āš ļø" + title = "NO RESULTS" + } + + sb.WriteString(fmt.Sprintf("## %s Load Test Results: %s\n\n", emoji, title)) + + if summary.TestType == "quick" { + sb.WriteString("> šŸš€ **Quick Test** (S1, S4, S6) ā€” Use `/loadtest` for full suite\n\n") + } + + passRate := 0 + if summary.TotalCount > 0 { + passRate = summary.PassCount * 100 / summary.TotalCount + } + sb.WriteString(fmt.Sprintf("**%d/%d passed** (%d%%)\n\n", summary.PassCount, summary.TotalCount, passRate)) + + sb.WriteString("| | Scenario | Description | Actions | Errors |\n") + sb.WriteString("|:-:|:--------:|-------------|:-------:|:------:|\n") + + for _, s := range summary.Scenarios { + icon := "āœ…" + if s.Status != "PASS" { + icon = "āŒ" + } + + desc := s.Description + if len(desc) > 45 { + desc = desc[:42] + "..." + } + + actions := fmt.Sprintf("%.0f", s.ActionTotal) + if s.ActionExp > 0 { + actions = fmt.Sprintf("%.0f/%.0f", s.ActionTotal, s.ActionExp) + } + + errors := fmt.Sprintf("%.0f", s.ErrorsTotal) + if s.ErrorsTotal > 0 { + errors = fmt.Sprintf("āš ļø %.0f", s.ErrorsTotal) + } + + sb.WriteString(fmt.Sprintf("| %s | **%s** | %s | %s | %s |\n", icon, s.ID, desc, actions, errors)) + } + + sb.WriteString("\nšŸ“¦ **[Download detailed results](../artifacts)**\n") + + return sb.String() +} From 76287e04202f6d590ad731368947cb0ebdaff9aa Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Thu, 8 Jan 2026 23:36:02 +0100 Subject: [PATCH 35/45] refactor: Cleanup logic for reloader in loadtests --- test/loadtest/internal/cmd/run.go | 25 ++++----------------- test/loadtest/internal/reloader/reloader.go | 21 +++++++++++++++++ 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/test/loadtest/internal/cmd/run.go b/test/loadtest/internal/cmd/run.go index 8c88b54..4a0f731 100644 --- a/test/loadtest/internal/cmd/run.go +++ b/test/loadtest/internal/cmd/run.go @@ -208,8 +208,8 @@ func runSequential(ctx context.Context, cfg RunConfig, scenariosToRun []string, log.Printf("========================================") cleanupTestNamespaces(ctx, "") - cleanupReloader(ctx, "old", "") - cleanupReloader(ctx, "new", "") + reloader.CleanupByVersion(ctx, "old", "") + reloader.CleanupByVersion(ctx, "new", "") if err := promMgr.Reset(ctx); err != nil { log.Printf("Warning: failed to reset Prometheus: %v", err) @@ -357,8 +357,8 @@ func runParallel(ctx context.Context, cfg RunConfig, scenariosToRun []string, ru log.Printf("[Worker %d] Starting scenario %s", w.id, scenarioID) cleanupTestNamespaces(ctx, w.kubeContext) - cleanupReloader(ctx, "old", w.kubeContext) - cleanupReloader(ctx, "new", w.kubeContext) + reloader.CleanupByVersion(ctx, "old", w.kubeContext) + reloader.CleanupByVersion(ctx, "new", w.kubeContext) if err := w.promMgr.Reset(ctx); err != nil { log.Printf("[Worker %d] Warning: failed to reset Prometheus: %v", w.id, err) @@ -629,20 +629,3 @@ func cleanupTestNamespaces(ctx context.Context, kubeContext string) { } } -func cleanupReloader(ctx context.Context, version string, kubeContext string) { - ns := fmt.Sprintf("reloader-%s", version) - - nsArgs := []string{"delete", "namespace", ns, "--wait=false", "--ignore-not-found"} - crArgs := []string{"delete", "clusterrole", fmt.Sprintf("reloader-%s", version), "--ignore-not-found"} - crbArgs := []string{"delete", "clusterrolebinding", fmt.Sprintf("reloader-%s", version), "--ignore-not-found"} - - if kubeContext != "" { - nsArgs = append([]string{"--context", kubeContext}, nsArgs...) - crArgs = append([]string{"--context", kubeContext}, crArgs...) - crbArgs = append([]string{"--context", kubeContext}, crbArgs...) - } - - exec.CommandContext(ctx, "kubectl", nsArgs...).Run() - exec.CommandContext(ctx, "kubectl", crArgs...).Run() - exec.CommandContext(ctx, "kubectl", crbArgs...).Run() -} diff --git a/test/loadtest/internal/reloader/reloader.go b/test/loadtest/internal/reloader/reloader.go index ff3cfdb..2667cd4 100644 --- a/test/loadtest/internal/reloader/reloader.go +++ b/test/loadtest/internal/reloader/reloader.go @@ -216,6 +216,27 @@ func (m *Manager) Cleanup(ctx context.Context) error { return nil } +// CleanupByVersion removes Reloader resources for a specific version without needing a Manager instance. +// This is useful for cleaning up from previous runs before creating a new Manager. +func CleanupByVersion(ctx context.Context, version, kubeContext string) { + ns := fmt.Sprintf("reloader-%s", version) + name := fmt.Sprintf("reloader-%s", version) + + nsArgs := []string{"delete", "namespace", ns, "--wait=false", "--ignore-not-found"} + crArgs := []string{"delete", "clusterrole", name, "--ignore-not-found"} + crbArgs := []string{"delete", "clusterrolebinding", name, "--ignore-not-found"} + + if kubeContext != "" { + nsArgs = append([]string{"--context", kubeContext}, nsArgs...) + crArgs = append([]string{"--context", kubeContext}, crArgs...) + crbArgs = append([]string{"--context", kubeContext}, crbArgs...) + } + + exec.CommandContext(ctx, "kubectl", nsArgs...).Run() + exec.CommandContext(ctx, "kubectl", crArgs...).Run() + exec.CommandContext(ctx, "kubectl", crbArgs...).Run() +} + // CollectLogs collects logs from the Reloader pod and writes them to the specified file. func (m *Manager) CollectLogs(ctx context.Context, logPath string) error { ns := m.namespace() From 2442eddd811aebe11e1463464f3e24ae0f9146a6 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:34:09 +0100 Subject: [PATCH 36/45] fix: Skip loading images when already done --- .github/actions/loadtest/action.yml | 1 + test/loadtest/internal/cluster/kind.go | 20 +++++-- test/loadtest/internal/cmd/run.go | 78 +++++++++++++++----------- 3 files changed, 60 insertions(+), 39 deletions(-) diff --git a/.github/actions/loadtest/action.yml b/.github/actions/loadtest/action.yml index 3b91957..86dcf49 100644 --- a/.github/actions/loadtest/action.yml +++ b/.github/actions/loadtest/action.yml @@ -162,6 +162,7 @@ runs: ARGS="$ARGS --scenario=${{ inputs.scenarios }}" ARGS="$ARGS --duration=${{ inputs.duration }}" ARGS="$ARGS --cluster-name=${{ steps.cluster.outputs.name }}" + ARGS="$ARGS --skip-image-load" if [ -n "${{ steps.images.outputs.old }}" ]; then ARGS="$ARGS --old-image=${{ steps.images.outputs.old }}" diff --git a/test/loadtest/internal/cluster/kind.go b/test/loadtest/internal/cluster/kind.go index bcd5e19..557e628 100644 --- a/test/loadtest/internal/cluster/kind.go +++ b/test/loadtest/internal/cluster/kind.go @@ -29,14 +29,24 @@ func NewManager(cfg Config) *Manager { } // DetectContainerRuntime finds available container runtime. +// It checks if the runtime daemon is actually running, not just if the binary exists. func DetectContainerRuntime() (string, error) { - if _, err := exec.LookPath("podman"); err == nil { - return "podman", nil - } + // Prefer docker as it's more commonly used with kind if _, err := exec.LookPath("docker"); err == nil { - return "docker", nil + // Verify docker daemon is running + cmd := exec.Command("docker", "info") + if err := cmd.Run(); err == nil { + return "docker", nil + } } - return "", fmt.Errorf("neither docker nor podman found in PATH") + if _, err := exec.LookPath("podman"); err == nil { + // Verify podman is functional (check if we can run a basic command) + cmd := exec.Command("podman", "info") + if err := cmd.Run(); err == nil { + return "podman", nil + } + } + return "", fmt.Errorf("neither docker nor podman is running") } // Exists checks if the cluster already exists. diff --git a/test/loadtest/internal/cmd/run.go b/test/loadtest/internal/cmd/run.go index 4a0f731..29bcb34 100644 --- a/test/loadtest/internal/cmd/run.go +++ b/test/loadtest/internal/cmd/run.go @@ -24,15 +24,16 @@ import ( // RunConfig holds CLI configuration for the run command. type RunConfig struct { - OldImage string - NewImage string - Scenario string - Duration int - SkipCluster bool - ClusterName string - ResultsDir string - ManifestsDir string - Parallelism int + OldImage string + NewImage string + Scenario string + Duration int + SkipCluster bool + SkipImageLoad bool + ClusterName string + ResultsDir string + ManifestsDir string + Parallelism int } // workerContext holds all resources for a single worker (cluster + prometheus). @@ -76,6 +77,7 @@ func init() { runCmd.Flags().IntVar(&runCfg.Duration, "duration", 60, "Test duration in seconds") runCmd.Flags().IntVar(&runCfg.Parallelism, "parallelism", 1, "Run N scenarios in parallel on N clusters") runCmd.Flags().BoolVar(&runCfg.SkipCluster, "skip-cluster", false, "Skip kind cluster creation (use existing)") + runCmd.Flags().BoolVar(&runCfg.SkipImageLoad, "skip-image-load", false, "Skip loading images into kind (use when images already loaded)") runCmd.Flags().StringVar(&runCfg.ClusterName, "cluster-name", DefaultClusterName, "Kind cluster name") runCmd.Flags().StringVar(&runCfg.ResultsDir, "results-dir", "./results", "Directory for results") runCmd.Flags().StringVar(&runCfg.ManifestsDir, "manifests-dir", "", "Directory containing manifests (auto-detected if not set)") @@ -179,23 +181,27 @@ func runSequential(ctx context.Context, cfg RunConfig, scenariosToRun []string, } defer promMgr.StopPortForward() - log.Println("Loading images into kind cluster...") - if runOld { - log.Printf("Loading old image: %s", cfg.OldImage) - if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { - log.Fatalf("Failed to load old image: %v", err) + if cfg.SkipImageLoad { + log.Println("Skipping image loading (--skip-image-load)") + } else { + log.Println("Loading images into kind cluster...") + if runOld { + log.Printf("Loading old image: %s", cfg.OldImage) + if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { + log.Fatalf("Failed to load old image: %v", err) + } } - } - if runNew { - log.Printf("Loading new image: %s", cfg.NewImage) - if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { - log.Fatalf("Failed to load new image: %v", err) + if runNew { + log.Printf("Loading new image: %s", cfg.NewImage) + if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { + log.Fatalf("Failed to load new image: %v", err) + } } - } - log.Println("Pre-loading test images...") - testImage := "gcr.io/google-containers/busybox:1.27" - clusterMgr.LoadImage(ctx, testImage) + log.Println("Pre-loading test images...") + testImage := "gcr.io/google-containers/busybox:1.27" + clusterMgr.LoadImage(ctx, testImage) + } kubeClient, err := getKubeClient("") if err != nil { @@ -422,20 +428,24 @@ func setupWorker(ctx context.Context, cfg RunConfig, workerID int, runtime strin return nil, fmt.Errorf("starting prometheus port-forward: %w", err) } - log.Printf("[Worker %d] Loading images...", workerID) - if runOld { - if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { - log.Printf("[Worker %d] Warning: failed to load old image: %v", workerID, err) + if cfg.SkipImageLoad { + log.Printf("[Worker %d] Skipping image loading (--skip-image-load)", workerID) + } else { + log.Printf("[Worker %d] Loading images...", workerID) + if runOld { + if err := clusterMgr.LoadImage(ctx, cfg.OldImage); err != nil { + log.Printf("[Worker %d] Warning: failed to load old image: %v", workerID, err) + } } - } - if runNew { - if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { - log.Printf("[Worker %d] Warning: failed to load new image: %v", workerID, err) + if runNew { + if err := clusterMgr.LoadImage(ctx, cfg.NewImage); err != nil { + log.Printf("[Worker %d] Warning: failed to load new image: %v", workerID, err) + } } - } - testImage := "gcr.io/google-containers/busybox:1.27" - clusterMgr.LoadImage(ctx, testImage) + testImage := "gcr.io/google-containers/busybox:1.27" + clusterMgr.LoadImage(ctx, testImage) + } kubeClient, err := getKubeClient(kubeContext) if err != nil { From c4f3255c78cddecdaae6feec533576ea9f64a174 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:34:25 +0100 Subject: [PATCH 37/45] ci: Disable tests temporarily --- .github/workflows/push.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index dda9a1c..092da7c 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -73,8 +73,8 @@ jobs: kind create cluster kubectl cluster-info - - name: Test - run: make test + #- name: Test + # run: make test - name: Set up QEMU uses: docker/setup-qemu-action@v3 From e56323d582ba629d4de1e39455414b6c5b06b2ba Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:37:34 +0100 Subject: [PATCH 38/45] ci: Disable tests in PR --- .github/workflows/pull_request.yaml | 4 ++-- .github/workflows/push.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index fbf59ab..2160c48 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -107,8 +107,8 @@ jobs: kubectl cluster-info - - name: Test - run: make test + #- name: Test + # run: make test - name: Run quick A/B load tests uses: ./.github/actions/loadtest diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 092da7c..dda9a1c 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -73,8 +73,8 @@ jobs: kind create cluster kubectl cluster-info - #- name: Test - # run: make test + - name: Test + run: make test - name: Set up QEMU uses: docker/setup-qemu-action@v3 From 2674f405ce76f3e0944ec76b1c777eee3b6959ed Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:44:43 +0100 Subject: [PATCH 39/45] fix: Issue with not parsing multiple scenario in args --- test/loadtest/internal/cmd/run.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/loadtest/internal/cmd/run.go b/test/loadtest/internal/cmd/run.go index 29bcb34..c78e579 100644 --- a/test/loadtest/internal/cmd/run.go +++ b/test/loadtest/internal/cmd/run.go @@ -133,9 +133,16 @@ func runCommand() { cancel() }() - scenariosToRun := []string{runCfg.Scenario} + var scenariosToRun []string if runCfg.Scenario == "all" { scenariosToRun = []string{"S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8", "S9", "S10", "S11", "S12", "S13"} + } else { + // Split comma-separated scenarios (e.g., "S1,S4,S6") + for _, s := range strings.Split(runCfg.Scenario, ",") { + if trimmed := strings.TrimSpace(s); trimmed != "" { + scenariosToRun = append(scenariosToRun, trimmed) + } + } } if runCfg.SkipCluster && runCfg.Parallelism > 1 { From a132ed8deaffc7e22455e1098d5456f1e0ef65c7 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:48:05 +0100 Subject: [PATCH 40/45] ci: Don't comment on forked PRs automatically --- .github/actions/loadtest/action.yml | 22 ++++++++++++++++------ .github/workflows/pull_request.yaml | 1 + 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/actions/loadtest/action.yml b/.github/actions/loadtest/action.yml index 86dcf49..3f71ae9 100644 --- a/.github/actions/loadtest/action.yml +++ b/.github/actions/loadtest/action.yml @@ -208,6 +208,7 @@ runs: - name: Post PR comment if: inputs.post-comment == 'true' && inputs.pr-number != '' + continue-on-error: true uses: actions/github-script@v7 with: github-token: ${{ inputs.github-token }} @@ -234,12 +235,21 @@ runs: `**Artifacts:** [Download](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})`, ].join('\n'); - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: ${{ inputs.pr-number }}, - body: body - }); + try { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ${{ inputs.pr-number }}, + body: body + }); + console.log('Comment posted successfully'); + } catch (error) { + if (error.status === 403) { + console.log('Could not post comment (fork PR with restricted permissions). Use /loadtest command to run with comment posting.'); + } else { + throw error; + } + } - name: Upload results uses: actions/upload-artifact@v4 diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 2160c48..1ed8939 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -36,6 +36,7 @@ jobs: permissions: contents: read pull-requests: write + issues: write runs-on: ubuntu-latest name: Build From ad6013adbf6eb456c88ac383fb04afb3f9fe5ce6 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 00:58:28 +0100 Subject: [PATCH 41/45] fix: Treat missing metrics as info --- test/loadtest/internal/cmd/report.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/loadtest/internal/cmd/report.go b/test/loadtest/internal/cmd/report.go index aa273a5..7bf4cc6 100644 --- a/test/loadtest/internal/cmd/report.go +++ b/test/loadtest/internal/cmd/report.go @@ -608,7 +608,11 @@ func compareMetricWithExpected(name string, oldValue, newValue, expected float64 oldMeetsExp := meetsExpected(oldValue, expected) newMeetsExp := meetsExpected(newValue, expected) - if expected > 0 && threshold.metricType == ShouldMatch { + isNewMetric := info.isCounter && oldValue == 0 && newValue > 0 && expected == 0 + + if isNewMetric { + status = "info" + } else if expected > 0 && threshold.metricType == ShouldMatch { if newMeetsExp == "āœ—" { status = "fail" } From 07f7365d63f16d9d67c3ef735eb955b12342d229 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 01:14:45 +0100 Subject: [PATCH 42/45] ci: Enable tests for PR again --- .github/workflows/pull_request.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 1ed8939..c428826 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -108,8 +108,8 @@ jobs: kubectl cluster-info - #- name: Test - # run: make test + - name: Test + run: make test - name: Run quick A/B load tests uses: ./.github/actions/loadtest From 1945a740d0fb6860f137f675b723f371a117c40f Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 01:22:24 +0100 Subject: [PATCH 43/45] chore: Format files# --- internal/pkg/metrics/prometheus.go | 38 ++++++++++-------------------- pkg/kube/resourcemapper.go | 6 ++--- 2 files changed, 15 insertions(+), 29 deletions(-) diff --git a/internal/pkg/metrics/prometheus.go b/internal/pkg/metrics/prometheus.go index e6f2f35..6d103a1 100644 --- a/internal/pkg/metrics/prometheus.go +++ b/internal/pkg/metrics/prometheus.go @@ -57,38 +57,24 @@ func init() { // Collectors holds all Prometheus metrics collectors for Reloader. type Collectors struct { - // Existing metrics (preserved for backward compatibility) Reloaded *prometheus.CounterVec ReloadedByNamespace *prometheus.CounterVec countByNamespace bool - // Reconcile/Handler metrics ReconcileTotal *prometheus.CounterVec // Total reconcile calls by result ReconcileDuration *prometheus.HistogramVec // Time spent in reconcile/handler - - // Action metrics - ActionTotal *prometheus.CounterVec // Total actions by workload kind and result - ActionLatency *prometheus.HistogramVec // Time from event to action applied - - // Skip metrics - SkippedTotal *prometheus.CounterVec // Skipped operations by reason - - // Queue metrics - QueueDepth prometheus.Gauge // Current queue depth - QueueAdds prometheus.Counter // Total items added to queue - QueueLatency *prometheus.HistogramVec // Time spent in queue - - // Error and retry metrics - ErrorsTotal *prometheus.CounterVec // Errors by type - RetriesTotal prometheus.Counter // Total retries - - // Event processing metrics - EventsReceived *prometheus.CounterVec // Events received by type (add/update/delete) - EventsProcessed *prometheus.CounterVec // Events processed by type and result - - // Resource discovery metrics - WorkloadsScanned *prometheus.CounterVec // Workloads scanned by kind - WorkloadsMatched *prometheus.CounterVec // Workloads matched for reload by kind + ActionTotal *prometheus.CounterVec // Total actions by workload kind and result + ActionLatency *prometheus.HistogramVec // Time from event to action applied + SkippedTotal *prometheus.CounterVec // Skipped operations by reason + QueueDepth prometheus.Gauge // Current queue depth + QueueAdds prometheus.Counter // Total items added to queue + QueueLatency *prometheus.HistogramVec // Time spent in queue + ErrorsTotal *prometheus.CounterVec // Errors by type + RetriesTotal prometheus.Counter // Total retries + EventsReceived *prometheus.CounterVec // Events received by type (add/update/delete) + EventsProcessed *prometheus.CounterVec // Events processed by type and result + WorkloadsScanned *prometheus.CounterVec // Workloads scanned by kind + WorkloadsMatched *prometheus.CounterVec // Workloads matched for reload by kind } // RecordReload records a reload event with the given success status and namespace. diff --git a/pkg/kube/resourcemapper.go b/pkg/kube/resourcemapper.go index 286d408..bdb7858 100644 --- a/pkg/kube/resourcemapper.go +++ b/pkg/kube/resourcemapper.go @@ -8,8 +8,8 @@ import ( // ResourceMap are resources from where changes are going to be detected var ResourceMap = map[string]runtime.Object{ - "configmaps": &v1.ConfigMap{}, - "secrets": &v1.Secret{}, - "namespaces": &v1.Namespace{}, + "configmaps": &v1.ConfigMap{}, + "secrets": &v1.Secret{}, + "namespaces": &v1.Namespace{}, "secretproviderclasspodstatuses": &csiv1.SecretProviderClassPodStatus{}, } From 1be910749b91728fdfd6d9573f10580e9925b454 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 01:34:16 +0100 Subject: [PATCH 44/45] chore: A lot of cleanup --- .github/workflows/loadtest.yml | 1 - internal/pkg/controller/controller.go | 12 +--- internal/pkg/metrics/prometheus.go | 3 - test/loadtest/cmd/loadtest/main.go | 1 - test/loadtest/internal/cluster/kind.go | 9 --- test/loadtest/internal/cmd/root.go | 1 - .../internal/prometheus/prometheus.go | 29 +-------- test/loadtest/internal/scenarios/scenarios.go | 65 +------------------ 8 files changed, 9 insertions(+), 112 deletions(-) diff --git a/.github/workflows/loadtest.yml b/.github/workflows/loadtest.yml index f4eb322..c997e13 100644 --- a/.github/workflows/loadtest.yml +++ b/.github/workflows/loadtest.yml @@ -10,7 +10,6 @@ permissions: issues: write jobs: - # Full load test suite triggered by /loadtest command loadtest: # Only run on PR comments with /loadtest command if: | diff --git a/internal/pkg/controller/controller.go b/internal/pkg/controller/controller.go index da2b2c7..1a51d9a 100644 --- a/internal/pkg/controller/controller.go +++ b/internal/pkg/controller/controller.go @@ -110,7 +110,6 @@ func NewController( // Add function to add a new object to the queue in case of creating a resource func (c *Controller) Add(obj interface{}) { - // Record event received c.collectors.RecordEventReceived("add", c.resource) switch object := obj.(type) { @@ -127,7 +126,7 @@ func (c *Controller) Add(obj interface{}) { Resource: obj, Collectors: c.collectors, Recorder: c.recorder, - EnqueueTime: time.Now(), // Track when item was enqueued + EnqueueTime: time.Now(), }) } else { c.collectors.RecordSkipped("ignored_or_not_selected") @@ -186,7 +185,6 @@ func (c *Controller) removeSelectedNamespaceFromCache(namespace v1.Namespace) { // Update function to add an old object and a new object to the queue in case of updating a resource func (c *Controller) Update(old interface{}, new interface{}) { - // Record event received c.collectors.RecordEventReceived("update", c.resource) switch new.(type) { @@ -200,7 +198,7 @@ func (c *Controller) Update(old interface{}, new interface{}) { OldResource: old, Collectors: c.collectors, Recorder: c.recorder, - EnqueueTime: time.Now(), // Track when item was enqueued + EnqueueTime: time.Now(), }) } else { c.collectors.RecordSkipped("ignored_or_not_selected") @@ -209,7 +207,6 @@ func (c *Controller) Update(old interface{}, new interface{}) { // Delete function to add an object to the queue in case of deleting a resource func (c *Controller) Delete(old interface{}) { - // Record event received c.collectors.RecordEventReceived("delete", c.resource) if _, ok := old.(*csiv1.SecretProviderClassPodStatus); ok { @@ -222,7 +219,7 @@ func (c *Controller) Delete(old interface{}) { Resource: old, Collectors: c.collectors, Recorder: c.recorder, - EnqueueTime: time.Now(), // Track when item was enqueued + EnqueueTime: time.Now(), }) } else { c.collectors.RecordSkipped("ignored_or_not_selected") @@ -285,7 +282,6 @@ func (c *Controller) processNextItem() bool { return false } - // Update queue depth after getting item c.collectors.SetQueueDepth(c.queue.Len()) // Tell the queue that we are done with processing this key. This unblocks the key for other workers @@ -307,7 +303,6 @@ func (c *Controller) processNextItem() bool { duration := time.Since(startTime) - // Record reconcile metrics if err != nil { c.collectors.RecordReconcile("error", duration) } else { @@ -355,7 +350,6 @@ func (c *Controller) handleErr(err error, key interface{}) { logrus.Errorf("Dropping key out of the queue: %v", err) logrus.Debugf("Dropping the key %q out of the queue: %v", key, err) - // Record failed event processing c.collectors.RecordEventProcessed("unknown", c.resource, "dropped") } diff --git a/internal/pkg/metrics/prometheus.go b/internal/pkg/metrics/prometheus.go index 6d103a1..4310393 100644 --- a/internal/pkg/metrics/prometheus.go +++ b/internal/pkg/metrics/prometheus.go @@ -219,8 +219,6 @@ func NewCollectors() Collectors { []string{"success", "namespace"}, ) - // === NEW: Comprehensive metrics === - reconcileTotal := prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: "reloader", @@ -372,7 +370,6 @@ func NewCollectors() Collectors { func SetupPrometheusEndpoint() Collectors { collectors := NewCollectors() - // Register all metrics prometheus.MustRegister(collectors.Reloaded) prometheus.MustRegister(collectors.ReconcileTotal) prometheus.MustRegister(collectors.ReconcileDuration) diff --git a/test/loadtest/cmd/loadtest/main.go b/test/loadtest/cmd/loadtest/main.go index a8925bc..510ce0b 100644 --- a/test/loadtest/cmd/loadtest/main.go +++ b/test/loadtest/cmd/loadtest/main.go @@ -1,4 +1,3 @@ -// Package main is the entrypoint for the load test CLI. package main import "github.com/stakater/Reloader/test/loadtest/internal/cmd" diff --git a/test/loadtest/internal/cluster/kind.go b/test/loadtest/internal/cluster/kind.go index 557e628..1fde314 100644 --- a/test/loadtest/internal/cluster/kind.go +++ b/test/loadtest/internal/cluster/kind.go @@ -31,16 +31,13 @@ func NewManager(cfg Config) *Manager { // DetectContainerRuntime finds available container runtime. // It checks if the runtime daemon is actually running, not just if the binary exists. func DetectContainerRuntime() (string, error) { - // Prefer docker as it's more commonly used with kind if _, err := exec.LookPath("docker"); err == nil { - // Verify docker daemon is running cmd := exec.Command("docker", "info") if err := cmd.Run(); err == nil { return "docker", nil } } if _, err := exec.LookPath("podman"); err == nil { - // Verify podman is functional (check if we can run a basic command) cmd := exec.Command("podman", "info") if err := cmd.Run(); err == nil { return "podman", nil @@ -85,7 +82,6 @@ func (m *Manager) Create(ctx context.Context) error { } } - // Calculate unique ports based on offset (for parallel clusters) httpPort := 8080 + m.cfg.PortOffset httpsPort := 8443 + m.cfg.PortOffset @@ -231,7 +227,6 @@ func (m *Manager) Name() string { // LoadImage loads a container image into the kind cluster. func (m *Manager) LoadImage(ctx context.Context, image string) error { - // First check if image exists locally if !m.imageExistsLocally(image) { fmt.Printf(" Image not found locally, pulling: %s\n", image) pullCmd := exec.CommandContext(ctx, m.cfg.ContainerRuntime, "pull", image) @@ -247,7 +242,6 @@ func (m *Manager) LoadImage(ctx context.Context, image string) error { fmt.Printf(" Copying image to kind cluster...\n") if m.cfg.ContainerRuntime == "podman" { - // For podman, save to archive and load tmpFile := fmt.Sprintf("/tmp/kind-image-%d.tar", time.Now().UnixNano()) defer os.Remove(tmpFile) @@ -276,19 +270,16 @@ func (m *Manager) LoadImage(ctx context.Context, image string) error { // imageExistsLocally checks if an image exists in the local container runtime. func (m *Manager) imageExistsLocally(image string) bool { - // Try "image exists" command (works for podman) cmd := exec.Command(m.cfg.ContainerRuntime, "image", "exists", image) if err := cmd.Run(); err == nil { return true } - // Try "image inspect" (works for both docker and podman) cmd = exec.Command(m.cfg.ContainerRuntime, "image", "inspect", image) if err := cmd.Run(); err == nil { return true } - // Try listing images and grep cmd = exec.Command(m.cfg.ContainerRuntime, "images", "--format", "{{.Repository}}:{{.Tag}}") out, err := cmd.Output() if err == nil { diff --git a/test/loadtest/internal/cmd/root.go b/test/loadtest/internal/cmd/root.go index 406d576..46e9be5 100644 --- a/test/loadtest/internal/cmd/root.go +++ b/test/loadtest/internal/cmd/root.go @@ -1,4 +1,3 @@ -// Package cmd implements the CLI commands for the load test tool. package cmd import ( diff --git a/test/loadtest/internal/prometheus/prometheus.go b/test/loadtest/internal/prometheus/prometheus.go index f16df78..b9bf755 100644 --- a/test/loadtest/internal/prometheus/prometheus.go +++ b/test/loadtest/internal/prometheus/prometheus.go @@ -21,14 +21,14 @@ type Manager struct { manifestPath string portForward *exec.Cmd localPort int - kubeContext string // Optional: use specific kubeconfig context + kubeContext string } // NewManager creates a new Prometheus manager. func NewManager(manifestPath string) *Manager { return &Manager{ manifestPath: manifestPath, - localPort: 9091, // Use 9091 to avoid conflicts + localPort: 9091, } } @@ -51,7 +51,6 @@ func (m *Manager) kubectl(args ...string) []string { // Deploy deploys Prometheus to the cluster. func (m *Manager) Deploy(ctx context.Context) error { - // Create namespace cmd := exec.CommandContext(ctx, "kubectl", m.kubectl("create", "namespace", "monitoring", "--dry-run=client", "-o", "yaml")...) out, err := cmd.Output() if err != nil { @@ -64,7 +63,6 @@ func (m *Manager) Deploy(ctx context.Context) error { return fmt.Errorf("applying namespace: %w", err) } - // Apply Prometheus manifest applyCmd = exec.CommandContext(ctx, "kubectl", m.kubectl("apply", "-f", m.manifestPath)...) applyCmd.Stdout = os.Stdout applyCmd.Stderr = os.Stderr @@ -72,7 +70,6 @@ func (m *Manager) Deploy(ctx context.Context) error { return fmt.Errorf("applying prometheus manifest: %w", err) } - // Wait for Prometheus to be ready fmt.Println("Waiting for Prometheus to be ready...") waitCmd := exec.CommandContext(ctx, "kubectl", m.kubectl("wait", "--for=condition=ready", "pod", "-l", "app=prometheus", "-n", "monitoring", "--timeout=120s")...) @@ -89,7 +86,6 @@ func (m *Manager) Deploy(ctx context.Context) error { func (m *Manager) StartPortForward(ctx context.Context) error { m.StopPortForward() - // Start port-forward m.portForward = exec.CommandContext(ctx, "kubectl", m.kubectl("port-forward", "-n", "monitoring", "svc/prometheus", fmt.Sprintf("%d:9090", m.localPort))...) @@ -97,7 +93,6 @@ func (m *Manager) StartPortForward(ctx context.Context) error { return fmt.Errorf("starting port-forward: %w", err) } - // Wait for port-forward to be ready for i := 0; i < 30; i++ { time.Sleep(time.Second) if m.isAccessible() { @@ -115,7 +110,6 @@ func (m *Manager) StopPortForward() { m.portForward.Process.Kill() m.portForward = nil } - // Also kill any lingering port-forwards exec.Command("pkill", "-f", fmt.Sprintf("kubectl port-forward.*prometheus.*%d", m.localPort)).Run() } @@ -123,12 +117,10 @@ func (m *Manager) StopPortForward() { func (m *Manager) Reset(ctx context.Context) error { m.StopPortForward() - // Delete Prometheus pod to reset metrics cmd := exec.CommandContext(ctx, "kubectl", m.kubectl("delete", "pod", "-n", "monitoring", "-l", "app=prometheus", "--grace-period=0", "--force")...) - cmd.Run() // Ignore errors + cmd.Run() - // Wait for new pod fmt.Println("Waiting for Prometheus to restart...") waitCmd := exec.CommandContext(ctx, "kubectl", m.kubectl("wait", "--for=condition=ready", "pod", "-l", "app=prometheus", "-n", "monitoring", "--timeout=120s")...) @@ -136,12 +128,10 @@ func (m *Manager) Reset(ctx context.Context) error { return fmt.Errorf("waiting for prometheus restart: %w", err) } - // Restart port-forward if err := m.StartPortForward(ctx); err != nil { return err } - // Wait for scraping to initialize fmt.Println("Waiting 5s for Prometheus to initialize scraping...") time.Sleep(5 * time.Second) @@ -155,7 +145,6 @@ func (m *Manager) isAccessible() bool { } conn.Close() - // Also try HTTP resp, err := http.Get(fmt.Sprintf("http://localhost:%d/api/v1/status/config", m.localPort)) if err != nil { return false @@ -186,7 +175,6 @@ func (m *Manager) WaitForTarget(ctx context.Context, job string, timeout time.Du } } - // Print debug info on timeout m.printTargetStatus(job) return fmt.Errorf("timeout waiting for Prometheus to scrape job '%s'", job) } @@ -338,7 +326,6 @@ func (m *Manager) CollectMetrics(ctx context.Context, job, outputDir, scenario s // For S6 (restart scenario), use increase() to handle counter resets useIncrease := scenario == "S6" - // Counter metrics counterMetrics := []string{ "reloader_reconcile_total", "reloader_action_total", @@ -363,7 +350,6 @@ func (m *Manager) CollectMetrics(ctx context.Context, job, outputDir, scenario s } } - // Histogram percentiles histogramMetrics := []struct { name string prefix string @@ -384,7 +370,6 @@ func (m *Manager) CollectMetrics(ctx context.Context, job, outputDir, scenario s } } - // REST client metrics restQueries := map[string]string{ "rest_client_requests_total.json": fmt.Sprintf(`sum(rest_client_requests_total{job="%s"})`, job), "rest_client_requests_get.json": fmt.Sprintf(`sum(rest_client_requests_total{job="%s",method="GET"})`, job), @@ -399,30 +384,23 @@ func (m *Manager) CollectMetrics(ctx context.Context, job, outputDir, scenario s } } - // Resource consumption metrics (memory, CPU, goroutines) resourceQueries := map[string]string{ - // Memory metrics (in bytes) "memory_rss_bytes_avg.json": fmt.Sprintf(`avg_over_time(process_resident_memory_bytes{job="%s"}[%s])`, job, timeRange), "memory_rss_bytes_max.json": fmt.Sprintf(`max_over_time(process_resident_memory_bytes{job="%s"}[%s])`, job, timeRange), "memory_rss_bytes_cur.json": fmt.Sprintf(`process_resident_memory_bytes{job="%s"}`, job), - // Heap memory (Go runtime) "memory_heap_bytes_avg.json": fmt.Sprintf(`avg_over_time(go_memstats_heap_alloc_bytes{job="%s"}[%s])`, job, timeRange), "memory_heap_bytes_max.json": fmt.Sprintf(`max_over_time(go_memstats_heap_alloc_bytes{job="%s"}[%s])`, job, timeRange), - // CPU metrics (rate of CPU seconds used) "cpu_usage_cores_avg.json": fmt.Sprintf(`rate(process_cpu_seconds_total{job="%s"}[%s])`, job, timeRange), "cpu_usage_cores_max.json": fmt.Sprintf(`max_over_time(rate(process_cpu_seconds_total{job="%s"}[1m])[%s:1m])`, job, timeRange), - // Goroutines (concurrency indicator) "goroutines_avg.json": fmt.Sprintf(`avg_over_time(go_goroutines{job="%s"}[%s])`, job, timeRange), "goroutines_max.json": fmt.Sprintf(`max_over_time(go_goroutines{job="%s"}[%s])`, job, timeRange), "goroutines_cur.json": fmt.Sprintf(`go_goroutines{job="%s"}`, job), - // GC metrics "gc_duration_seconds_p99.json": fmt.Sprintf(`histogram_quantile(0.99, sum(rate(go_gc_duration_seconds_bucket{job="%s"}[%s])) by (le))`, job, timeRange), - // Threads "threads_cur.json": fmt.Sprintf(`go_threads{job="%s"}`, job), } @@ -438,7 +416,6 @@ func (m *Manager) CollectMetrics(ctx context.Context, job, outputDir, scenario s func (m *Manager) queryAndSave(ctx context.Context, query, outputPath string) error { result, err := m.Query(ctx, query) if err != nil { - // Write empty result on error emptyResult := `{"status":"success","data":{"resultType":"vector","result":[]}}` return os.WriteFile(outputPath, []byte(emptyResult), 0644) } diff --git a/test/loadtest/internal/scenarios/scenarios.go b/test/loadtest/internal/scenarios/scenarios.go index 794f1a6..4909feb 100644 --- a/test/loadtest/internal/scenarios/scenarios.go +++ b/test/loadtest/internal/scenarios/scenarios.go @@ -197,7 +197,6 @@ func (s *FanOutScenario) Run(ctx context.Context, client kubernetes.Interface, n log.Println("S2: Updating shared ConfigMap...") - // Check context state before starting update loop if ctx.Err() != nil { log.Printf("S2: WARNING - Context already done before update loop: %v", ctx.Err()) } @@ -503,14 +502,7 @@ func (s *WorkloadChurnScenario) Run(ctx context.Context, client kubernetes.Inter wg.Wait() log.Printf("S5: Created %d, deleted %d deployments, %d CM updates", deployCounter, deleteCounter, cmUpdateCount) - // S5 does NOT set expected values for action_total/reload_executed_total because: - // - There are ~10 active deployments at any time (creates new, deletes old) - // - Each CM update triggers reloads on ALL active deployments - // - Exact counts depend on timing of creates/deletes vs CM updates - // - "Not found" errors are expected when a deployment is deleted during processing - // Instead, S5 pass/fail compares old vs new (both should be similar) return ExpectedMetrics{ - // No expected values - churn makes exact counts unpredictable Description: fmt.Sprintf("S5: Churn test - %d deploys created, %d deleted, %d CM updates, ~10 active deploys at any time", deployCounter, deleteCounter, cmUpdateCount), }, nil } @@ -765,8 +757,6 @@ func (s *LargeObjectScenario) Run(ctx context.Context, client kubernetes.Interfa }, nil } -// Helper functions - func waitForDeploymentsReady(ctx context.Context, client kubernetes.Interface, namespace string, timeout time.Duration) error { log.Printf("Waiting for all deployments in %s to be ready (timeout: %v)...", namespace, timeout) @@ -1051,7 +1041,6 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int setupCtx := context.Background() - // Create Secrets for i := 0; i < numSecrets; i++ { secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -1067,7 +1056,6 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int } } - // Create ConfigMaps for i := 0; i < numConfigMaps; i++ { cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -1083,7 +1071,6 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int } } - // Create Secret-only deployments for i := 0; i < numSecretOnlyDeploys; i++ { deploy := createDeploymentWithSecret( fmt.Sprintf("secret-only-deploy-%d", i), @@ -1095,7 +1082,6 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int } } - // Create ConfigMap-only deployments for i := 0; i < numConfigMapOnlyDeploys; i++ { deploy := createDeployment( fmt.Sprintf("cm-only-deploy-%d", i), @@ -1107,7 +1093,6 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int } } - // Create mixed deployments (using both Secret and ConfigMap) for i := 0; i < numMixedDeploys; i++ { deploy := createDeploymentWithBoth( fmt.Sprintf("mixed-deploy-%d", i), @@ -1131,7 +1116,7 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int ticker := time.NewTicker(500 * time.Millisecond) defer ticker.Stop() - updateSecret := true // Alternate between Secret and ConfigMap updates + updateSecret := true endTime := time.Now().Add(duration - 5*time.Second) for time.Now().Before(endTime) { @@ -1140,7 +1125,6 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int return s.calculateExpected(secretUpdateCount, cmUpdateCount, numSecrets, numConfigMaps, numSecretOnlyDeploys, numConfigMapOnlyDeploys, numMixedDeploys), nil case <-ticker.C: if updateSecret { - // Update a random Secret secretIndex := rand.Intn(numSecrets) secret, err := client.CoreV1().Secrets(namespace).Get(setupCtx, fmt.Sprintf("mixed-secret-%d", secretIndex), metav1.GetOptions{}) if err != nil { @@ -1153,7 +1137,6 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int secretUpdateCount++ } } else { - // Update a random ConfigMap cmIndex := rand.Intn(numConfigMaps) cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("mixed-cm-%d", cmIndex), metav1.GetOptions{}) if err != nil { @@ -1173,11 +1156,9 @@ func (s *SecretsAndMixedScenario) Run(ctx context.Context, client kubernetes.Int } func (s *SecretsAndMixedScenario) calculateExpected(secretUpdates, cmUpdates, numSecrets, numConfigMaps, secretOnlyDeploys, cmOnlyDeploys, mixedDeploys int) ExpectedMetrics { - // Average deploys triggered per random secret update avgSecretReloads := float64(secretOnlyDeploys)/float64(numSecrets) + float64(mixedDeploys)/float64(numSecrets) secretTriggeredReloads := int(float64(secretUpdates) * avgSecretReloads) - // Average deploys triggered per random CM update avgCMReloads := float64(cmOnlyDeploys)/float64(numConfigMaps) + float64(mixedDeploys)/float64(numConfigMaps) cmTriggeredReloads := int(float64(cmUpdates) * avgCMReloads) @@ -1208,7 +1189,6 @@ func (s *MultiWorkloadTypeScenario) Run(ctx context.Context, client kubernetes.I setupCtx := context.Background() - // Create shared ConfigMap cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "multi-type-cm", @@ -1220,7 +1200,6 @@ func (s *MultiWorkloadTypeScenario) Run(ctx context.Context, client kubernetes.I return ExpectedMetrics{}, fmt.Errorf("failed to create shared ConfigMap: %w", err) } - // Create Deployments for i := 0; i < numDeployments; i++ { deploy := createDeployment(fmt.Sprintf("multi-deploy-%d", i), namespace, "multi-type-cm") if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { @@ -1228,7 +1207,6 @@ func (s *MultiWorkloadTypeScenario) Run(ctx context.Context, client kubernetes.I } } - // Create StatefulSets for i := 0; i < numStatefulSets; i++ { sts := createStatefulSet(fmt.Sprintf("multi-sts-%d", i), namespace, "multi-type-cm") if _, err := client.AppsV1().StatefulSets(namespace).Create(setupCtx, sts, metav1.CreateOptions{}); err != nil { @@ -1236,7 +1214,6 @@ func (s *MultiWorkloadTypeScenario) Run(ctx context.Context, client kubernetes.I } } - // Create DaemonSets for i := 0; i < numDaemonSets; i++ { ds := createDaemonSet(fmt.Sprintf("multi-ds-%d", i), namespace, "multi-type-cm") if _, err := client.AppsV1().DaemonSets(namespace).Create(setupCtx, ds, metav1.CreateOptions{}); err != nil { @@ -1244,7 +1221,6 @@ func (s *MultiWorkloadTypeScenario) Run(ctx context.Context, client kubernetes.I } } - // Wait for workloads to be ready if err := waitForDeploymentsReady(setupCtx, client, namespace, 3*time.Minute); err != nil { log.Printf("Warning: %v - continuing anyway", err) } @@ -1286,7 +1262,6 @@ func (s *MultiWorkloadTypeScenario) Run(ctx context.Context, client kubernetes.I } func (s *MultiWorkloadTypeScenario) calculateExpected(updateCount, numDeployments, numStatefulSets, numDaemonSets int) ExpectedMetrics { - // Each CM update triggers reload on all workloads totalWorkloads := numDeployments + numStatefulSets + numDaemonSets expectedReloads := updateCount * totalWorkloads @@ -1376,7 +1351,6 @@ func createDaemonSet(name, namespace, configMapName string) *appsv1.DaemonSet { }, Spec: corev1.PodSpec{ TerminationGracePeriodSeconds: &terminationGracePeriod, - // Use tolerations to run on all nodes including control-plane Tolerations: []corev1.Toleration{ { Key: "node-role.kubernetes.io/control-plane", @@ -1509,7 +1483,6 @@ func (s *ComplexReferencesScenario) Run(ctx context.Context, client kubernetes.I setupCtx := context.Background() - // Create ConfigMaps with multiple keys for i := 0; i < numConfigMaps; i++ { cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -1527,9 +1500,7 @@ func (s *ComplexReferencesScenario) Run(ctx context.Context, client kubernetes.I } } - // Create complex deployments with various reference types for i := 0; i < numDeployments; i++ { - // Each deployment references multiple ConfigMaps in different ways primaryCM := fmt.Sprintf("complex-cm-%d", i) secondaryCM := fmt.Sprintf("complex-cm-%d", (i+1)%numConfigMaps) @@ -1560,7 +1531,6 @@ func (s *ComplexReferencesScenario) Run(ctx context.Context, client kubernetes.I case <-ctx.Done(): return s.calculateExpected(updateCount, numConfigMaps, numDeployments), nil case <-ticker.C: - // Update a random ConfigMap cmIndex := rand.Intn(numConfigMaps) cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("complex-cm-%d", cmIndex), metav1.GetOptions{}) if err != nil { @@ -1582,11 +1552,6 @@ func (s *ComplexReferencesScenario) Run(ctx context.Context, client kubernetes.I } func (s *ComplexReferencesScenario) calculateExpected(updateCount, numConfigMaps, numDeployments int) ExpectedMetrics { - // Each ConfigMap is referenced by: - // - 1 deployment as primary (envFrom in init + valueFrom in main + volume mount) - // - 1 deployment as secondary (projected volume) - // So each CM update triggers 2 deployments (on average with random updates) - // But since we're randomly updating, each update affects those 2 deployments expectedReloadsPerUpdate := 2 expectedReloads := updateCount * expectedReloadsPerUpdate @@ -1616,7 +1581,6 @@ func (s *PauseResumeScenario) Run(ctx context.Context, client kubernetes.Interfa setupCtx := context.Background() - // Create ConfigMaps for i := 0; i < numConfigMaps; i++ { cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -1630,7 +1594,6 @@ func (s *PauseResumeScenario) Run(ctx context.Context, client kubernetes.Interfa } } - // Create Deployments with pause-period annotation for i := 0; i < numDeployments; i++ { deploy := createDeploymentWithPause( fmt.Sprintf("pause-deploy-%d", i), @@ -1659,7 +1622,6 @@ func (s *PauseResumeScenario) Run(ctx context.Context, client kubernetes.Interfa case <-ctx.Done(): return s.calculateExpected(updateCount, duration, updateInterval, pausePeriod), nil case <-ticker.C: - // Update a random ConfigMap cmIndex := rand.Intn(numConfigMaps) cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("pause-cm-%d", cmIndex), metav1.GetOptions{}) if err != nil { @@ -1679,12 +1641,6 @@ func (s *PauseResumeScenario) Run(ctx context.Context, client kubernetes.Interfa } func (s *PauseResumeScenario) calculateExpected(updateCount int, duration, updateInterval, pausePeriod time.Duration) ExpectedMetrics { - // With pause-period, we expect fewer reloads than updates - // Each deployment gets updates at random, and pause-period prevents rapid consecutive reloads - // The exact count depends on the distribution of updates across ConfigMaps - // Rough estimate: each CM gets updated ~(updateCount/10) times - // With 15s pause and 2s interval, we get roughly 1 reload per pause period per CM - // So expected reloads ā‰ˆ duration / pausePeriod per deployment = (duration/pausePeriod) * numDeployments // This is an approximation - the actual value depends on random distribution expectedCycles := int(duration / pausePeriod) @@ -1693,8 +1649,6 @@ func (s *PauseResumeScenario) calculateExpected(updateCount int, duration, updat } return ExpectedMetrics{ - // Don't set exact expected values since pause-period makes counts unpredictable - // The scenario validates that reloads << updates due to pause behavior Description: fmt.Sprintf("S12: %d updates with %v pause-period (expect ~%d reload cycles, actual reloads << updates)", updateCount, pausePeriod, expectedCycles), } @@ -1703,7 +1657,6 @@ func (s *PauseResumeScenario) calculateExpected(updateCount int, duration, updat // AnnotationStrategyScenario - Tests annotation-based reload strategy. // This scenario deploys its own Reloader instance with --reload-strategy=annotations. type AnnotationStrategyScenario struct { - // Image is the Reloader image to use. Must be set before running. Image string } @@ -1719,7 +1672,6 @@ func (s *AnnotationStrategyScenario) Run(ctx context.Context, client kubernetes. log.Println("S11: Deploying Reloader with --reload-strategy=annotations...") - // Deploy S11's own Reloader instance reloaderNS := "reloader-s11" mgr := reloader.NewManager(reloader.Config{ Version: "s11", @@ -1732,7 +1684,6 @@ func (s *AnnotationStrategyScenario) Run(ctx context.Context, client kubernetes. return ExpectedMetrics{}, fmt.Errorf("deploying S11 reloader: %w", err) } - // Ensure cleanup on exit defer func() { log.Println("S11: Cleaning up S11-specific Reloader...") cleanupCtx := context.Background() @@ -1748,7 +1699,6 @@ func (s *AnnotationStrategyScenario) Run(ctx context.Context, client kubernetes. setupCtx := context.Background() - // Create ConfigMaps for i := 0; i < numConfigMaps; i++ { cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -1762,7 +1712,6 @@ func (s *AnnotationStrategyScenario) Run(ctx context.Context, client kubernetes. } } - // Create Deployments for i := 0; i < numDeployments; i++ { deploy := createDeployment(fmt.Sprintf("annot-deploy-%d", i), namespace, fmt.Sprintf("annot-cm-%d", i)) if _, err := client.AppsV1().Deployments(namespace).Create(setupCtx, deploy, metav1.CreateOptions{}); err != nil { @@ -1781,13 +1730,12 @@ func (s *AnnotationStrategyScenario) Run(ctx context.Context, client kubernetes. ticker := time.NewTicker(500 * time.Millisecond) defer ticker.Stop() - endTime := time.Now().Add(duration - 10*time.Second) // Extra time for cleanup + endTime := time.Now().Add(duration - 10*time.Second) for time.Now().Before(endTime) { select { case <-ctx.Done(): return s.calculateExpected(updateCount, annotationUpdatesSeen), nil case <-ticker.C: - // Update a random ConfigMap cmIndex := rand.Intn(numConfigMaps) cm, err := client.CoreV1().ConfigMaps(namespace).Get(setupCtx, fmt.Sprintf("annot-cm-%d", cmIndex), metav1.GetOptions{}) if err != nil { @@ -1800,7 +1748,6 @@ func (s *AnnotationStrategyScenario) Run(ctx context.Context, client kubernetes. updateCount++ } - // Periodically check for annotation updates on deployments if updateCount%10 == 0 { deploy, err := client.AppsV1().Deployments(namespace).Get(setupCtx, fmt.Sprintf("annot-deploy-%d", cmIndex), metav1.GetOptions{}) if err == nil { @@ -1812,9 +1759,8 @@ func (s *AnnotationStrategyScenario) Run(ctx context.Context, client kubernetes. } } - // Final check: verify annotation strategy is working log.Println("S11: Verifying annotation-based reload...") - time.Sleep(5 * time.Second) // Allow time for final updates to propagate + time.Sleep(5 * time.Second) deploysWithAnnotation := 0 for i := 0; i < numDeployments; i++ { @@ -1945,7 +1891,6 @@ func createComplexDeployment(name, namespace, primaryCM, secondaryCM string) *ap }, Spec: corev1.PodSpec{ TerminationGracePeriodSeconds: &terminationGracePeriod, - // Init container using envFrom InitContainers: []corev1.Container{ { Name: "init", @@ -1973,7 +1918,6 @@ func createComplexDeployment(name, namespace, primaryCM, secondaryCM string) *ap }, }, Containers: []corev1.Container{ - // Main container using valueFrom (individual keys) { Name: "main", Image: "gcr.io/google-containers/busybox:1.27", @@ -2013,7 +1957,6 @@ func createComplexDeployment(name, namespace, primaryCM, secondaryCM string) *ap }, }, }, - // Sidecar using volume mount { Name: "sidecar", Image: "gcr.io/google-containers/busybox:1.27", @@ -2041,7 +1984,6 @@ func createComplexDeployment(name, namespace, primaryCM, secondaryCM string) *ap }, }, Volumes: []corev1.Volume{ - // Regular ConfigMap volume { Name: "config-volume", VolumeSource: corev1.VolumeSource{ @@ -2052,7 +1994,6 @@ func createComplexDeployment(name, namespace, primaryCM, secondaryCM string) *ap }, }, }, - // Projected volume combining multiple ConfigMaps { Name: "projected-volume", VolumeSource: corev1.VolumeSource{ From 16ff7f6ac972eb01577fc96d37dce3d2eb8febe7 Mon Sep 17 00:00:00 2001 From: TheiLLeniumStudios <104288623+TheiLLeniumStudios@users.noreply.github.com> Date: Fri, 9 Jan 2026 15:27:19 +0100 Subject: [PATCH 45/45] fix: Default reconcile metric result to error for panic safety --- internal/pkg/handler/create.go | 33 +++++++++++++------------- internal/pkg/handler/delete.go | 33 +++++++++++++------------- internal/pkg/handler/update.go | 42 +++++++++++++++++----------------- 3 files changed, 53 insertions(+), 55 deletions(-) diff --git a/internal/pkg/handler/create.go b/internal/pkg/handler/create.go index 5fd3014..d676610 100644 --- a/internal/pkg/handler/create.go +++ b/internal/pkg/handler/create.go @@ -27,7 +27,7 @@ func (r ResourceCreatedHandler) GetEnqueueTime() time.Time { // Handle processes the newly created resource func (r ResourceCreatedHandler) Handle() error { startTime := time.Now() - result := "success" + result := "error" defer func() { r.Collectors.RecordReconcile(result, time.Since(startTime)) @@ -35,25 +35,24 @@ func (r ResourceCreatedHandler) Handle() error { if r.Resource == nil { logrus.Errorf("Resource creation handler received nil resource") - result = "error" - } else { - config, _ := r.GetConfig() - // Send webhook - if options.WebhookUrl != "" { - err := sendUpgradeWebhook(config, options.WebhookUrl) - if err != nil { - result = "error" - } - return err - } - // process resource based on its type - err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy) - if err != nil { - result = "error" + return nil + } + + config, _ := r.GetConfig() + // Send webhook + if options.WebhookUrl != "" { + err := sendUpgradeWebhook(config, options.WebhookUrl) + if err == nil { + result = "success" } return err } - return nil + // process resource based on its type + err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy) + if err == nil { + result = "success" + } + return err } // GetConfig gets configurations containing SHA, annotations, namespace and resource name diff --git a/internal/pkg/handler/delete.go b/internal/pkg/handler/delete.go index 243602c..34e032b 100644 --- a/internal/pkg/handler/delete.go +++ b/internal/pkg/handler/delete.go @@ -35,7 +35,7 @@ func (r ResourceDeleteHandler) GetEnqueueTime() time.Time { // Handle processes resources being deleted func (r ResourceDeleteHandler) Handle() error { startTime := time.Now() - result := "success" + result := "error" defer func() { r.Collectors.RecordReconcile(result, time.Since(startTime)) @@ -43,25 +43,24 @@ func (r ResourceDeleteHandler) Handle() error { if r.Resource == nil { logrus.Errorf("Resource delete handler received nil resource") - result = "error" - } else { - config, _ := r.GetConfig() - // Send webhook - if options.WebhookUrl != "" { - err := sendUpgradeWebhook(config, options.WebhookUrl) - if err != nil { - result = "error" - } - return err - } - // process resource based on its type - err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeDeleteStrategy) - if err != nil { - result = "error" + return nil + } + + config, _ := r.GetConfig() + // Send webhook + if options.WebhookUrl != "" { + err := sendUpgradeWebhook(config, options.WebhookUrl) + if err == nil { + result = "success" } return err } - return nil + // process resource based on its type + err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeDeleteStrategy) + if err == nil { + result = "success" + } + return err } // GetConfig gets configurations containing SHA, annotations, namespace and resource name diff --git a/internal/pkg/handler/update.go b/internal/pkg/handler/update.go index 1d0403b..3fde98e 100644 --- a/internal/pkg/handler/update.go +++ b/internal/pkg/handler/update.go @@ -30,7 +30,7 @@ func (r ResourceUpdatedHandler) GetEnqueueTime() time.Time { // Handle processes the updated resource func (r ResourceUpdatedHandler) Handle() error { startTime := time.Now() - result := "success" + result := "error" defer func() { r.Collectors.RecordReconcile(result, time.Since(startTime)) @@ -38,30 +38,30 @@ func (r ResourceUpdatedHandler) Handle() error { if r.Resource == nil || r.OldResource == nil { logrus.Errorf("Resource update handler received nil resource") - result = "error" - } else { - config, oldSHAData := r.GetConfig() - if config.SHAValue != oldSHAData { - // Send a webhook if update - if options.WebhookUrl != "" { - err := sendUpgradeWebhook(config, options.WebhookUrl) - if err != nil { - result = "error" - } - return err - } - // process resource based on its type - err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy) - if err != nil { - result = "error" + return nil + } + + config, oldSHAData := r.GetConfig() + if config.SHAValue != oldSHAData { + // Send a webhook if update + if options.WebhookUrl != "" { + err := sendUpgradeWebhook(config, options.WebhookUrl) + if err == nil { + result = "success" } return err - } else { - // No data change - skip - result = "skipped" - r.Collectors.RecordSkipped("no_data_change") } + // process resource based on its type + err := doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy) + if err == nil { + result = "success" + } + return err } + + // No data change - skip + result = "skipped" + r.Collectors.RecordSkipped("no_data_change") return nil }