diff --git a/cmd/reloader/main.go b/cmd/reloader/main.go index 83b78c4d..35151f9b 100644 --- a/cmd/reloader/main.go +++ b/cmd/reloader/main.go @@ -19,6 +19,7 @@ import ( "github.com/stakater/Reloader/internal/pkg/config" "github.com/stakater/Reloader/internal/pkg/controller" + "github.com/stakater/Reloader/internal/pkg/csi" "github.com/stakater/Reloader/internal/pkg/metadata" "github.com/stakater/Reloader/internal/pkg/metrics" "github.com/stakater/Reloader/internal/pkg/openshift" @@ -108,17 +109,30 @@ func run(cmd *cobra.Command, args []string) error { collectors := metrics.SetupPrometheusEndpoint() + restConfig := controllerruntime.GetConfigOrDie() + discoveryClient, discErr := discovery.NewDiscoveryClientForConfig(restConfig) + if discErr != nil { + log.V(1).Info("Failed to create discovery client", "error", discErr) + } + if config.ShouldAutoDetectOpenShift() { - restConfig := controllerruntime.GetConfigOrDie() - discoveryClient, err := discovery.NewDiscoveryClientForConfig(restConfig) - if err != nil { - log.V(1).Info("Failed to create discovery client for DeploymentConfig detection", "error", err) - } else if openshift.HasDeploymentConfigSupport(discoveryClient, log) { + if discoveryClient != nil && openshift.HasDeploymentConfigSupport(discoveryClient, log) { cfg.DeploymentConfigEnabled = true } } - controller.AddOptionalSchemes(cfg.ArgoRolloutsEnabled, cfg.DeploymentConfigEnabled) + // CSI: require both the flag AND CRD presence (parity with master's + // shouldRunCSIController). If the flag is set but CRDs are missing, log and + // disable so the watch is not added (controller-runtime would crash on an + // absent CRD). + if cfg.CSIIntegrationEnabled { + if discoveryClient == nil || !csi.HasCSISupport(discoveryClient, log) { + log.Info("Disabling CSI integration: secrets-store CSI driver CRDs not detected") + cfg.CSIIntegrationEnabled = false + } + } + + controller.AddOptionalSchemes(cfg.ArgoRolloutsEnabled, cfg.DeploymentConfigEnabled, cfg.CSIIntegrationEnabled) mgr, err := controller.NewManager( controller.ManagerOptions{ diff --git a/deployments/kubernetes/chart/reloader/templates/clusterrole.yaml b/deployments/kubernetes/chart/reloader/templates/clusterrole.yaml index 11e8a5d4..8f31764f 100644 --- a/deployments/kubernetes/chart/reloader/templates/clusterrole.yaml +++ b/deployments/kubernetes/chart/reloader/templates/clusterrole.yaml @@ -110,6 +110,17 @@ rules: - create - get - update +{{- end}} +{{- if .Values.reloader.enableCSIIntegration }} + - apiGroups: + - "secrets-store.csi.x-k8s.io" + resources: + - secretproviderclasspodstatuses + - secretproviderclasses + verbs: + - list + - get + - watch {{- end}} - apiGroups: - "" diff --git a/deployments/kubernetes/chart/reloader/templates/deployment.yaml b/deployments/kubernetes/chart/reloader/templates/deployment.yaml index c1b14502..4b0e9e57 100644 --- a/deployments/kubernetes/chart/reloader/templates/deployment.yaml +++ b/deployments/kubernetes/chart/reloader/templates/deployment.yaml @@ -215,7 +215,7 @@ spec: {{- . | toYaml | nindent 10 }} {{- end }} {{- end }} - {{- if or (.Values.reloader.logFormat) (.Values.reloader.logLevel) (.Values.reloader.ignoreSecrets) (.Values.reloader.ignoreNamespaces) (include "reloader-namespaceSelector" .) (.Values.reloader.resourceLabelSelector) (.Values.reloader.ignoreConfigMaps) (.Values.reloader.custom_annotations) (eq .Values.reloader.isArgoRollouts true) (eq .Values.reloader.reloadOnCreate true) (eq .Values.reloader.reloadOnDelete true) (ne .Values.reloader.reloadStrategy "default") (.Values.reloader.enableHA) (.Values.reloader.autoReloadAll) (.Values.reloader.ignoreJobs) (.Values.reloader.ignoreCronJobs)}} + {{- if or (.Values.reloader.logFormat) (.Values.reloader.logLevel) (.Values.reloader.ignoreSecrets) (.Values.reloader.ignoreNamespaces) (include "reloader-namespaceSelector" .) (.Values.reloader.resourceLabelSelector) (.Values.reloader.ignoreConfigMaps) (.Values.reloader.custom_annotations) (eq .Values.reloader.isArgoRollouts true) (eq .Values.reloader.reloadOnCreate true) (eq .Values.reloader.reloadOnDelete true) (ne .Values.reloader.reloadStrategy "default") (.Values.reloader.enableHA) (.Values.reloader.autoReloadAll) (.Values.reloader.ignoreJobs) (.Values.reloader.ignoreCronJobs) (.Values.reloader.enableCSIIntegration)}} args: {{- if .Values.reloader.logFormat }} - "--log-format={{ .Values.reloader.logFormat }}" @@ -251,6 +251,9 @@ spec: - "--pprof-addr={{ .Values.reloader.pprofAddr }}" {{- end }} {{- end }} + {{- if .Values.reloader.enableCSIIntegration }} + - "--enable-csi-integration" + {{- end }} {{- if .Values.reloader.custom_annotations }} {{- if .Values.reloader.custom_annotations.configmap }} - "--configmap-annotation" diff --git a/deployments/kubernetes/chart/reloader/templates/role.yaml b/deployments/kubernetes/chart/reloader/templates/role.yaml index c6cfed64..495a3855 100644 --- a/deployments/kubernetes/chart/reloader/templates/role.yaml +++ b/deployments/kubernetes/chart/reloader/templates/role.yaml @@ -97,6 +97,17 @@ rules: - create - get - update +{{- end}} +{{- if .Values.reloader.enableCSIIntegration }} + - apiGroups: + - "secrets-store.csi.x-k8s.io" + resources: + - secretproviderclasspodstatuses + - secretproviderclasses + verbs: + - list + - get + - watch {{- end}} - apiGroups: - "" diff --git a/deployments/kubernetes/chart/reloader/values.yaml b/deployments/kubernetes/chart/reloader/values.yaml index 1e57c5d6..bbd28800 100644 --- a/deployments/kubernetes/chart/reloader/values.yaml +++ b/deployments/kubernetes/chart/reloader/values.yaml @@ -49,6 +49,8 @@ reloader: enableHA: false # Set to true to enable pprof for profiling enablePProf: false + # Set to true to enable CSI / SecretProviderClass integration + enableCSIIntegration: false # Address to start pprof server on. Default is ":6060" pprofAddr: ":6060" # Set to true if you have a pod security policy that enforces readOnlyRootFilesystem diff --git a/go.mod b/go.mod index e8703a8d..a6d752df 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( k8s.io/client-go v0.36.0 k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 sigs.k8s.io/controller-runtime v0.24.1 + sigs.k8s.io/secrets-store-csi-driver v1.5.5 ) require ( @@ -80,7 +81,6 @@ require ( github.com/dlclark/regexp2 v1.11.5 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/ettle/strcase v0.2.0 // indirect - github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/fatih/color v1.18.0 // indirect github.com/fatih/structtag v1.2.0 // indirect @@ -135,6 +135,7 @@ require ( github.com/google/pprof v0.0.0-20260106004452-d7df1bf2cac7 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gordonklaus/ineffassign v0.2.0 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/gostaticanalysis/analysisutil v0.7.1 // indirect github.com/gostaticanalysis/comment v1.5.0 // indirect github.com/gostaticanalysis/forcetypeassert v0.2.0 // indirect @@ -177,6 +178,7 @@ require ( github.com/mgechev/revive v1.13.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/spdystream v0.5.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect @@ -268,6 +270,7 @@ require ( k8s.io/apiextensions-apiserver v0.36.0 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31 // indirect + k8s.io/streaming v0.36.0 // indirect mvdan.cc/gofumpt v0.9.2 // indirect mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect diff --git a/go.sum b/go.sum index 458c1664..be593f22 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,8 @@ github.com/alingse/nilnesserr v0.2.0 h1:raLem5KG7EFVb4UIDAXgrv3N2JIaffeKNtcEXkEW github.com/alingse/nilnesserr v0.2.0/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg= github.com/argoproj/argo-rollouts v1.9.0 h1:bXgBpwCByXyAUcgBnyP0fxkSW2CEot78InTFjFlag5g= github.com/argoproj/argo-rollouts v1.9.0/go.mod h1:jOalqf2kDSmCp7eQpFF4i3kHnlEqNE/Yjwz1q7CpPIU= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/ashanbrown/forbidigo/v2 v2.3.0 h1:OZZDOchCgsX5gvToVtEBoV2UWbFfI6RKQTir2UZzSxo= github.com/ashanbrown/forbidigo/v2 v2.3.0/go.mod h1:5p6VmsG5/1xx3E785W9fouMxIOkvY2rRV9nMdWadd6c= github.com/ashanbrown/makezero/v2 v2.1.0 h1:snuKYMbqosNokUKm+R6/+vOPs8yVAi46La7Ck6QYSaE= @@ -266,6 +268,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gordonklaus/ineffassign v0.2.0 h1:Uths4KnmwxNJNzq87fwQQDDnbNb7De00VOk9Nu0TySs= github.com/gordonklaus/ineffassign v0.2.0/go.mod h1:TIpymnagPSexySzs7F9FnO1XFTy8IT3a59vmZp5Y9Lw= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/gostaticanalysis/analysisutil v0.7.1 h1:ZMCjoue3DtDWQ5WyU16YbjbQEQ3VuzwxALrpYd+HeKk= github.com/gostaticanalysis/analysisutil v0.7.1/go.mod h1:v21E3hY37WKMGSnbsw2S/ojApNWb6C1//mXO48CXbVc= github.com/gostaticanalysis/comment v1.4.2/go.mod h1:KLUTGDv6HOCotCH8h2erHKmpci2ZoR8VPu34YA2uzdM= @@ -378,6 +382,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= +github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -737,6 +743,8 @@ k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31 h1:V+sn9a/1fEYDGwnllCmqXBk8x7obZ+hl869Q3Abumkg= k8s.io/kube-openapi v0.0.0-20260330154417-16be699c7b31/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/streaming v0.36.0 h1:agnTxU+NFulUrtYzXUGKO3ndEa8jKwht1Kwn9nu9x+4= +k8s.io/streaming v0.36.0/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 h1:kBawHLSnx/mYHmRnNUf9d4CpjREbeZuxoSGOX/J+aYM= k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= mvdan.cc/gofumpt v0.9.2 h1:zsEMWL8SVKGHNztrx6uZrXdp7AX8r421Vvp23sz7ik4= @@ -757,6 +765,8 @@ sigs.k8s.io/kustomize/kyaml v0.20.1 h1:PCMnA2mrVbRP3NIB6v9kYCAc38uvFLVs8j/CD567A sigs.k8s.io/kustomize/kyaml v0.20.1/go.mod h1:0EmkQHRUsJxY8Ug9Niig1pUMSCGHxQ5RklbpV/Ri6po= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/secrets-store-csi-driver v1.5.5 h1:LJDpDL5TILhlP68nGvtGSlJFxSDgAD2m148NT0Ts7os= +sigs.k8s.io/secrets-store-csi-driver v1.5.5/go.mod h1:i2WqLicYH00hrTG3JAzICPMF4HL4KMEORlDt9UQoZLk= sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index c33b78ad..b63611e7 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -32,6 +32,7 @@ type Config struct { ArgoRolloutsEnabled bool `json:"argoRolloutsEnabled"` ArgoRolloutStrategy ArgoRolloutStrategy `json:"argoRolloutStrategy"` DeploymentConfigEnabled bool `json:"deploymentConfigEnabled"` + CSIIntegrationEnabled bool `json:"csiIntegrationEnabled"` ReloadOnCreate bool `json:"reloadOnCreate"` ReloadOnDelete bool `json:"reloadOnDelete"` SyncAfterRestart bool `json:"syncAfterRestart"` @@ -61,21 +62,24 @@ type Config struct { // AnnotationConfig holds customizable annotation keys. type AnnotationConfig struct { - Prefix string `json:"prefix"` - Auto string `json:"auto"` - ConfigmapAuto string `json:"configmapAuto"` - SecretAuto string `json:"secretAuto"` - ConfigmapReload string `json:"configmapReload"` - SecretReload string `json:"secretReload"` - ConfigmapExclude string `json:"configmapExclude"` - SecretExclude string `json:"secretExclude"` - Ignore string `json:"ignore"` - Search string `json:"search"` - Match string `json:"match"` - RolloutStrategy string `json:"rolloutStrategy"` - PausePeriod string `json:"pausePeriod"` - PausedAt string `json:"pausedAt"` - LastReloadedFrom string `json:"lastReloadedFrom"` + Prefix string `json:"prefix"` + Auto string `json:"auto"` + ConfigmapAuto string `json:"configmapAuto"` + SecretAuto string `json:"secretAuto"` + ConfigmapReload string `json:"configmapReload"` + SecretReload string `json:"secretReload"` + ConfigmapExclude string `json:"configmapExclude"` + SecretExclude string `json:"secretExclude"` + SecretProviderClassAuto string `json:"secretProviderClassAuto"` + SecretProviderClassReload string `json:"secretProviderClassReload"` + SecretProviderClassExclude string `json:"secretProviderClassExclude"` + Ignore string `json:"ignore"` + Search string `json:"search"` + Match string `json:"match"` + RolloutStrategy string `json:"rolloutStrategy"` + PausePeriod string `json:"pausePeriod"` + PausedAt string `json:"pausedAt"` + LastReloadedFrom string `json:"lastReloadedFrom"` } // AlertingConfig holds configuration for alerting integrations. @@ -108,6 +112,7 @@ func NewDefault() *Config { ArgoRolloutsEnabled: false, ArgoRolloutStrategy: ArgoRolloutStrategyRollout, DeploymentConfigEnabled: false, + CSIIntegrationEnabled: false, ReloadOnCreate: false, ReloadOnDelete: false, SyncAfterRestart: false, @@ -140,21 +145,24 @@ func NewDefault() *Config { // DefaultAnnotations returns the default annotation configuration. func DefaultAnnotations() AnnotationConfig { return AnnotationConfig{ - Prefix: "reloader.stakater.com", - Auto: "reloader.stakater.com/auto", - ConfigmapAuto: "configmap.reloader.stakater.com/auto", - SecretAuto: "secret.reloader.stakater.com/auto", - ConfigmapReload: "configmap.reloader.stakater.com/reload", - SecretReload: "secret.reloader.stakater.com/reload", - ConfigmapExclude: "configmaps.exclude.reloader.stakater.com/reload", - SecretExclude: "secrets.exclude.reloader.stakater.com/reload", - Ignore: "reloader.stakater.com/ignore", - Search: "reloader.stakater.com/search", - Match: "reloader.stakater.com/match", - RolloutStrategy: "reloader.stakater.com/rollout-strategy", - PausePeriod: "deployment.reloader.stakater.com/pause-period", - PausedAt: "deployment.reloader.stakater.com/paused-at", - LastReloadedFrom: "reloader.stakater.com/last-reloaded-from", + Prefix: "reloader.stakater.com", + Auto: "reloader.stakater.com/auto", + ConfigmapAuto: "configmap.reloader.stakater.com/auto", + SecretAuto: "secret.reloader.stakater.com/auto", + ConfigmapReload: "configmap.reloader.stakater.com/reload", + SecretReload: "secret.reloader.stakater.com/reload", + ConfigmapExclude: "configmaps.exclude.reloader.stakater.com/reload", + SecretExclude: "secrets.exclude.reloader.stakater.com/reload", + SecretProviderClassAuto: "secretproviderclass.reloader.stakater.com/auto", + SecretProviderClassReload: "secretproviderclass.reloader.stakater.com/reload", + SecretProviderClassExclude: "secretproviderclasses.exclude.reloader.stakater.com/reload", + Ignore: "reloader.stakater.com/ignore", + Search: "reloader.stakater.com/search", + Match: "reloader.stakater.com/match", + RolloutStrategy: "reloader.stakater.com/rollout-strategy", + PausePeriod: "deployment.reloader.stakater.com/pause-period", + PausedAt: "deployment.reloader.stakater.com/paused-at", + LastReloadedFrom: "reloader.stakater.com/last-reloaded-from", } } diff --git a/internal/pkg/config/config_test.go b/internal/pkg/config/config_test.go index f117ad60..bc6ea0a1 100644 --- a/internal/pkg/config/config_test.go +++ b/internal/pkg/config/config_test.go @@ -173,6 +173,25 @@ func TestConfig_IsWorkloadIgnored(t *testing.T) { } } +func TestDefaultAnnotationsSecretProviderClass(t *testing.T) { + a := DefaultAnnotations() + if a.SecretProviderClassAuto != "secretproviderclass.reloader.stakater.com/auto" { + t.Fatalf("SecretProviderClassAuto = %q", a.SecretProviderClassAuto) + } + if a.SecretProviderClassReload != "secretproviderclass.reloader.stakater.com/reload" { + t.Fatalf("SecretProviderClassReload = %q", a.SecretProviderClassReload) + } + if a.SecretProviderClassExclude != "secretproviderclasses.exclude.reloader.stakater.com/reload" { + t.Fatalf("SecretProviderClassExclude = %q", a.SecretProviderClassExclude) + } +} + +func TestNewDefaultCSIDisabled(t *testing.T) { + if NewDefault().CSIIntegrationEnabled { + t.Fatal("CSIIntegrationEnabled should default to false") + } +} + func TestConfig_IsNamespaceIgnored(t *testing.T) { cfg := NewDefault() cfg.IgnoredNamespaces = []string{"kube-system", "kube-public"} diff --git a/internal/pkg/config/flags.go b/internal/pkg/config/flags.go index 195b84ef..7f3008a6 100644 --- a/internal/pkg/config/flags.go +++ b/internal/pkg/config/flags.go @@ -47,6 +47,12 @@ func BindFlags(fs *pflag.FlagSet, cfg *Config) { "Enable OpenShift DeploymentConfig support (true/false/auto). Empty or 'auto' enables auto-detection", ) + // CSI integration + fs.Bool( + "enable-csi-integration", cfg.CSIIntegrationEnabled, + "Enable CSI SecretProviderClass integration (requires secrets-store CSI driver CRDs)", + ) + // Event watching fs.String( "reload-on-create", "false", @@ -241,6 +247,7 @@ func ApplyFlags(cfg *Config) error { cfg.SyncAfterRestart = v.GetBool("sync-after-restart") cfg.EnableHA = v.GetBool("enable-ha") cfg.EnablePProf = v.GetBool("enable-pprof") + cfg.CSIIntegrationEnabled = v.GetBool("enable-csi-integration") // Boolean string flags (legacy format: "true"/"false" strings) cfg.ArgoRolloutsEnabled = parseBoolString(v.GetString("is-Argo-Rollouts")) @@ -287,6 +294,18 @@ func ApplyFlags(cfg *Config) error { cfg.Annotations.PausePeriod = v.GetString("pause-deployment-annotation") cfg.Annotations.PausedAt = v.GetString("pause-deployment-time-annotation") + // SecretProviderClass annotations have no dedicated CLI flag (parity with + // master); keep the configured defaults. + if cfg.Annotations.SecretProviderClassAuto == "" { + cfg.Annotations.SecretProviderClassAuto = DefaultAnnotations().SecretProviderClassAuto + } + if cfg.Annotations.SecretProviderClassReload == "" { + cfg.Annotations.SecretProviderClassReload = DefaultAnnotations().SecretProviderClassReload + } + if cfg.Annotations.SecretProviderClassExclude == "" { + cfg.Annotations.SecretProviderClassExclude = DefaultAnnotations().SecretProviderClassExclude + } + // Alerting cfg.Alerting.Enabled = v.GetBool("alert-on-reload") cfg.Alerting.WebhookURL = v.GetString("alert-webhook-url") diff --git a/internal/pkg/config/flags_test.go b/internal/pkg/config/flags_test.go index 6a5b3fb2..2d18ab75 100644 --- a/internal/pkg/config/flags_test.go +++ b/internal/pkg/config/flags_test.go @@ -26,6 +26,8 @@ func TestBindFlags(t *testing.T) { "auto-reload-all", "reload-strategy", "is-Argo-Rollouts", + "is-openshift", + "enable-csi-integration", "reload-on-create", "reload-on-delete", "sync-after-restart", @@ -370,6 +372,22 @@ func TestApplyFlags_LegacyProxyEnvVar(t *testing.T) { } } +func TestApplyFlagsCSIIntegration(t *testing.T) { + resetViper() + cfg := NewDefault() + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + BindFlags(fs, cfg) + if err := fs.Parse([]string{"--enable-csi-integration=true"}); err != nil { + t.Fatal(err) + } + if err := ApplyFlags(cfg); err != nil { + t.Fatal(err) + } + if !cfg.CSIIntegrationEnabled { + t.Fatal("expected CSIIntegrationEnabled=true") + } +} + func TestParseBoolString(t *testing.T) { tests := []struct { input string diff --git a/internal/pkg/controller/manager.go b/internal/pkg/controller/manager.go index 2190b8b3..f6f81d97 100644 --- a/internal/pkg/controller/manager.go +++ b/internal/pkg/controller/manager.go @@ -15,6 +15,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/healthz" ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics/server" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" "github.com/stakater/Reloader/internal/pkg/alerting" "github.com/stakater/Reloader/internal/pkg/config" @@ -31,14 +32,17 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(runtimeScheme)) } -// AddOptionalSchemes adds optional workload type schemes if enabled. -func AddOptionalSchemes(argoRolloutsEnabled, deploymentConfigEnabled bool) { +// AddOptionalSchemes adds optional workload/resource type schemes if enabled. +func AddOptionalSchemes(argoRolloutsEnabled, deploymentConfigEnabled, csiEnabled bool) { if argoRolloutsEnabled { utilruntime.Must(argorolloutsv1alpha1.AddToScheme(runtimeScheme)) } if deploymentConfigEnabled { utilruntime.Must(openshiftv1.AddToScheme(runtimeScheme)) } + if csiEnabled { + utilruntime.Must(csiv1.AddToScheme(runtimeScheme)) + } } // ManagerOptions contains options for creating a new Manager. @@ -224,6 +228,30 @@ func SetupReconcilers(mgr ctrl.Manager, cfg *config.Config, log logr.Logger, col } } + // Setup SecretProviderClass reconciler (CSI integration) + if cfg.CSIIntegrationEnabled { + spcReconciler := NewSecretProviderClassReconciler( + ResourceReconcilerDeps{ + Client: mgr.GetClient(), + Log: log.WithName("secretproviderclass-reconciler"), + Config: cfg, + ReloadService: reloadService, + Registry: registry, + Collectors: collectors, + EventRecorder: eventRecorder, + WebhookClient: webhookClient, + Alerter: alerter, + PauseHandler: pauseHandler, + NamespaceCache: nsCache, + }, + mgr.GetAPIReader(), + ) + if err := spcReconciler.SetupWithManager(mgr); err != nil { + return fmt.Errorf("setting up secretproviderclass reconciler: %w", err) + } + log.Info("CSI SecretProviderClass reconciler enabled") + } + // Setup Deployment reconciler for pause handling if err := (&DeploymentReconciler{ Client: mgr.GetClient(), diff --git a/internal/pkg/controller/manager_test.go b/internal/pkg/controller/manager_test.go new file mode 100644 index 00000000..7b671c00 --- /dev/null +++ b/internal/pkg/controller/manager_test.go @@ -0,0 +1,27 @@ +package controller + +import ( + "testing" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" +) + +func TestAddOptionalSchemesRegistersCSI(t *testing.T) { + // Reset to a clean scheme for the test. + runtimeScheme = runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(runtimeScheme)) + + AddOptionalSchemes(false, false, true) + + gvk := schema.GroupVersionKind{ + Group: "secrets-store.csi.x-k8s.io", + Version: "v1", + Kind: "SecretProviderClassPodStatus", + } + if !runtimeScheme.Recognizes(gvk) { + t.Fatal("expected CSI SecretProviderClassPodStatus to be registered in scheme") + } +} diff --git a/internal/pkg/controller/secretproviderclass_reconciler.go b/internal/pkg/controller/secretproviderclass_reconciler.go new file mode 100644 index 00000000..1c2691cf --- /dev/null +++ b/internal/pkg/controller/secretproviderclass_reconciler.go @@ -0,0 +1,169 @@ +package controller + +import ( + "context" + "time" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" + + "github.com/stakater/Reloader/internal/pkg/reload" + "github.com/stakater/Reloader/internal/pkg/workload" +) + +// SecretProviderClassReconciler watches SecretProviderClassPodStatus objects and +// triggers workload reloads when the secret versions they track change. +// +// It watches SecretProviderClassPodStatus (the per-pod status written by the CSI +// driver) rather than SecretProviderClass directly, because only the pod status +// carries the current object IDs and versions that indicate a secret rotation. +type SecretProviderClassReconciler struct { + ResourceReconcilerDeps + + // apiReader is a direct API client (not cached) used to look up the parent + // SecretProviderClass object. In tests this is set to the fake client. + apiReader client.Reader + + handler *ReloadHandler +} + +// NewSecretProviderClassReconciler creates a new SecretProviderClassReconciler. +func NewSecretProviderClassReconciler(deps ResourceReconcilerDeps, apiReader client.Reader) *SecretProviderClassReconciler { + return &SecretProviderClassReconciler{ + ResourceReconcilerDeps: deps, + apiReader: apiReader, + } +} + +// Reconcile handles a SecretProviderClassPodStatus event. +func (r *SecretProviderClassReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + startTime := time.Now() + resourceType := string(reload.ResourceTypeSecretProviderClass) + log := r.Log.WithValues("secretproviderclasspodstatus", req.NamespacedName) + + r.Collectors.RecordEventReceived("reconcile", resourceType) + + spcps := &csiv1.SecretProviderClassPodStatus{} + if err := r.Client.Get(ctx, req.NamespacedName, spcps); err != nil { + if errors.IsNotFound(err) { + r.Collectors.RecordSkipped("not_found") + r.Collectors.RecordReconcile("success", time.Since(startTime)) + return ctrl.Result{}, nil + } + log.Error(err, "failed to get SecretProviderClassPodStatus") + r.Collectors.RecordError("get_secretproviderclasspodstatus") + r.Collectors.RecordReconcile("error", time.Since(startTime)) + return ctrl.Result{}, err + } + + namespace := spcps.GetNamespace() + if r.Config.IsNamespaceIgnored(namespace) { + log.V(1).Info("skipping SecretProviderClassPodStatus in ignored namespace") + r.Collectors.RecordSkipped("ignored_namespace") + r.Collectors.RecordReconcile("success", time.Since(startTime)) + return ctrl.Result{}, nil + } + + if r.NamespaceCache != nil && r.NamespaceCache.IsEnabled() && !r.NamespaceCache.Contains(namespace) { + log.V(1).Info("skipping SecretProviderClassPodStatus in namespace not matching selector", "namespace", namespace) + r.Collectors.RecordSkipped("namespace_selector") + r.Collectors.RecordReconcile("success", time.Since(startTime)) + return ctrl.Result{}, nil + } + + spcName, spcAnnotations := r.resolveSPCAnnotations(ctx, spcps) + if spcName == "" { + r.Collectors.RecordSkipped("no_spc_name") + r.Collectors.RecordReconcile("success", time.Since(startTime)) + return ctrl.Result{}, nil + } + + change := reload.SecretProviderClassChange{ + Name: spcName, + Namespace: namespace, + Annotations: spcAnnotations, + Status: spcps.Status, + EventType: reload.EventTypeUpdate, + } + + result, err := r.reloadHandler().Process( + ctx, namespace, spcName, reload.ResourceTypeSecretProviderClass, + func(workloads []workload.Workload) []reload.ReloadDecision { + return r.ReloadService.Process(change, workloads) + }, log, + ) + + if err != nil { + r.Collectors.RecordReconcile("error", time.Since(startTime)) + } else { + r.Collectors.RecordReconcile("success", time.Since(startTime)) + } + return result, err +} + +// resolveSPCAnnotations looks up the SecretProviderClass referenced by the +// given pod status and returns its name and annotations. It never returns an +// error: on any Get failure it logs and returns the SPC name (from the pod +// status) with an empty annotations map, so callers can still process workloads +// that match via their own auto/named annotations. This matches master's +// behaviour in populateAnnotationsFromSecretProviderClass. +func (r *SecretProviderClassReconciler) resolveSPCAnnotations( + ctx context.Context, + spcps *csiv1.SecretProviderClassPodStatus, +) (string, map[string]string) { + spcName := spcps.Status.SecretProviderClassName + spc := &csiv1.SecretProviderClass{} + if err := r.apiReader.Get(ctx, types.NamespacedName{ + Name: spcName, + Namespace: spcps.GetNamespace(), + }, spc); err != nil { + if errors.IsNotFound(err) { + r.Log.WithValues("spc", spcName).Info("SecretProviderClass not found; proceeding without its annotations") + } else { + r.Log.V(1).Error(err, "failed to get SecretProviderClass; proceeding without its annotations", "spc", spcName) + } + return spcName, map[string]string{} + } + annotations := spc.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + return spc.Name, annotations +} + +func (r *SecretProviderClassReconciler) reloadHandler() *ReloadHandler { + if r.handler == nil { + r.handler = &ReloadHandler{ + Client: r.Client, + Lister: workload.NewLister(r.Client, r.Registry, r.Config), + ReloadService: r.ReloadService, + WebhookClient: r.WebhookClient, + Collectors: r.Collectors, + EventRecorder: r.EventRecorder, + Alerter: r.Alerter, + PauseHandler: r.PauseHandler, + } + } + return r.handler +} + +// SetupWithManager wires the reconciler to watch SecretProviderClassPodStatus. +func (r *SecretProviderClassReconciler) SetupWithManager(mgr ctrl.Manager) error { + var nsChecker reload.NamespaceChecker + if r.NamespaceCache != nil { + nsChecker = r.NamespaceCache + } + return ctrl.NewControllerManagedBy(mgr). + For(&csiv1.SecretProviderClassPodStatus{}). + WithEventFilter(reload.CombinedPredicates( + reload.NamespaceFilterPredicateWithCache(r.Config, nsChecker), + reload.SecretProviderClassPodStatusPredicates(r.Config, r.ReloadService.Hasher()), + )). + Complete(r) +} + +var _ reconcile.Reconciler = &SecretProviderClassReconciler{} diff --git a/internal/pkg/controller/secretproviderclass_reconciler_test.go b/internal/pkg/controller/secretproviderclass_reconciler_test.go new file mode 100644 index 00000000..8664996a --- /dev/null +++ b/internal/pkg/controller/secretproviderclass_reconciler_test.go @@ -0,0 +1,171 @@ +package controller_test + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" + + "github.com/stakater/Reloader/internal/pkg/config" + "github.com/stakater/Reloader/internal/pkg/controller" + "github.com/stakater/Reloader/internal/pkg/reload" + "github.com/stakater/Reloader/internal/pkg/testutil" +) + +// newSecretProviderClassReconcilerWithClient creates a SecretProviderClassReconciler for +// testing and returns both the reconciler and the fake client (for assertions). +func newSecretProviderClassReconcilerWithClient(t *testing.T, cfg *config.Config, objects ...client.Object) (*controller.SecretProviderClassReconciler, client.Client) { + t.Helper() + // Convert client.Object slice to runtime.Object slice for newTestDeps. + // We create the fake client ourselves to hold the reference. + deps := newTestDeps(t, cfg) + for _, obj := range objects { + deps.client = deps.client.WithObjects(obj) + } + cl := deps.client.Build() + reconciler := controller.NewSecretProviderClassReconciler( + controller.ResourceReconcilerDeps{ + Client: cl, + Log: deps.log, + Config: deps.cfg, + ReloadService: deps.reloadService, + Registry: deps.registry, + Collectors: deps.collectors, + EventRecorder: deps.eventRecorder, + WebhookClient: deps.webhookClient, + Alerter: deps.alerter, + PauseHandler: reload.NewPauseHandler(cfg), + NamespaceCache: nil, + }, + cl, // APIReader = same fake client in tests + ) + return reconciler, cl +} + +// TestSecretProviderClassReconciler_NotFound ensures that reconciling a +// nonexistent SecretProviderClassPodStatus returns cleanly without error. +func TestSecretProviderClassReconciler_NotFound(t *testing.T) { + cfg := config.NewDefault() + reconciler, _ := newSecretProviderClassReconcilerWithClient(t, cfg) + assertReconcileSuccess(t, reconciler, reconcileRequest("nonexistent", "default")) +} + +// TestSecretProviderClassReconciler_MatchingDeployment_AutoAnnotation tests the +// core happy path: a SPCPS update resolves the SPC, creates a change event, and +// the deployment gets the STAKATER_MY_SPC_SECRETPROVIDERCLASS env var. +func TestSecretProviderClassReconciler_MatchingDeployment_AutoAnnotation(t *testing.T) { + cfg := config.NewDefault() + + deployment := testutil.NewDeployment("test-deployment", "default", map[string]string{ + cfg.Annotations.SecretProviderClassAuto: "true", + }) + + spc := &csiv1.SecretProviderClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-spc", + Namespace: "default", + }, + } + + spcps := &csiv1.SecretProviderClassPodStatus{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-spcps", + Namespace: "default", + }, + Status: csiv1.SecretProviderClassPodStatusStatus{ + SecretProviderClassName: "my-spc", + Objects: []csiv1.SecretProviderClassObject{ + {ID: "a", Version: "1"}, + }, + }, + } + + reconciler, cl := newSecretProviderClassReconcilerWithClient(t, cfg, deployment, spc, spcps) + assertReconcileSuccess(t, reconciler, reconcileRequest("test-pod-spcps", "default")) + + // Verify the deployment was updated with the expected env var. + updated := &appsv1.Deployment{} + if err := cl.Get(context.Background(), types.NamespacedName{Name: "test-deployment", Namespace: "default"}, updated); err != nil { + t.Fatalf("failed to get updated deployment: %v", err) + } + + if len(updated.Spec.Template.Spec.Containers) == 0 { + t.Fatal("deployment has no containers") + } + + const expectedEnvVar = "STAKATER_MY_SPC_SECRETPROVIDERCLASS" + found := false + for _, env := range updated.Spec.Template.Spec.Containers[0].Env { + if env.Name == expectedEnvVar && env.Value != "" { + found = true + break + } + } + if !found { + t.Errorf("expected env var %q to be set on deployment container; got envs: %v", + expectedEnvVar, updated.Spec.Template.Spec.Containers[0].Env) + } +} + +// TestSecretProviderClassReconciler_SPCNotFound ensures that if the SPCPS +// references a SecretProviderClass that does not exist (e.g. it was deleted +// while secrets are still rotating), the reconciler still reloads any workload +// annotated with secretproviderclass auto/named annotations. The SPC object's +// annotations are only needed for ignore-check and search/match; auto and +// named-reload matching is driven by the workload's own annotations alone. +func TestSecretProviderClassReconciler_SPCNotFound(t *testing.T) { + cfg := config.NewDefault() + + // A deployment annotated for auto-reload on any SPC change. + deployment := testutil.NewDeployment("test-deployment", "default", map[string]string{ + cfg.Annotations.SecretProviderClassAuto: "true", + }) + + // Only add the SPCPS; the SPC ("missing-spc") is intentionally absent from + // the fake client to simulate deletion. + spcps := &csiv1.SecretProviderClassPodStatus{ + ObjectMeta: metav1.ObjectMeta{ + Name: "orphaned-spcps", + Namespace: "default", + }, + Status: csiv1.SecretProviderClassPodStatusStatus{ + SecretProviderClassName: "missing-spc", + Objects: []csiv1.SecretProviderClassObject{ + {ID: "b", Version: "2"}, + }, + }, + } + + reconciler, cl := newSecretProviderClassReconcilerWithClient(t, cfg, deployment, spcps) + + // Reconcile must succeed with no error and no requeue. + assertReconcileSuccess(t, reconciler, reconcileRequest("orphaned-spcps", "default")) + + // The deployment must have been reloaded: container[0].Env must contain + // STAKATER_MISSING_SPC_SECRETPROVIDERCLASS with a non-empty SHA value. + updated := &appsv1.Deployment{} + if err := cl.Get(context.Background(), types.NamespacedName{Name: "test-deployment", Namespace: "default"}, updated); err != nil { + t.Fatalf("failed to get updated deployment: %v", err) + } + + if len(updated.Spec.Template.Spec.Containers) == 0 { + t.Fatal("deployment has no containers") + } + + const expectedEnvVar = "STAKATER_MISSING_SPC_SECRETPROVIDERCLASS" + found := false + for _, env := range updated.Spec.Template.Spec.Containers[0].Env { + if env.Name == expectedEnvVar && env.Value != "" { + found = true + break + } + } + if !found { + t.Errorf("expected env var %q to be set on deployment container after SPC-not-found reconcile; got envs: %v", + expectedEnvVar, updated.Spec.Template.Spec.Containers[0].Env) + } +} diff --git a/internal/pkg/csi/detect.go b/internal/pkg/csi/detect.go new file mode 100644 index 00000000..7a7725bc --- /dev/null +++ b/internal/pkg/csi/detect.go @@ -0,0 +1,36 @@ +// Package csi provides detection of the secrets-store CSI driver CRDs. +package csi + +import ( + "github.com/go-logr/logr" + "k8s.io/client-go/discovery" +) + +const ( + // CSIAPIGroup is the API group for the secrets-store CSI driver. + CSIAPIGroup = "secrets-store.csi.x-k8s.io" + // CSIAPIVersion is the API version used by Reloader. + CSIAPIVersion = "v1" + // CSIPodStatusResource is the watched resource. + CSIPodStatusResource = "secretproviderclasspodstatuses" +) + +// HasCSISupport reports whether the cluster has the secrets-store CSI driver +// CRDs installed (specifically the SecretProviderClassPodStatus resource). +func HasCSISupport(client discovery.DiscoveryInterface, log logr.Logger) bool { + resources, err := client.ServerResourcesForGroupVersion(CSIAPIGroup + "/" + CSIAPIVersion) + if err != nil { + log.V(1).Info("CSI API not available", "error", err) + return false + } + + for _, r := range resources.APIResources { + if r.Name == CSIPodStatusResource { + log.Info("CSI provider detected, enabling SecretProviderClass support") + return true + } + } + + log.V(1).Info("CSI resource not found in " + CSIAPIGroup + "/" + CSIAPIVersion) + return false +} diff --git a/internal/pkg/csi/detect_test.go b/internal/pkg/csi/detect_test.go new file mode 100644 index 00000000..0bf557a8 --- /dev/null +++ b/internal/pkg/csi/detect_test.go @@ -0,0 +1,38 @@ +package csi + +import ( + "testing" + + "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + fake "k8s.io/client-go/discovery/fake" + clienttesting "k8s.io/client-go/testing" +) + +func newFakeDiscovery(resources []*metav1.APIResourceList) *fake.FakeDiscovery { + return &fake.FakeDiscovery{ + Fake: &clienttesting.Fake{Resources: resources}, + } +} + +func TestHasCSISupportTrue(t *testing.T) { + d := newFakeDiscovery([]*metav1.APIResourceList{ + { + GroupVersion: "secrets-store.csi.x-k8s.io/v1", + APIResources: []metav1.APIResource{ + {Name: "secretproviderclasspodstatuses"}, + {Name: "secretproviderclasses"}, + }, + }, + }) + if !HasCSISupport(d, logr.Discard()) { + t.Fatal("expected CSI support detected") + } +} + +func TestHasCSISupportFalse(t *testing.T) { + d := newFakeDiscovery(nil) + if HasCSISupport(d, logr.Discard()) { + t.Fatal("expected CSI support not detected") + } +} diff --git a/internal/pkg/reload/change.go b/internal/pkg/reload/change.go index b7fa4443..2b6a2f09 100644 --- a/internal/pkg/reload/change.go +++ b/internal/pkg/reload/change.go @@ -2,6 +2,7 @@ package reload import ( corev1 "k8s.io/api/core/v1" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" ) // EventType represents the type of change event. @@ -54,3 +55,28 @@ func (c SecretChange) GetNamespace() string { return c.Secret.Names func (c SecretChange) GetAnnotations() map[string]string { return c.Secret.Annotations } func (c SecretChange) GetResourceType() ResourceType { return ResourceTypeSecret } func (c SecretChange) ComputeHash(h *Hasher) string { return h.HashSecret(c.Secret) } + +// SecretProviderClassChange represents a change event derived from a +// SecretProviderClassPodStatus update. Name/Annotations refer to the resolved +// SecretProviderClass; Status carries the SPCPS status used for hashing. +type SecretProviderClassChange struct { + Name string + Namespace string + Annotations map[string]string + Status csiv1.SecretProviderClassPodStatusStatus + EventType EventType +} + +func (c SecretProviderClassChange) IsNil() bool { return c.Name == "" } +func (c SecretProviderClassChange) GetEventType() EventType { return c.EventType } +func (c SecretProviderClassChange) GetName() string { return c.Name } +func (c SecretProviderClassChange) GetNamespace() string { return c.Namespace } +func (c SecretProviderClassChange) GetAnnotations() map[string]string { + return c.Annotations +} +func (c SecretProviderClassChange) GetResourceType() ResourceType { + return ResourceTypeSecretProviderClass +} +func (c SecretProviderClassChange) ComputeHash(h *Hasher) string { + return h.HashSecretProviderClass(c.Status) +} diff --git a/internal/pkg/reload/change_test.go b/internal/pkg/reload/change_test.go new file mode 100644 index 00000000..a17b45a4 --- /dev/null +++ b/internal/pkg/reload/change_test.go @@ -0,0 +1,51 @@ +package reload + +import ( + "testing" + + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" +) + +func TestSecretProviderClassChange(t *testing.T) { + status := csiv1.SecretProviderClassPodStatusStatus{ + SecretProviderClassName: "my-spc", + Objects: []csiv1.SecretProviderClassObject{{ID: "a", Version: "1"}}, + } + c := SecretProviderClassChange{ + Name: "my-spc", + Namespace: "ns1", + Annotations: map[string]string{"k": "v"}, + Status: status, + EventType: EventTypeUpdate, + } + + if c.IsNil() { + t.Fatal("IsNil() = true, want false") + } + if c.GetName() != "my-spc" { + t.Fatalf("GetName() = %q", c.GetName()) + } + if c.GetNamespace() != "ns1" { + t.Fatalf("GetNamespace() = %q", c.GetNamespace()) + } + if c.GetResourceType() != ResourceTypeSecretProviderClass { + t.Fatalf("GetResourceType() = %q", c.GetResourceType()) + } + if c.GetEventType() != EventTypeUpdate { + t.Fatalf("GetEventType() = %q", c.GetEventType()) + } + if c.GetAnnotations()["k"] != "v" { + t.Fatalf("GetAnnotations() missing key") + } + h := NewHasher() + if c.ComputeHash(h) != h.HashSecretProviderClass(status) { + t.Fatalf("ComputeHash mismatch") + } +} + +func TestSecretProviderClassChangeIsNil(t *testing.T) { + c := SecretProviderClassChange{Name: ""} + if !c.IsNil() { + t.Fatal("IsNil() = false, want true for empty name") + } +} diff --git a/internal/pkg/reload/csi_dep_check_test.go b/internal/pkg/reload/csi_dep_check_test.go new file mode 100644 index 00000000..1bb9dc21 --- /dev/null +++ b/internal/pkg/reload/csi_dep_check_test.go @@ -0,0 +1,24 @@ +package reload + +import ( + "testing" + + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" +) + +// TestCSIDependencyAvailable ensures the CSI types are importable and the +// fields this feature depends on exist. +func TestCSIDependencyAvailable(t *testing.T) { + status := csiv1.SecretProviderClassPodStatusStatus{ + SecretProviderClassName: "spc", + Objects: []csiv1.SecretProviderClassObject{ + {ID: "secret/data/foo", Version: "1"}, + }, + } + if status.SecretProviderClassName != "spc" { + t.Fatalf("unexpected SecretProviderClassName") + } + if len(status.Objects) != 1 || status.Objects[0].ID != "secret/data/foo" { + t.Fatalf("unexpected Objects") + } +} diff --git a/internal/pkg/reload/hasher.go b/internal/pkg/reload/hasher.go index 0d259ac3..1f46c5fc 100644 --- a/internal/pkg/reload/hasher.go +++ b/internal/pkg/reload/hasher.go @@ -10,6 +10,7 @@ import ( "strings" corev1 "k8s.io/api/core/v1" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" ) // Hasher computes content hashes for ConfigMaps and Secrets. @@ -68,6 +69,19 @@ func (h *Hasher) computeSHA(data string) string { return fmt.Sprintf("%x", hasher.Sum(nil)) } +// HashSecretProviderClass computes a SHA1 hash of a SecretProviderClassPodStatus +// status: the sorted set of object ID=Version entries plus the SPC name. +// This mirrors master's util.GetSHAfromSecretProviderClassPodStatus exactly. +func (h *Hasher) HashSecretProviderClass(status csiv1.SecretProviderClassPodStatusStatus) string { + values := make([]string, 0, len(status.Objects)+1) + for _, obj := range status.Objects { + values = append(values, obj.ID+"="+obj.Version) + } + values = append(values, "SecretProviderClassName="+status.SecretProviderClassName) + sort.Strings(values) + return h.computeSHA(strings.Join(values, ";")) +} + // EmptyHash returns an empty string to signal resource deletion. func (h *Hasher) EmptyHash() string { return "" diff --git a/internal/pkg/reload/hasher_test.go b/internal/pkg/reload/hasher_test.go index ff5693ff..09c44af0 100644 --- a/internal/pkg/reload/hasher_test.go +++ b/internal/pkg/reload/hasher_test.go @@ -4,6 +4,7 @@ import ( "testing" corev1 "k8s.io/api/core/v1" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" ) func TestHasher_HashConfigMap(t *testing.T) { @@ -234,3 +235,45 @@ func TestHasher_NilInput(t *testing.T) { t.Error("nil Secret should return a valid hash") } } + +func TestHashSecretProviderClass(t *testing.T) { + h := NewHasher() + status := csiv1.SecretProviderClassPodStatusStatus{ + SecretProviderClassName: "my-spc", + Objects: []csiv1.SecretProviderClassObject{ + {ID: "secret/data/b", Version: "2"}, + {ID: "secret/data/a", Version: "1"}, + }, + } + + // Expected = SHA1 hex of the sorted, ';'-joined string, matching master. + expectedInput := "SecretProviderClassName=my-spc;secret/data/a=1;secret/data/b=2" + expected := h.computeSHA(expectedInput) + + got := h.HashSecretProviderClass(status) + if got != expected { + t.Fatalf("HashSecretProviderClass = %q, want %q", got, expected) + } + + // Order independence: shuffling objects must not change the hash. + statusReordered := csiv1.SecretProviderClassPodStatusStatus{ + SecretProviderClassName: "my-spc", + Objects: []csiv1.SecretProviderClassObject{ + {ID: "secret/data/a", Version: "1"}, + {ID: "secret/data/b", Version: "2"}, + }, + } + if h.HashSecretProviderClass(statusReordered) != got { + t.Fatalf("hash not order-independent") + } +} + +func TestHashSecretProviderClassEmpty(t *testing.T) { + h := NewHasher() + status := csiv1.SecretProviderClassPodStatusStatus{SecretProviderClassName: "empty"} + got := h.HashSecretProviderClass(status) + want := h.computeSHA("SecretProviderClassName=empty") + if got != want { + t.Fatalf("HashSecretProviderClass(empty) = %q, want %q", got, want) + } +} diff --git a/internal/pkg/reload/matcher.go b/internal/pkg/reload/matcher.go index e817f7f5..9fb8ba11 100644 --- a/internal/pkg/reload/matcher.go +++ b/internal/pkg/reload/matcher.go @@ -144,6 +144,8 @@ func (m *Matcher) isResourceExcluded(resourceName string, resourceType ResourceT excludeAnn = m.cfg.Annotations.ConfigmapExclude case ResourceTypeSecret: excludeAnn = m.cfg.Annotations.SecretExclude + case ResourceTypeSecretProviderClass: + excludeAnn = m.cfg.Annotations.SecretProviderClassExclude } excludeList, ok := annotations[excludeAnn] @@ -242,6 +244,8 @@ func (m *Matcher) getExplicitAnnotation(resourceType ResourceType) string { return m.cfg.Annotations.ConfigmapReload case ResourceTypeSecret: return m.cfg.Annotations.SecretReload + case ResourceTypeSecretProviderClass: + return m.cfg.Annotations.SecretProviderClassReload default: return "" } @@ -253,6 +257,8 @@ func (m *Matcher) getTypedAutoAnnotation(resourceType ResourceType) string { return m.cfg.Annotations.ConfigmapAuto case ResourceTypeSecret: return m.cfg.Annotations.SecretAuto + case ResourceTypeSecretProviderClass: + return m.cfg.Annotations.SecretProviderClassAuto default: return "" } diff --git a/internal/pkg/reload/matcher_test.go b/internal/pkg/reload/matcher_test.go index 1c58fd30..b683ffb1 100644 --- a/internal/pkg/reload/matcher_test.go +++ b/internal/pkg/reload/matcher_test.go @@ -404,6 +404,52 @@ func TestMatcher_AutoDoesNotIgnoreExplicit(t *testing.T) { t.Log("✓ Explicit reload annotation works even when auto is enabled") } +func TestMatcherSecretProviderClassExplicit(t *testing.T) { + cfg := config.NewDefault() + m := NewMatcher(cfg) + res := m.ShouldReload(MatchInput{ + ResourceName: "my-spc", + ResourceType: ResourceTypeSecretProviderClass, + WorkloadAnnotations: map[string]string{ + "secretproviderclass.reloader.stakater.com/reload": "my-spc", + }, + }) + if !res.ShouldReload || res.AutoReload { + t.Fatalf("explicit SPC reload: got %+v", res) + } +} + +func TestMatcherSecretProviderClassAuto(t *testing.T) { + cfg := config.NewDefault() + m := NewMatcher(cfg) + res := m.ShouldReload(MatchInput{ + ResourceName: "my-spc", + ResourceType: ResourceTypeSecretProviderClass, + WorkloadAnnotations: map[string]string{ + "secretproviderclass.reloader.stakater.com/auto": "true", + }, + }) + if !res.ShouldReload || !res.AutoReload { + t.Fatalf("auto SPC reload: got %+v", res) + } +} + +func TestMatcherSecretProviderClassExcluded(t *testing.T) { + cfg := config.NewDefault() + m := NewMatcher(cfg) + res := m.ShouldReload(MatchInput{ + ResourceName: "my-spc", + ResourceType: ResourceTypeSecretProviderClass, + WorkloadAnnotations: map[string]string{ + "secretproviderclass.reloader.stakater.com/auto": "true", + "secretproviderclasses.exclude.reloader.stakater.com/reload": "my-spc", + }, + }) + if res.ShouldReload { + t.Fatalf("excluded SPC should not reload: got %+v", res) + } +} + // TestMatcher_PrecedenceOrder verifies the correct order of precedence: // 1. Ignore annotation → skip // 2. Exclude annotation → skip diff --git a/internal/pkg/reload/predicate.go b/internal/pkg/reload/predicate.go index c866364a..38fa5841 100644 --- a/internal/pkg/reload/predicate.go +++ b/internal/pkg/reload/predicate.go @@ -5,6 +5,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" "github.com/stakater/Reloader/internal/pkg/config" ) @@ -157,3 +158,22 @@ func IgnoreAnnotationPredicate(cfg *config.Config) predicate.Predicate { func CombinedPredicates(predicates ...predicate.Predicate) predicate.Predicate { return predicate.And(predicates...) } + +// SecretProviderClassPodStatusPredicates filters SecretProviderClassPodStatus events. +// Create and Delete are ignored (matching master); Update passes only when the +// hashed status (object IDs/versions + SPC name) changes. +func SecretProviderClassPodStatusPredicates(cfg *config.Config, hasher *Hasher) predicate.Predicate { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { return false }, + DeleteFunc: func(e event.DeleteEvent) bool { return false }, + GenericFunc: func(e event.GenericEvent) bool { return false }, + UpdateFunc: func(e event.UpdateEvent) bool { + oldObj, okOld := e.ObjectOld.(*csiv1.SecretProviderClassPodStatus) + newObj, okNew := e.ObjectNew.(*csiv1.SecretProviderClassPodStatus) + if !okOld || !okNew { + return false + } + return hasher.HashSecretProviderClass(oldObj.Status) != hasher.HashSecretProviderClass(newObj.Status) + }, + } +} diff --git a/internal/pkg/reload/predicate_test.go b/internal/pkg/reload/predicate_test.go index b6d48340..85aacaa4 100644 --- a/internal/pkg/reload/predicate_test.go +++ b/internal/pkg/reload/predicate_test.go @@ -7,6 +7,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/event" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" "github.com/stakater/Reloader/internal/pkg/config" ) @@ -934,3 +935,37 @@ func TestLabelsSet(t *testing.T) { t.Errorf("Get(nonexistent) = %v, want empty string", ls.Get("nonexistent")) } } + +func TestSecretProviderClassPodStatusPredicates(t *testing.T) { + cfg := config.NewDefault() + p := SecretProviderClassPodStatusPredicates(cfg, NewHasher()) + + oldObj := &csiv1.SecretProviderClassPodStatus{ + Status: csiv1.SecretProviderClassPodStatusStatus{ + SecretProviderClassName: "spc", + Objects: []csiv1.SecretProviderClassObject{{ID: "a", Version: "1"}}, + }, + } + newObjChanged := &csiv1.SecretProviderClassPodStatus{ + Status: csiv1.SecretProviderClassPodStatusStatus{ + SecretProviderClassName: "spc", + Objects: []csiv1.SecretProviderClassObject{{ID: "a", Version: "2"}}, + }, + } + newObjSame := oldObj.DeepCopy() + + // Create and Delete are always ignored for SPCPS. + if p.Create(event.CreateEvent{Object: oldObj}) { + t.Fatal("CreateFunc should return false") + } + if p.Delete(event.DeleteEvent{Object: oldObj}) { + t.Fatal("DeleteFunc should return false") + } + // Update only when the status hash changes. + if !p.Update(event.UpdateEvent{ObjectOld: oldObj, ObjectNew: newObjChanged}) { + t.Fatal("UpdateFunc should return true on changed status") + } + if p.Update(event.UpdateEvent{ObjectOld: oldObj, ObjectNew: newObjSame}) { + t.Fatal("UpdateFunc should return false on unchanged status") + } +} diff --git a/internal/pkg/reload/resource_type.go b/internal/pkg/reload/resource_type.go index 0404e815..5c086831 100644 --- a/internal/pkg/reload/resource_type.go +++ b/internal/pkg/reload/resource_type.go @@ -8,6 +8,8 @@ const ( ResourceTypeConfigMap ResourceType = "configmap" // ResourceTypeSecret represents a Secret resource. ResourceTypeSecret ResourceType = "secret" + // ResourceTypeSecretProviderClass represents a CSI SecretProviderClass resource. + ResourceTypeSecretProviderClass ResourceType = "secretproviderclass" ) // Kind returns the capitalized Kubernetes Kind (e.g., "ConfigMap", "Secret"). @@ -17,6 +19,8 @@ func (r ResourceType) Kind() string { return "ConfigMap" case ResourceTypeSecret: return "Secret" + case ResourceTypeSecretProviderClass: + return "SecretProviderClass" default: return string(r) } diff --git a/internal/pkg/reload/resource_type_test.go b/internal/pkg/reload/resource_type_test.go index e577e82b..51e2f1b5 100644 --- a/internal/pkg/reload/resource_type_test.go +++ b/internal/pkg/reload/resource_type_test.go @@ -26,3 +26,12 @@ func TestResourceType_Kind(t *testing.T) { ) } } + +func TestResourceTypeSecretProviderClassKind(t *testing.T) { + if got := ResourceTypeSecretProviderClass.Kind(); got != "SecretProviderClass" { + t.Fatalf("Kind() = %q, want SecretProviderClass", got) + } + if string(ResourceTypeSecretProviderClass) != "secretproviderclass" { + t.Fatalf("value = %q, want secretproviderclass", ResourceTypeSecretProviderClass) + } +} diff --git a/internal/pkg/reload/service.go b/internal/pkg/reload/service.go index 076cf786..e712f90d 100644 --- a/internal/pkg/reload/service.go +++ b/internal/pkg/reload/service.go @@ -83,6 +83,10 @@ func (s *Service) processResource( usesResource = wl.UsesConfigMap(resourceName) case ResourceTypeSecret: usesResource = wl.UsesSecret(resourceName) + case ResourceTypeSecretProviderClass: + // Annotation-only matching (parity with master): the workload's + // annotations alone decide the reload; no volume-uses scan. + usesResource = true } input := MatchInput{ diff --git a/internal/pkg/reload/service_test.go b/internal/pkg/reload/service_test.go index 24356b1e..6141e2c5 100644 --- a/internal/pkg/reload/service_test.go +++ b/internal/pkg/reload/service_test.go @@ -7,6 +7,7 @@ import ( "github.com/go-logr/logr/testr" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" "github.com/stakater/Reloader/internal/pkg/config" "github.com/stakater/Reloader/internal/pkg/testutil" @@ -1332,6 +1333,31 @@ func TestService_ProcessNilChange(t *testing.T) { } } +func TestServiceProcessSecretProviderClassAuto(t *testing.T) { + cfg := config.NewDefault() + svc := NewService(cfg, testr.New(t)) + + deploy := testutil.NewDeployment("test-deploy", "default", map[string]string{ + "secretproviderclass.reloader.stakater.com/auto": "true", + }) + workloads := []workload.Workload{workload.NewDeploymentWorkload(deploy)} + + change := SecretProviderClassChange{ + Name: "my-spc", + Namespace: "default", + Status: csiv1.SecretProviderClassPodStatusStatus{ + SecretProviderClassName: "my-spc", + Objects: []csiv1.SecretProviderClassObject{{ID: "a", Version: "1"}}, + }, + EventType: EventTypeUpdate, + } + + decisions := svc.Process(change, workloads) + if len(decisions) != 1 || !decisions[0].ShouldReload { + t.Fatalf("expected auto SPC reload, got %+v", decisions) + } +} + func TestService_ProcessCreateEventDisabled(t *testing.T) { cfg := config.NewDefault() cfg.ReloadOnCreate = false diff --git a/internal/pkg/reload/strategy.go b/internal/pkg/reload/strategy.go index 4386bf4b..3c3d9b7b 100644 --- a/internal/pkg/reload/strategy.go +++ b/internal/pkg/reload/strategy.go @@ -19,6 +19,8 @@ const ( ConfigmapEnvVarPostfix = "CONFIGMAP" // SecretEnvVarPostfix is the postfix for Secret environment variables. SecretEnvVarPostfix = "SECRET" + // SecretProviderClassEnvVarPostfix is the postfix for SecretProviderClass environment variables. + SecretProviderClassEnvVarPostfix = "SECRETPROVIDERCLASS" ) // Strategy defines how workload restarts are triggered. @@ -108,6 +110,8 @@ func (s *EnvVarStrategy) envVarName(resourceName string, resourceType ResourceTy postfix = ConfigmapEnvVarPostfix case ResourceTypeSecret: postfix = SecretEnvVarPostfix + case ResourceTypeSecretProviderClass: + postfix = SecretProviderClassEnvVarPostfix } return EnvVarPrefix + convertToEnvVarName(resourceName) + "_" + postfix } diff --git a/internal/pkg/reload/strategy_test.go b/internal/pkg/reload/strategy_test.go index 3ea4f245..57451597 100644 --- a/internal/pkg/reload/strategy_test.go +++ b/internal/pkg/reload/strategy_test.go @@ -291,3 +291,12 @@ func TestNewStrategy(t *testing.T) { } }) } + +func TestEnvVarNameSecretProviderClass(t *testing.T) { + s := NewEnvVarStrategy() + got := s.envVarName("my-vault-spc", ResourceTypeSecretProviderClass) + want := "STAKATER_MY_VAULT_SPC_SECRETPROVIDERCLASS" + if got != want { + t.Fatalf("envVarName = %q, want %q", got, want) + } +} diff --git a/internal/pkg/testutil/fixtures.go b/internal/pkg/testutil/fixtures.go index 6ba8587a..b9044da5 100644 --- a/internal/pkg/testutil/fixtures.go +++ b/internal/pkg/testutil/fixtures.go @@ -7,6 +7,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" ) // NewDeploymentConfig creates a minimal DeploymentConfig for unit testing. @@ -72,6 +73,7 @@ func NewScheme() *runtime.Scheme { _ = appsv1.AddToScheme(scheme) _ = batchv1.AddToScheme(scheme) _ = openshiftv1.AddToScheme(scheme) + _ = csiv1.AddToScheme(scheme) return scheme } diff --git a/test/e2e/csi/csi_suite_test.go b/test/e2e/csi/csi_suite_test.go new file mode 100644 index 00000000..f2e809cf --- /dev/null +++ b/test/e2e/csi/csi_suite_test.go @@ -0,0 +1,95 @@ +package csi + +import ( + "context" + "encoding/json" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + csiclient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned" + + "github.com/stakater/Reloader/test/e2e/utils" +) + +var ( + kubeClient kubernetes.Interface + csiClient csiclient.Interface + restConfig *rest.Config + testNamespace string + ctx context.Context + testEnv *utils.TestEnvironment +) + +func TestCSI(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "CSI SecretProviderClass E2E Suite") +} + +// SynchronizedBeforeSuite ensures only process 1 deploys Reloader. +// Process 1 also checks prerequisites (CSI driver, Vault) and calls Skip if +// they are not installed — Ginkgo propagates the skip to all processes. +var _ = SynchronizedBeforeSuite( + // Process 1 only: check prerequisites, create namespace, deploy Reloader. + func() []byte { + setupEnv, err := utils.SetupTestEnvironment(context.Background(), "reloader-csi-test") + Expect(err).NotTo(HaveOccurred(), "Failed to setup test environment") + // Ensure the namespace is deleted even if DeployAndWait fails, so + // orphaned namespaces don't accumulate on long-lived clusters. + DeferCleanup(setupEnv.CleanupOnFailure) + + if !utils.IsCSIDriverInstalled(context.Background(), setupEnv.CSIClient) { + Skip("CSI secrets store driver not installed - skipping CSI suite") + } + if !utils.IsVaultProviderInstalled(context.Background(), setupEnv.KubeClient) { + Skip("Vault CSI provider not installed - skipping CSI suite") + } + + Expect(setupEnv.DeployAndWait(map[string]string{ + "reloader.reloadStrategy": "annotations", + "reloader.watchGlobally": "false", + "reloader.enableCSIIntegration": "true", + })).To(Succeed(), "Failed to deploy Reloader") + + data, err := json.Marshal(utils.SharedEnvData{ + Namespace: setupEnv.Namespace, + ReleaseName: setupEnv.ReleaseName, + }) + Expect(err).NotTo(HaveOccurred()) + return data + }, + // All processes (including #1): connect to the shared environment. + func(data []byte) { + var shared utils.SharedEnvData + Expect(json.Unmarshal(data, &shared)).To(Succeed()) + + var err error + testEnv, err = utils.SetupSharedTestEnvironment(context.Background(), shared.Namespace, shared.ReleaseName) + Expect(err).NotTo(HaveOccurred(), "Failed to setup shared test environment") + + kubeClient = testEnv.KubeClient + csiClient = testEnv.CSIClient + restConfig = testEnv.RestConfig + testNamespace = testEnv.Namespace + ctx = testEnv.Ctx + }, +) + +var _ = SynchronizedAfterSuite( + // All processes: cancel the per-process context. + func() { + if testEnv != nil { + testEnv.Cancel() + } + }, + // Process 1 only (runs last): undeploy Reloader and delete namespace. + func() { + if testEnv != nil { + err := testEnv.Cleanup() + Expect(err).NotTo(HaveOccurred(), "Failed to cleanup test environment") + } + GinkgoWriter.Println("CSI E2E Suite cleanup complete") + }, +) diff --git a/test/e2e/csi/csi_test.go b/test/e2e/csi/csi_test.go new file mode 100644 index 00000000..ef55f2bd --- /dev/null +++ b/test/e2e/csi/csi_test.go @@ -0,0 +1,330 @@ +package csi + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/stakater/Reloader/test/e2e/utils" +) + +var _ = Describe("CSI SecretProviderClass Tests", Label("csi"), Serial, func() { + var ( + deploymentName string + configMapName string + spcName string + vaultSecretPath string + adapter *utils.DeploymentAdapter + ) + + BeforeEach(func() { + deploymentName = utils.RandName("deploy") + configMapName = utils.RandName("cm") + spcName = utils.RandName("spc") + vaultSecretPath = fmt.Sprintf("secret/%s", utils.RandName("test")) + adapter = utils.NewDeploymentAdapter(kubeClient) + }) + + AfterEach(func() { + _ = utils.DeleteDeployment(ctx, kubeClient, testNamespace, deploymentName) + _ = utils.DeleteConfigMap(ctx, kubeClient, testNamespace, configMapName) + _ = utils.DeleteSecretProviderClass(ctx, csiClient, testNamespace, spcName) + _ = utils.DeleteVaultSecret(ctx, kubeClient, restConfig, vaultSecretPath) + }) + + Context("Real Vault Integration Tests", func() { + It("should reload when Vault secret changes", func() { + By("Creating a secret in Vault") + err := utils.CreateVaultSecret( + ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"api_key": "initial-value-v1"}) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a SecretProviderClass pointing to Vault secret") + _, err = utils.CreateSecretProviderClassWithSecret( + ctx, csiClient, testNamespace, spcName, + vaultSecretPath, "api_key", + ) + Expect(err).NotTo(HaveOccurred()) + + By("Creating Deployment with CSI volume and SPC reload annotation") + _, err = utils.CreateDeployment( + ctx, kubeClient, testNamespace, deploymentName, + utils.WithCSIVolume(spcName), + utils.WithAnnotations(utils.BuildSecretProviderClassReloadAnnotation(spcName)), + ) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for Deployment to be ready") + err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout) + Expect(err).NotTo(HaveOccurred()) + + By("Finding the SPCPS created by CSI driver") + spcpsName, err := utils.FindSPCPSForDeployment( + ctx, csiClient, kubeClient, testNamespace, deploymentName, utils.WorkloadReadyTimeout, + ) + Expect(err).NotTo(HaveOccurred()) + GinkgoWriter.Printf("Found SPCPS: %s\n", spcpsName) + + By("Getting initial SPCPS version") + initialVersion, err := utils.GetSPCPSVersion(ctx, csiClient, testNamespace, spcpsName) + Expect(err).NotTo(HaveOccurred()) + GinkgoWriter.Printf("Initial SPCPS version: %s\n", initialVersion) + + By("Updating the Vault secret") + err = utils.UpdateVaultSecret( + ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"api_key": "updated-value-v2"}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for CSI driver to sync the new secret version") + err = utils.WaitForSPCPSVersionChange(ctx, csiClient, testNamespace, spcpsName, initialVersion, 10*time.Second) + Expect(err).NotTo(HaveOccurred()) + GinkgoWriter.Println("CSI driver synced new secret version") + + By("Waiting for Deployment to be reloaded by Reloader") + reloaded, err := adapter.WaitReloaded( + ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, utils.ReloadTimeout, + ) + Expect(err).NotTo(HaveOccurred()) + Expect(reloaded).To(BeTrue(), "Deployment should have been reloaded after Vault secret change") + }) + + It("should handle multiple Vault secret updates", func() { + By("Creating a secret in Vault") + err := utils.CreateVaultSecret( + ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"password": "pass-v1"}) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a SecretProviderClass pointing to Vault secret") + _, err = utils.CreateSecretProviderClassWithSecret( + ctx, csiClient, testNamespace, spcName, + vaultSecretPath, "password", + ) + Expect(err).NotTo(HaveOccurred()) + + By("Creating Deployment with CSI volume") + _, err = utils.CreateDeployment( + ctx, kubeClient, testNamespace, deploymentName, + utils.WithCSIVolume(spcName), + utils.WithAnnotations(utils.BuildSecretProviderClassReloadAnnotation(spcName)), + ) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for Deployment to be ready") + err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout) + Expect(err).NotTo(HaveOccurred()) + + By("Finding the SPCPS") + spcpsName, err := utils.FindSPCPSForDeployment( + ctx, csiClient, kubeClient, testNamespace, deploymentName, utils.WorkloadReadyTimeout, + ) + Expect(err).NotTo(HaveOccurred()) + + By("First update to Vault secret") + initialVersion, _ := utils.GetSPCPSVersion(ctx, csiClient, testNamespace, spcpsName) + err = utils.UpdateVaultSecret( + ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"password": "pass-v2"}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for first CSI sync") + err = utils.WaitForSPCPSVersionChange(ctx, csiClient, testNamespace, spcpsName, initialVersion, 10*time.Second) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for first reload") + reloaded, err := adapter.WaitReloaded( + ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, utils.ReloadTimeout, + ) + Expect(err).NotTo(HaveOccurred()) + Expect(reloaded).To(BeTrue()) + + By("Getting annotation value after first reload") + deploy, err := utils.GetDeployment(ctx, kubeClient, testNamespace, deploymentName) + Expect(err).NotTo(HaveOccurred()) + firstReloadValue := deploy.Spec.Template.Annotations[utils.AnnotationLastReloadedFrom] + Expect(firstReloadValue).NotTo(BeEmpty()) + + By("Waiting for Deployment to stabilize") + err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout) + Expect(err).NotTo(HaveOccurred()) + + By("Finding the NEW SPCPS after first reload (new pod = new SPCPS)") + newSpcpsName, err := utils.FindSPCPSForDeployment( + ctx, csiClient, kubeClient, testNamespace, deploymentName, utils.WorkloadReadyTimeout, + ) + Expect(err).NotTo(HaveOccurred()) + GinkgoWriter.Printf("New SPCPS after first reload: %s\n", newSpcpsName) + + By("Second update to Vault secret") + err = utils.UpdateVaultSecret( + ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"password": "pass-v3"}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for second reload with different annotation value") + Eventually(func() string { + deploy, err := utils.GetDeployment(ctx, kubeClient, testNamespace, deploymentName) + if err != nil { + return "" + } + return deploy.Spec.Template.Annotations[utils.AnnotationLastReloadedFrom] + }, utils.ReloadTimeout).ShouldNot(Equal(firstReloadValue), "Annotation should change after second Vault secret update") + }) + }) + + Context("Typed Auto Annotation Tests", func() { + It("should reload only SPC changes with secretproviderclass auto annotation, not ConfigMap", func() { + By("Creating a ConfigMap") + _, err := utils.CreateConfigMap( + ctx, kubeClient, testNamespace, configMapName, + map[string]string{"key": "initial"}, nil, + ) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a secret in Vault") + err = utils.CreateVaultSecret( + ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"token": "token-v1"}) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a SecretProviderClass pointing to Vault secret") + _, err = utils.CreateSecretProviderClassWithSecret( + ctx, csiClient, testNamespace, spcName, + vaultSecretPath, "token", + ) + Expect(err).NotTo(HaveOccurred()) + + By("Creating Deployment with ConfigMap envFrom AND CSI volume, but only SPC auto annotation") + _, err = utils.CreateDeployment( + ctx, kubeClient, testNamespace, deploymentName, + utils.WithConfigMapEnvFrom(configMapName), + utils.WithCSIVolume(spcName), + utils.WithAnnotations(utils.BuildSecretProviderClassAutoAnnotation()), + ) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for Deployment to be ready") + err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout) + Expect(err).NotTo(HaveOccurred()) + + By("Updating the ConfigMap (should NOT trigger reload)") + err = utils.UpdateConfigMap( + ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying Deployment was NOT reloaded for ConfigMap change") + time.Sleep(utils.NegativeTestWait) + reloaded, err := adapter.WaitReloaded( + ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, utils.ShortTimeout, + ) + Expect(err).NotTo(HaveOccurred()) + Expect(reloaded).To(BeFalse(), "SPC auto annotation should not trigger reload for ConfigMap changes") + + By("Finding the SPCPS") + spcpsName, err := utils.FindSPCPSForDeployment( + ctx, csiClient, kubeClient, testNamespace, deploymentName, utils.WorkloadReadyTimeout, + ) + Expect(err).NotTo(HaveOccurred()) + + By("Getting SPCPS version before Vault update") + initialVersion, _ := utils.GetSPCPSVersion(ctx, csiClient, testNamespace, spcpsName) + + By("Updating the Vault secret (should trigger reload)") + err = utils.UpdateVaultSecret( + ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"token": "token-v2"}) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for CSI driver to sync") + err = utils.WaitForSPCPSVersionChange(ctx, csiClient, testNamespace, spcpsName, initialVersion, 10*time.Second) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying Deployment WAS reloaded for Vault secret change") + reloaded, err = adapter.WaitReloaded( + ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, utils.ReloadTimeout, + ) + Expect(err).NotTo(HaveOccurred()) + Expect(reloaded).To(BeTrue(), "SPC auto annotation should trigger reload for Vault secret changes") + }) + + It("should reload for both ConfigMap and SPC when using combined auto=true", func() { + By("Creating a ConfigMap") + _, err := utils.CreateConfigMap( + ctx, kubeClient, testNamespace, configMapName, + map[string]string{"key": "initial"}, nil, + ) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a secret in Vault") + err = utils.CreateVaultSecret( + ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"secret": "secret-v1"}) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a SecretProviderClass pointing to Vault secret") + _, err = utils.CreateSecretProviderClassWithSecret( + ctx, csiClient, testNamespace, spcName, + vaultSecretPath, "secret", + ) + Expect(err).NotTo(HaveOccurred()) + + By("Creating Deployment with ConfigMap envFrom AND CSI volume with combined auto=true") + _, err = utils.CreateDeployment( + ctx, kubeClient, testNamespace, deploymentName, + utils.WithConfigMapEnvFrom(configMapName), + utils.WithCSIVolume(spcName), + utils.WithAnnotations(utils.BuildAutoTrueAnnotation()), + ) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for Deployment to be ready") + err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout) + Expect(err).NotTo(HaveOccurred()) + + By("Updating the ConfigMap (should trigger reload with auto=true)") + err = utils.UpdateConfigMap( + ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying Deployment WAS reloaded for ConfigMap change") + reloaded, err := adapter.WaitReloaded( + ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, utils.ReloadTimeout, + ) + Expect(err).NotTo(HaveOccurred()) + Expect(reloaded).To(BeTrue(), "Combined auto=true should trigger reload for ConfigMap changes") + + By("Waiting for Deployment to stabilize") + err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout) + Expect(err).NotTo(HaveOccurred()) + + By("Getting current annotation value") + deploy, err := utils.GetDeployment(ctx, kubeClient, testNamespace, deploymentName) + Expect(err).NotTo(HaveOccurred()) + firstReloadValue := deploy.Spec.Template.Annotations[utils.AnnotationLastReloadedFrom] + + By("Finding the NEW SPCPS after ConfigMap reload (new pod = new SPCPS)") + newSpcpsName, err := utils.FindSPCPSForDeployment( + ctx, csiClient, kubeClient, testNamespace, deploymentName, utils.WorkloadReadyTimeout, + ) + Expect(err).NotTo(HaveOccurred()) + GinkgoWriter.Printf("New SPCPS after ConfigMap reload: %s\n", newSpcpsName) + + By("Updating the Vault secret (should also trigger reload with auto=true)") + err = utils.UpdateVaultSecret( + ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"secret": "secret-v2"}) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying Deployment WAS reloaded for Vault secret change") + Eventually(func() string { + deploy, err := utils.GetDeployment(ctx, kubeClient, testNamespace, deploymentName) + if err != nil { + return "" + } + return deploy.Spec.Template.Annotations[utils.AnnotationLastReloadedFrom] + }, utils.ReloadTimeout).ShouldNot(Equal(firstReloadValue), + "Combined auto=true should trigger reload for Vault secret changes", + ) + }) + }) +}) diff --git a/test/e2e/utils/annotations.go b/test/e2e/utils/annotations.go index 1be04157..891c14e9 100644 --- a/test/e2e/utils/annotations.go +++ b/test/e2e/utils/annotations.go @@ -20,6 +20,11 @@ const ( // Value: comma-separated list of Secret names, e.g., "secret1,secret2" AnnotationSecretReload = "secret.reloader.stakater.com/reload" + // AnnotationSecretProviderClassReload triggers reload when specified SecretProviderClass(es) change. + // Value: comma-separated list of SecretProviderClass names, e.g., "spc1,spc2" + // Note: Reloader actually watches SecretProviderClassPodStatus resources, not SecretProviderClass. + AnnotationSecretProviderClassReload = "secretproviderclass.reloader.stakater.com/reload" + // ============================================================ // Auto-reload annotations // ============================================================ @@ -36,6 +41,10 @@ const ( // Value: "true" or "false" AnnotationSecretAuto = "secret.reloader.stakater.com/auto" + // AnnotationSecretProviderClassAuto enables auto-reload for all referenced SecretProviderClasses only. + // Value: "true" or "false" + AnnotationSecretProviderClassAuto = "secretproviderclass.reloader.stakater.com/auto" + // ============================================================ // Exclude annotations (used with auto=true to exclude specific resources) // ============================================================ @@ -48,6 +57,10 @@ const ( // Value: comma-separated list of Secret names AnnotationSecretExclude = "secrets.exclude.reloader.stakater.com/reload" + // AnnotationSecretProviderClassExclude excludes specified SecretProviderClasses from auto-reload. + // Value: comma-separated list of SecretProviderClass names + AnnotationSecretProviderClassExclude = "secretproviderclasses.exclude.reloader.stakater.com/reload" + // ============================================================ // Search annotations (for regex matching) // ============================================================ @@ -117,6 +130,13 @@ func BuildSecretReloadAnnotation(secretNames ...string) map[string]string { } } +// BuildSecretProviderClassReloadAnnotation creates an annotation map for SecretProviderClass reload. +func BuildSecretProviderClassReloadAnnotation(spcNames ...string) map[string]string { + return map[string]string{ + AnnotationSecretProviderClassReload: joinNames(spcNames), + } +} + // BuildAutoTrueAnnotation creates an annotation map with auto=true. func BuildAutoTrueAnnotation() map[string]string { return map[string]string{ @@ -145,6 +165,13 @@ func BuildSecretAutoAnnotation() map[string]string { } } +// BuildSecretProviderClassAutoAnnotation creates an annotation map with secretproviderclass auto=true. +func BuildSecretProviderClassAutoAnnotation() map[string]string { + return map[string]string{ + AnnotationSecretProviderClassAuto: AnnotationValueTrue, + } +} + // BuildSearchAnnotation creates an annotation map to enable search mode. func BuildSearchAnnotation() map[string]string { return map[string]string{ diff --git a/test/e2e/utils/conditions.go b/test/e2e/utils/conditions.go index 7afd146f..2e886379 100644 --- a/test/e2e/utils/conditions.go +++ b/test/e2e/utils/conditions.go @@ -6,6 +6,7 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" ) // PodTemplateAccessor extracts PodTemplateSpec from a workload. @@ -215,3 +216,35 @@ func IsTriggeredJobForCronJob(cronJobName string) Condition[*batchv1.Job] { return false } } + +// SPCPSVersionChanged returns a condition that checks if the SPCPS version has changed +// from the initial version and the SPCPS is mounted. +func SPCPSVersionChanged(initialVersion string) Condition[*csiv1.SecretProviderClassPodStatus] { + return func(spcps *csiv1.SecretProviderClassPodStatus) bool { + if !spcps.Status.Mounted || len(spcps.Status.Objects) == 0 { + return false + } + for _, obj := range spcps.Status.Objects { + if obj.Version != initialVersion { + return true + } + } + return false + } +} + +// SPCPSForSPC returns a condition that checks if the SPCPS references a specific +// SecretProviderClass and is mounted. +func SPCPSForSPC(spcName string) Condition[*csiv1.SecretProviderClassPodStatus] { + return func(spcps *csiv1.SecretProviderClassPodStatus) bool { + return spcps.Status.SecretProviderClassName == spcName && spcps.Status.Mounted + } +} + +// SPCPSForPods returns a condition that checks if the SPCPS references any of the +// specified pods and is mounted. +func SPCPSForPods(podNames map[string]bool) Condition[*csiv1.SecretProviderClassPodStatus] { + return func(spcps *csiv1.SecretProviderClassPodStatus) bool { + return podNames[spcps.Status.PodName] && spcps.Status.Mounted + } +} diff --git a/test/e2e/utils/csi.go b/test/e2e/utils/csi.go new file mode 100644 index 00000000..3a34ff2a --- /dev/null +++ b/test/e2e/utils/csi.go @@ -0,0 +1,338 @@ +package utils + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" + csiclient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned" +) + +// CSI Driver constants +const ( + // CSIDriverName is the name of the secrets-store CSI driver + CSIDriverName = "secrets-store.csi.k8s.io" + + // DefaultCSIProvider is the default provider name for testing (Vault) + DefaultCSIProvider = "vault" + + // VaultAddress is the default Vault address in the cluster + VaultAddress = "http://vault.vault:8200" + + // VaultRole is the Kubernetes auth role configured in Vault for testing + VaultRole = "test-role" + + // VaultNamespace is the namespace where Vault is deployed + VaultNamespace = "vault" + + // VaultPodName is the name of the Vault pod (dev mode) + VaultPodName = "vault-0" + + // CSIVolumeName is the default volume name for CSI volumes in tests + CSIVolumeName = "csi-secrets-store" + + // CSIMountPath is the default mount path for CSI volumes in tests + CSIMountPath = "/mnt/secrets-store" + + // CSIRotationPollInterval is how often CSI driver checks for secret changes + CSIRotationPollInterval = 2 * time.Second +) + +// NewCSIClient creates a new CSI client using the default kubeconfig. +func NewCSIClient() (csiclient.Interface, error) { + kubeconfig := GetKubeconfig() + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return nil, fmt.Errorf("building config from kubeconfig: %w", err) + } + return NewCSIClientFromConfig(config) +} + +// NewCSIClientFromConfig creates a new CSI client from a rest.Config. +func NewCSIClientFromConfig(config *rest.Config) (csiclient.Interface, error) { + client, err := csiclient.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("creating CSI client: %w", err) + } + return client, nil +} + +// IsCSIDriverInstalled checks if the CSI secrets store driver CRDs are available in the cluster. +// This checks for the SecretProviderClass CRD which is required for CSI tests. +func IsCSIDriverInstalled(ctx context.Context, client csiclient.Interface) bool { + if client == nil { + return false + } + + // Try to list SecretProviderClasses - if CRD doesn't exist, this will fail + _, err := client.SecretsstoreV1().SecretProviderClasses("default").List(ctx, metav1.ListOptions{Limit: 1}) + return err == nil +} + +// IsVaultProviderInstalled checks if Vault CSI provider is installed by checking for the vault-csi-provider DaemonSet. +// This is used to determine if CSI tests with actual volume mounting can run. +func IsVaultProviderInstalled(ctx context.Context, kubeClient kubernetes.Interface) bool { + if kubeClient == nil { + return false + } + + // Check if vault-csi-provider DaemonSet exists in vault namespace + _, err := kubeClient.AppsV1().DaemonSets("vault").Get(ctx, "vault-csi-provider", metav1.GetOptions{}) + return err == nil +} + +// CreateSecretProviderClass creates a SecretProviderClass in the given namespace. +// If params is nil, it creates a Vault-compatible SecretProviderClass with default test settings. +func CreateSecretProviderClass(ctx context.Context, client csiclient.Interface, namespace, name string, params map[string]string) ( + *csiv1.SecretProviderClass, error, +) { + if params == nil { + params = map[string]string{ + "vaultAddress": VaultAddress, + "roleName": VaultRole, + "objects": `- objectName: "test-secret" + secretPath: "secret/data/test" + secretKey: "username"`, + } + } + + spc := &csiv1.SecretProviderClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: csiv1.SecretProviderClassSpec{ + Provider: DefaultCSIProvider, + Parameters: params, + }, + } + + created, err := client.SecretsstoreV1().SecretProviderClasses(namespace).Create(ctx, spc, metav1.CreateOptions{}) + if err != nil { + return nil, fmt.Errorf("creating SecretProviderClass %s/%s: %w", namespace, name, err) + } + return created, nil +} + +// CreateSecretProviderClassWithSecret creates a SecretProviderClass that fetches a specific secret from Vault. +// secretPath should be like "secret/mysecret" (the function converts it to KV v2 format "secret/data/mysecret"). +// secretKey is the key within that secret to fetch. +func CreateSecretProviderClassWithSecret(ctx context.Context, client csiclient.Interface, namespace, name, secretPath, secretKey string) ( + *csiv1.SecretProviderClass, error, +) { + kvV2Path := secretPath + if strings.HasPrefix(secretPath, "secret/") && !strings.HasPrefix(secretPath, "secret/data/") { + kvV2Path = strings.Replace(secretPath, "secret/", "secret/data/", 1) + } + + params := map[string]string{ + "vaultAddress": VaultAddress, + "roleName": VaultRole, + "objects": fmt.Sprintf( + `- objectName: "%s" + secretPath: "%s" + secretKey: "%s"`, secretKey, kvV2Path, secretKey, + ), + } + return CreateSecretProviderClass(ctx, client, namespace, name, params) +} + +// DeleteSecretProviderClass deletes a SecretProviderClass by name. +func DeleteSecretProviderClass(ctx context.Context, client csiclient.Interface, namespace, name string) error { + err := client.SecretsstoreV1().SecretProviderClasses(namespace).Delete(ctx, name, metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("deleting SecretProviderClass %s/%s: %w", namespace, name, err) + } + return nil +} + +// UpdateSecretProviderClassPodStatusLabels updates only the labels on a SecretProviderClassPodStatus. +// This should NOT trigger a reload (used for negative testing to verify Reloader ignores label-only changes). +func UpdateSecretProviderClassPodStatusLabels(ctx context.Context, client csiclient.Interface, namespace, name string, labels map[string]string) error { + spcps, err := client.SecretsstoreV1().SecretProviderClassPodStatuses(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("getting SecretProviderClassPodStatus %s/%s: %w", namespace, name, err) + } + + if spcps.Labels == nil { + spcps.Labels = make(map[string]string) + } + for k, v := range labels { + spcps.Labels[k] = v + } + + _, err = client.SecretsstoreV1().SecretProviderClassPodStatuses(namespace).Update(ctx, spcps, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("updating SecretProviderClassPodStatus labels %s/%s: %w", namespace, name, err) + } + return nil +} + +// ============================================================================= +// Vault Integration Helpers +// ============================================================================= + +// CreateVaultSecret creates a new secret in Vault. +// secretPath should be like "secret/test" (without "data" prefix - it's added automatically). +// data is a map of key-value pairs to store in the secret. +func CreateVaultSecret(ctx context.Context, kubeClient kubernetes.Interface, restConfig *rest.Config, secretPath string, data map[string]string) error { + return UpdateVaultSecret(ctx, kubeClient, restConfig, secretPath, data) +} + +// UpdateVaultSecret updates a secret in Vault. This triggers the CSI driver to +// sync the new secret version, which creates/updates the SecretProviderClassPodStatus. +// secretPath should be like "secret/test" (without "data" prefix - it's added automatically). +// data is a map of key-value pairs to store in the secret. +func UpdateVaultSecret(ctx context.Context, kubeClient kubernetes.Interface, restConfig *rest.Config, secretPath string, data map[string]string) error { + args := []string{"kv", "put", secretPath} + for k, v := range data { + args = append(args, fmt.Sprintf("%s=%s", k, v)) + } + + if err := execInVaultPod(ctx, kubeClient, restConfig, args); err != nil { + return fmt.Errorf("updating Vault secret %s: %w", secretPath, err) + } + return nil +} + +// DeleteVaultSecret deletes a secret from Vault. +// secretPath should be like "secret/test". +func DeleteVaultSecret(ctx context.Context, kubeClient kubernetes.Interface, restConfig *rest.Config, secretPath string) error { + args := []string{"kv", "metadata", "delete", secretPath} + if err := execInVaultPod(ctx, kubeClient, restConfig, args); err != nil { + if strings.Contains(err.Error(), "No value found") { + return nil + } + return fmt.Errorf("deleting Vault secret %s: %w", secretPath, err) + } + return nil +} + +// execInVaultPod executes a vault command in the Vault pod. +func execInVaultPod(ctx context.Context, kubeClient kubernetes.Interface, restConfig *rest.Config, args []string) error { + req := kubeClient.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(VaultPodName). + Namespace(VaultNamespace). + SubResource("exec"). + VersionedParams( + &corev1.PodExecOptions{ + Container: "vault", + Command: append([]string{"vault"}, args...), + Stdout: true, + Stderr: true, + }, scheme.ParameterCodec, + ) + + exec, err := remotecommand.NewSPDYExecutor(restConfig, "POST", req.URL()) + if err != nil { + return fmt.Errorf("creating executor: %w", err) + } + + var stdout, stderr bytes.Buffer + err = exec.StreamWithContext( + ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }, + ) + if err != nil { + return fmt.Errorf("executing command: %w (stderr: %s)", err, stderr.String()) + } + + return nil +} + +// WaitForSPCPSVersionChange waits for the SecretProviderClassPodStatus version to change +// from the initial version using watches. This is used after updating a Vault secret to +// wait for CSI driver to sync the new version. +func WaitForSPCPSVersionChange(ctx context.Context, client csiclient.Interface, namespace, spcpsName, initialVersion string, timeout time.Duration) error { + watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + return client.SecretsstoreV1().SecretProviderClassPodStatuses(namespace).Watch(ctx, opts) + } + + _, err := WatchUntil(ctx, watchFunc, spcpsName, SPCPSVersionChanged(initialVersion), timeout) + if errors.Is(err, ErrWatchTimeout) { + return fmt.Errorf("timeout waiting for SecretProviderClassPodStatus %s/%s version to change from %s", namespace, spcpsName, initialVersion) + } + return err +} + +// FindSPCPSForDeployment finds the SecretProviderClassPodStatus created by CSI driver +// for pods of a given deployment using watches. Returns the first matching SPCPS name. +func FindSPCPSForDeployment(ctx context.Context, csiClient csiclient.Interface, kubeClient kubernetes.Interface, namespace, deploymentName string, timeout time.Duration) ( + string, error, +) { + pods, err := kubeClient.CoreV1().Pods(namespace).List( + ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("app=%s", deploymentName), + }, + ) + if err != nil { + return "", fmt.Errorf("listing pods for deployment %s: %w", deploymentName, err) + } + + podNames := make(map[string]bool) + for _, pod := range pods.Items { + podNames[pod.Name] = true + } + + watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + return csiClient.SecretsstoreV1().SecretProviderClassPodStatuses(namespace).Watch(ctx, opts) + } + + spcps, err := WatchUntil(ctx, watchFunc, "", SPCPSForPods(podNames), timeout) + if errors.Is(err, ErrWatchTimeout) { + return "", fmt.Errorf("timeout finding SecretProviderClassPodStatus for deployment %s/%s", namespace, deploymentName) + } + if err != nil { + return "", err + } + return spcps.Name, nil +} + +// FindSPCPSForSPC finds the SecretProviderClassPodStatus created by CSI driver +// that references a specific SecretProviderClass using watches. Returns the first matching SPCPS name. +func FindSPCPSForSPC(ctx context.Context, csiClient csiclient.Interface, namespace, spcName string, timeout time.Duration) (string, error) { + watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + return csiClient.SecretsstoreV1().SecretProviderClassPodStatuses(namespace).Watch(ctx, opts) + } + + spcps, err := WatchUntil(ctx, watchFunc, "", SPCPSForSPC(spcName), timeout) + if errors.Is(err, ErrWatchTimeout) { + return "", fmt.Errorf("timeout finding SecretProviderClassPodStatus for SPC %s/%s", namespace, spcName) + } + if err != nil { + return "", err + } + return spcps.Name, nil +} + +// GetSPCPSVersion gets the current version string from a SecretProviderClassPodStatus. +// Returns the version of the first object, or empty string if not found. +func GetSPCPSVersion(ctx context.Context, client csiclient.Interface, namespace, name string) (string, error) { + spcps, err := client.SecretsstoreV1().SecretProviderClassPodStatuses(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("getting SecretProviderClassPodStatus %s/%s: %w", namespace, name, err) + } + if len(spcps.Status.Objects) == 0 { + return "", nil + } + var versions []string + for _, obj := range spcps.Status.Objects { + versions = append(versions, obj.Version) + } + return strings.Join(versions, ","), nil +} diff --git a/test/e2e/utils/resources.go b/test/e2e/utils/resources.go index 36ae3be7..960dc6b9 100644 --- a/test/e2e/utils/resources.go +++ b/test/e2e/utils/resources.go @@ -516,6 +516,43 @@ func WithInitContainerProjectedVolume(cmName, secretName string) DeploymentOptio } } +// WithCSIVolume adds a CSI volume referencing a SecretProviderClass to a Deployment. +func WithCSIVolume(spcName string) DeploymentOption { + return func(d *appsv1.Deployment) { + volumeName := csiVolumeName(spcName) + mountPath := csiMountPath(spcName) + + d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + CSI: &corev1.CSIVolumeSource{ + Driver: CSIDriverName, + ReadOnly: ptr.To(true), + VolumeAttributes: map[string]string{ + "secretProviderClass": spcName, + }, + }, + }, + }) + d.Spec.Template.Spec.Containers[0].VolumeMounts = append( + d.Spec.Template.Spec.Containers[0].VolumeMounts, + corev1.VolumeMount{ + Name: volumeName, + MountPath: mountPath, + ReadOnly: true, + }, + ) + } +} + +func csiVolumeName(spcName string) string { + return fmt.Sprintf("csi-%s", spcName) +} + +func csiMountPath(spcName string) string { + return fmt.Sprintf("/mnt/secrets-store/%s", spcName) +} + func baseDeploymentResource(namespace, name string) *appsv1.Deployment { labels := map[string]string{"app": name} return &appsv1.Deployment{ diff --git a/test/e2e/utils/testenv.go b/test/e2e/utils/testenv.go index 6f73bd06..a8be4551 100644 --- a/test/e2e/utils/testenv.go +++ b/test/e2e/utils/testenv.go @@ -13,6 +13,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + csiclient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned" ) // TestEnvironment holds the common test environment state. @@ -21,6 +22,7 @@ type TestEnvironment struct { Cancel context.CancelFunc KubeClient kubernetes.Interface DiscoveryClient discovery.DiscoveryInterface + CSIClient csiclient.Interface RolloutsClient rolloutsclient.Interface OpenShiftClient openshiftclient.Interface RestConfig *rest.Config @@ -82,6 +84,9 @@ func SetupSharedTestEnvironment(ctx context.Context, namespace, releaseName stri } // Optional clients — failures are non-fatal. + if env.CSIClient, err = csiclient.NewForConfig(config); err != nil { + env.CSIClient = nil + } if env.RolloutsClient, err = rolloutsclient.NewForConfig(config); err != nil { env.RolloutsClient = nil } @@ -130,6 +135,12 @@ func SetupTestEnvironment(ctx context.Context, namespacePrefix string) (*TestEnv return nil, fmt.Errorf("creating discovery client: %w", err) } + env.CSIClient, err = csiclient.NewForConfig(config) + if err != nil { + ginkgo.GinkgoWriter.Printf("Warning: Could not create CSI client: %v (CSI tests will be skipped)\n", err) + env.CSIClient = nil + } + // Try to create Argo Rollouts client (optional - may not be installed) env.RolloutsClient, err = rolloutsclient.NewForConfig(config) if err != nil {