diff --git a/go.mod b/go.mod index 05edecc..1b51b65 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( k8s.io/client-go v0.32.3 k8s.io/kubectl v0.32.3 k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 + sigs.k8s.io/secrets-store-csi-driver v1.5.4 ) require ( diff --git a/go.sum b/go.sum index 59339ea..05738d7 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,7 @@ github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= @@ -192,6 +193,8 @@ sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/secrets-store-csi-driver v1.5.4 h1:enl+v1+JbKDyVjdfT/7CillZsc4rLAM9tTHyf7GeLxc= +sigs.k8s.io/secrets-store-csi-driver v1.5.4/go.mod h1:Ct85xqsKLk/dxkj8inRjWA3RJsXXkPLjNSAJ0db5vKs= sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/internal/pkg/cmd/reloader.go b/internal/pkg/cmd/reloader.go index f54d757..f20e0b8 100644 --- a/internal/pkg/cmd/reloader.go +++ b/internal/pkg/cmd/reloader.go @@ -160,6 +160,16 @@ func startReloader(cmd *cobra.Command, args []string) { var controllers []*controller.Controller for k := range kube.ResourceMap { + if k == "secretproviderclasspodstatuses" { + if !options.EnableCSIIntegration { + continue + } + if !kube.IsCSIInstalled { + logrus.Infof("Can't run secretproviderclasspodstatuses controller as CSI CRDs are not installed") + continue + } + } + if ignoredResourcesList.Contains(k) || (len(namespaceLabelSelector) == 0 && k == "namespaces") { continue } diff --git a/internal/pkg/constants/constants.go b/internal/pkg/constants/constants.go index 18d1cc7..0d1f1c7 100644 --- a/internal/pkg/constants/constants.go +++ b/internal/pkg/constants/constants.go @@ -8,6 +8,8 @@ const ( ConfigmapEnvVarPostfix = "CONFIGMAP" // SecretEnvVarPostfix is a postfix for secret envVar SecretEnvVarPostfix = "SECRET" + // SecretProviderClassEnvVarPostfix is a postfix for secretproviderclasspodstatus envVar + SecretProviderClassEnvVarPostfix = "SECRETPROVIDERCLASS" // EnvVarPrefix is a Prefix for environment variable EnvVarPrefix = "STAKATER_" diff --git a/internal/pkg/controller/controller.go b/internal/pkg/controller/controller.go index 15b2e0f..a670d81 100644 --- a/internal/pkg/controller/controller.go +++ b/internal/pkg/controller/controller.go @@ -22,6 +22,7 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/kubectl/pkg/scheme" "k8s.io/utils/strings/slices" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" ) // Controller for checking events @@ -79,7 +80,16 @@ func NewController( } } - listWatcher := cache.NewFilteredListWatchFromClient(client.CoreV1().RESTClient(), resource, namespace, optionsModifier) + getterRESTClient := client.CoreV1().RESTClient() + if resource == "secretproviderclasspodstatuses" { + csiClient, err := kube.GetCSIClient() + if err != nil { + logrus.Fatal(err) + } + getterRESTClient = csiClient.SecretsstoreV1().RESTClient() + } + + listWatcher := cache.NewFilteredListWatchFromClient(getterRESTClient, resource, namespace, optionsModifier) _, informer := cache.NewInformerWithOptions(cache.InformerOptions{ ListerWatcher: listWatcher, @@ -108,6 +118,8 @@ func (c *Controller) Add(obj interface{}) { case *v1.Namespace: c.addSelectedNamespaceToCache(*object) return + case *csiv1.SecretProviderClassPodStatus: + return } if options.ReloadOnCreate == "true" { @@ -122,11 +134,13 @@ func (c *Controller) Add(obj interface{}) { } func (c *Controller) resourceInIgnoredNamespace(raw interface{}) bool { - switch object := raw.(type) { + switch obj := raw.(type) { case *v1.ConfigMap: - return c.ignoredNamespaces.Contains(object.Namespace) + return c.ignoredNamespaces.Contains(obj.Namespace) case *v1.Secret: - return c.ignoredNamespaces.Contains(object.Namespace) + return c.ignoredNamespaces.Contains(obj.Namespace) + case *csiv1.SecretProviderClassPodStatus: + return c.ignoredNamespaces.Contains(obj.Namespace) } return false } @@ -145,6 +159,10 @@ func (c *Controller) resourceInSelectedNamespaces(raw interface{}) bool { if slices.Contains(selectedNamespacesCache, object.GetNamespace()) { return true } + case *csiv1.SecretProviderClassPodStatus: + if slices.Contains(selectedNamespacesCache, object.GetNamespace()) { + return true + } } return false } @@ -183,6 +201,9 @@ func (c *Controller) Update(old interface{}, new interface{}) { // Delete function to add an object to the queue in case of deleting a resource func (c *Controller) Delete(old interface{}) { + if _, ok := old.(*csiv1.SecretProviderClassPodStatus); ok { + return + } if options.ReloadOnDelete == "true" { if !c.resourceInIgnoredNamespace(old) && c.resourceInSelectedNamespaces(old) && secretControllerInitialized && configmapControllerInitialized { diff --git a/internal/pkg/controller/controller_test.go b/internal/pkg/controller/controller_test.go index 63e6be3..0399933 100644 --- a/internal/pkg/controller/controller_test.go +++ b/internal/pkg/controller/controller_test.go @@ -26,14 +26,15 @@ import ( ) var ( - clients = kube.GetClients() - namespace = "test-reloader-" + testutil.RandSeq(5) - configmapNamePrefix = "testconfigmap-reloader" - secretNamePrefix = "testsecret-reloader" - data = "dGVzdFNlY3JldEVuY29kaW5nRm9yUmVsb2FkZXI=" - newData = "dGVzdE5ld1NlY3JldEVuY29kaW5nRm9yUmVsb2FkZXI=" - updatedData = "dGVzdFVwZGF0ZWRTZWNyZXRFbmNvZGluZ0ZvclJlbG9hZGVy" - collectors = metrics.NewCollectors() + clients = kube.GetClients() + namespace = "test-reloader-" + testutil.RandSeq(5) + configmapNamePrefix = "testconfigmap-reloader" + secretNamePrefix = "testsecret-reloader" + secretProviderClassPodStatusPrefix = "testsecretproviderclasspodstatus-reloader" + data = "dGVzdFNlY3JldEVuY29kaW5nRm9yUmVsb2FkZXI=" + newData = "dGVzdE5ld1NlY3JldEVuY29kaW5nRm9yUmVsb2FkZXI=" + updatedData = "dGVzdFVwZGF0ZWRTZWNyZXRFbmNvZGluZ0ZvclJlbG9hZGVy" + collectors = metrics.NewCollectors() ) const ( @@ -46,6 +47,10 @@ func TestMain(m *testing.M) { logrus.Infof("Creating controller") for k := range kube.ResourceMap { + // Don't create controller if CSI provider is not installed + if k == "secretproviderclasspodstatuses" && !kube.IsCSIInstalled { + continue + } if k == "namespaces" { continue } @@ -579,6 +584,217 @@ func TestControllerUpdatingSecretLabelsShouldNotCreateOrUpdatePodAnnotationInDep time.Sleep(sleepDuration) } +// Perform rolling upgrade on deployment and create pod annotation var upon updating the secretclassproviderpodstatus +func TestControllerUpdatingSecretProviderClassPodStatusShouldCreatePodAnnotationInDeployment(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + + if !kube.IsCSIInstalled { + return + } + + // Creating secretproviderclass + secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) + _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretproviderclass %v", err) + } + + // Creating secretproviderclasspodstatus + spcpsClient, err := testutil.CreateSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclasssproviderpodstatus %v", err) + } + + // Creating deployment + _, err = testutil.CreateDeployment(clients.KubernetesClient, secretproviderclasspodstatusName, namespace, true) + if err != nil { + t.Errorf("Error in deployment creation: %v", err) + } + + // Updating secretproviderclasspodstatus for first time + updateErr := testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "", newData) + if updateErr != nil { + t.Errorf("Secretproviderclasspodstatus was not updated") + } + + // Verifying deployment update + logrus.Infof("Verifying pod annotation has been created") + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, namespace, secretproviderclasspodstatusName, newData) + config := common.Config{ + Namespace: namespace, + ResourceName: secretproviderclasspodstatusName, + SHAValue: shaData, + Annotation: options.SecretProviderClassUpdateOnChangeAnnotation, + } + deploymentFuncs := handler.GetDeploymentRollingUpgradeFuncs() + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + time.Sleep(sleepDuration) + + // Deleting deployment + err = testutil.DeleteDeployment(clients.KubernetesClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the deployment %v", err) + } + + // Deleting secretproviderclass + err = testutil.DeleteSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass %v", err) + } + + // Deleting secretproviderclasspodstatus + err = testutil.DeleteSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclasspodstatus %v", err) + } + time.Sleep(sleepDuration) +} + +// Perform rolling upgrade on deployment and update pod annotation var upon updating the secretproviderclasspodstatus +func TestControllerUpdatingSecretProviderClassPodStatusShouldUpdatePodAnnotationInDeployment(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + + if !kube.IsCSIInstalled { + return + } + + // Creating secretproviderclass + secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) + _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretproviderclass %v", err) + } + + // Creating secretproviderclasspodstatus + spcpsClient, err := testutil.CreateSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclasssproviderpodstatus %v", err) + } + + // Creating deployment + _, err = testutil.CreateDeployment(clients.KubernetesClient, secretproviderclasspodstatusName, namespace, true) + if err != nil { + t.Errorf("Error in deployment creation: %v", err) + } + + // Updating Secret + err = testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "", newData) + if err != nil { + t.Errorf("Error while updating secretproviderclasspodstatus %v", err) + } + + // Updating Secret + err = testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "", updatedData) + if err != nil { + t.Errorf("Error while updating secretproviderclasspodstatus %v", err) + } + + // Verifying Upgrade + logrus.Infof("Verifying pod annotation has been updated") + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, namespace, secretproviderclasspodstatusName, updatedData) + config := common.Config{ + Namespace: namespace, + ResourceName: secretproviderclasspodstatusName, + SHAValue: shaData, + Annotation: options.SecretProviderClassUpdateOnChangeAnnotation, + } + deploymentFuncs := handler.GetDeploymentRollingUpgradeFuncs() + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + // Deleting Deployment + err = testutil.DeleteDeployment(clients.KubernetesClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the deployment %v", err) + } + + // Deleting secretproviderclass + err = testutil.DeleteSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass %v", err) + } + + // Deleting secretproviderclasspodstatus + err = testutil.DeleteSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclasspodstatus %v", err) + } + time.Sleep(sleepDuration) + +} + +// Do not Perform rolling upgrade on pod and create or update a pod annotation upon updating the label in secretproviderclasspodstatus +func TestControllerUpdatingSecretProviderClassPodStatusWithSameDataShouldNotCreateOrUpdatePodAnnotationInDeployment(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + + if !kube.IsCSIInstalled { + return + } + + // Creating secretproviderclass + secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) + _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretproviderclass %v", err) + } + + // Creating secretproviderclasspodstatus + spcpsClient, err := testutil.CreateSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclasssproviderpodstatus %v", err) + } + + // Creating deployment + _, err = testutil.CreateDeployment(clients.KubernetesClient, secretproviderclasspodstatusName, namespace, true) + if err != nil { + t.Errorf("Error in deployment creation: %v", err) + } + + err = testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "", data) + if err != nil { + t.Errorf("Error while updating secretproviderclasspodstatus %v", err) + } + + // Verifying Upgrade + logrus.Infof("Verifying pod annotation has been created") + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, namespace, secretproviderclasspodstatusName, data) + config := common.Config{ + Namespace: namespace, + ResourceName: secretproviderclasspodstatusName, + SHAValue: shaData, + Annotation: options.SecretProviderClassUpdateOnChangeAnnotation, + } + deploymentFuncs := handler.GetDeploymentRollingUpgradeFuncs() + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if updated { + t.Errorf("Deployment should not be updated by changing in secretproviderclasspodstatus") + } + + // Deleting Deployment + err = testutil.DeleteDeployment(clients.KubernetesClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the deployment %v", err) + } + + // Deleting secretproviderclass + err = testutil.DeleteSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass %v", err) + } + + // Deleting secretproviderclasspodstatus + err = testutil.DeleteSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclasspodstatus %v", err) + } + time.Sleep(sleepDuration) +} + // Perform rolling upgrade on DaemonSet and create pod annotation var upon updating the configmap func TestControllerUpdatingConfigmapShouldCreatePodAnnotationInDaemonSet(t *testing.T) { options.ReloadStrategy = constants.AnnotationsReloadStrategy @@ -1531,6 +1747,215 @@ func TestControllerUpdatingSecretLabelsShouldNotCreateOrUpdateEnvInDeployment(t time.Sleep(sleepDuration) } +// Perform rolling upgrade on pod and create a env var upon updating the secretproviderclasspodstatus +func TestControllerUpdatingSecretProviderClassPodStatusShouldCreateEnvInDeployment(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + + if !kube.IsCSIInstalled { + return + } + + // Creating secretproviderclass + secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) + _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretproviderclass %v", err) + } + + // Creating secretproviderclasspodstatus + spcpsClient, err := testutil.CreateSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclasssproviderpodstatus %v", err) + } + + // Creating deployment + _, err = testutil.CreateDeployment(clients.KubernetesClient, secretproviderclasspodstatusName, namespace, true) + if err != nil { + t.Errorf("Error in deployment creation: %v", err) + } + + // Updating Secret + err = testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "", newData) + if err != nil { + t.Errorf("Error while updating secretproviderclasspodstatus %v", err) + } + + // Verifying Upgrade + logrus.Infof("Verifying env var has been created") + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, namespace, secretproviderclasspodstatusName, newData) + config := common.Config{ + Namespace: namespace, + ResourceName: secretproviderclasspodstatusName, + SHAValue: shaData, + Annotation: options.SecretProviderClassUpdateOnChangeAnnotation, + } + deploymentFuncs := handler.GetDeploymentRollingUpgradeFuncs() + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, constants.SecretProviderClassEnvVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + // Deleting Deployment + err = testutil.DeleteDeployment(clients.KubernetesClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the deployment %v", err) + } + + // Deleting secretproviderclass + err = testutil.DeleteSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass %v", err) + } + + // Deleting secretproviderclasspodstatus + err = testutil.DeleteSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclasspodstatus %v", err) + } + time.Sleep(sleepDuration) +} + +// Perform rolling upgrade on deployment and update env var upon updating the secretproviderclasspodstatus +func TestControllerUpdatingSecretProviderClassPodStatusShouldUpdateEnvInDeployment(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + + if !kube.IsCSIInstalled { + return + } + + // Creating secretproviderclass + secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) + _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretproviderclass %v", err) + } + + // Creating secretproviderclasspodstatus + spcpsClient, err := testutil.CreateSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclasssproviderpodstatus %v", err) + } + + // Creating deployment + _, err = testutil.CreateDeployment(clients.KubernetesClient, secretproviderclasspodstatusName, namespace, true) + if err != nil { + t.Errorf("Error in deployment creation: %v", err) + } + + // Updating secretproviderclasspodstatus + err = testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "", newData) + if err != nil { + t.Errorf("Error while updating secretproviderclasspodstatus %v", err) + } + + // Updating secretproviderclasspodstatus + err = testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "", updatedData) + if err != nil { + t.Errorf("Error while updating secretproviderclasspodstatus %v", err) + } + + // Verifying Upgrade + logrus.Infof("Verifying env var has been updated") + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, namespace, secretproviderclasspodstatusName, updatedData) + config := common.Config{ + Namespace: namespace, + ResourceName: secretproviderclasspodstatusName, + SHAValue: shaData, + Annotation: options.SecretProviderClassUpdateOnChangeAnnotation, + } + deploymentFuncs := handler.GetDeploymentRollingUpgradeFuncs() + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, constants.SecretProviderClassEnvVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + // Deleting Deployment + err = testutil.DeleteDeployment(clients.KubernetesClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the deployment %v", err) + } + + // Deleting secretproviderclass + err = testutil.DeleteSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass %v", err) + } + + // Deleting secretproviderclasspodstatus + err = testutil.DeleteSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclasspodstatus %v", err) + } + time.Sleep(sleepDuration) +} + +// Do not Perform rolling upgrade on pod and create or update a env var upon updating the label in secretclasssproviderpodstatus +func TestControllerUpdatingSecretProviderClassPodStatusLabelsShouldNotCreateOrUpdateEnvInDeployment(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + + if !kube.IsCSIInstalled { + return + } + + // Creating secretproviderclass + secretproviderclasspodstatusName := secretProviderClassPodStatusPrefix + "-update-" + testutil.RandSeq(5) + _, err := testutil.CreateSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretproviderclass %v", err) + } + + // Creating secretproviderclasspodstatus + spcpsClient, err := testutil.CreateSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName, data) + if err != nil { + t.Errorf("Error while creating the secretclasssproviderpodstatus %v", err) + } + + // Creating deployment + _, err = testutil.CreateDeployment(clients.KubernetesClient, secretproviderclasspodstatusName, namespace, true) + if err != nil { + t.Errorf("Error in deployment creation: %v", err) + } + + err = testutil.UpdateSecretProviderClassPodStatus(spcpsClient, namespace, secretproviderclasspodstatusName, "test", data) + if err != nil { + t.Errorf("Error while updating secretproviderclasspodstatus %v", err) + } + + // Verifying Upgrade + logrus.Infof("Verifying env var has been created") + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, namespace, secretproviderclasspodstatusName, data) + config := common.Config{ + Namespace: namespace, + ResourceName: secretproviderclasspodstatusName, + SHAValue: shaData, + Annotation: options.SecretProviderClassUpdateOnChangeAnnotation, + } + deploymentFuncs := handler.GetDeploymentRollingUpgradeFuncs() + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, constants.SecretProviderClassEnvVarPostfix, deploymentFuncs) + if updated { + t.Errorf("Deployment should not be updated by changing label in secretproviderclasspodstatus") + } + + // Deleting Deployment + err = testutil.DeleteDeployment(clients.KubernetesClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the deployment %v", err) + } + + // Deleting secretproviderclass + err = testutil.DeleteSecretProviderClass(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass %v", err) + } + + // Deleting secretproviderclasspodstatus + err = testutil.DeleteSecretProviderClassPodStatus(clients.CSIClient, namespace, secretproviderclasspodstatusName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclasspodstatus %v", err) + } + time.Sleep(sleepDuration) +} + // Perform rolling upgrade on DaemonSet and create env var upon updating the configmap func TestControllerUpdatingConfigmapShouldCreateEnvInDaemonSet(t *testing.T) { options.ReloadStrategy = constants.EnvVarsReloadStrategy diff --git a/internal/pkg/handler/update.go b/internal/pkg/handler/update.go index ae0bb1e..cc1e16b 100644 --- a/internal/pkg/handler/update.go +++ b/internal/pkg/handler/update.go @@ -8,6 +8,7 @@ import ( "github.com/stakater/Reloader/pkg/common" v1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/record" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" ) // ResourceUpdatedHandler contains updated objects @@ -40,14 +41,18 @@ func (r ResourceUpdatedHandler) Handle() error { func (r ResourceUpdatedHandler) GetConfig() (common.Config, string) { var oldSHAData string var config common.Config - if _, ok := r.Resource.(*v1.ConfigMap); ok { + switch res := r.Resource.(type) { + case *v1.ConfigMap: oldSHAData = util.GetSHAfromConfigmap(r.OldResource.(*v1.ConfigMap)) - config = common.GetConfigmapConfig(r.Resource.(*v1.ConfigMap)) - } else if _, ok := r.Resource.(*v1.Secret); ok { + config = common.GetConfigmapConfig(res) + case *v1.Secret: oldSHAData = util.GetSHAfromSecret(r.OldResource.(*v1.Secret).Data) - config = common.GetSecretConfig(r.Resource.(*v1.Secret)) - } else { - logrus.Warnf("Invalid resource: Resource should be 'Secret' or 'Configmap' but found, %v", r.Resource) + config = common.GetSecretConfig(res) + case *csiv1.SecretProviderClassPodStatus: + oldSHAData = util.GetSHAfromSecretProviderClassPodStatus(r.OldResource.(*csiv1.SecretProviderClassPodStatus).Status) + config = common.GetSecretProviderClassPodStatusConfig(res) + default: + logrus.Warnf("Invalid resource: Resource should be 'Secret', 'Configmap' or 'SecretProviderClassPodStatus' but found, %v", r.Resource) } return config, oldSHAData } diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index 6f185f1..f5b7ead 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -2,11 +2,13 @@ package handler import ( "bytes" + "context" "encoding/json" "errors" "fmt" "io" "os" + "strings" "github.com/parnurzeal/gorequest" "github.com/prometheus/client_golang/prometheus" @@ -23,6 +25,7 @@ import ( v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" patchtypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -283,6 +286,10 @@ func upgradeResource(clients kube.Clients, config common.Config, upgradeFuncs ca return err } } + if config.Type == constants.SecretProviderClassEnvVarPostfix { + populateAnnotationsFromSecretProviderClass(clients, &config) + } + annotations := upgradeFuncs.AnnotationsFunc(resource) podAnnotations := upgradeFuncs.PodAnnotationsFunc(resource) result := common.ShouldReload(config, upgradeFuncs.ResourceType, annotations, podAnnotations, common.GetCommandLineOptions()) @@ -380,6 +387,10 @@ func getVolumeMountName(volumes []v1.Volume, mountType string, volumeName string } } } + case constants.SecretProviderClassEnvVarPostfix: + if volumes[i].CSI != nil && volumes[i].CSI.VolumeAttributes["secretProviderClass"] == volumeName { + return volumes[i].Name + } } } @@ -516,6 +527,10 @@ func updatePodAnnotations(upgradeFuncs callbacks.RollingUpgradeFuncs, item runti return InvokeStrategyResult{constants.NotUpdated, nil} } + if config.Type == constants.SecretProviderClassEnvVarPostfix && secretProviderClassAnnotationReloaded(pa, config) { + return InvokeStrategyResult{constants.NotUpdated, nil} + } + for k, v := range annotations { pa[k] = v } @@ -523,6 +538,11 @@ func updatePodAnnotations(upgradeFuncs callbacks.RollingUpgradeFuncs, item runti return InvokeStrategyResult{constants.Updated, &Patch{Type: patchtypes.StrategicMergePatchType, Bytes: patch}} } +func secretProviderClassAnnotationReloaded(oldAnnotations map[string]string, newConfig common.Config) bool { + annotaion := oldAnnotations[getReloaderAnnotationKey()] + return strings.Contains(annotaion, newConfig.ResourceName) && strings.Contains(annotaion, newConfig.SHAValue) +} + func getReloaderAnnotationKey() string { return fmt.Sprintf("%s/%s", constants.ReloaderAnnotationPrefix, @@ -573,6 +593,10 @@ func updateContainerEnvVars(upgradeFuncs callbacks.RollingUpgradeFuncs, item run return InvokeStrategyResult{constants.NoContainerFound, nil} } + if config.Type == constants.SecretProviderClassEnvVarPostfix && secretProviderClassEnvReloaded(upgradeFuncs.ContainersFunc(item), envVar, config.SHAValue) { + return InvokeStrategyResult{constants.NotUpdated, nil} + } + //update if env var exists updateResult := updateEnvVar(container, envVar, config.SHAValue) @@ -609,6 +633,29 @@ func updateEnvVar(container *v1.Container, envVar string, shaData string) consta return constants.NoEnvVarFound } +func secretProviderClassEnvReloaded(containers []v1.Container, envVar string, shaData string) bool { + for i := range containers { + envs := containers[i].Env + for j := range envs { + if envs[j].Name == envVar { + return envs[j].Value == shaData + } + } + } + return false +} + +func populateAnnotationsFromSecretProviderClass(clients kube.Clients, config *common.Config) { + obj, err := clients.CSIClient.SecretsstoreV1().SecretProviderClasses(config.Namespace).Get(context.TODO(), config.ResourceName, metav1.GetOptions{}) + annotations := make(map[string]string) + if err != nil { + logrus.Infof("Couldn't find secretproviderclass '%s' in '%s' namespace for typed annotation", config.ResourceName, config.Namespace) + } else if obj.Annotations != nil { + annotations = obj.Annotations + } + config.ResourceAnnotations = annotations +} + func jsonEscape(toEscape string) (string, error) { bytes, err := json.Marshal(toEscape) if err != nil { diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index 9a0e945..5bf490f 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -27,10 +27,14 @@ import ( "k8s.io/apimachinery/pkg/runtime" patchtypes "k8s.io/apimachinery/pkg/types" testclient "k8s.io/client-go/kubernetes/fake" + csitestclient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned/fake" ) var ( - clients = kube.Clients{KubernetesClient: testclient.NewSimpleClientset()} + clients = kube.Clients{ + KubernetesClient: testclient.NewSimpleClientset(), + CSIClient: csitestclient.NewSimpleClientset(), + } arsNamespace = "test-handler-" + testutil.RandSeq(5) arsConfigmapName = "testconfigmap-handler-" + testutil.RandSeq(5) @@ -59,6 +63,14 @@ var ( arsSecretWithIgnoreAnnotation = "testsecretWithIgnoreAnnotation-handler-" + testutil.RandSeq(5) arsConfigmapWithPausedDeployment = "testconfigmapWithPausedDeployment-handler-" + testutil.RandSeq(5) + // Secret provider class + arsSecretProviderClassName = "testsecretproviderclass-handler-" + testutil.RandSeq(5) + arsSecretProviderClassWithInitContainer = "testsecretproviderclassWithInitContainer-handler-" + testutil.RandSeq(5) + arsSecretProviderClassWithSPCAutoAnnotation = "testsecretproviderclasswithspcautoannotationdeployment-handler-" + testutil.RandSeq(5) + arsSecretProviderClassWithExcludeSPCAnnotation = "testsecretproviderclasswithspcexcludeannotationdeployment-handler-" + testutil.RandSeq(5) + arsSecretProviderClassReloadedWithSameConfig = "testsecretproviderclassreloadedwithsameconfig-handler-" + testutil.RandSeq(5) + arsSecretProviderClassReloadedWithDifferentConfig = "testsecretproviderclassreloadedwithdifferentconfig-handler-" + testutil.RandSeq(5) + ersNamespace = "test-handler-" + testutil.RandSeq(5) ersConfigmapName = "testconfigmap-handler-" + testutil.RandSeq(5) ersSecretName = "testsecret-handler-" + testutil.RandSeq(5) @@ -84,6 +96,15 @@ var ( ersConfigmapWithIgnoreAnnotation = "testconfigmapWithIgnoreAnnotation-handler-" + testutil.RandSeq(5) ersSecretWithIgnoreAnnotation = "testsecretWithIgnoreAnnotation-handler-" + testutil.RandSeq(5) ersConfigmapWithPausedDeployment = "testconfigmapWithPausedDeployment-handler-" + testutil.RandSeq(5) + + // SecretProviderClass + ersSecretProviderClassName = "testsecretproviderclass-handler-" + testutil.RandSeq(5) + ersSecretProviderClassWithInitContainer = "testsecretproviderclassWithInitContainer-handler-" + testutil.RandSeq(5) + + ersSecretProviderClassWithSPCAutoAnnotation = "testsecretproviderclasswithspcautoannotationdeployment-handler-" + testutil.RandSeq(5) + ersSecretProviderClassWithExcludeSPCAnnotation = "testsecretproviderclasswithspcexcludeannotationdeployment-handler-" + testutil.RandSeq(5) + ersSecretProviderClassReloadedWithSameConfig = "testsecretproviderclassreloadedwithsameconfig-handler-" + testutil.RandSeq(5) + ersSecretProviderClassReloadedWithDifferentConfig = "testsecretproviderclassreloadedwithdifferentconfig-handler-" + testutil.RandSeq(5) ) func TestMain(m *testing.M) { @@ -122,6 +143,12 @@ func setupArs() { logrus.Errorf("Error in secret creation: %v", err) } + // Creating secretproviderclass + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassName, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + // Creating configmap will be used in projected volume _, err = testutil.CreateConfigMap(clients.KubernetesClient, arsNamespace, arsProjectedConfigMapName, "www.google.com") if err != nil { @@ -190,6 +217,12 @@ func setupArs() { logrus.Errorf("Error in secret creation: %v", err) } + // Creating secretproviderclass + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassWithInitContainer, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + _, err = testutil.CreateConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapWithPodAnnotations, "www.google.com") if err != nil { logrus.Errorf("Error in configmap creation: %v", err) @@ -206,6 +239,12 @@ func setupArs() { logrus.Errorf("Error in secret creation: %v", err) } + // Creating secretproviderclass used with secretproviderclass auto annotation + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassWithSPCAutoAnnotation, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + // Creating configmap used with configmap auto annotation _, err = testutil.CreateConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapWithConfigMapAutoAnnotation, "www.google.com") if err != nil { @@ -224,6 +263,24 @@ func setupArs() { logrus.Errorf("Error in secret creation: %v", err) } + // Creating secretproviderclass used with secret auto annotation + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassWithExcludeSPCAnnotation, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + + // Creating secretproviderclass to reload with same config + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassReloadedWithSameConfig, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + + // Creating secretproviderclass to reload with different config + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassReloadedWithDifferentConfig, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + // Creating configmap used with configmap auto annotation _, err = testutil.CreateConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapWithExcludeConfigMapAnnotation, "www.google.com") if err != nil { @@ -301,6 +358,12 @@ func setupArs() { logrus.Errorf("Error in Deployment with secret creation: %v", err) } + // Creating Deployment with secretproviderclass mounted in init container + _, err = testutil.CreateDeploymentWithInitContainer(clients.KubernetesClient, arsSecretProviderClassWithInitContainer, arsNamespace, true) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass creation: %v", err) + } + // Creating Deployment with configmap mounted as Env in init container _, err = testutil.CreateDeploymentWithInitContainer(clients.KubernetesClient, arsConfigmapWithInitEnv, arsNamespace, false) if err != nil { @@ -319,6 +382,12 @@ func setupArs() { logrus.Errorf("Error in Deployment with secret creation: %v", err) } + // Creating Deployment with secretproviderclass + _, err = testutil.CreateDeployment(clients.KubernetesClient, arsSecretProviderClassName, arsNamespace, true) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass creation: %v", err) + } + // Creating Deployment with env var source as configmap _, err = testutil.CreateDeployment(clients.KubernetesClient, arsConfigmapWithEnvName, arsNamespace, false) if err != nil { @@ -366,6 +435,12 @@ func setupArs() { logrus.Errorf("Error in Deployment with secret and with secret auto annotation: %v", err) } + // Creating Deployment with secretproviderclass and with secretproviderclass auto annotation + _, err = testutil.CreateDeploymentWithTypedAutoAnnotation(clients.KubernetesClient, arsSecretProviderClassWithSPCAutoAnnotation, arsNamespace, testutil.SecretProviderClassPodStatusResourceType) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass and with secretproviderclass auto annotation: %v", err) + } + // Creating Deployment with secret and with secret auto annotation _, err = testutil.CreateDeploymentWithTypedAutoAnnotation(clients.KubernetesClient, arsConfigmapWithConfigMapAutoAnnotation, arsNamespace, testutil.ConfigmapResourceType) if err != nil { @@ -378,6 +453,24 @@ func setupArs() { logrus.Errorf("Error in Deployment with secret and with secret exclude annotation: %v", err) } + // Creating Deployment with secretproviderclass and exclude secretproviderclass annotation + _, err = testutil.CreateDeploymentWithExcludeAnnotation(clients.KubernetesClient, arsSecretProviderClassWithExcludeSPCAnnotation, arsNamespace, testutil.SecretProviderClassPodStatusResourceType) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass and with secretproviderclass exclude annotation: %v", err) + } + + // Creating Deployment with secretproviderclass to reload with same config + _, err = testutil.CreateDeploymentWithTypedAutoAnnotation(clients.KubernetesClient, arsSecretProviderClassReloadedWithSameConfig, arsNamespace, testutil.SecretProviderClassPodStatusResourceType) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass to reload with same config: %v", err) + } + + // Creating Deployment with secretproviderclass to reload with different config + _, err = testutil.CreateDeploymentWithTypedAutoAnnotation(clients.KubernetesClient, arsSecretProviderClassReloadedWithDifferentConfig, arsNamespace, testutil.SecretProviderClassPodStatusResourceType) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass to reload with different config: %v", err) + } + // Creating Deployment with secret and exclude configmap annotation _, err = testutil.CreateDeploymentWithExcludeAnnotation(clients.KubernetesClient, arsConfigmapWithExcludeConfigMapAnnotation, arsNamespace, testutil.ConfigmapResourceType) if err != nil { @@ -396,6 +489,12 @@ func setupArs() { logrus.Errorf("Error in DaemonSet with secret creation: %v", err) } + // Creating DaemonSet with secretproviderclass + _, err = testutil.CreateDaemonSet(clients.KubernetesClient, arsSecretProviderClassName, arsNamespace, true) + if err != nil { + logrus.Errorf("Error in DaemonSet with secretproviderclass creation: %v", err) + } + // Creating DaemonSet with configmap in projected volume _, err = testutil.CreateDaemonSet(clients.KubernetesClient, arsProjectedConfigMapName, arsNamespace, true) if err != nil { @@ -432,6 +531,12 @@ func setupArs() { logrus.Errorf("Error in StatefulSet with secret creation: %v", err) } + // Creating StatefulSet with secretproviderclass + _, err = testutil.CreateStatefulSet(clients.KubernetesClient, arsSecretProviderClassName, arsNamespace, true) + if err != nil { + logrus.Errorf("Error in StatefulSet with secretproviderclass creation: %v", err) + } + // Creating StatefulSet with configmap in projected volume _, err = testutil.CreateStatefulSet(clients.KubernetesClient, arsProjectedConfigMapName, arsNamespace, true) if err != nil { @@ -489,6 +594,12 @@ func teardownArs() { logrus.Errorf("Error while deleting deployment with secret %v", deploymentError) } + // Deleting Deployment with secretproviderclass + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsSecretProviderClassName) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass %v", deploymentError) + } + // Deleting Deployment with configmap in projected volume deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsProjectedConfigMapName) if deploymentError != nil { @@ -537,6 +648,12 @@ func teardownArs() { logrus.Errorf("Error while deleting deployment with secret mounted in init container %v", deploymentError) } + // Deleting Deployment with secretproviderclass mounted in init container + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsSecretProviderClassWithInitContainer) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass mounted in init container %v", deploymentError) + } + // Deleting Deployment with configmap mounted as env in init container deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsConfigmapWithInitEnv) if deploymentError != nil { @@ -585,6 +702,12 @@ func teardownArs() { logrus.Errorf("Error while deleting deployment with secret auto annotation %v", deploymentError) } + // Deleting Deployment with secretproviderclass and secretproviderclass auto annotation + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsSecretProviderClassWithSPCAutoAnnotation) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass auto annotation %v", deploymentError) + } + // Deleting Deployment with configmap and configmap auto annotation deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsConfigmapWithConfigMapAutoAnnotation) if deploymentError != nil { @@ -597,6 +720,24 @@ func teardownArs() { logrus.Errorf("Error while deleting deployment with secret auto annotation %v", deploymentError) } + // Deleting Deployment with secretproviderclass and exclude secretproviderclass annotation + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsSecretProviderClassWithExcludeSPCAnnotation) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass auto annotation %v", deploymentError) + } + + // Deleting Deployment with secretproviderclass to reload with same config + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsSecretProviderClassReloadedWithSameConfig) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass to reload with same config %v", deploymentError) + } + + // Deleting Deployment with secretproviderclass to reload with different config + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsSecretProviderClassReloadedWithDifferentConfig) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass to reload with different config %v", deploymentError) + } + // Deleting Deployment with configmap and exclude configmap annotation deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsConfigmapWithExcludeConfigMapAnnotation) if deploymentError != nil { @@ -609,12 +750,18 @@ func teardownArs() { logrus.Errorf("Error while deleting daemonSet with configmap %v", daemonSetError) } - // Deleting Deployment with secret + // Deleting DeamonSet with secret daemonSetError = testutil.DeleteDaemonSet(clients.KubernetesClient, arsNamespace, arsSecretName) if daemonSetError != nil { logrus.Errorf("Error while deleting daemonSet with secret %v", daemonSetError) } + // Deleting DeamonSet with secretproviderclass + daemonSetError = testutil.DeleteDaemonSet(clients.KubernetesClient, arsNamespace, arsSecretProviderClassName) + if daemonSetError != nil { + logrus.Errorf("Error while deleting daemonSet with secretproviderclass %v", daemonSetError) + } + // Deleting DaemonSet with configmap in projected volume daemonSetError = testutil.DeleteDaemonSet(clients.KubernetesClient, arsNamespace, arsProjectedConfigMapName) if daemonSetError != nil { @@ -645,12 +792,18 @@ func teardownArs() { logrus.Errorf("Error while deleting statefulSet with configmap %v", statefulSetError) } - // Deleting Deployment with secret + // Deleting StatefulSet with secret statefulSetError = testutil.DeleteStatefulSet(clients.KubernetesClient, arsNamespace, arsSecretName) if statefulSetError != nil { logrus.Errorf("Error while deleting statefulSet with secret %v", statefulSetError) } + // Deleting StatefulSet with secretproviderclass + statefulSetError = testutil.DeleteStatefulSet(clients.KubernetesClient, arsNamespace, arsSecretProviderClassName) + if statefulSetError != nil { + logrus.Errorf("Error while deleting statefulSet with secretproviderclass %v", statefulSetError) + } + // Deleting StatefulSet with configmap in projected volume statefulSetError = testutil.DeleteStatefulSet(clients.KubernetesClient, arsNamespace, arsProjectedConfigMapName) if statefulSetError != nil { @@ -693,6 +846,12 @@ func teardownArs() { logrus.Errorf("Error while deleting the secret %v", err) } + // Deleting Secretproviderclass + err = testutil.DeleteSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass %v", err) + } + // Deleting configmap used in projected volume err = testutil.DeleteConfigMap(clients.KubernetesClient, arsNamespace, arsProjectedConfigMapName) if err != nil { @@ -741,6 +900,12 @@ func teardownArs() { logrus.Errorf("Error while deleting the secret used in init container %v", err) } + // Deleting Secretproviderclass used in init container + err = testutil.DeleteSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassWithInitContainer) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used in init container %v", err) + } + // Deleting Configmap used as env var source err = testutil.DeleteConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapWithEnvFromName) if err != nil { @@ -776,6 +941,12 @@ func teardownArs() { logrus.Errorf("Error while deleting the secret used with secret auto annotations: %v", err) } + // Deleting SecretProviderClass used with secretproviderclass auto annotation + err = testutil.DeleteSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassWithSPCAutoAnnotation) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used with secretproviderclass auto annotations: %v", err) + } + // Deleting ConfigMap used with configmap auto annotation err = testutil.DeleteConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapWithConfigMapAutoAnnotation) if err != nil { @@ -788,6 +959,24 @@ func teardownArs() { logrus.Errorf("Error while deleting the secret used with secret auto annotations: %v", err) } + // Deleting Secretproviderclass used with exclude secretproviderclass annotation + err = testutil.DeleteSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassWithExcludeSPCAnnotation) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used with secretproviderclass auto annotations: %v", err) + } + + // Deleting SecretProviderClass used with secretproviderclass to reload with same config + err = testutil.DeleteSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassReloadedWithSameConfig) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used with secretproviderclass to reload with same config: %v", err) + } + + // Deleting SecretProviderClass used with secretproviderclass to reload with different config + err = testutil.DeleteSecretProviderClass(clients.CSIClient, arsNamespace, arsSecretProviderClassReloadedWithDifferentConfig) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used with secretproviderclass to reload with different config: %v", err) + } + // Deleting ConfigMap used with exclude configmap annotation err = testutil.DeleteConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapWithExcludeConfigMapAnnotation) if err != nil { @@ -819,6 +1008,12 @@ func setupErs() { logrus.Errorf("Error in secret creation: %v", err) } + // Creating secretproviderclass + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassName, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + // Creating configmap will be used in projected volume _, err = testutil.CreateConfigMap(clients.KubernetesClient, ersNamespace, ersProjectedConfigMapName, "www.google.com") if err != nil { @@ -893,6 +1088,12 @@ func setupErs() { logrus.Errorf("Error in secret creation: %v", err) } + // Creating secretproviderclass + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassWithInitContainer, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + _, err = testutil.CreateConfigMap(clients.KubernetesClient, ersNamespace, ersConfigmapWithPodAnnotations, "www.google.com") if err != nil { logrus.Errorf("Error in configmap creation: %v", err) @@ -910,6 +1111,12 @@ func setupErs() { logrus.Errorf("Error in configmap creation: %v", err) } + // Creating secretproviderclass used with secretproviderclass auto annotation + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassWithSPCAutoAnnotation, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + // Creating secret used with secret exclude annotation _, err = testutil.CreateSecret(clients.KubernetesClient, ersNamespace, ersSecretWithSecretExcludeAnnotation, data) if err != nil { @@ -922,6 +1129,24 @@ func setupErs() { logrus.Errorf("Error in configmap creation: %v", err) } + // Creating secretproviderclass used with secret exclude annotation + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassWithExcludeSPCAnnotation, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + + // Creating secretproviderclass to reload with same config + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassReloadedWithSameConfig, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + + // Creating secretproviderclass to reload with different config + _, err = testutil.CreateSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassReloadedWithDifferentConfig, "testing") + if err != nil { + logrus.Errorf("Error in secretproviderclass creation: %v", err) + } + // Creating configmap with ignore annotation _, err = testutil.CreateConfigMap(clients.KubernetesClient, ersNamespace, ersConfigmapWithIgnoreAnnotation, "www.google.com") if err != nil { @@ -992,6 +1217,12 @@ func setupErs() { logrus.Errorf("Error in Deployment with secret creation: %v", err) } + // Creating Deployment with secretproviderclass mounted in init container + _, err = testutil.CreateDeploymentWithInitContainer(clients.KubernetesClient, ersSecretProviderClassWithInitContainer, ersNamespace, true) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass creation: %v", err) + } + // Creating Deployment with configmap mounted as Env in init container _, err = testutil.CreateDeploymentWithInitContainer(clients.KubernetesClient, ersConfigmapWithInitEnv, ersNamespace, false) if err != nil { @@ -1010,6 +1241,12 @@ func setupErs() { logrus.Errorf("Error in Deployment with secret creation: %v", err) } + // Creating Deployment with secretproviderclass + _, err = testutil.CreateDeployment(clients.KubernetesClient, ersSecretProviderClassName, ersNamespace, true) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass creation: %v", err) + } + // Creating Deployment with env var source as configmap _, err = testutil.CreateDeployment(clients.KubernetesClient, ersConfigmapWithEnvName, ersNamespace, false) if err != nil { @@ -1057,6 +1294,12 @@ func setupErs() { logrus.Errorf("Error in Deployment with configmap and with configmap auto annotation: %v", err) } + // Creating Deployment with secretproviderclass and with secretproviderclass auto annotation + _, err = testutil.CreateDeploymentWithTypedAutoAnnotation(clients.KubernetesClient, ersSecretProviderClassWithSPCAutoAnnotation, ersNamespace, testutil.SecretProviderClassPodStatusResourceType) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass and with secretproviderclass auto annotation: %v", err) + } + // Creating Deployment with secret and with secret exclude annotation _, err = testutil.CreateDeploymentWithExcludeAnnotation(clients.KubernetesClient, ersSecretWithSecretExcludeAnnotation, ersNamespace, testutil.SecretResourceType) if err != nil { @@ -1069,6 +1312,12 @@ func setupErs() { logrus.Errorf("Error in Deployment with configmap and with configmap exclude annotation: %v", err) } + // Creating Deployment with secretproviderclass and with secretproviderclass exclude annotation + _, err = testutil.CreateDeploymentWithExcludeAnnotation(clients.KubernetesClient, ersSecretProviderClassWithExcludeSPCAnnotation, ersNamespace, testutil.SecretProviderClassPodStatusResourceType) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass and with secretproviderclass exclude annotation: %v", err) + } + // Creating Deployment with pause annotation _, err = testutil.CreateDeploymentWithAnnotations(clients.KubernetesClient, ersConfigmapWithPausedDeployment, ersNamespace, map[string]string{options.PauseDeploymentAnnotation: "10s"}, false) if err != nil { @@ -1087,6 +1336,12 @@ func setupErs() { logrus.Errorf("Error in DaemonSet with secret creation: %v", err) } + // Creating DaemonSet with secretproviderclass + _, err = testutil.CreateDaemonSet(clients.KubernetesClient, ersSecretProviderClassName, ersNamespace, true) + if err != nil { + logrus.Errorf("Error in DaemonSet with secretproviderclass creation: %v", err) + } + // Creating DaemonSet with configmap in projected volume _, err = testutil.CreateDaemonSet(clients.KubernetesClient, ersProjectedConfigMapName, ersNamespace, true) if err != nil { @@ -1123,6 +1378,12 @@ func setupErs() { logrus.Errorf("Error in StatefulSet with secret creation: %v", err) } + // Creating StatefulSet with secretproviderclass + _, err = testutil.CreateStatefulSet(clients.KubernetesClient, ersSecretProviderClassName, ersNamespace, true) + if err != nil { + logrus.Errorf("Error in StatefulSet with secretproviderclass creation: %v", err) + } + // Creating StatefulSet with configmap in projected volume _, err = testutil.CreateStatefulSet(clients.KubernetesClient, ersProjectedConfigMapName, ersNamespace, true) if err != nil { @@ -1158,6 +1419,18 @@ func setupErs() { if err != nil { logrus.Errorf("Error in Deployment with both annotations: %v", err) } + + // Creating Deployment with secretproviderclass to reload with same config + _, err = testutil.CreateDeploymentWithTypedAutoAnnotation(clients.KubernetesClient, ersSecretProviderClassReloadedWithSameConfig, ersNamespace, testutil.SecretProviderClassPodStatusResourceType) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass to reload with same config: %v", err) + } + + // Creating Deployment with secretproviderclass to reload with different config + _, err = testutil.CreateDeploymentWithTypedAutoAnnotation(clients.KubernetesClient, ersSecretProviderClassReloadedWithDifferentConfig, ersNamespace, testutil.SecretProviderClassPodStatusResourceType) + if err != nil { + logrus.Errorf("Error in Deployment with secretproviderclass to reload with different config: %v", err) + } } func teardownErs() { @@ -1173,6 +1446,12 @@ func teardownErs() { logrus.Errorf("Error while deleting deployment with secret %v", deploymentError) } + // Deleting Deployment with secretproviderclass + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersSecretProviderClassName) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretprovider class %v", deploymentError) + } + // Deleting Deployment with configmap in projected volume deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersProjectedConfigMapName) if deploymentError != nil { @@ -1221,6 +1500,12 @@ func teardownErs() { logrus.Errorf("Error while deleting deployment with secret mounted in init container %v", deploymentError) } + // Deleting Deployment with secretproviderclass mounted in init container + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersSecretProviderClassWithInitContainer) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass mounted in init container %v", deploymentError) + } + // Deleting Deployment with configmap mounted as env in init container deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersConfigmapWithInitEnv) if deploymentError != nil { @@ -1275,6 +1560,12 @@ func teardownErs() { logrus.Errorf("Error while deleting deployment with configmap auto annotation %v", deploymentError) } + // Deleting Deployment with secretproviderclass and secretproviderclass auto annotation + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersSecretProviderClassWithSPCAutoAnnotation) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass auto annotation %v", deploymentError) + } + // Deleting Deployment with secret and secret exclude annotation deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersSecretWithSecretExcludeAnnotation) if deploymentError != nil { @@ -1287,18 +1578,42 @@ func teardownErs() { logrus.Errorf("Error while deleting deployment with configmap exclude annotation %v", deploymentError) } + // Deleting Deployment with secretproviderclass and secretproviderclass exclude annotation + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersSecretProviderClassWithExcludeSPCAnnotation) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass exclude annotation %v", deploymentError) + } + + // Deleting Deployment with secretproviderclass to reload with same config + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersSecretProviderClassReloadedWithSameConfig) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass to reload with same config %v", deploymentError) + } + + // Deleting Deployment with secretproviderclass to reload with different config + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersSecretProviderClassReloadedWithDifferentConfig) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secretproviderclass to reload with different config %v", deploymentError) + } + // Deleting DaemonSet with configmap daemonSetError := testutil.DeleteDaemonSet(clients.KubernetesClient, ersNamespace, ersConfigmapName) if daemonSetError != nil { logrus.Errorf("Error while deleting daemonSet with configmap %v", daemonSetError) } - // Deleting Deployment with secret + // Deleting DaemonSet with secret daemonSetError = testutil.DeleteDaemonSet(clients.KubernetesClient, ersNamespace, ersSecretName) if daemonSetError != nil { logrus.Errorf("Error while deleting daemonSet with secret %v", daemonSetError) } + // Deleting DaemonSet with secretproviderclass + daemonSetError = testutil.DeleteDaemonSet(clients.KubernetesClient, ersNamespace, ersSecretProviderClassName) + if daemonSetError != nil { + logrus.Errorf("Error while deleting daemonSet with secretproviderclass %v", daemonSetError) + } + // Deleting DaemonSet with configmap in projected volume daemonSetError = testutil.DeleteDaemonSet(clients.KubernetesClient, ersNamespace, ersProjectedConfigMapName) if daemonSetError != nil { @@ -1329,12 +1644,18 @@ func teardownErs() { logrus.Errorf("Error while deleting statefulSet with configmap %v", statefulSetError) } - // Deleting Deployment with secret + // Deleting StatefulSet with secret statefulSetError = testutil.DeleteStatefulSet(clients.KubernetesClient, ersNamespace, ersSecretName) if statefulSetError != nil { logrus.Errorf("Error while deleting statefulSet with secret %v", statefulSetError) } + // Deleting StatefulSet with secretproviderclass + statefulSetError = testutil.DeleteStatefulSet(clients.KubernetesClient, ersNamespace, ersSecretProviderClassName) + if statefulSetError != nil { + logrus.Errorf("Error while deleting statefulSet with secretproviderclass %v", statefulSetError) + } + // Deleting StatefulSet with configmap in projected volume statefulSetError = testutil.DeleteStatefulSet(clients.KubernetesClient, ersNamespace, ersProjectedConfigMapName) if statefulSetError != nil { @@ -1377,6 +1698,12 @@ func teardownErs() { logrus.Errorf("Error while deleting the secret %v", err) } + // Deleting SecretProviderClass + err = testutil.DeleteSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassName) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass %v", err) + } + // Deleting configmap used in projected volume err = testutil.DeleteConfigMap(clients.KubernetesClient, ersNamespace, ersProjectedConfigMapName) if err != nil { @@ -1425,6 +1752,12 @@ func teardownErs() { logrus.Errorf("Error while deleting the secret used in init container %v", err) } + // Deleting SecretProviderClass used in init container + err = testutil.DeleteSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassWithInitContainer) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used in init container %v", err) + } + // Deleting Configmap used as env var source err = testutil.DeleteConfigMap(clients.KubernetesClient, ersNamespace, ersConfigmapWithEnvFromName) if err != nil { @@ -1466,6 +1799,12 @@ func teardownErs() { logrus.Errorf("Error while deleting the configmap used with configmap auto annotation: %v", err) } + // Deleting SecretProviderClass used with secretproviderclass auto annotation + err = testutil.DeleteSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassWithSPCAutoAnnotation) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used with secretproviderclass auto annotation: %v", err) + } + // Deleting Secret used with secret exclude annotation err = testutil.DeleteSecret(clients.KubernetesClient, ersNamespace, ersSecretWithSecretExcludeAnnotation) if err != nil { @@ -1478,6 +1817,23 @@ func teardownErs() { logrus.Errorf("Error while deleting the configmap used with configmap exclude annotation: %v", err) } + // Deleting SecretProviderClass used with secretproviderclass exclude annotation + err = testutil.DeleteSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassWithExcludeSPCAnnotation) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used with secretproviderclass exclude annotation: %v", err) + } + + // Deleting SecretProviderClass used with secretproviderclass to reload with same config + err = testutil.DeleteSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassReloadedWithSameConfig) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used with secretproviderclass to reload with same config: %v", err) + } + + // Deleting SecretProviderClass used with secretproviderclass to reload with different config + err = testutil.DeleteSecretProviderClass(clients.CSIClient, ersNamespace, ersSecretProviderClassReloadedWithDifferentConfig) + if err != nil { + logrus.Errorf("Error while deleting the secretproviderclass used with secretproviderclass to reload with different config: %v", err) + } // Deleting ConfigMap for testing pausing deployments err = testutil.DeleteConfigMap(clients.KubernetesClient, ersNamespace, ersConfigmapWithPausedDeployment) if err != nil { @@ -2042,6 +2398,38 @@ func TestRollingUpgradeForDeploymentWithSecretUsingArs(t *testing.T) { testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForDeploymentWithSecretProviderClassUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassName, "testing1") + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassName, shaData, options.SecretProviderClassUpdateOnChangeAnnotation, options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with SecretProviderClass") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": arsNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForDeploymentWithSecretInProjectedVolumeUsingArs(t *testing.T) { options.ReloadStrategy = constants.AnnotationsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -2106,6 +2494,38 @@ func TestRollingUpgradeForDeploymentWithSecretinInitContainerUsingArs(t *testing testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForDeploymentWithSecretproviderclassInInitContainerUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassWithInitContainer, "testing1") + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassWithInitContainer, shaData, options.SecretProviderClassUpdateOnChangeAnnotation, options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with SecretProviderClass") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": arsNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForDeploymentWithSecretInProjectedVolumeinInitContainerUsingArs(t *testing.T) { options.ReloadStrategy = constants.AnnotationsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -2254,6 +2674,100 @@ func TestRollingUpgradeForDeploymentWithSecretExcludeAnnotationUsingArs(t *testi } } +func TestRollingUpgradeForDeploymentWithSecretproviderclassExcludeAnnotationUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassWithExcludeSPCAnnotation, "testing1") + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassWithExcludeSPCAnnotation, shaData, "", options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with SecretProviderClass") + } + + logrus.Infof("Verifying deployment did not update") + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if updated { + t.Errorf("Deployment which had to be exluded was updated") + } +} + +func TestRollingUpgradeForDeploymentWithSecretProviderClassReloadedWithSameConfigUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassReloadedWithSameConfig, "testing1") + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassReloadedWithSameConfig, shaData, "", options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with same config") + } + + logrus.Infof("Verifying deployment did update") + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + logrus.Infof("Performing reload using same config") + err = PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Second rolling upgrade failed for Deployment with same config") + } + + logrus.Infof("Verifying second reload did not reload") + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 && + promtestutil.ToFloat64(collectors.Reloaded.With(labelFailed)) != 0 { + t.Errorf("Second reload with same config updated Deployment") + } +} + +func TestRollingUpgradeForDeploymentWithSecretProviderClassReloadedWithDifferentConfigUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassReloadedWithDifferentConfig, "testing1") + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassReloadedWithDifferentConfig, shaData, "", options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with different config") + } + + logrus.Infof("Verifying deployment did update") + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + logrus.Infof("Applying different config") + shaData = testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassReloadedWithDifferentConfig, "testing2") + config.SHAValue = shaData + + err = PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Second rolling upgrade failed for Deployment with different config") + } + + logrus.Infof("Verifying deployment did update") + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 2 && + promtestutil.ToFloat64(collectors.Reloaded.With(labelFailed)) != 0 { + t.Errorf("Second reload with different config did not update Deployment") + } +} + func TestRollingUpgradeForDeploymentWithSecretAutoAnnotationUsingArs(t *testing.T) { options.ReloadStrategy = constants.AnnotationsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -2286,6 +2800,38 @@ func TestRollingUpgradeForDeploymentWithSecretAutoAnnotationUsingArs(t *testing. testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForDeploymentWithSecretProviderClassAutoAnnotationUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassWithSPCAutoAnnotation, "testing1") + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassWithSPCAutoAnnotation, shaData, "", options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with SecretProviderClass") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": arsNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForDeploymentWithExcludeConfigMapAnnotationUsingArs(t *testing.T) { options.ReloadStrategy = constants.AnnotationsReloadStrategy envVarPostfix := constants.ConfigmapEnvVarPostfix @@ -2538,6 +3084,38 @@ func TestRollingUpgradeForDaemonSetWithSecretUsingArs(t *testing.T) { testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, daemonSetFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForDaemonSetWithSecretProviderClassUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassName, "testing1") + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassName, shaData, options.SecretProviderClassUpdateOnChangeAnnotation, options.SecretProviderClassReloaderAutoAnnotation) + daemonSetFuncs := GetDaemonSetRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, daemonSetFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for DaemonSet with SecretProviderClass") + } + + logrus.Infof("Verifying daemonSet update") + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, daemonSetFuncs) + if !updated { + t.Errorf("DaemonSet was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": arsNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, daemonSetFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForDaemonSetWithSecretInProjectedVolumeUsingArs(t *testing.T) { options.ReloadStrategy = constants.AnnotationsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -2737,6 +3315,38 @@ func TestRollingUpgradeForStatefulSetWithSecretUsingArs(t *testing.T) { testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, statefulSetFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForStatefulSetWithSecretProviderClassUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, arsNamespace, arsSecretProviderClassName, "testing1") + config := getConfigWithAnnotations(envVarPostfix, arsSecretProviderClassName, shaData, options.SecretProviderClassUpdateOnChangeAnnotation, options.SecretProviderClassReloaderAutoAnnotation) + statefulSetFuncs := GetStatefulSetRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, statefulSetFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for StatefulSet with SecretProviderClass") + } + + logrus.Infof("Verifying statefulSet update") + updated := testutil.VerifyResourceAnnotationUpdate(clients, config, statefulSetFuncs) + if !updated { + t.Errorf("StatefulSet was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": arsNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyArs(t, clients, config, statefulSetFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForStatefulSetWithSecretInProjectedVolumeUsingArs(t *testing.T) { options.ReloadStrategy = constants.AnnotationsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -3349,6 +3959,38 @@ func TestRollingUpgradeForDeploymentWithSecretUsingErs(t *testing.T) { testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForDeploymentWithSecretProviderClassUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassName, "testing1") + config := getConfigWithAnnotations(envVarPostfix, ersSecretProviderClassName, shaData, options.SecretProviderClassUpdateOnChangeAnnotation, options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with SecretProviderClass") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, envVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": ersNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForDeploymentWithSecretInProjectedVolumeUsingErs(t *testing.T) { options.ReloadStrategy = constants.EnvVarsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -3413,6 +4055,38 @@ func TestRollingUpgradeForDeploymentWithSecretinInitContainerUsingErs(t *testing testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForDeploymentWithSecretProviderClassinInitContainerUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassWithInitContainer, "testing1") + config := getConfigWithAnnotations(envVarPostfix, ersSecretProviderClassWithInitContainer, shaData, options.SecretProviderClassUpdateOnChangeAnnotation, options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with SecretProviderClass") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, envVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": ersNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForDeploymentWithSecretInProjectedVolumeinInitContainerUsingErs(t *testing.T) { options.ReloadStrategy = constants.EnvVarsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -3563,6 +4237,101 @@ func TestRollingUpgradeForDeploymentWithSecretExcludeAnnotationUsingErs(t *testi } } +func TestRollingUpgradeForDeploymentWithSecretProviderClassExcludeAnnotationUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassWithExcludeSPCAnnotation, "testing1") + config := getConfigWithAnnotations(envVarPostfix, ersSecretProviderClassWithExcludeSPCAnnotation, shaData, "", options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with exclude SecretProviderClass") + } + + logrus.Infof("Verifying deployment did not update") + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, envVarPostfix, deploymentFuncs) + if updated { + t.Errorf("Deployment that had to be excluded was updated") + } +} + +func TestRollingUpgradeForDeploymentWithSecretProviderClassReloadedWithSameConfigUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassReloadedWithSameConfig, "testing1") + config := getConfigWithAnnotations(envVarPostfix, ersSecretProviderClassReloadedWithSameConfig, shaData, "", options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with same config") + } + + logrus.Infof("Verifying deployment did update") + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, envVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + logrus.Infof("Performing reload using same config") + err = PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Second rolling upgrade failed for Deployment with same config") + } + + logrus.Infof("Verifying second reload did not reload") + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 && + promtestutil.ToFloat64(collectors.Reloaded.With(labelFailed)) != 0 { + t.Errorf("Second reload with same config updated Deployment") + } +} + +func TestRollingUpgradeForDeploymentWithSecretProviderClassReloadedWithDifferentConfigUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassReloadedWithDifferentConfig, "testing1") + config := getConfigWithAnnotations(envVarPostfix, ersSecretProviderClassReloadedWithDifferentConfig, shaData, "", options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with different config") + } + + logrus.Infof("Verifying deployment did update") + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, envVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + logrus.Infof("Applying different config") + shaData = testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassReloadedWithDifferentConfig, "testing2") + config.SHAValue = shaData + + err = PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Second rolling upgrade failed for Deployment with different config") + } + + logrus.Infof("Verifying deployment did update") + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 2 && + promtestutil.ToFloat64(collectors.Reloaded.With(labelFailed)) != 0 { + t.Errorf("Second reload with different config did not update Deployment") + } +} + func TestRollingUpgradeForDeploymentWithSecretAutoAnnotationUsingErs(t *testing.T) { options.ReloadStrategy = constants.EnvVarsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -3595,6 +4364,38 @@ func TestRollingUpgradeForDeploymentWithSecretAutoAnnotationUsingErs(t *testing. testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForDeploymentWithSecretProviderClassAutoAnnotationUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassWithSPCAutoAnnotation, "testing1") + config := getConfigWithAnnotations(envVarPostfix, ersSecretProviderClassWithSPCAutoAnnotation, shaData, "", options.SecretProviderClassReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with SecretProviderClass") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, envVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": ersNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, deploymentFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForDeploymentWithConfigMapExcludeAnnotationUsingErs(t *testing.T) { options.ReloadStrategy = constants.EnvVarsReloadStrategy envVarPostfix := constants.ConfigmapEnvVarPostfix @@ -3820,6 +4621,38 @@ func TestRollingUpgradeForDaemonSetWithSecretUsingErs(t *testing.T) { testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, daemonSetFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForDaemonSetWithSecretProviderClassUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassName, "testing1") + config := getConfigWithAnnotations(envVarPostfix, ersSecretProviderClassName, shaData, options.SecretProviderClassUpdateOnChangeAnnotation, options.SecretProviderClassReloaderAutoAnnotation) + daemonSetFuncs := GetDaemonSetRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, daemonSetFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for DaemonSet with SecretProviderClass") + } + + logrus.Infof("Verifying daemonSet update") + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, envVarPostfix, daemonSetFuncs) + if !updated { + t.Errorf("DaemonSet was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": ersNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, daemonSetFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForDaemonSetWithSecretInProjectedVolumeUsingErs(t *testing.T) { options.ReloadStrategy = constants.EnvVarsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix @@ -3991,6 +4824,38 @@ func TestRollingUpgradeForStatefulSetWithSecretUsingErs(t *testing.T) { testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, statefulSetFuncs, collectors, envVarPostfix) } +func TestRollingUpgradeForStatefulSetWithSecretProviderClassUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + envVarPostfix := constants.SecretProviderClassEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.SecretProviderClassPodStatusResourceType, ersNamespace, ersSecretProviderClassName, "testing1") + config := getConfigWithAnnotations(envVarPostfix, ersSecretProviderClassName, shaData, options.SecretProviderClassUpdateOnChangeAnnotation, options.SecretProviderClassReloaderAutoAnnotation) + statefulSetFuncs := GetStatefulSetRollingUpgradeFuncs() + collectors := getCollectors() + + err := PerformAction(clients, config, statefulSetFuncs, collectors, nil, invokeReloadStrategy) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for StatefulSet with SecretProviderClass") + } + + logrus.Infof("Verifying statefulSet update") + updated := testutil.VerifyResourceEnvVarUpdate(clients, config, envVarPostfix, statefulSetFuncs) + if !updated { + t.Errorf("StatefulSet was not updated") + } + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": ersNamespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + testRollingUpgradeInvokeDeleteStrategyErs(t, clients, config, statefulSetFuncs, collectors, envVarPostfix) +} + func TestRollingUpgradeForStatefulSetWithSecretInProjectedVolumeUsingErs(t *testing.T) { options.ReloadStrategy = constants.EnvVarsReloadStrategy envVarPostfix := constants.SecretEnvVarPostfix diff --git a/internal/pkg/options/flags.go b/internal/pkg/options/flags.go index 0f99be8..62f2853 100644 --- a/internal/pkg/options/flags.go +++ b/internal/pkg/options/flags.go @@ -20,6 +20,9 @@ var ( // SecretUpdateOnChangeAnnotation is an annotation to detect changes in // secrets specified by name SecretUpdateOnChangeAnnotation = "secret.reloader.stakater.com/reload" + // SecretProviderClassUpdateOnChangeAnnotation is an annotation to detect changes in + // secretproviderclasses specified by name + SecretProviderClassUpdateOnChangeAnnotation = "secretproviderclass.reloader.stakater.com/reload" // ReloaderAutoAnnotation is an annotation to detect changes in secrets/configmaps ReloaderAutoAnnotation = "reloader.stakater.com/auto" // IgnoreResourceAnnotation is an annotation to ignore changes in secrets/configmaps @@ -28,10 +31,14 @@ var ( ConfigmapReloaderAutoAnnotation = "configmap.reloader.stakater.com/auto" // SecretReloaderAutoAnnotation is an annotation to detect changes in secrets SecretReloaderAutoAnnotation = "secret.reloader.stakater.com/auto" + // SecretProviderClassReloaderAutoAnnotation is an annotation to detect changes in secretproviderclasses + SecretProviderClassReloaderAutoAnnotation = "secretproviderclass.reloader.stakater.com/auto" // ConfigmapReloaderAutoAnnotation is a comma separated list of configmaps that excludes detecting changes on cms ConfigmapExcludeReloaderAnnotation = "configmaps.exclude.reloader.stakater.com/reload" // SecretExcludeReloaderAnnotation is a comma separated list of secrets that excludes detecting changes on secrets SecretExcludeReloaderAnnotation = "secrets.exclude.reloader.stakater.com/reload" + // SecretProviderClassExcludeReloaderAnnotation is a comma separated list of secret provider classes that excludes detecting changes on secret provider class + SecretProviderClassExcludeReloaderAnnotation = "secretproviderclasses.exclude.reloader.stakater.com/reload" // AutoSearchAnnotation is an annotation to detect changes in // configmaps or triggers with the SearchMatchAnnotation AutoSearchAnnotation = "reloader.stakater.com/search" @@ -63,6 +70,8 @@ var ( EnableHA = false // Url to send a request to instead of triggering a reload WebhookUrl = "" + // EnableCSIIntegration Adds support to watch SecretProviderClassPodStatus and restart deployment based on it + EnableCSIIntegration = false // ResourcesToIgnore is a list of resources to ignore when watching for changes ResourcesToIgnore = []string{} // WorkloadTypesToIgnore is a list of workload types to ignore when watching for changes diff --git a/internal/pkg/testutil/kube.go b/internal/pkg/testutil/kube.go index 1ad43e1..4901d9a 100644 --- a/internal/pkg/testutil/kube.go +++ b/internal/pkg/testutil/kube.go @@ -30,6 +30,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" core_v1 "k8s.io/client-go/kubernetes/typed/core/v1" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" + csiclient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned" + csiclient_v1 "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned/typed/apis/v1" ) var ( @@ -38,6 +41,8 @@ var ( ConfigmapResourceType = "configMaps" // SecretResourceType is a resource type which controller watches for changes SecretResourceType = "secrets" + // SecretproviderclasspodstatusResourceType is a resource type which controller watches for changes + SecretProviderClassPodStatusResourceType = "secretproviderclasspodstatuses" ) var ( @@ -73,16 +78,16 @@ func DeleteNamespace(namespace string, client kubernetes.Interface) { } } -func getObjectMeta(namespace string, name string, autoReload bool, secretAutoReload bool, configmapAutoReload bool, extraAnnotations map[string]string) metav1.ObjectMeta { +func getObjectMeta(namespace string, name string, autoReload bool, secretAutoReload bool, configmapAutoReload bool, secretproviderclass bool, extraAnnotations map[string]string) metav1.ObjectMeta { return metav1.ObjectMeta{ Name: name, Namespace: namespace, Labels: map[string]string{"firstLabel": "temp"}, - Annotations: getAnnotations(name, autoReload, secretAutoReload, configmapAutoReload, extraAnnotations), + Annotations: getAnnotations(name, autoReload, secretAutoReload, configmapAutoReload, secretproviderclass, extraAnnotations), } } -func getAnnotations(name string, autoReload bool, secretAutoReload bool, configmapAutoReload bool, extraAnnotations map[string]string) map[string]string { +func getAnnotations(name string, autoReload bool, secretAutoReload bool, configmapAutoReload bool, secretproviderclass bool, extraAnnotations map[string]string) map[string]string { annotations := make(map[string]string) if autoReload { annotations[options.ReloaderAutoAnnotation] = "true" @@ -93,11 +98,16 @@ func getAnnotations(name string, autoReload bool, secretAutoReload bool, configm if configmapAutoReload { annotations[options.ConfigmapReloaderAutoAnnotation] = "true" } + if secretproviderclass { + annotations[options.SecretProviderClassReloaderAutoAnnotation] = "true" + } if len(annotations) == 0 { annotations = map[string]string{ - options.ConfigmapUpdateOnChangeAnnotation: name, - options.SecretUpdateOnChangeAnnotation: name} + options.ConfigmapUpdateOnChangeAnnotation: name, + options.SecretUpdateOnChangeAnnotation: name, + options.SecretProviderClassUpdateOnChangeAnnotation: name, + } } for k, v := range extraAnnotations { annotations[k] = v @@ -176,6 +186,15 @@ func getVolumes(name string) []v1.Volume { }, }, }, + { + Name: "secretproviderclass", + VolumeSource: v1.VolumeSource{ + CSI: &v1.CSIVolumeSource{ + Driver: "secrets-store.csi.k8s.io", + VolumeAttributes: map[string]string{"secretProviderClass": name}, + }, + }, + }, } } @@ -189,6 +208,10 @@ func getVolumeMounts() []v1.VolumeMount { MountPath: "etc/sec", Name: "secret", }, + { + MountPath: "etc/spc", + Name: "secretproviderclass", + }, { MountPath: "etc/projectedconfig", Name: "projectedconfigmap", @@ -348,7 +371,7 @@ func getPodTemplateSpecWithInitContainerAndEnv(name string) v1.PodTemplateSpec { func GetDeployment(namespace string, deploymentName string) *appsv1.Deployment { replicaset := int32(1) return &appsv1.Deployment{ - ObjectMeta: getObjectMeta(namespace, deploymentName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, deploymentName, false, false, false, false, map[string]string{}), Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -367,7 +390,7 @@ func GetDeploymentConfig(namespace string, deploymentConfigName string) *openshi replicaset := int32(1) podTemplateSpecWithVolume := getPodTemplateSpecWithVolumes(deploymentConfigName) return &openshiftv1.DeploymentConfig{ - ObjectMeta: getObjectMeta(namespace, deploymentConfigName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, deploymentConfigName, false, false, false, false, map[string]string{}), Spec: openshiftv1.DeploymentConfigSpec{ Replicas: replicaset, Strategy: openshiftv1.DeploymentStrategy{ @@ -382,7 +405,7 @@ func GetDeploymentConfig(namespace string, deploymentConfigName string) *openshi func GetDeploymentWithInitContainer(namespace string, deploymentName string) *appsv1.Deployment { replicaset := int32(1) return &appsv1.Deployment{ - ObjectMeta: getObjectMeta(namespace, deploymentName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, deploymentName, false, false, false, false, map[string]string{}), Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -400,7 +423,7 @@ func GetDeploymentWithInitContainer(namespace string, deploymentName string) *ap func GetDeploymentWithInitContainerAndEnv(namespace string, deploymentName string) *appsv1.Deployment { replicaset := int32(1) return &appsv1.Deployment{ - ObjectMeta: getObjectMeta(namespace, deploymentName, true, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, deploymentName, true, false, false, false, map[string]string{}), Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -417,7 +440,7 @@ func GetDeploymentWithInitContainerAndEnv(namespace string, deploymentName strin func GetDeploymentWithEnvVars(namespace string, deploymentName string) *appsv1.Deployment { replicaset := int32(1) return &appsv1.Deployment{ - ObjectMeta: getObjectMeta(namespace, deploymentName, true, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, deploymentName, true, false, false, false, map[string]string{}), Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -435,7 +458,7 @@ func GetDeploymentConfigWithEnvVars(namespace string, deploymentConfigName strin replicaset := int32(1) podTemplateSpecWithEnvVars := getPodTemplateSpecWithEnvVars(deploymentConfigName) return &openshiftv1.DeploymentConfig{ - ObjectMeta: getObjectMeta(namespace, deploymentConfigName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, deploymentConfigName, false, false, false, false, map[string]string{}), Spec: openshiftv1.DeploymentConfigSpec{ Replicas: replicaset, Strategy: openshiftv1.DeploymentStrategy{ @@ -449,7 +472,7 @@ func GetDeploymentConfigWithEnvVars(namespace string, deploymentConfigName strin func GetDeploymentWithEnvVarSources(namespace string, deploymentName string) *appsv1.Deployment { replicaset := int32(1) return &appsv1.Deployment{ - ObjectMeta: getObjectMeta(namespace, deploymentName, true, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, deploymentName, true, false, false, false, map[string]string{}), Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -466,7 +489,7 @@ func GetDeploymentWithEnvVarSources(namespace string, deploymentName string) *ap func GetDeploymentWithPodAnnotations(namespace string, deploymentName string, both bool) *appsv1.Deployment { replicaset := int32(1) deployment := &appsv1.Deployment{ - ObjectMeta: getObjectMeta(namespace, deploymentName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, deploymentName, false, false, false, false, map[string]string{}), Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -481,7 +504,7 @@ func GetDeploymentWithPodAnnotations(namespace string, deploymentName string, bo if !both { deployment.Annotations = nil } - deployment.Spec.Template.Annotations = getAnnotations(deploymentName, true, false, false, map[string]string{}) + deployment.Spec.Template.Annotations = getAnnotations(deploymentName, true, false, false, false, map[string]string{}) return deployment } @@ -490,9 +513,11 @@ func GetDeploymentWithTypedAutoAnnotation(namespace string, deploymentName strin var objectMeta metav1.ObjectMeta switch resourceType { case SecretResourceType: - objectMeta = getObjectMeta(namespace, deploymentName, false, true, false, map[string]string{}) + objectMeta = getObjectMeta(namespace, deploymentName, false, true, false, false, map[string]string{}) case ConfigmapResourceType: - objectMeta = getObjectMeta(namespace, deploymentName, false, false, true, map[string]string{}) + objectMeta = getObjectMeta(namespace, deploymentName, false, false, true, false, map[string]string{}) + case SecretProviderClassPodStatusResourceType: + objectMeta = getObjectMeta(namespace, deploymentName, false, false, false, true, map[string]string{}) } return &appsv1.Deployment{ @@ -520,6 +545,8 @@ func GetDeploymentWithExcludeAnnotation(namespace string, deploymentName string, annotation[options.SecretExcludeReloaderAnnotation] = deploymentName case ConfigmapResourceType: annotation[options.ConfigmapExcludeReloaderAnnotation] = deploymentName + case SecretProviderClassPodStatusResourceType: + annotation[options.SecretProviderClassExcludeReloaderAnnotation] = deploymentName } return &appsv1.Deployment{ @@ -545,7 +572,7 @@ func GetDeploymentWithExcludeAnnotation(namespace string, deploymentName string, // GetDaemonSet provides daemonset for testing func GetDaemonSet(namespace string, daemonsetName string) *appsv1.DaemonSet { return &appsv1.DaemonSet{ - ObjectMeta: getObjectMeta(namespace, daemonsetName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, daemonsetName, false, false, false, false, map[string]string{}), Spec: appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -560,7 +587,7 @@ func GetDaemonSet(namespace string, daemonsetName string) *appsv1.DaemonSet { func GetDaemonSetWithEnvVars(namespace string, daemonSetName string) *appsv1.DaemonSet { return &appsv1.DaemonSet{ - ObjectMeta: getObjectMeta(namespace, daemonSetName, true, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, daemonSetName, true, false, false, false, map[string]string{}), Spec: appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -576,7 +603,7 @@ func GetDaemonSetWithEnvVars(namespace string, daemonSetName string) *appsv1.Dae // GetStatefulSet provides statefulset for testing func GetStatefulSet(namespace string, statefulsetName string) *appsv1.StatefulSet { return &appsv1.StatefulSet{ - ObjectMeta: getObjectMeta(namespace, statefulsetName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, statefulsetName, false, false, false, false, map[string]string{}), Spec: appsv1.StatefulSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -592,7 +619,7 @@ func GetStatefulSet(namespace string, statefulsetName string) *appsv1.StatefulSe // GetStatefulSet provides statefulset for testing func GetStatefulSetWithEnvVar(namespace string, statefulsetName string) *appsv1.StatefulSet { return &appsv1.StatefulSet{ - ObjectMeta: getObjectMeta(namespace, statefulsetName, true, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, statefulsetName, true, false, false, false, map[string]string{}), Spec: appsv1.StatefulSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -617,6 +644,42 @@ func GetConfigmap(namespace string, configmapName string, testData string) *v1.C } } +func GetSecretProviderClass(namespace string, secretProviderClassName string, data string) *csiv1.SecretProviderClass { + return &csiv1.SecretProviderClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretProviderClassName, + Namespace: namespace, + }, + Spec: csiv1.SecretProviderClassSpec{ + Provider: "Test", + Parameters: map[string]string{ + "parameter1": data, + }, + }, + } +} + +func GetSecretProviderClassPodStatus(namespace string, secretProviderClassPodStatusName string, data string) *csiv1.SecretProviderClassPodStatus { + return &csiv1.SecretProviderClassPodStatus{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretProviderClassPodStatusName, + Namespace: namespace, + }, + Status: csiv1.SecretProviderClassPodStatusStatus{ + PodName: "test123", + SecretProviderClassName: secretProviderClassPodStatusName, + TargetPath: "/var/lib/kubelet/d8771ddf-935a-4199-a20b-f35f71c1d9e7/volumes/kubernetes.io~csi/secrets-store-inline/mount", + Mounted: true, + Objects: []csiv1.SecretProviderClassObject{ + { + ID: "parameter1", + Version: data, + }, + }, + }, + } +} + // GetConfigmapWithUpdatedLabel provides configmap for testing func GetConfigmapWithUpdatedLabel(namespace string, configmapName string, testLabel string, testData string) *v1.ConfigMap { return &v1.ConfigMap{ @@ -643,7 +706,7 @@ func GetSecret(namespace string, secretName string, data string) *v1.Secret { func GetCronJob(namespace string, cronJobName string) *batchv1.CronJob { return &batchv1.CronJob{ - ObjectMeta: getObjectMeta(namespace, cronJobName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, cronJobName, false, false, false, false, map[string]string{}), Spec: batchv1.CronJobSpec{ Schedule: "*/5 * * * *", // Run every 5 minutes JobTemplate: batchv1.JobTemplateSpec{ @@ -660,7 +723,7 @@ func GetCronJob(namespace string, cronJobName string) *batchv1.CronJob { func GetJob(namespace string, jobName string) *batchv1.Job { return &batchv1.Job{ - ObjectMeta: getObjectMeta(namespace, jobName, false, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, jobName, false, false, false, false, map[string]string{}), Spec: batchv1.JobSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -672,7 +735,7 @@ func GetJob(namespace string, jobName string) *batchv1.Job { func GetCronJobWithEnvVar(namespace string, cronJobName string) *batchv1.CronJob { return &batchv1.CronJob{ - ObjectMeta: getObjectMeta(namespace, cronJobName, true, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, cronJobName, true, false, false, false, map[string]string{}), Spec: batchv1.CronJobSpec{ Schedule: "*/5 * * * *", // Run every 5 minutes JobTemplate: batchv1.JobTemplateSpec{ @@ -689,7 +752,7 @@ func GetCronJobWithEnvVar(namespace string, cronJobName string) *batchv1.CronJob func GetJobWithEnvVar(namespace string, jobName string) *batchv1.Job { return &batchv1.Job{ - ObjectMeta: getObjectMeta(namespace, jobName, true, false, false, map[string]string{}), + ObjectMeta: getObjectMeta(namespace, jobName, true, false, false, false, map[string]string{}), Spec: batchv1.JobSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, @@ -746,7 +809,7 @@ func GetResourceSHAFromAnnotation(podAnnotations map[string]string) string { return last.Hash } -// ConvertResourceToSHA generates SHA from secret or configmap data +// ConvertResourceToSHA generates SHA from secret, configmap or secretproviderclasspodstatus data func ConvertResourceToSHA(resourceType string, namespace string, resourceName string, data string) string { values := []string{} switch resourceType { @@ -760,6 +823,12 @@ func ConvertResourceToSHA(resourceType string, namespace string, resourceName st for k, v := range configmap.Data { values = append(values, k+"="+v) } + case SecretProviderClassPodStatusResourceType: + secretproviderclasspodstatus := GetSecretProviderClassPodStatus(namespace, resourceName, data) + for _, v := range secretproviderclasspodstatus.Status.Objects { + values = append(values, v.ID+"="+v.Version) + } + values = append(values, "SecretProviderClassName="+secretproviderclasspodstatus.Status.SecretProviderClassName) } sort.Strings(values) return crypto.GenerateSHA(strings.Join(values, ";")) @@ -774,6 +843,25 @@ func CreateConfigMap(client kubernetes.Interface, namespace string, configmapNam return configmapClient, err } +// CreateSecretProviderClass creates a SecretProviderClass in given namespace and returns the SecretProviderClassInterface +func CreateSecretProviderClass(client csiclient.Interface, namespace string, secretProviderClassName string, data string) (csiclient_v1.SecretProviderClassInterface, error) { + logrus.Infof("Creating SecretProviderClass") + secretProviderClassClient := client.SecretsstoreV1().SecretProviderClasses(namespace) + _, err := secretProviderClassClient.Create(context.TODO(), GetSecretProviderClass(namespace, secretProviderClassName, data), metav1.CreateOptions{}) + time.Sleep(3 * time.Second) + return secretProviderClassClient, err +} + +// CreateSecretProviderClassPodStatus creates a SecretProviderClassPodStatus in given namespace and returns the SecretProviderClassPodStatusInterface +func CreateSecretProviderClassPodStatus(client csiclient.Interface, namespace string, secretProviderClassPodStatusName string, data string) (csiclient_v1.SecretProviderClassPodStatusInterface, error) { + logrus.Infof("Creating SecretProviderClassPodStatus") + secretProviderClassPodStatusClient := client.SecretsstoreV1().SecretProviderClassPodStatuses(namespace) + secretProviderClassPodStatus := GetSecretProviderClassPodStatus(namespace, secretProviderClassPodStatusName, data) + _, err := secretProviderClassPodStatusClient.Create(context.TODO(), secretProviderClassPodStatus, metav1.CreateOptions{}) + time.Sleep(3 * time.Second) + return secretProviderClassPodStatusClient, err +} + // CreateSecret creates a secret in given namespace and returns the SecretInterface func CreateSecret(client kubernetes.Interface, namespace string, secretName string, data string) (core_v1.SecretInterface, error) { logrus.Infof("Creating secret") @@ -1036,6 +1124,27 @@ func UpdateSecret(secretClient core_v1.SecretInterface, namespace string, secret return updateErr } +// UpdateSecretProviderClassPodStatus updates a secretproviderclasspodstatus in given namespace and returns the error if any +func UpdateSecretProviderClassPodStatus(spcpsClient csiclient_v1.SecretProviderClassPodStatusInterface, namespace string, spcpsName string, label string, data string) error { + logrus.Infof("Updating secretproviderclasspodstatus %q.\n", spcpsName) + updatedStatus := GetSecretProviderClassPodStatus(namespace, spcpsName, data).Status + secretproviderclasspodstatus, err := spcpsClient.Get(context.TODO(), spcpsName, metav1.GetOptions{}) + if err != nil { + return err + } + secretproviderclasspodstatus.Status = updatedStatus + if label != "" { + labels := secretproviderclasspodstatus.Labels + if labels == nil { + labels = make(map[string]string) + } + labels["firstLabel"] = label + } + _, updateErr := spcpsClient.Update(context.TODO(), secretproviderclasspodstatus, metav1.UpdateOptions{}) + time.Sleep(3 * time.Second) + return updateErr +} + // DeleteConfigMap deletes a configmap in given namespace and returns the error if any func DeleteConfigMap(client kubernetes.Interface, namespace string, configmapName string) error { logrus.Infof("Deleting configmap %q.\n", configmapName) @@ -1052,6 +1161,22 @@ func DeleteSecret(client kubernetes.Interface, namespace string, secretName stri return err } +// DeleteSecretProviderClass deletes a secretproviderclass in given namespace and returns the error if any +func DeleteSecretProviderClass(client csiclient.Interface, namespace string, secretProviderClassName string) error { + logrus.Infof("Deleting secretproviderclass %q.\n", secretProviderClassName) + err := client.SecretsstoreV1().SecretProviderClasses(namespace).Delete(context.TODO(), secretProviderClassName, metav1.DeleteOptions{}) + time.Sleep(3 * time.Second) + return err +} + +// DeleteSecretProviderClassPodStatus deletes a secretproviderclasspodstatus in given namespace and returns the error if any +func DeleteSecretProviderClassPodStatus(client csiclient.Interface, namespace string, secretProviderClassPodStatusName string) error { + logrus.Infof("Deleting secretproviderclasspodstatus %q.\n", secretProviderClassPodStatusName) + err := client.SecretsstoreV1().SecretProviderClassPodStatuses(namespace).Delete(context.TODO(), secretProviderClassPodStatusName, metav1.DeleteOptions{}) + time.Sleep(3 * time.Second) + return err +} + // RandSeq generates a random sequence func RandSeq(n int) string { b := make([]rune, n) @@ -1209,7 +1334,7 @@ func GetSHAfromEmptyData() string { func GetRollout(namespace string, rolloutName string, annotations map[string]string) *argorolloutv1alpha1.Rollout { replicaset := int32(1) return &argorolloutv1alpha1.Rollout{ - ObjectMeta: getObjectMeta(namespace, rolloutName, false, false, false, annotations), + ObjectMeta: getObjectMeta(namespace, rolloutName, false, false, false, false, annotations), Spec: argorolloutv1alpha1.RolloutSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"secondLabel": "temp"}, diff --git a/internal/pkg/util/util.go b/internal/pkg/util/util.go index ec86d1c..53846f3 100644 --- a/internal/pkg/util/util.go +++ b/internal/pkg/util/util.go @@ -13,6 +13,7 @@ import ( "github.com/stakater/Reloader/internal/pkg/crypto" "github.com/stakater/Reloader/internal/pkg/options" v1 "k8s.io/api/core/v1" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" ) // ConvertToEnvVarName converts the given text into a usable env var @@ -57,6 +58,16 @@ func GetSHAfromSecret(data map[string][]byte) string { return crypto.GenerateSHA(strings.Join(values, ";")) } +func GetSHAfromSecretProviderClassPodStatus(data csiv1.SecretProviderClassPodStatusStatus) string { + values := []string{} + for _, v := range data.Objects { + values = append(values, v.ID+"="+v.Version) + } + values = append(values, "SecretProviderClassName="+data.SecretProviderClassName) + sort.Strings(values) + return crypto.GenerateSHA(strings.Join(values, ";")) +} + type List []string func (l *List) Contains(s string) bool { diff --git a/pkg/common/common.go b/pkg/common/common.go index 84d9827..b6fe3b5 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -32,6 +32,8 @@ type ReloaderOptions struct { ConfigmapUpdateOnChangeAnnotation string `json:"configmapUpdateOnChangeAnnotation"` // SecretUpdateOnChangeAnnotation is the annotation key used to detect changes in Secrets specified by name SecretUpdateOnChangeAnnotation string `json:"secretUpdateOnChangeAnnotation"` + // SecretProviderClassUpdateOnChangeAnnotation is the annotation key used to detect changes in SecretProviderClasses specified by name + SecretProviderClassUpdateOnChangeAnnotation string `json:"secretProviderClassUpdateOnChangeAnnotation"` // ReloaderAutoAnnotation is the annotation key used to detect changes in any referenced ConfigMaps or Secrets ReloaderAutoAnnotation string `json:"reloaderAutoAnnotation"` // IgnoreResourceAnnotation is the annotation key used to ignore resources from being watched @@ -40,10 +42,14 @@ type ReloaderOptions struct { ConfigmapReloaderAutoAnnotation string `json:"configmapReloaderAutoAnnotation"` // SecretReloaderAutoAnnotation is the annotation key used to detect changes in Secrets only SecretReloaderAutoAnnotation string `json:"secretReloaderAutoAnnotation"` + // SecretProviderClassReloaderAutoAnnotation is the annotation key used to detect changes in SecretProviderClasses only + SecretProviderClassReloaderAutoAnnotation string `json:"secretProviderClassReloaderAutoAnnotation"` // ConfigmapExcludeReloaderAnnotation is the annotation key containing comma-separated list of ConfigMaps to exclude from watching ConfigmapExcludeReloaderAnnotation string `json:"configmapExcludeReloaderAnnotation"` // SecretExcludeReloaderAnnotation is the annotation key containing comma-separated list of Secrets to exclude from watching SecretExcludeReloaderAnnotation string `json:"secretExcludeReloaderAnnotation"` + // SecretProviderClassExcludeReloaderAnnotation is the annotation key containing comma-separated list of SecretProviderClasses to exclude from watching + SecretProviderClassExcludeReloaderAnnotation string `json:"secretProviderClassExcludeReloaderAnnotation"` // AutoSearchAnnotation is the annotation key used to detect changes in ConfigMaps/Secrets tagged with SearchMatchAnnotation AutoSearchAnnotation string `json:"autoSearchAnnotation"` // SearchMatchAnnotation is the annotation key used to tag ConfigMaps/Secrets to be found by AutoSearchAnnotation @@ -71,6 +77,8 @@ type ReloaderOptions struct { SyncAfterRestart bool `json:"syncAfterRestart"` // EnableHA indicates whether High Availability mode is enabled with leader election EnableHA bool `json:"enableHA"` + // EnableCSIIntegration indicates whether CSI integration is enabled to watch SecretProviderClassPodStatus + EnableCSIIntegration bool `json:"enableCSIIntegration"` // WebhookUrl is the URL to send webhook notifications to instead of performing reloads WebhookUrl string `json:"webhookUrl"` // ResourcesToIgnore is a list of resource types to ignore (e.g., "configmaps" or "secrets") @@ -224,6 +232,7 @@ func ShouldReload(config Config, resourceType string, annotations Map, podAnnota typedAutoAnnotationEnabledValue, foundTypedAuto := annotations[config.TypedAutoAnnotation] excludeConfigmapAnnotationValue, foundExcludeConfigmap := annotations[options.ConfigmapExcludeReloaderAnnotation] excludeSecretAnnotationValue, foundExcludeSecret := annotations[options.SecretExcludeReloaderAnnotation] + excludeSecretProviderClassProviderAnnotationValue, foundExcludeSecretProviderClass := annotations[options.SecretProviderClassExcludeReloaderAnnotation] if !found && !foundAuto && !foundTypedAuto && !foundSearchAnn { annotations = podAnnotations @@ -244,6 +253,11 @@ func ShouldReload(config Config, resourceType string, annotations Map, podAnnota if foundExcludeSecret { isResourceExcluded = checkIfResourceIsExcluded(config.ResourceName, excludeSecretAnnotationValue) } + + case constants.SecretProviderClassEnvVarPostfix: + if foundExcludeSecretProviderClass { + isResourceExcluded = checkIfResourceIsExcluded(config.ResourceName, excludeSecretProviderClassProviderAnnotationValue) + } } if isResourceExcluded { @@ -315,12 +329,15 @@ func GetCommandLineOptions() *ReloaderOptions { CommandLineOptions.AutoReloadAll = options.AutoReloadAll CommandLineOptions.ConfigmapUpdateOnChangeAnnotation = options.ConfigmapUpdateOnChangeAnnotation CommandLineOptions.SecretUpdateOnChangeAnnotation = options.SecretUpdateOnChangeAnnotation + CommandLineOptions.SecretProviderClassUpdateOnChangeAnnotation = options.SecretProviderClassUpdateOnChangeAnnotation CommandLineOptions.ReloaderAutoAnnotation = options.ReloaderAutoAnnotation CommandLineOptions.IgnoreResourceAnnotation = options.IgnoreResourceAnnotation CommandLineOptions.ConfigmapReloaderAutoAnnotation = options.ConfigmapReloaderAutoAnnotation CommandLineOptions.SecretReloaderAutoAnnotation = options.SecretReloaderAutoAnnotation + CommandLineOptions.SecretProviderClassReloaderAutoAnnotation = options.SecretProviderClassReloaderAutoAnnotation CommandLineOptions.ConfigmapExcludeReloaderAnnotation = options.ConfigmapExcludeReloaderAnnotation CommandLineOptions.SecretExcludeReloaderAnnotation = options.SecretExcludeReloaderAnnotation + CommandLineOptions.SecretProviderClassExcludeReloaderAnnotation = options.SecretProviderClassExcludeReloaderAnnotation CommandLineOptions.AutoSearchAnnotation = options.AutoSearchAnnotation CommandLineOptions.SearchMatchAnnotation = options.SearchMatchAnnotation CommandLineOptions.RolloutStrategyAnnotation = options.RolloutStrategyAnnotation @@ -331,6 +348,7 @@ func GetCommandLineOptions() *ReloaderOptions { CommandLineOptions.ReloadStrategy = options.ReloadStrategy CommandLineOptions.SyncAfterRestart = options.SyncAfterRestart CommandLineOptions.EnableHA = options.EnableHA + CommandLineOptions.EnableCSIIntegration = options.EnableCSIIntegration CommandLineOptions.WebhookUrl = options.WebhookUrl CommandLineOptions.ResourcesToIgnore = options.ResourcesToIgnore CommandLineOptions.WorkloadTypesToIgnore = options.WorkloadTypesToIgnore diff --git a/pkg/common/config.go b/pkg/common/config.go index 4227c2b..4421fa5 100644 --- a/pkg/common/config.go +++ b/pkg/common/config.go @@ -5,6 +5,7 @@ import ( "github.com/stakater/Reloader/internal/pkg/options" "github.com/stakater/Reloader/internal/pkg/util" v1 "k8s.io/api/core/v1" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" ) // Config contains rolling upgrade configuration parameters @@ -46,3 +47,16 @@ func GetSecretConfig(secret *v1.Secret) Config { Labels: secret.Labels, } } + +func GetSecretProviderClassPodStatusConfig(podStatus *csiv1.SecretProviderClassPodStatus) Config { + // As csi injects SecretProviderClass, we will create config for it instead of SecretProviderClassPodStatus + // ResourceAnnotations will be retrieved during PerformAction call + return Config{ + Namespace: podStatus.Namespace, + ResourceName: podStatus.Status.SecretProviderClassName, + Annotation: options.SecretProviderClassUpdateOnChangeAnnotation, + TypedAutoAnnotation: options.SecretProviderClassReloaderAutoAnnotation, + SHAValue: util.GetSHAfromSecretProviderClassPodStatus(podStatus.Status), + Type: constants.SecretProviderClassEnvVarPostfix, + } +} diff --git a/pkg/kube/client.go b/pkg/kube/client.go index 4230063..9582929 100644 --- a/pkg/kube/client.go +++ b/pkg/kube/client.go @@ -11,6 +11,7 @@ import ( "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + csiclient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned" ) // Clients struct exposes interfaces for kubernetes as well as openshift if available @@ -18,11 +19,14 @@ type Clients struct { KubernetesClient kubernetes.Interface OpenshiftAppsClient appsclient.Interface ArgoRolloutClient argorollout.Interface + CSIClient csiclient.Interface } var ( // IsOpenshift is true if environment is Openshift, it is false if environment is Kubernetes IsOpenshift = isOpenshift() + // IsCSIEnabled is true if environment has CSI provider installed, otherwise false + IsCSIInstalled = isCSIInstalled() ) // GetClients returns a `Clients` object containing both openshift and kubernetes clients with an openshift identifier @@ -48,10 +52,20 @@ func GetClients() Clients { logrus.Warnf("Unable to create ArgoRollout client error = %v", err) } + var csiClient *csiclient.Clientset + + if IsCSIInstalled { + csiClient, err = GetCSIClient() + if err != nil { + logrus.Warnf("Unable to create CSI client error = %v", err) + } + } + return Clients{ KubernetesClient: client, OpenshiftAppsClient: appsClient, ArgoRolloutClient: rolloutClient, + CSIClient: csiClient, } } @@ -63,6 +77,28 @@ func GetArgoRolloutClient() (*argorollout.Clientset, error) { return argorollout.NewForConfig(config) } +func isCSIInstalled() bool { + client, err := GetKubernetesClient() + if err != nil { + logrus.Fatalf("Unable to create Kubernetes client error = %v", err) + } + _, err = client.RESTClient().Get().AbsPath("/apis/secrets-store.csi.x-k8s.io/v1").Do(context.TODO()).Raw() + if err == nil { + logrus.Info("CSI provider is installed") + return true + } + logrus.Info("CSI provider is not installed") + return false +} + +func GetCSIClient() (*csiclient.Clientset, error) { + config, err := getConfig() + if err != nil { + return nil, err + } + return csiclient.NewForConfig(config) +} + func isOpenshift() bool { client, err := GetKubernetesClient() if err != nil { diff --git a/pkg/kube/resourcemapper.go b/pkg/kube/resourcemapper.go index 89ac2af..286d408 100644 --- a/pkg/kube/resourcemapper.go +++ b/pkg/kube/resourcemapper.go @@ -3,6 +3,7 @@ package kube import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1" ) // ResourceMap are resources from where changes are going to be detected @@ -10,4 +11,5 @@ var ResourceMap = map[string]runtime.Object{ "configmaps": &v1.ConfigMap{}, "secrets": &v1.Secret{}, "namespaces": &v1.Namespace{}, + "secretproviderclasspodstatuses": &csiv1.SecretProviderClassPodStatus{}, }