wip: port csi provider to v2 sa-8436

This commit is contained in:
Safwan
2026-06-22 16:42:01 +05:00
parent 14117194ac
commit 73729c951e
40 changed files with 1821 additions and 40 deletions
+95
View File
@@ -0,0 +1,95 @@
package csi
import (
"context"
"encoding/json"
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
csiclient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned"
"github.com/stakater/Reloader/test/e2e/utils"
)
var (
kubeClient kubernetes.Interface
csiClient csiclient.Interface
restConfig *rest.Config
testNamespace string
ctx context.Context
testEnv *utils.TestEnvironment
)
func TestCSI(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "CSI SecretProviderClass E2E Suite")
}
// SynchronizedBeforeSuite ensures only process 1 deploys Reloader.
// Process 1 also checks prerequisites (CSI driver, Vault) and calls Skip if
// they are not installed — Ginkgo propagates the skip to all processes.
var _ = SynchronizedBeforeSuite(
// Process 1 only: check prerequisites, create namespace, deploy Reloader.
func() []byte {
setupEnv, err := utils.SetupTestEnvironment(context.Background(), "reloader-csi-test")
Expect(err).NotTo(HaveOccurred(), "Failed to setup test environment")
// Ensure the namespace is deleted even if DeployAndWait fails, so
// orphaned namespaces don't accumulate on long-lived clusters.
DeferCleanup(setupEnv.CleanupOnFailure)
if !utils.IsCSIDriverInstalled(context.Background(), setupEnv.CSIClient) {
Skip("CSI secrets store driver not installed - skipping CSI suite")
}
if !utils.IsVaultProviderInstalled(context.Background(), setupEnv.KubeClient) {
Skip("Vault CSI provider not installed - skipping CSI suite")
}
Expect(setupEnv.DeployAndWait(map[string]string{
"reloader.reloadStrategy": "annotations",
"reloader.watchGlobally": "false",
"reloader.enableCSIIntegration": "true",
})).To(Succeed(), "Failed to deploy Reloader")
data, err := json.Marshal(utils.SharedEnvData{
Namespace: setupEnv.Namespace,
ReleaseName: setupEnv.ReleaseName,
})
Expect(err).NotTo(HaveOccurred())
return data
},
// All processes (including #1): connect to the shared environment.
func(data []byte) {
var shared utils.SharedEnvData
Expect(json.Unmarshal(data, &shared)).To(Succeed())
var err error
testEnv, err = utils.SetupSharedTestEnvironment(context.Background(), shared.Namespace, shared.ReleaseName)
Expect(err).NotTo(HaveOccurred(), "Failed to setup shared test environment")
kubeClient = testEnv.KubeClient
csiClient = testEnv.CSIClient
restConfig = testEnv.RestConfig
testNamespace = testEnv.Namespace
ctx = testEnv.Ctx
},
)
var _ = SynchronizedAfterSuite(
// All processes: cancel the per-process context.
func() {
if testEnv != nil {
testEnv.Cancel()
}
},
// Process 1 only (runs last): undeploy Reloader and delete namespace.
func() {
if testEnv != nil {
err := testEnv.Cleanup()
Expect(err).NotTo(HaveOccurred(), "Failed to cleanup test environment")
}
GinkgoWriter.Println("CSI E2E Suite cleanup complete")
},
)
+330
View File
@@ -0,0 +1,330 @@
package csi
import (
"fmt"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/stakater/Reloader/test/e2e/utils"
)
var _ = Describe("CSI SecretProviderClass Tests", Label("csi"), Serial, func() {
var (
deploymentName string
configMapName string
spcName string
vaultSecretPath string
adapter *utils.DeploymentAdapter
)
BeforeEach(func() {
deploymentName = utils.RandName("deploy")
configMapName = utils.RandName("cm")
spcName = utils.RandName("spc")
vaultSecretPath = fmt.Sprintf("secret/%s", utils.RandName("test"))
adapter = utils.NewDeploymentAdapter(kubeClient)
})
AfterEach(func() {
_ = utils.DeleteDeployment(ctx, kubeClient, testNamespace, deploymentName)
_ = utils.DeleteConfigMap(ctx, kubeClient, testNamespace, configMapName)
_ = utils.DeleteSecretProviderClass(ctx, csiClient, testNamespace, spcName)
_ = utils.DeleteVaultSecret(ctx, kubeClient, restConfig, vaultSecretPath)
})
Context("Real Vault Integration Tests", func() {
It("should reload when Vault secret changes", func() {
By("Creating a secret in Vault")
err := utils.CreateVaultSecret(
ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"api_key": "initial-value-v1"})
Expect(err).NotTo(HaveOccurred())
By("Creating a SecretProviderClass pointing to Vault secret")
_, err = utils.CreateSecretProviderClassWithSecret(
ctx, csiClient, testNamespace, spcName,
vaultSecretPath, "api_key",
)
Expect(err).NotTo(HaveOccurred())
By("Creating Deployment with CSI volume and SPC reload annotation")
_, err = utils.CreateDeployment(
ctx, kubeClient, testNamespace, deploymentName,
utils.WithCSIVolume(spcName),
utils.WithAnnotations(utils.BuildSecretProviderClassReloadAnnotation(spcName)),
)
Expect(err).NotTo(HaveOccurred())
By("Waiting for Deployment to be ready")
err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout)
Expect(err).NotTo(HaveOccurred())
By("Finding the SPCPS created by CSI driver")
spcpsName, err := utils.FindSPCPSForDeployment(
ctx, csiClient, kubeClient, testNamespace, deploymentName, utils.WorkloadReadyTimeout,
)
Expect(err).NotTo(HaveOccurred())
GinkgoWriter.Printf("Found SPCPS: %s\n", spcpsName)
By("Getting initial SPCPS version")
initialVersion, err := utils.GetSPCPSVersion(ctx, csiClient, testNamespace, spcpsName)
Expect(err).NotTo(HaveOccurred())
GinkgoWriter.Printf("Initial SPCPS version: %s\n", initialVersion)
By("Updating the Vault secret")
err = utils.UpdateVaultSecret(
ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"api_key": "updated-value-v2"})
Expect(err).NotTo(HaveOccurred())
By("Waiting for CSI driver to sync the new secret version")
err = utils.WaitForSPCPSVersionChange(ctx, csiClient, testNamespace, spcpsName, initialVersion, 10*time.Second)
Expect(err).NotTo(HaveOccurred())
GinkgoWriter.Println("CSI driver synced new secret version")
By("Waiting for Deployment to be reloaded by Reloader")
reloaded, err := adapter.WaitReloaded(
ctx, testNamespace, deploymentName,
utils.AnnotationLastReloadedFrom, utils.ReloadTimeout,
)
Expect(err).NotTo(HaveOccurred())
Expect(reloaded).To(BeTrue(), "Deployment should have been reloaded after Vault secret change")
})
It("should handle multiple Vault secret updates", func() {
By("Creating a secret in Vault")
err := utils.CreateVaultSecret(
ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"password": "pass-v1"})
Expect(err).NotTo(HaveOccurred())
By("Creating a SecretProviderClass pointing to Vault secret")
_, err = utils.CreateSecretProviderClassWithSecret(
ctx, csiClient, testNamespace, spcName,
vaultSecretPath, "password",
)
Expect(err).NotTo(HaveOccurred())
By("Creating Deployment with CSI volume")
_, err = utils.CreateDeployment(
ctx, kubeClient, testNamespace, deploymentName,
utils.WithCSIVolume(spcName),
utils.WithAnnotations(utils.BuildSecretProviderClassReloadAnnotation(spcName)),
)
Expect(err).NotTo(HaveOccurred())
By("Waiting for Deployment to be ready")
err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout)
Expect(err).NotTo(HaveOccurred())
By("Finding the SPCPS")
spcpsName, err := utils.FindSPCPSForDeployment(
ctx, csiClient, kubeClient, testNamespace, deploymentName, utils.WorkloadReadyTimeout,
)
Expect(err).NotTo(HaveOccurred())
By("First update to Vault secret")
initialVersion, _ := utils.GetSPCPSVersion(ctx, csiClient, testNamespace, spcpsName)
err = utils.UpdateVaultSecret(
ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"password": "pass-v2"})
Expect(err).NotTo(HaveOccurred())
By("Waiting for first CSI sync")
err = utils.WaitForSPCPSVersionChange(ctx, csiClient, testNamespace, spcpsName, initialVersion, 10*time.Second)
Expect(err).NotTo(HaveOccurred())
By("Waiting for first reload")
reloaded, err := adapter.WaitReloaded(
ctx, testNamespace, deploymentName,
utils.AnnotationLastReloadedFrom, utils.ReloadTimeout,
)
Expect(err).NotTo(HaveOccurred())
Expect(reloaded).To(BeTrue())
By("Getting annotation value after first reload")
deploy, err := utils.GetDeployment(ctx, kubeClient, testNamespace, deploymentName)
Expect(err).NotTo(HaveOccurred())
firstReloadValue := deploy.Spec.Template.Annotations[utils.AnnotationLastReloadedFrom]
Expect(firstReloadValue).NotTo(BeEmpty())
By("Waiting for Deployment to stabilize")
err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout)
Expect(err).NotTo(HaveOccurred())
By("Finding the NEW SPCPS after first reload (new pod = new SPCPS)")
newSpcpsName, err := utils.FindSPCPSForDeployment(
ctx, csiClient, kubeClient, testNamespace, deploymentName, utils.WorkloadReadyTimeout,
)
Expect(err).NotTo(HaveOccurred())
GinkgoWriter.Printf("New SPCPS after first reload: %s\n", newSpcpsName)
By("Second update to Vault secret")
err = utils.UpdateVaultSecret(
ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"password": "pass-v3"})
Expect(err).NotTo(HaveOccurred())
By("Waiting for second reload with different annotation value")
Eventually(func() string {
deploy, err := utils.GetDeployment(ctx, kubeClient, testNamespace, deploymentName)
if err != nil {
return ""
}
return deploy.Spec.Template.Annotations[utils.AnnotationLastReloadedFrom]
}, utils.ReloadTimeout).ShouldNot(Equal(firstReloadValue), "Annotation should change after second Vault secret update")
})
})
Context("Typed Auto Annotation Tests", func() {
It("should reload only SPC changes with secretproviderclass auto annotation, not ConfigMap", func() {
By("Creating a ConfigMap")
_, err := utils.CreateConfigMap(
ctx, kubeClient, testNamespace, configMapName,
map[string]string{"key": "initial"}, nil,
)
Expect(err).NotTo(HaveOccurred())
By("Creating a secret in Vault")
err = utils.CreateVaultSecret(
ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"token": "token-v1"})
Expect(err).NotTo(HaveOccurred())
By("Creating a SecretProviderClass pointing to Vault secret")
_, err = utils.CreateSecretProviderClassWithSecret(
ctx, csiClient, testNamespace, spcName,
vaultSecretPath, "token",
)
Expect(err).NotTo(HaveOccurred())
By("Creating Deployment with ConfigMap envFrom AND CSI volume, but only SPC auto annotation")
_, err = utils.CreateDeployment(
ctx, kubeClient, testNamespace, deploymentName,
utils.WithConfigMapEnvFrom(configMapName),
utils.WithCSIVolume(spcName),
utils.WithAnnotations(utils.BuildSecretProviderClassAutoAnnotation()),
)
Expect(err).NotTo(HaveOccurred())
By("Waiting for Deployment to be ready")
err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout)
Expect(err).NotTo(HaveOccurred())
By("Updating the ConfigMap (should NOT trigger reload)")
err = utils.UpdateConfigMap(
ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"})
Expect(err).NotTo(HaveOccurred())
By("Verifying Deployment was NOT reloaded for ConfigMap change")
time.Sleep(utils.NegativeTestWait)
reloaded, err := adapter.WaitReloaded(
ctx, testNamespace, deploymentName,
utils.AnnotationLastReloadedFrom, utils.ShortTimeout,
)
Expect(err).NotTo(HaveOccurred())
Expect(reloaded).To(BeFalse(), "SPC auto annotation should not trigger reload for ConfigMap changes")
By("Finding the SPCPS")
spcpsName, err := utils.FindSPCPSForDeployment(
ctx, csiClient, kubeClient, testNamespace, deploymentName, utils.WorkloadReadyTimeout,
)
Expect(err).NotTo(HaveOccurred())
By("Getting SPCPS version before Vault update")
initialVersion, _ := utils.GetSPCPSVersion(ctx, csiClient, testNamespace, spcpsName)
By("Updating the Vault secret (should trigger reload)")
err = utils.UpdateVaultSecret(
ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"token": "token-v2"})
Expect(err).NotTo(HaveOccurred())
By("Waiting for CSI driver to sync")
err = utils.WaitForSPCPSVersionChange(ctx, csiClient, testNamespace, spcpsName, initialVersion, 10*time.Second)
Expect(err).NotTo(HaveOccurred())
By("Verifying Deployment WAS reloaded for Vault secret change")
reloaded, err = adapter.WaitReloaded(
ctx, testNamespace, deploymentName,
utils.AnnotationLastReloadedFrom, utils.ReloadTimeout,
)
Expect(err).NotTo(HaveOccurred())
Expect(reloaded).To(BeTrue(), "SPC auto annotation should trigger reload for Vault secret changes")
})
It("should reload for both ConfigMap and SPC when using combined auto=true", func() {
By("Creating a ConfigMap")
_, err := utils.CreateConfigMap(
ctx, kubeClient, testNamespace, configMapName,
map[string]string{"key": "initial"}, nil,
)
Expect(err).NotTo(HaveOccurred())
By("Creating a secret in Vault")
err = utils.CreateVaultSecret(
ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"secret": "secret-v1"})
Expect(err).NotTo(HaveOccurred())
By("Creating a SecretProviderClass pointing to Vault secret")
_, err = utils.CreateSecretProviderClassWithSecret(
ctx, csiClient, testNamespace, spcName,
vaultSecretPath, "secret",
)
Expect(err).NotTo(HaveOccurred())
By("Creating Deployment with ConfigMap envFrom AND CSI volume with combined auto=true")
_, err = utils.CreateDeployment(
ctx, kubeClient, testNamespace, deploymentName,
utils.WithConfigMapEnvFrom(configMapName),
utils.WithCSIVolume(spcName),
utils.WithAnnotations(utils.BuildAutoTrueAnnotation()),
)
Expect(err).NotTo(HaveOccurred())
By("Waiting for Deployment to be ready")
err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout)
Expect(err).NotTo(HaveOccurred())
By("Updating the ConfigMap (should trigger reload with auto=true)")
err = utils.UpdateConfigMap(
ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"})
Expect(err).NotTo(HaveOccurred())
By("Verifying Deployment WAS reloaded for ConfigMap change")
reloaded, err := adapter.WaitReloaded(
ctx, testNamespace, deploymentName,
utils.AnnotationLastReloadedFrom, utils.ReloadTimeout,
)
Expect(err).NotTo(HaveOccurred())
Expect(reloaded).To(BeTrue(), "Combined auto=true should trigger reload for ConfigMap changes")
By("Waiting for Deployment to stabilize")
err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout)
Expect(err).NotTo(HaveOccurred())
By("Getting current annotation value")
deploy, err := utils.GetDeployment(ctx, kubeClient, testNamespace, deploymentName)
Expect(err).NotTo(HaveOccurred())
firstReloadValue := deploy.Spec.Template.Annotations[utils.AnnotationLastReloadedFrom]
By("Finding the NEW SPCPS after ConfigMap reload (new pod = new SPCPS)")
newSpcpsName, err := utils.FindSPCPSForDeployment(
ctx, csiClient, kubeClient, testNamespace, deploymentName, utils.WorkloadReadyTimeout,
)
Expect(err).NotTo(HaveOccurred())
GinkgoWriter.Printf("New SPCPS after ConfigMap reload: %s\n", newSpcpsName)
By("Updating the Vault secret (should also trigger reload with auto=true)")
err = utils.UpdateVaultSecret(
ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"secret": "secret-v2"})
Expect(err).NotTo(HaveOccurred())
By("Verifying Deployment WAS reloaded for Vault secret change")
Eventually(func() string {
deploy, err := utils.GetDeployment(ctx, kubeClient, testNamespace, deploymentName)
if err != nil {
return ""
}
return deploy.Spec.Template.Annotations[utils.AnnotationLastReloadedFrom]
}, utils.ReloadTimeout).ShouldNot(Equal(firstReloadValue),
"Combined auto=true should trigger reload for Vault secret changes",
)
})
})
})
+27
View File
@@ -20,6 +20,11 @@ const (
// Value: comma-separated list of Secret names, e.g., "secret1,secret2"
AnnotationSecretReload = "secret.reloader.stakater.com/reload"
// AnnotationSecretProviderClassReload triggers reload when specified SecretProviderClass(es) change.
// Value: comma-separated list of SecretProviderClass names, e.g., "spc1,spc2"
// Note: Reloader actually watches SecretProviderClassPodStatus resources, not SecretProviderClass.
AnnotationSecretProviderClassReload = "secretproviderclass.reloader.stakater.com/reload"
// ============================================================
// Auto-reload annotations
// ============================================================
@@ -36,6 +41,10 @@ const (
// Value: "true" or "false"
AnnotationSecretAuto = "secret.reloader.stakater.com/auto"
// AnnotationSecretProviderClassAuto enables auto-reload for all referenced SecretProviderClasses only.
// Value: "true" or "false"
AnnotationSecretProviderClassAuto = "secretproviderclass.reloader.stakater.com/auto"
// ============================================================
// Exclude annotations (used with auto=true to exclude specific resources)
// ============================================================
@@ -48,6 +57,10 @@ const (
// Value: comma-separated list of Secret names
AnnotationSecretExclude = "secrets.exclude.reloader.stakater.com/reload"
// AnnotationSecretProviderClassExclude excludes specified SecretProviderClasses from auto-reload.
// Value: comma-separated list of SecretProviderClass names
AnnotationSecretProviderClassExclude = "secretproviderclasses.exclude.reloader.stakater.com/reload"
// ============================================================
// Search annotations (for regex matching)
// ============================================================
@@ -117,6 +130,13 @@ func BuildSecretReloadAnnotation(secretNames ...string) map[string]string {
}
}
// BuildSecretProviderClassReloadAnnotation creates an annotation map for SecretProviderClass reload.
func BuildSecretProviderClassReloadAnnotation(spcNames ...string) map[string]string {
return map[string]string{
AnnotationSecretProviderClassReload: joinNames(spcNames),
}
}
// BuildAutoTrueAnnotation creates an annotation map with auto=true.
func BuildAutoTrueAnnotation() map[string]string {
return map[string]string{
@@ -145,6 +165,13 @@ func BuildSecretAutoAnnotation() map[string]string {
}
}
// BuildSecretProviderClassAutoAnnotation creates an annotation map with secretproviderclass auto=true.
func BuildSecretProviderClassAutoAnnotation() map[string]string {
return map[string]string{
AnnotationSecretProviderClassAuto: AnnotationValueTrue,
}
}
// BuildSearchAnnotation creates an annotation map to enable search mode.
func BuildSearchAnnotation() map[string]string {
return map[string]string{
+33
View File
@@ -6,6 +6,7 @@ import (
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1"
)
// PodTemplateAccessor extracts PodTemplateSpec from a workload.
@@ -215,3 +216,35 @@ func IsTriggeredJobForCronJob(cronJobName string) Condition[*batchv1.Job] {
return false
}
}
// SPCPSVersionChanged returns a condition that checks if the SPCPS version has changed
// from the initial version and the SPCPS is mounted.
func SPCPSVersionChanged(initialVersion string) Condition[*csiv1.SecretProviderClassPodStatus] {
return func(spcps *csiv1.SecretProviderClassPodStatus) bool {
if !spcps.Status.Mounted || len(spcps.Status.Objects) == 0 {
return false
}
for _, obj := range spcps.Status.Objects {
if obj.Version != initialVersion {
return true
}
}
return false
}
}
// SPCPSForSPC returns a condition that checks if the SPCPS references a specific
// SecretProviderClass and is mounted.
func SPCPSForSPC(spcName string) Condition[*csiv1.SecretProviderClassPodStatus] {
return func(spcps *csiv1.SecretProviderClassPodStatus) bool {
return spcps.Status.SecretProviderClassName == spcName && spcps.Status.Mounted
}
}
// SPCPSForPods returns a condition that checks if the SPCPS references any of the
// specified pods and is mounted.
func SPCPSForPods(podNames map[string]bool) Condition[*csiv1.SecretProviderClassPodStatus] {
return func(spcps *csiv1.SecretProviderClassPodStatus) bool {
return podNames[spcps.Status.PodName] && spcps.Status.Mounted
}
}
+338
View File
@@ -0,0 +1,338 @@
package utils
import (
"bytes"
"context"
"errors"
"fmt"
"strings"
"time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/tools/remotecommand"
csiv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1"
csiclient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned"
)
// CSI Driver constants
const (
// CSIDriverName is the name of the secrets-store CSI driver
CSIDriverName = "secrets-store.csi.k8s.io"
// DefaultCSIProvider is the default provider name for testing (Vault)
DefaultCSIProvider = "vault"
// VaultAddress is the default Vault address in the cluster
VaultAddress = "http://vault.vault:8200"
// VaultRole is the Kubernetes auth role configured in Vault for testing
VaultRole = "test-role"
// VaultNamespace is the namespace where Vault is deployed
VaultNamespace = "vault"
// VaultPodName is the name of the Vault pod (dev mode)
VaultPodName = "vault-0"
// CSIVolumeName is the default volume name for CSI volumes in tests
CSIVolumeName = "csi-secrets-store"
// CSIMountPath is the default mount path for CSI volumes in tests
CSIMountPath = "/mnt/secrets-store"
// CSIRotationPollInterval is how often CSI driver checks for secret changes
CSIRotationPollInterval = 2 * time.Second
)
// NewCSIClient creates a new CSI client using the default kubeconfig.
func NewCSIClient() (csiclient.Interface, error) {
kubeconfig := GetKubeconfig()
config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
if err != nil {
return nil, fmt.Errorf("building config from kubeconfig: %w", err)
}
return NewCSIClientFromConfig(config)
}
// NewCSIClientFromConfig creates a new CSI client from a rest.Config.
func NewCSIClientFromConfig(config *rest.Config) (csiclient.Interface, error) {
client, err := csiclient.NewForConfig(config)
if err != nil {
return nil, fmt.Errorf("creating CSI client: %w", err)
}
return client, nil
}
// IsCSIDriverInstalled checks if the CSI secrets store driver CRDs are available in the cluster.
// This checks for the SecretProviderClass CRD which is required for CSI tests.
func IsCSIDriverInstalled(ctx context.Context, client csiclient.Interface) bool {
if client == nil {
return false
}
// Try to list SecretProviderClasses - if CRD doesn't exist, this will fail
_, err := client.SecretsstoreV1().SecretProviderClasses("default").List(ctx, metav1.ListOptions{Limit: 1})
return err == nil
}
// IsVaultProviderInstalled checks if Vault CSI provider is installed by checking for the vault-csi-provider DaemonSet.
// This is used to determine if CSI tests with actual volume mounting can run.
func IsVaultProviderInstalled(ctx context.Context, kubeClient kubernetes.Interface) bool {
if kubeClient == nil {
return false
}
// Check if vault-csi-provider DaemonSet exists in vault namespace
_, err := kubeClient.AppsV1().DaemonSets("vault").Get(ctx, "vault-csi-provider", metav1.GetOptions{})
return err == nil
}
// CreateSecretProviderClass creates a SecretProviderClass in the given namespace.
// If params is nil, it creates a Vault-compatible SecretProviderClass with default test settings.
func CreateSecretProviderClass(ctx context.Context, client csiclient.Interface, namespace, name string, params map[string]string) (
*csiv1.SecretProviderClass, error,
) {
if params == nil {
params = map[string]string{
"vaultAddress": VaultAddress,
"roleName": VaultRole,
"objects": `- objectName: "test-secret"
secretPath: "secret/data/test"
secretKey: "username"`,
}
}
spc := &csiv1.SecretProviderClass{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: csiv1.SecretProviderClassSpec{
Provider: DefaultCSIProvider,
Parameters: params,
},
}
created, err := client.SecretsstoreV1().SecretProviderClasses(namespace).Create(ctx, spc, metav1.CreateOptions{})
if err != nil {
return nil, fmt.Errorf("creating SecretProviderClass %s/%s: %w", namespace, name, err)
}
return created, nil
}
// CreateSecretProviderClassWithSecret creates a SecretProviderClass that fetches a specific secret from Vault.
// secretPath should be like "secret/mysecret" (the function converts it to KV v2 format "secret/data/mysecret").
// secretKey is the key within that secret to fetch.
func CreateSecretProviderClassWithSecret(ctx context.Context, client csiclient.Interface, namespace, name, secretPath, secretKey string) (
*csiv1.SecretProviderClass, error,
) {
kvV2Path := secretPath
if strings.HasPrefix(secretPath, "secret/") && !strings.HasPrefix(secretPath, "secret/data/") {
kvV2Path = strings.Replace(secretPath, "secret/", "secret/data/", 1)
}
params := map[string]string{
"vaultAddress": VaultAddress,
"roleName": VaultRole,
"objects": fmt.Sprintf(
`- objectName: "%s"
secretPath: "%s"
secretKey: "%s"`, secretKey, kvV2Path, secretKey,
),
}
return CreateSecretProviderClass(ctx, client, namespace, name, params)
}
// DeleteSecretProviderClass deletes a SecretProviderClass by name.
func DeleteSecretProviderClass(ctx context.Context, client csiclient.Interface, namespace, name string) error {
err := client.SecretsstoreV1().SecretProviderClasses(namespace).Delete(ctx, name, metav1.DeleteOptions{})
if err != nil {
return fmt.Errorf("deleting SecretProviderClass %s/%s: %w", namespace, name, err)
}
return nil
}
// UpdateSecretProviderClassPodStatusLabels updates only the labels on a SecretProviderClassPodStatus.
// This should NOT trigger a reload (used for negative testing to verify Reloader ignores label-only changes).
func UpdateSecretProviderClassPodStatusLabels(ctx context.Context, client csiclient.Interface, namespace, name string, labels map[string]string) error {
spcps, err := client.SecretsstoreV1().SecretProviderClassPodStatuses(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("getting SecretProviderClassPodStatus %s/%s: %w", namespace, name, err)
}
if spcps.Labels == nil {
spcps.Labels = make(map[string]string)
}
for k, v := range labels {
spcps.Labels[k] = v
}
_, err = client.SecretsstoreV1().SecretProviderClassPodStatuses(namespace).Update(ctx, spcps, metav1.UpdateOptions{})
if err != nil {
return fmt.Errorf("updating SecretProviderClassPodStatus labels %s/%s: %w", namespace, name, err)
}
return nil
}
// =============================================================================
// Vault Integration Helpers
// =============================================================================
// CreateVaultSecret creates a new secret in Vault.
// secretPath should be like "secret/test" (without "data" prefix - it's added automatically).
// data is a map of key-value pairs to store in the secret.
func CreateVaultSecret(ctx context.Context, kubeClient kubernetes.Interface, restConfig *rest.Config, secretPath string, data map[string]string) error {
return UpdateVaultSecret(ctx, kubeClient, restConfig, secretPath, data)
}
// UpdateVaultSecret updates a secret in Vault. This triggers the CSI driver to
// sync the new secret version, which creates/updates the SecretProviderClassPodStatus.
// secretPath should be like "secret/test" (without "data" prefix - it's added automatically).
// data is a map of key-value pairs to store in the secret.
func UpdateVaultSecret(ctx context.Context, kubeClient kubernetes.Interface, restConfig *rest.Config, secretPath string, data map[string]string) error {
args := []string{"kv", "put", secretPath}
for k, v := range data {
args = append(args, fmt.Sprintf("%s=%s", k, v))
}
if err := execInVaultPod(ctx, kubeClient, restConfig, args); err != nil {
return fmt.Errorf("updating Vault secret %s: %w", secretPath, err)
}
return nil
}
// DeleteVaultSecret deletes a secret from Vault.
// secretPath should be like "secret/test".
func DeleteVaultSecret(ctx context.Context, kubeClient kubernetes.Interface, restConfig *rest.Config, secretPath string) error {
args := []string{"kv", "metadata", "delete", secretPath}
if err := execInVaultPod(ctx, kubeClient, restConfig, args); err != nil {
if strings.Contains(err.Error(), "No value found") {
return nil
}
return fmt.Errorf("deleting Vault secret %s: %w", secretPath, err)
}
return nil
}
// execInVaultPod executes a vault command in the Vault pod.
func execInVaultPod(ctx context.Context, kubeClient kubernetes.Interface, restConfig *rest.Config, args []string) error {
req := kubeClient.CoreV1().RESTClient().Post().
Resource("pods").
Name(VaultPodName).
Namespace(VaultNamespace).
SubResource("exec").
VersionedParams(
&corev1.PodExecOptions{
Container: "vault",
Command: append([]string{"vault"}, args...),
Stdout: true,
Stderr: true,
}, scheme.ParameterCodec,
)
exec, err := remotecommand.NewSPDYExecutor(restConfig, "POST", req.URL())
if err != nil {
return fmt.Errorf("creating executor: %w", err)
}
var stdout, stderr bytes.Buffer
err = exec.StreamWithContext(
ctx, remotecommand.StreamOptions{
Stdout: &stdout,
Stderr: &stderr,
},
)
if err != nil {
return fmt.Errorf("executing command: %w (stderr: %s)", err, stderr.String())
}
return nil
}
// WaitForSPCPSVersionChange waits for the SecretProviderClassPodStatus version to change
// from the initial version using watches. This is used after updating a Vault secret to
// wait for CSI driver to sync the new version.
func WaitForSPCPSVersionChange(ctx context.Context, client csiclient.Interface, namespace, spcpsName, initialVersion string, timeout time.Duration) error {
watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
return client.SecretsstoreV1().SecretProviderClassPodStatuses(namespace).Watch(ctx, opts)
}
_, err := WatchUntil(ctx, watchFunc, spcpsName, SPCPSVersionChanged(initialVersion), timeout)
if errors.Is(err, ErrWatchTimeout) {
return fmt.Errorf("timeout waiting for SecretProviderClassPodStatus %s/%s version to change from %s", namespace, spcpsName, initialVersion)
}
return err
}
// FindSPCPSForDeployment finds the SecretProviderClassPodStatus created by CSI driver
// for pods of a given deployment using watches. Returns the first matching SPCPS name.
func FindSPCPSForDeployment(ctx context.Context, csiClient csiclient.Interface, kubeClient kubernetes.Interface, namespace, deploymentName string, timeout time.Duration) (
string, error,
) {
pods, err := kubeClient.CoreV1().Pods(namespace).List(
ctx, metav1.ListOptions{
LabelSelector: fmt.Sprintf("app=%s", deploymentName),
},
)
if err != nil {
return "", fmt.Errorf("listing pods for deployment %s: %w", deploymentName, err)
}
podNames := make(map[string]bool)
for _, pod := range pods.Items {
podNames[pod.Name] = true
}
watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
return csiClient.SecretsstoreV1().SecretProviderClassPodStatuses(namespace).Watch(ctx, opts)
}
spcps, err := WatchUntil(ctx, watchFunc, "", SPCPSForPods(podNames), timeout)
if errors.Is(err, ErrWatchTimeout) {
return "", fmt.Errorf("timeout finding SecretProviderClassPodStatus for deployment %s/%s", namespace, deploymentName)
}
if err != nil {
return "", err
}
return spcps.Name, nil
}
// FindSPCPSForSPC finds the SecretProviderClassPodStatus created by CSI driver
// that references a specific SecretProviderClass using watches. Returns the first matching SPCPS name.
func FindSPCPSForSPC(ctx context.Context, csiClient csiclient.Interface, namespace, spcName string, timeout time.Duration) (string, error) {
watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
return csiClient.SecretsstoreV1().SecretProviderClassPodStatuses(namespace).Watch(ctx, opts)
}
spcps, err := WatchUntil(ctx, watchFunc, "", SPCPSForSPC(spcName), timeout)
if errors.Is(err, ErrWatchTimeout) {
return "", fmt.Errorf("timeout finding SecretProviderClassPodStatus for SPC %s/%s", namespace, spcName)
}
if err != nil {
return "", err
}
return spcps.Name, nil
}
// GetSPCPSVersion gets the current version string from a SecretProviderClassPodStatus.
// Returns the version of the first object, or empty string if not found.
func GetSPCPSVersion(ctx context.Context, client csiclient.Interface, namespace, name string) (string, error) {
spcps, err := client.SecretsstoreV1().SecretProviderClassPodStatuses(namespace).Get(ctx, name, metav1.GetOptions{})
if err != nil {
return "", fmt.Errorf("getting SecretProviderClassPodStatus %s/%s: %w", namespace, name, err)
}
if len(spcps.Status.Objects) == 0 {
return "", nil
}
var versions []string
for _, obj := range spcps.Status.Objects {
versions = append(versions, obj.Version)
}
return strings.Join(versions, ","), nil
}
+37
View File
@@ -516,6 +516,43 @@ func WithInitContainerProjectedVolume(cmName, secretName string) DeploymentOptio
}
}
// WithCSIVolume adds a CSI volume referencing a SecretProviderClass to a Deployment.
func WithCSIVolume(spcName string) DeploymentOption {
return func(d *appsv1.Deployment) {
volumeName := csiVolumeName(spcName)
mountPath := csiMountPath(spcName)
d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, corev1.Volume{
Name: volumeName,
VolumeSource: corev1.VolumeSource{
CSI: &corev1.CSIVolumeSource{
Driver: CSIDriverName,
ReadOnly: ptr.To(true),
VolumeAttributes: map[string]string{
"secretProviderClass": spcName,
},
},
},
})
d.Spec.Template.Spec.Containers[0].VolumeMounts = append(
d.Spec.Template.Spec.Containers[0].VolumeMounts,
corev1.VolumeMount{
Name: volumeName,
MountPath: mountPath,
ReadOnly: true,
},
)
}
}
func csiVolumeName(spcName string) string {
return fmt.Sprintf("csi-%s", spcName)
}
func csiMountPath(spcName string) string {
return fmt.Sprintf("/mnt/secrets-store/%s", spcName)
}
func baseDeploymentResource(namespace, name string) *appsv1.Deployment {
labels := map[string]string{"app": name}
return &appsv1.Deployment{
+11
View File
@@ -13,6 +13,7 @@ import (
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
csiclient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned"
)
// TestEnvironment holds the common test environment state.
@@ -21,6 +22,7 @@ type TestEnvironment struct {
Cancel context.CancelFunc
KubeClient kubernetes.Interface
DiscoveryClient discovery.DiscoveryInterface
CSIClient csiclient.Interface
RolloutsClient rolloutsclient.Interface
OpenShiftClient openshiftclient.Interface
RestConfig *rest.Config
@@ -82,6 +84,9 @@ func SetupSharedTestEnvironment(ctx context.Context, namespace, releaseName stri
}
// Optional clients — failures are non-fatal.
if env.CSIClient, err = csiclient.NewForConfig(config); err != nil {
env.CSIClient = nil
}
if env.RolloutsClient, err = rolloutsclient.NewForConfig(config); err != nil {
env.RolloutsClient = nil
}
@@ -130,6 +135,12 @@ func SetupTestEnvironment(ctx context.Context, namespacePrefix string) (*TestEnv
return nil, fmt.Errorf("creating discovery client: %w", err)
}
env.CSIClient, err = csiclient.NewForConfig(config)
if err != nil {
ginkgo.GinkgoWriter.Printf("Warning: Could not create CSI client: %v (CSI tests will be skipped)\n", err)
env.CSIClient = nil
}
// Try to create Argo Rollouts client (optional - may not be installed)
env.RolloutsClient, err = rolloutsclient.NewForConfig(config)
if err != nil {