github/Reloader
1
0
Fork 0
mirror of https://github.com/stakater/Reloader.git synced 2026-08-23 22:16:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(e2e): fix TOCTOU race in CSI reload waits

Browse Source 
The CSI e2e tests wait for the SPCPS version change before calling
WaitReloaded/WaitEnvVar, but Reloader reacts to that same SPCPS update.
When Reloader won the race, WaitReloaded captured the already-reloaded
annotation as its baseline and then timed out waiting for a further
change (seen in CI: "Init container with CSI volume should reload...").

Add WaitReloadedFrom/WaitEnvVarFrom adapter variants that take a
caller-supplied baseline, and have the CSI tests capture that baseline
before updating the Vault secret. Negative tests also benefit: an
erroneous reload that lands during the CSI sync wait is now detected
instead of silently absorbed into the baseline.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Michał Marszałek
2026-07-06 14:10:38 +02:00
co-authored by Claude Fable 5
parent 53ae379242
commit 0e45c6b24d
13 changed files with 201 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/e2e/utils/workload_adapter.go
+15
View File
@@ -69,12 +69,27 @@ type WorkloadAdapter interface {
// WaitReloaded waits for the workload to have the reload annotation.
// Returns true if the annotation was found, false if timeout occurred.
// It captures the baseline annotation value at call time, which races with Reloader
// if the reload trigger happened earlier — prefer WaitReloadedFrom in that case.
WaitReloaded(ctx context.Context, namespace, name, annotationKey string, timeout time.Duration) (bool, error)
// WaitReloadedFrom waits for the reload annotation to be present with a value different
// from priorValue. Capture priorValue (via GetPodTemplateAnnotation) BEFORE performing
// the change that triggers the reload; capturing it afterwards can observe the already
// reloaded value and then wait for a further change that never comes.
WaitReloadedFrom(ctx context.Context, namespace, name, annotationKey, priorValue string, timeout time.Duration) (bool, error)
// WaitEnvVar waits for the workload to have a STAKATER_ env var (for envvars strategy).
// Returns true if the env var was found, false if timeout occurred.
// It captures the baseline env var value at call time, which races with Reloader
// if the reload trigger happened earlier — prefer WaitEnvVarFrom in that case.
WaitEnvVar(ctx context.Context, namespace, name, prefix string, timeout time.Duration) (bool, error)
// WaitEnvVarFrom waits for a STAKATER_ env var whose value differs from priorValue.
// Capture priorValue BEFORE performing the change that triggers the reload
// (an empty priorValue means the env var is expected to appear).
WaitEnvVarFrom(ctx context.Context, namespace, name, prefix, priorValue string, timeout time.Duration) (bool, error)
// SupportsEnvVarStrategy returns true if the workload supports env var reload strategy.
// CronJob does not support this as it uses job creation instead.
SupportsEnvVarStrategy() bool
test/e2e/utils/workload_argo.go
+12
View File
@@ -58,6 +58,12 @@ func (a *ArgoRolloutAdapter) WaitReady(ctx context.Context, namespace, name stri
// Captures the current annotation value first to avoid false positives from prior reloads.
func (a *ArgoRolloutAdapter) WaitReloaded(ctx context.Context, namespace, name, annotationKey string, timeout time.Duration) (bool, error) {
priorValue, _ := a.GetPodTemplateAnnotation(ctx, namespace, name, annotationKey)
return a.WaitReloadedFrom(ctx, namespace, name, annotationKey, priorValue, timeout)
}
// WaitReloadedFrom waits for the reload annotation to be present with a value different from
// priorValue, which the caller captured before triggering the reload.
func (a *ArgoRolloutAdapter) WaitReloadedFrom(ctx context.Context, namespace, name, annotationKey, priorValue string, timeout time.Duration) (bool, error) {
watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
return a.rolloutsClient.ArgoprojV1alpha1().Rollouts(namespace).Watch(ctx, opts)
}
@@ -72,6 +78,12 @@ func (a *ArgoRolloutAdapter) WaitEnvVar(ctx context.Context, namespace, name, pr
if r, err := a.rolloutsClient.ArgoprojV1alpha1().Rollouts(namespace).Get(ctx, name, metav1.GetOptions{}); err == nil {
priorValue = GetEnvVarValueByPrefix(r.Spec.Template.Spec.Containers, prefix)
}
return a.WaitEnvVarFrom(ctx, namespace, name, prefix, priorValue, timeout)
}
// WaitEnvVarFrom waits for a STAKATER_ env var whose value differs from priorValue, which the
// caller captured before triggering the reload.
func (a *ArgoRolloutAdapter) WaitEnvVarFrom(ctx context.Context, namespace, name, prefix, priorValue string, timeout time.Duration) (bool, error) {
watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
return a.rolloutsClient.ArgoprojV1alpha1().Rollouts(namespace).Watch(ctx, opts)
}
test/e2e/utils/workload_cronjob.go
+11
View File
@@ -50,6 +50,12 @@ func (a *CronJobAdapter) WaitReady(ctx context.Context, namespace, name string,
// Captures the current annotation value first to avoid false positives from prior reloads.
func (a *CronJobAdapter) WaitReloaded(ctx context.Context, namespace, name, annotationKey string, timeout time.Duration) (bool, error) {
priorValue, _ := a.GetPodTemplateAnnotation(ctx, namespace, name, annotationKey)
return a.WaitReloadedFrom(ctx, namespace, name, annotationKey, priorValue, timeout)
}
// WaitReloadedFrom waits for the reload annotation to be present with a value different from
// priorValue, which the caller captured before triggering the reload.
func (a *CronJobAdapter) WaitReloadedFrom(ctx context.Context, namespace, name, annotationKey, priorValue string, timeout time.Duration) (bool, error) {
watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
return a.client.BatchV1().CronJobs(namespace).Watch(ctx, opts)
}
@@ -62,6 +68,11 @@ func (a *CronJobAdapter) WaitEnvVar(ctx context.Context, namespace, name, prefix
return false, ErrUnsupportedOperation
}
// WaitEnvVarFrom returns an error because CronJobs don't support env var reload strategy.
func (a *CronJobAdapter) WaitEnvVarFrom(ctx context.Context, namespace, name, prefix, priorValue string, timeout time.Duration) (bool, error) {
return false, ErrUnsupportedOperation
}
// SupportsEnvVarStrategy returns false as CronJobs don't support env var reload strategy.
func (a *CronJobAdapter) SupportsEnvVarStrategy() bool {
return false
test/e2e/utils/workload_daemonset.go
+12
View File
@@ -50,6 +50,12 @@ func (a *DaemonSetAdapter) WaitReady(ctx context.Context, namespace, name string
// Captures the current annotation value first to avoid false positives from prior reloads.
func (a *DaemonSetAdapter) WaitReloaded(ctx context.Context, namespace, name, annotationKey string, timeout time.Duration) (bool, error) {
priorValue, _ := a.GetPodTemplateAnnotation(ctx, namespace, name, annotationKey)
return a.WaitReloadedFrom(ctx, namespace, name, annotationKey, priorValue, timeout)
}
// WaitReloadedFrom waits for the reload annotation to be present with a value different from
// priorValue, which the caller captured before triggering the reload.
func (a *DaemonSetAdapter) WaitReloadedFrom(ctx context.Context, namespace, name, annotationKey, priorValue string, timeout time.Duration) (bool, error) {
watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
return a.client.AppsV1().DaemonSets(namespace).Watch(ctx, opts)
}
@@ -64,6 +70,12 @@ func (a *DaemonSetAdapter) WaitEnvVar(ctx context.Context, namespace, name, pref
if ds, err := a.client.AppsV1().DaemonSets(namespace).Get(ctx, name, metav1.GetOptions{}); err == nil {
priorValue = GetEnvVarValueByPrefix(ds.Spec.Template.Spec.Containers, prefix)
}
return a.WaitEnvVarFrom(ctx, namespace, name, prefix, priorValue, timeout)
}
// WaitEnvVarFrom waits for a STAKATER_ env var whose value differs from priorValue, which the
// caller captured before triggering the reload.
func (a *DaemonSetAdapter) WaitEnvVarFrom(ctx context.Context, namespace, name, prefix, priorValue string, timeout time.Duration) (bool, error) {
watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
return a.client.AppsV1().DaemonSets(namespace).Watch(ctx, opts)
}
test/e2e/utils/workload_deployment.go
+12
View File
@@ -51,6 +51,12 @@ func (a *DeploymentAdapter) WaitReady(ctx context.Context, namespace, name strin
// does not cause a false positive — the condition triggers only when the value changes.
func (a *DeploymentAdapter) WaitReloaded(ctx context.Context, namespace, name, annotationKey string, timeout time.Duration) (bool, error) {
priorValue, _ := a.GetPodTemplateAnnotation(ctx, namespace, name, annotationKey)
return a.WaitReloadedFrom(ctx, namespace, name, annotationKey, priorValue, timeout)
}
// WaitReloadedFrom waits for the reload annotation to be present with a value different from
// priorValue, which the caller captured before triggering the reload.
func (a *DeploymentAdapter) WaitReloadedFrom(ctx context.Context, namespace, name, annotationKey, priorValue string, timeout time.Duration) (bool, error) {
watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
return a.client.AppsV1().Deployments(namespace).Watch(ctx, opts)
}
@@ -66,6 +72,12 @@ func (a *DeploymentAdapter) WaitEnvVar(ctx context.Context, namespace, name, pre
if d, err := a.client.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}); err == nil {
priorValue = GetEnvVarValueByPrefix(d.Spec.Template.Spec.Containers, prefix)
}
return a.WaitEnvVarFrom(ctx, namespace, name, prefix, priorValue, timeout)
}
// WaitEnvVarFrom waits for a STAKATER_ env var whose value differs from priorValue, which the
// caller captured before triggering the reload.
func (a *DeploymentAdapter) WaitEnvVarFrom(ctx context.Context, namespace, name, prefix, priorValue string, timeout time.Duration) (bool, error) {
watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
return a.client.AppsV1().Deployments(namespace).Watch(ctx, opts)
}
test/e2e/utils/workload_job.go
+11
View File
@@ -54,11 +54,22 @@ func (a *JobAdapter) WaitReloaded(ctx context.Context, namespace, name, annotati
return false, ErrUnsupportedOperation
}
// WaitReloadedFrom returns an error because Jobs are recreated, not updated.
// Use the Recreatable interface (GetOriginalUID + WaitRecreated) instead.
func (a *JobAdapter) WaitReloadedFrom(ctx context.Context, namespace, name, annotationKey, priorValue string, timeout time.Duration) (bool, error) {
return false, ErrUnsupportedOperation
}
// WaitEnvVar returns an error because Jobs don't support env var reload strategy.
func (a *JobAdapter) WaitEnvVar(ctx context.Context, namespace, name, prefix string, timeout time.Duration) (bool, error) {
return false, ErrUnsupportedOperation
}
// WaitEnvVarFrom returns an error because Jobs don't support env var reload strategy.
func (a *JobAdapter) WaitEnvVarFrom(ctx context.Context, namespace, name, prefix, priorValue string, timeout time.Duration) (bool, error) {
return false, ErrUnsupportedOperation
}
// WaitRecreated waits for the Job to be recreated with a different UID using watches.
func (a *JobAdapter) WaitRecreated(ctx context.Context, namespace, name, originalUID string, timeout time.Duration) (string, bool, error) {
watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
test/e2e/utils/workload_openshift.go
+12
View File
@@ -60,6 +60,12 @@ func (a *DeploymentConfigAdapter) WaitReady(ctx context.Context, namespace, name
// Captures the current annotation value first to avoid false positives from prior reloads.
func (a *DeploymentConfigAdapter) WaitReloaded(ctx context.Context, namespace, name, annotationKey string, timeout time.Duration) (bool, error) {
priorValue, _ := a.GetPodTemplateAnnotation(ctx, namespace, name, annotationKey)
return a.WaitReloadedFrom(ctx, namespace, name, annotationKey, priorValue, timeout)
}
// WaitReloadedFrom waits for the reload annotation to be present with a value different from
// priorValue, which the caller captured before triggering the reload.
func (a *DeploymentConfigAdapter) WaitReloadedFrom(ctx context.Context, namespace, name, annotationKey, priorValue string, timeout time.Duration) (bool, error) {
watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
return a.openshiftClient.AppsV1().DeploymentConfigs(namespace).Watch(ctx, opts)
}
@@ -74,6 +80,12 @@ func (a *DeploymentConfigAdapter) WaitEnvVar(ctx context.Context, namespace, nam
if dc, err := a.openshiftClient.AppsV1().DeploymentConfigs(namespace).Get(ctx, name, metav1.GetOptions{}); err == nil && dc.Spec.Template != nil {
priorValue = GetEnvVarValueByPrefix(dc.Spec.Template.Spec.Containers, prefix)
}
return a.WaitEnvVarFrom(ctx, namespace, name, prefix, priorValue, timeout)
}
// WaitEnvVarFrom waits for a STAKATER_ env var whose value differs from priorValue, which the
// caller captured before triggering the reload.
func (a *DeploymentConfigAdapter) WaitEnvVarFrom(ctx context.Context, namespace, name, prefix, priorValue string, timeout time.Duration) (bool, error) {
watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
return a.openshiftClient.AppsV1().DeploymentConfigs(namespace).Watch(ctx, opts)
}
test/e2e/utils/workload_statefulset.go
+12
View File
@@ -50,6 +50,12 @@ func (a *StatefulSetAdapter) WaitReady(ctx context.Context, namespace, name stri
// Captures the current annotation value first to avoid false positives from prior reloads.
func (a *StatefulSetAdapter) WaitReloaded(ctx context.Context, namespace, name, annotationKey string, timeout time.Duration) (bool, error) {
priorValue, _ := a.GetPodTemplateAnnotation(ctx, namespace, name, annotationKey)
return a.WaitReloadedFrom(ctx, namespace, name, annotationKey, priorValue, timeout)
}
// WaitReloadedFrom waits for the reload annotation to be present with a value different from
// priorValue, which the caller captured before triggering the reload.
func (a *StatefulSetAdapter) WaitReloadedFrom(ctx context.Context, namespace, name, annotationKey, priorValue string, timeout time.Duration) (bool, error) {
watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
return a.client.AppsV1().StatefulSets(namespace).Watch(ctx, opts)
}
@@ -64,6 +70,12 @@ func (a *StatefulSetAdapter) WaitEnvVar(ctx context.Context, namespace, name, pr
if sts, err := a.client.AppsV1().StatefulSets(namespace).Get(ctx, name, metav1.GetOptions{}); err == nil {
priorValue = GetEnvVarValueByPrefix(sts.Spec.Template.Spec.Containers, prefix)
}
return a.WaitEnvVarFrom(ctx, namespace, name, prefix, priorValue, timeout)
}
// WaitEnvVarFrom waits for a STAKATER_ env var whose value differs from priorValue, which the
// caller captured before triggering the reload.
func (a *StatefulSetAdapter) WaitEnvVarFrom(ctx context.Context, namespace, name, prefix, priorValue string, timeout time.Duration) (bool, error) {
watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
return a.client.AppsV1().StatefulSets(namespace).Watch(ctx, opts)
}