diff --git a/test/e2e/advanced/multi_container_test.go b/test/e2e/advanced/multi_container_test.go index cef05184..776d5773 100644 --- a/test/e2e/advanced/multi_container_test.go +++ b/test/e2e/advanced/multi_container_test.go @@ -47,13 +47,18 @@ var _ = Describe("Multi-Container Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"shared-key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment with multiple containers should be reloaded") }) @@ -81,13 +86,18 @@ var _ = Describe("Multi-Container Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the first ConfigMap") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key1": "updated1"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should be reloaded when first container's ConfigMap changes") }) diff --git a/test/e2e/advanced/regex_test.go b/test/e2e/advanced/regex_test.go index 989bf0ab..9c1e996d 100644 --- a/test/e2e/advanced/regex_test.go +++ b/test/e2e/advanced/regex_test.go @@ -53,13 +53,18 @@ var _ = Describe("Regex Pattern Tests", func() { err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the matching ConfigMap") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, matchingCM, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should be reloaded when matching ConfigMap changes") }) @@ -87,14 +92,19 @@ var _ = Describe("Regex Pattern Tests", func() { err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the non-matching ConfigMap") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, nonMatchingCM, map[string]string{"other": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (pattern mismatch)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment should NOT reload when non-matching ConfigMap changes") }) @@ -120,13 +130,18 @@ var _ = Describe("Regex Pattern Tests", func() { err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the matching Secret") err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, matchingSecret, map[string]string{"password": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should be reloaded when matching Secret changes") }) diff --git a/test/e2e/annotations/auto_reload_test.go b/test/e2e/annotations/auto_reload_test.go index af4b7ac3..5961feb4 100644 --- a/test/e2e/annotations/auto_reload_test.go +++ b/test/e2e/annotations/auto_reload_test.go @@ -47,12 +47,16 @@ var _ = Describe("Auto Reload Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap data") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment with auto=true should have been reloaded") }) @@ -75,12 +79,16 @@ var _ = Describe("Auto Reload Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the Secret data") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName, map[string]string{"password": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment with auto=true should have been reloaded for Secret change") }) @@ -108,12 +116,16 @@ var _ = Describe("Auto Reload Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"config": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment with auto=true should have been reloaded for ConfigMap change") }) @@ -145,12 +157,16 @@ var _ = Describe("Auto Reload Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"config": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should have been reloaded for ConfigMap change") }) @@ -180,12 +196,16 @@ var _ = Describe("Auto Reload Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the Secret") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName, map[string]string{"secret": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should have been reloaded for Secret change") }) @@ -221,12 +241,16 @@ var _ = Describe("Auto Reload Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the second ConfigMap (auto-detected)") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName2, map[string]string{"key2": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should have been reloaded for auto-detected ConfigMap change") }) diff --git a/test/e2e/annotations/combination_test.go b/test/e2e/annotations/combination_test.go index e7f02efa..dab5a8fd 100644 --- a/test/e2e/annotations/combination_test.go +++ b/test/e2e/annotations/combination_test.go @@ -61,12 +61,16 @@ var _ = Describe("Combination Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the auto-detected ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should reload when auto-detected ConfigMap changes") }) @@ -95,12 +99,16 @@ var _ = Describe("Combination Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the explicitly listed ConfigMap (not mounted)") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName2, map[string]string{"extra": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should reload when explicitly listed ConfigMap changes") }) @@ -129,12 +137,16 @@ var _ = Describe("Combination Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the explicitly listed Secret") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName2, map[string]string{"api-key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should reload when explicitly listed Secret changes") }) @@ -166,13 +178,17 @@ var _ = Describe("Combination Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the excluded ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName2, map[string]string{"excluded": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (negative test)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment should NOT reload when excluded ConfigMap changes") }) @@ -202,12 +218,16 @@ var _ = Describe("Combination Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the non-excluded ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should reload when non-excluded ConfigMap changes") }) @@ -237,13 +257,17 @@ var _ = Describe("Combination Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the excluded Secret") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName2, map[string]string{"excluded": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (negative test)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment should NOT reload when excluded Secret changes") }) @@ -270,12 +294,16 @@ var _ = Describe("Combination Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the second ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName2, map[string]string{"key2": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should reload when any of the listed ConfigMaps changes") }) @@ -300,12 +328,16 @@ var _ = Describe("Combination Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the first Secret") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName, map[string]string{"key1": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should reload when any of the listed Secrets changes") }) @@ -333,12 +365,16 @@ var _ = Describe("Combination Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the Secret") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName, map[string]string{"password": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should reload when Secret changes with both annotations present") }) diff --git a/test/e2e/annotations/exclude_test.go b/test/e2e/annotations/exclude_test.go index 5a336514..bce14ee6 100644 --- a/test/e2e/annotations/exclude_test.go +++ b/test/e2e/annotations/exclude_test.go @@ -66,13 +66,17 @@ var _ = Describe("Exclude Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the excluded ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (excluded ConfigMap)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment should NOT reload when excluded ConfigMap changes") }) @@ -103,12 +107,16 @@ var _ = Describe("Exclude Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the non-excluded ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName2, map[string]string{"key2": "updated2"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should reload when non-excluded ConfigMap changes") }) @@ -141,13 +149,17 @@ var _ = Describe("Exclude Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the excluded Secret") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName, map[string]string{"password": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (excluded Secret)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment should NOT reload when excluded Secret changes") }) @@ -178,12 +190,16 @@ var _ = Describe("Exclude Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the non-excluded Secret") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName2, map[string]string{"password2": "updated2"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should reload when non-excluded Secret changes") }) @@ -225,13 +241,17 @@ var _ = Describe("Exclude Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the excluded ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying workload was NOT reloaded (excluded ConfigMap)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "%s should NOT reload with exclude on pod template", workloadType) }, diff --git a/test/e2e/annotations/pause_period_test.go b/test/e2e/annotations/pause_period_test.go index 869aed1d..6e80b55f 100644 --- a/test/e2e/annotations/pause_period_test.go +++ b/test/e2e/annotations/pause_period_test.go @@ -49,12 +49,16 @@ var _ = Describe("Pause Period Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap data") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should have been reloaded") @@ -83,12 +87,16 @@ var _ = Describe("Pause Period Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap data") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should have been reloaded") @@ -124,12 +132,16 @@ var _ = Describe("Pause Period Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap data") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should have been reloaded") diff --git a/test/e2e/annotations/resource_ignore_test.go b/test/e2e/annotations/resource_ignore_test.go index 132c91a6..0d97724d 100644 --- a/test/e2e/annotations/resource_ignore_test.go +++ b/test/e2e/annotations/resource_ignore_test.go @@ -50,13 +50,17 @@ var _ = Describe("Resource Ignore Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap data") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (negative test)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment should NOT reload when ConfigMap has ignore=true") }) @@ -80,13 +84,17 @@ var _ = Describe("Resource Ignore Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the Secret data") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName, map[string]string{"password": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (negative test)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment should NOT reload when Secret has ignore=true") }) diff --git a/test/e2e/annotations/search_match_test.go b/test/e2e/annotations/search_match_test.go index 02a1153c..dac03577 100644 --- a/test/e2e/annotations/search_match_test.go +++ b/test/e2e/annotations/search_match_test.go @@ -50,12 +50,16 @@ var _ = Describe("Search and Match Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap data") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment with search annotation should reload when ConfigMap has match annotation") }) @@ -78,13 +82,17 @@ var _ = Describe("Search and Match Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap data") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (negative test)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment should NOT reload when ConfigMap lacks match annotation") }) @@ -106,13 +114,17 @@ var _ = Describe("Search and Match Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap data") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (negative test)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment without search annotation should NOT reload even when ConfigMap has match") }) @@ -149,18 +161,24 @@ var _ = Describe("Search and Match Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap data") + // Capture the reload-annotation baselines before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload1, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + priorReload2, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName2, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for first Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload1, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment with search annotation should reload") By("Verifying second Deployment was NOT reloaded") - reloaded2, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName2, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded2, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName2, + utils.AnnotationLastReloadedFrom, priorReload2, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded2).To(BeFalse(), "Deployment without search annotation should NOT reload") }) @@ -196,12 +214,16 @@ var _ = Describe("Search and Match Annotation Tests", func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s should reload with search annotation on pod template", workloadType) }, diff --git a/test/e2e/argo/rollout_test.go b/test/e2e/argo/rollout_test.go index 019df62b..e6edf153 100644 --- a/test/e2e/argo/rollout_test.go +++ b/test/e2e/argo/rollout_test.go @@ -47,13 +47,18 @@ var _ = Describe("Argo Rollout Strategy Tests", func() { err = adapter.WaitReady(ctx, testNamespace, rolloutName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, rolloutName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Rollout to be reloaded with annotation") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, rolloutName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, rolloutName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Argo Rollout should be reloaded with default rollout strategy") }) diff --git a/test/e2e/core/reference_methods_test.go b/test/e2e/core/reference_methods_test.go index f3c0b8fd..60bf129e 100644 --- a/test/e2e/core/reference_methods_test.go +++ b/test/e2e/core/reference_methods_test.go @@ -58,13 +58,17 @@ var _ = Describe("Reference Method Tests", func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"config_key": "updated_value"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s with valueFrom.configMapKeyRef should reload", workloadType) }, @@ -107,13 +111,17 @@ var _ = Describe("Reference Method Tests", func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the Secret") err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName, map[string]string{"secret_key": "updated_secret"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s with valueFrom.secretKeyRef should reload", workloadType) }, @@ -154,13 +162,17 @@ var _ = Describe("Reference Method Tests", func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"config.yaml": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s with projected ConfigMap volume should reload", workloadType) }, @@ -196,13 +208,17 @@ var _ = Describe("Reference Method Tests", func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the Secret") err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName, map[string]string{"credentials": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s with projected Secret volume should reload", workloadType) }, @@ -246,13 +262,17 @@ var _ = Describe("Reference Method Tests", func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"config.yaml": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s should reload when ConfigMap in mixed projected volume changes", workloadType) }, @@ -296,13 +316,17 @@ var _ = Describe("Reference Method Tests", func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the Secret") err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName, map[string]string{"credentials": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s should reload when Secret in mixed projected volume changes", workloadType) }, @@ -343,13 +367,17 @@ var _ = Describe("Reference Method Tests", func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"INIT_VAR": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s with init container ConfigMap should reload", workloadType) }, @@ -385,13 +413,17 @@ var _ = Describe("Reference Method Tests", func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the Secret") err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName, map[string]string{"INIT_SECRET": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s with init container Secret should reload", workloadType) }, @@ -429,13 +461,17 @@ var _ = Describe("Reference Method Tests", func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"config.yaml": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s with init container ConfigMap volume should reload", workloadType) }, @@ -471,13 +507,17 @@ var _ = Describe("Reference Method Tests", func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the Secret") err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName, map[string]string{"credentials": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s with init container Secret volume should reload", workloadType) }, @@ -520,13 +560,17 @@ var _ = Describe("Reference Method Tests", func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"auto_config_key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s with auto=true and valueFrom should reload", workloadType) }, diff --git a/test/e2e/core/workloads_test.go b/test/e2e/core/workloads_test.go index fdbe98ec..b6a1a195 100644 --- a/test/e2e/core/workloads_test.go +++ b/test/e2e/core/workloads_test.go @@ -64,13 +64,17 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap data") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, - utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s should have been reloaded", workloadType) }, @@ -106,13 +110,17 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the Secret data") err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName, map[string]string{"password": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, - utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s should have been reloaded", workloadType) }, @@ -149,13 +157,17 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap data") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s with auto=true should have been reloaded", workloadType) }, @@ -192,14 +204,18 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating only the ConfigMap labels (no data change)") err = utils.UpdateConfigMapLabels(ctx, kubeClient, testNamespace, configMapName, map[string]string{"new-label": "new-value"}) Expect(err).NotTo(HaveOccurred()) By("Verifying workload was NOT reloaded (negative test)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "%s should NOT reload when only ConfigMap labels change", workloadType) }, @@ -235,14 +251,18 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating only the Secret labels (no data change)") err = utils.UpdateSecretLabels(ctx, kubeClient, testNamespace, secretName, map[string]string{"new-label": "new-value"}) Expect(err).NotTo(HaveOccurred()) By("Verifying workload was NOT reloaded (negative test)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "%s should NOT reload when only Secret labels change", workloadType) }, @@ -369,13 +389,17 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap data") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"config.yaml": "setting: updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, - utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s with volume-mounted ConfigMap should have been reloaded", workloadType) }, @@ -410,13 +434,17 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the Secret data") err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName, map[string]string{"credentials.yaml": "secret: updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, - utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s with volume-mounted Secret should have been reloaded", workloadType) }, @@ -451,14 +479,18 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap data") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying workload is NOT reloaded (negative test)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, - utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "%s without Reloader annotation should NOT be reloaded", workloadType) }, @@ -507,13 +539,19 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger: Reloader can react + // to the ConfigMap update before WaitReloaded captures its baseline, which would + // otherwise record the already-reloaded value and time out (TOCTOU race). + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the second ConfigMap") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName2, map[string]string{"key2": "updated-value2"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s should reload when second ConfigMap changes", workloadType) }, @@ -556,13 +594,18 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the TOCTOU + // race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the second Secret") err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName2, map[string]string{"key2": "updated-value2"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s should reload when second Secret changes", workloadType) }, @@ -598,13 +641,17 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("First update to ConfigMap") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "v2"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for first reload") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue()) @@ -666,13 +713,17 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the Secret") err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName, map[string]string{"secret": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s should reload when Secret changes", workloadType) }, @@ -708,14 +759,18 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap data") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying workload is NOT reloaded (auto=false)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "%s with auto=false should NOT be reloaded", workloadType) }, @@ -759,13 +814,17 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s should reload with pod template annotation", workloadType) }, @@ -801,13 +860,17 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the Secret") err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName, map[string]string{"password": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s should reload with pod template annotation", workloadType) }, @@ -843,13 +906,17 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s with auto=true on pod template should reload", workloadType) }, @@ -886,13 +953,17 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap") err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "%s should reload with annotations on both locations", workloadType) }, @@ -932,14 +1003,18 @@ var _ = Describe("Workload Reload Tests", Serial, func() { err = adapter.WaitReady(ctx, testNamespace, workloadName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the Secret (not the ConfigMap)") err = utils.UpdateSecretFromStrings(ctx, kubeClient, testNamespace, secretName, map[string]string{"password": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying workload was NOT reloaded (negative test)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, workloadName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, workloadName, utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "%s should NOT reload when updating different resource than annotated", workloadType) }, @@ -1016,8 +1091,7 @@ var _ = Describe("Workload Reload Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to have STAKATER_ env var") - found, err := adapter.WaitEnvVar(ctx, testNamespace, workloadName, utils.StakaterEnvVarPrefix, - utils.ReloadTimeout) + found, err := adapter.WaitEnvVarFrom(ctx, testNamespace, workloadName, utils.StakaterEnvVarPrefix, "", utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(found).To(BeTrue(), "%s should have STAKATER_ env var after ConfigMap change", workloadType) }, @@ -1061,8 +1135,7 @@ var _ = Describe("Workload Reload Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Waiting for workload to have STAKATER_ env var") - found, err := adapter.WaitEnvVar(ctx, testNamespace, workloadName, utils.StakaterEnvVarPrefix, - utils.ReloadTimeout) + found, err := adapter.WaitEnvVarFrom(ctx, testNamespace, workloadName, utils.StakaterEnvVarPrefix, "", utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(found).To(BeTrue(), "%s should have STAKATER_ env var after Secret change", workloadType) }, @@ -1109,8 +1182,7 @@ var _ = Describe("Workload Reload Tests", Serial, func() { By("Verifying workload does NOT have STAKATER_ env var") time.Sleep(utils.NegativeTestWait) - found, err := adapter.WaitEnvVar(ctx, testNamespace, workloadName, utils.StakaterEnvVarPrefix, - utils.ShortTimeout) + found, err := adapter.WaitEnvVarFrom(ctx, testNamespace, workloadName, utils.StakaterEnvVarPrefix, "", utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(found).To(BeFalse(), "%s should NOT have STAKATER_ env var for label-only change", workloadType) }, @@ -1154,8 +1226,7 @@ var _ = Describe("Workload Reload Tests", Serial, func() { By("Verifying workload does NOT have STAKATER_ env var") time.Sleep(utils.NegativeTestWait) - found, err := adapter.WaitEnvVar(ctx, testNamespace, workloadName, utils.StakaterEnvVarPrefix, - utils.ShortTimeout) + found, err := adapter.WaitEnvVarFrom(ctx, testNamespace, workloadName, utils.StakaterEnvVarPrefix, "", utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(found).To(BeFalse(), "%s should NOT have STAKATER_ env var for label-only change", workloadType) }, diff --git a/test/e2e/csi/csi_test.go b/test/e2e/csi/csi_test.go index ef55f2bd..08604d61 100644 --- a/test/e2e/csi/csi_test.go +++ b/test/e2e/csi/csi_test.go @@ -72,6 +72,11 @@ var _ = Describe("CSI SecretProviderClass Tests", Label("csi"), Serial, func() { Expect(err).NotTo(HaveOccurred()) GinkgoWriter.Printf("Initial SPCPS version: %s\n", initialVersion) + // Capture the reload-annotation baseline before the trigger: Reloader reacts to the + // same SPCPS update the test waits on below, so it may reload before WaitReloaded runs. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the Vault secret") err = utils.UpdateVaultSecret( ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"api_key": "updated-value-v2"}) @@ -83,9 +88,9 @@ var _ = Describe("CSI SecretProviderClass Tests", Label("csi"), Serial, func() { GinkgoWriter.Println("CSI driver synced new secret version") By("Waiting for Deployment to be reloaded by Reloader") - reloaded, err := adapter.WaitReloaded( + reloaded, err := adapter.WaitReloadedFrom( ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout, ) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should have been reloaded after Vault secret change") @@ -124,6 +129,10 @@ var _ = Describe("CSI SecretProviderClass Tests", Label("csi"), Serial, func() { By("First update to Vault secret") initialVersion, _ := utils.GetSPCPSVersion(ctx, csiClient, testNamespace, spcpsName) + // Capture the baseline before the trigger to avoid racing Reloader's own reaction + // to the SPCPS update below. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateVaultSecret( ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"password": "pass-v2"}) Expect(err).NotTo(HaveOccurred()) @@ -133,9 +142,9 @@ var _ = Describe("CSI SecretProviderClass Tests", Label("csi"), Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Waiting for first reload") - reloaded, err := adapter.WaitReloaded( + reloaded, err := adapter.WaitReloadedFrom( ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout, ) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue()) @@ -207,6 +216,11 @@ var _ = Describe("CSI SecretProviderClass Tests", Label("csi"), Serial, func() { err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap (should NOT trigger reload)") err = utils.UpdateConfigMap( ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) @@ -214,9 +228,9 @@ var _ = Describe("CSI SecretProviderClass Tests", Label("csi"), Serial, func() { By("Verifying Deployment was NOT reloaded for ConfigMap change") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded( + reloaded, err := adapter.WaitReloadedFrom( ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout, ) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "SPC auto annotation should not trigger reload for ConfigMap changes") @@ -230,6 +244,11 @@ var _ = Describe("CSI SecretProviderClass Tests", Label("csi"), Serial, func() { By("Getting SPCPS version before Vault update") initialVersion, _ := utils.GetSPCPSVersion(ctx, csiClient, testNamespace, spcpsName) + // Capture the baseline before the trigger to avoid racing Reloader's own reaction + // to the SPCPS update below. + priorReload, err = adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the Vault secret (should trigger reload)") err = utils.UpdateVaultSecret( ctx, kubeClient, restConfig, vaultSecretPath, map[string]string{"token": "token-v2"}) @@ -240,9 +259,9 @@ var _ = Describe("CSI SecretProviderClass Tests", Label("csi"), Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment WAS reloaded for Vault secret change") - reloaded, err = adapter.WaitReloaded( + reloaded, err = adapter.WaitReloadedFrom( ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout, ) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "SPC auto annotation should trigger reload for Vault secret changes") @@ -281,15 +300,20 @@ var _ = Describe("CSI SecretProviderClass Tests", Label("csi"), Serial, func() { err = adapter.WaitReady(ctx, testNamespace, deploymentName, utils.WorkloadReadyTimeout) Expect(err).NotTo(HaveOccurred()) + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) + By("Updating the ConfigMap (should trigger reload with auto=true)") err = utils.UpdateConfigMap( ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment WAS reloaded for ConfigMap change") - reloaded, err := adapter.WaitReloaded( + reloaded, err := adapter.WaitReloadedFrom( ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout, ) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Combined auto=true should trigger reload for ConfigMap changes") diff --git a/test/e2e/flags/auto_reload_all_test.go b/test/e2e/flags/auto_reload_all_test.go index f4cda1cd..839e5e0b 100644 --- a/test/e2e/flags/auto_reload_all_test.go +++ b/test/e2e/flags/auto_reload_all_test.go @@ -65,12 +65,16 @@ var _ = Describe("Auto Reload All Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, autoNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, autoNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded (autoReloadAll=true)") - reloaded, err := adapter.WaitReloaded(ctx, autoNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, autoNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment without annotations should reload when autoReloadAll=true") }) @@ -93,13 +97,17 @@ var _ = Describe("Auto Reload All Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, autoNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, autoNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (auto=false overrides autoReloadAll)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, autoNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, autoNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment with auto=false should NOT reload even with autoReloadAll=true") }) diff --git a/test/e2e/flags/ignore_resources_test.go b/test/e2e/flags/ignore_resources_test.go index 44eb539d..e28f8885 100644 --- a/test/e2e/flags/ignore_resources_test.go +++ b/test/e2e/flags/ignore_resources_test.go @@ -69,13 +69,17 @@ var _ = Describe("Ignore Resources Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the Secret") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, ignoreNS, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateSecretFromStrings(ctx, kubeClient, ignoreNS, secretName, map[string]string{"password": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (ignoreSecrets=true)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, ignoreNS, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, ignoreNS, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment should NOT reload when ignoreSecrets=true") }) @@ -98,12 +102,16 @@ var _ = Describe("Ignore Resources Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, ignoreNS, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, ignoreNS, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded (ConfigMap should still work)") - reloaded, err := adapter.WaitReloaded(ctx, ignoreNS, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, ignoreNS, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "ConfigMap changes should still trigger reload with ignoreSecrets=true") }) @@ -146,13 +154,17 @@ var _ = Describe("Ignore Resources Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, ignoreNS, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, ignoreNS, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (ignoreConfigMaps=true)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, ignoreNS, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, ignoreNS, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment should NOT reload when ignoreConfigMaps=true") }) @@ -175,12 +187,16 @@ var _ = Describe("Ignore Resources Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the Secret") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, ignoreNS, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateSecretFromStrings(ctx, kubeClient, ignoreNS, secretName, map[string]string{"password": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded (Secret should still work)") - reloaded, err := adapter.WaitReloaded(ctx, ignoreNS, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, ignoreNS, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Secret changes should still trigger reload with ignoreConfigMaps=true") }) diff --git a/test/e2e/flags/ignored_workloads_test.go b/test/e2e/flags/ignored_workloads_test.go index 7a5185ce..0bd4449c 100644 --- a/test/e2e/flags/ignored_workloads_test.go +++ b/test/e2e/flags/ignored_workloads_test.go @@ -64,13 +64,17 @@ var _ = Describe("Ignored Workloads Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := cronJobAdapter.GetPodTemplateAnnotation(ctx, ignoreNS, cronJobName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, ignoreNS, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying CronJob was NOT reloaded (ignoreCronJobs=true)") time.Sleep(utils.NegativeTestWait) - reloaded, err := cronJobAdapter.WaitReloaded(ctx, ignoreNS, cronJobName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := cronJobAdapter.WaitReloadedFrom(ctx, ignoreNS, cronJobName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "CronJob should NOT reload when ignoreCronJobs=true") }) @@ -98,12 +102,16 @@ var _ = Describe("Ignored Workloads Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := deploymentAdapter.GetPodTemplateAnnotation(ctx, ignoreNS, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, ignoreNS, configMapName, map[string]string{"key": "updated-deploy"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded (Deployment should still work)") - reloaded, err := deploymentAdapter.WaitReloaded(ctx, ignoreNS, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := deploymentAdapter.WaitReloadedFrom(ctx, ignoreNS, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should still reload with ignoreCronJobs=true") }) @@ -143,13 +151,17 @@ var _ = Describe("Ignored Workloads Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := cronJobAdapter.GetPodTemplateAnnotation(ctx, ignoreNS, cronJobName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, ignoreNS, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying CronJob was NOT reloaded") time.Sleep(utils.NegativeTestWait) - reloaded, err := cronJobAdapter.WaitReloaded(ctx, ignoreNS, cronJobName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := cronJobAdapter.WaitReloadedFrom(ctx, ignoreNS, cronJobName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "CronJob should NOT reload when ignoreCronJobs=true and ignoreJobs=true") }) diff --git a/test/e2e/flags/namespace_ignore_test.go b/test/e2e/flags/namespace_ignore_test.go index 39467909..d3599540 100644 --- a/test/e2e/flags/namespace_ignore_test.go +++ b/test/e2e/flags/namespace_ignore_test.go @@ -73,13 +73,17 @@ var _ = Describe("Namespace Ignore Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, ignoredNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, ignoredNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (ignored namespace)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, ignoredNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, ignoredNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment in ignored namespace should NOT be reloaded") }) @@ -102,12 +106,16 @@ var _ = Describe("Namespace Ignore Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, watchedNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, watchedNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, watchedNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, watchedNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment in non-ignored namespace should be reloaded") }) diff --git a/test/e2e/flags/namespace_selector_test.go b/test/e2e/flags/namespace_selector_test.go index 4ac49cbe..a58da579 100644 --- a/test/e2e/flags/namespace_selector_test.go +++ b/test/e2e/flags/namespace_selector_test.go @@ -74,12 +74,16 @@ var _ = Describe("Namespace Selector Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, matchingNS, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, matchingNS, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, matchingNS, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, matchingNS, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment in matching namespace should be reloaded") }) @@ -102,13 +106,17 @@ var _ = Describe("Namespace Selector Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, nonMatchingNS, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, nonMatchingNS, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (non-matching namespace)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, nonMatchingNS, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, nonMatchingNS, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment in non-matching namespace should NOT be reloaded") }) diff --git a/test/e2e/flags/reload_on_create_test.go b/test/e2e/flags/reload_on_create_test.go index 63fec0bb..b6946ef3 100644 --- a/test/e2e/flags/reload_on_create_test.go +++ b/test/e2e/flags/reload_on_create_test.go @@ -60,13 +60,17 @@ var _ = Describe("Reload On Create Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Creating the ConfigMap that the Deployment references") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, createNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) _, err = utils.CreateConfigMap(ctx, kubeClient, createNamespace, configMapName, map[string]string{"key": "value"}, nil) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded (reloadOnCreate=true)") - reloaded, err := adapter.WaitReloaded(ctx, createNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, createNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should reload when referenced ConfigMap is created") }) @@ -86,13 +90,17 @@ var _ = Describe("Reload On Create Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Creating the Secret that the Deployment references") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, createNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) _, err = utils.CreateSecretFromStrings(ctx, kubeClient, createNamespace, secretName, map[string]string{"password": "secret"}, nil) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded (reloadOnCreate=true)") - reloaded, err := adapter.WaitReloaded(ctx, createNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, createNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should reload when referenced Secret is created") }) @@ -127,14 +135,18 @@ var _ = Describe("Reload On Create Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Creating the ConfigMap that the Deployment references") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, createNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) _, err = utils.CreateConfigMap(ctx, kubeClient, createNamespace, configMapName, map[string]string{"key": "value"}, nil) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (reloadOnCreate=false)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, createNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, createNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment should NOT reload on create when reloadOnCreate=false") }) diff --git a/test/e2e/flags/reload_on_delete_test.go b/test/e2e/flags/reload_on_delete_test.go index ed400e35..88fda766 100644 --- a/test/e2e/flags/reload_on_delete_test.go +++ b/test/e2e/flags/reload_on_delete_test.go @@ -65,12 +65,16 @@ var _ = Describe("Reload On Delete Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Deleting the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, deleteNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.DeleteConfigMap(ctx, kubeClient, deleteNamespace, configMapName) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded (reloadOnDelete=true)") - reloaded, err := adapter.WaitReloaded(ctx, deleteNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, deleteNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should reload when referenced ConfigMap is deleted") }) @@ -94,12 +98,16 @@ var _ = Describe("Reload On Delete Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Deleting the Secret") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, deleteNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.DeleteSecret(ctx, kubeClient, deleteNamespace, secretName) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded (reloadOnDelete=true)") - reloaded, err := adapter.WaitReloaded(ctx, deleteNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, deleteNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should reload when referenced Secret is deleted") }) @@ -139,13 +147,17 @@ var _ = Describe("Reload On Delete Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Deleting the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, deleteNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.DeleteConfigMap(ctx, kubeClient, deleteNamespace, configMapName) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (reloadOnDelete=false)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, deleteNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, deleteNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment should NOT reload on delete when reloadOnDelete=false") }) diff --git a/test/e2e/flags/resource_selector_test.go b/test/e2e/flags/resource_selector_test.go index cc94612a..af31a958 100644 --- a/test/e2e/flags/resource_selector_test.go +++ b/test/e2e/flags/resource_selector_test.go @@ -70,12 +70,16 @@ var _ = Describe("Resource Label Selector Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the labeled ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, resourceNS, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, resourceNS, matchingCM, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded") - reloaded, err := adapter.WaitReloaded(ctx, resourceNS, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, resourceNS, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment should be reloaded when labeled ConfigMap changes") }) @@ -98,13 +102,17 @@ var _ = Describe("Resource Label Selector Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the unlabeled ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, resourceNS, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, resourceNS, nonMatchingCM, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (unlabeled ConfigMap)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, resourceNS, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, resourceNS, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment should NOT reload when unlabeled ConfigMap changes") }) diff --git a/test/e2e/flags/watch_globally_test.go b/test/e2e/flags/watch_globally_test.go index 96e3fb2d..204a3d65 100644 --- a/test/e2e/flags/watch_globally_test.go +++ b/test/e2e/flags/watch_globally_test.go @@ -68,12 +68,16 @@ var _ = Describe("Watch Globally Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, testNamespace, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, testNamespace, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded (same namespace should work)") - reloaded, err := adapter.WaitReloaded(ctx, testNamespace, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, testNamespace, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment in Reloader's namespace should reload with watchGlobally=false") }) @@ -96,13 +100,17 @@ var _ = Describe("Watch Globally Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap in the other namespace") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, otherNS, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, otherNS, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Verifying Deployment was NOT reloaded (different namespace with watchGlobally=false)") time.Sleep(utils.NegativeTestWait) - reloaded, err := adapter.WaitReloaded(ctx, otherNS, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ShortTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, otherNS, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ShortTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeFalse(), "Deployment in other namespace should NOT reload with watchGlobally=false") }) @@ -151,12 +159,16 @@ var _ = Describe("Watch Globally Flag Tests", Serial, func() { Expect(err).NotTo(HaveOccurred()) By("Updating the ConfigMap") + // Capture the reload-annotation baseline before the trigger to avoid the + // TOCTOU race where Reloader reloads before WaitReloaded records its baseline. + priorReload, err := adapter.GetPodTemplateAnnotation(ctx, globalNS, deploymentName, utils.AnnotationLastReloadedFrom) + Expect(err).NotTo(HaveOccurred()) err = utils.UpdateConfigMap(ctx, kubeClient, globalNS, configMapName, map[string]string{"key": "updated"}) Expect(err).NotTo(HaveOccurred()) By("Waiting for Deployment to be reloaded (watchGlobally=true)") - reloaded, err := adapter.WaitReloaded(ctx, globalNS, deploymentName, - utils.AnnotationLastReloadedFrom, utils.ReloadTimeout) + reloaded, err := adapter.WaitReloadedFrom(ctx, globalNS, deploymentName, + utils.AnnotationLastReloadedFrom, priorReload, utils.ReloadTimeout) Expect(err).NotTo(HaveOccurred()) Expect(reloaded).To(BeTrue(), "Deployment in any namespace should reload with watchGlobally=true") }) diff --git a/test/e2e/utils/workload_adapter.go b/test/e2e/utils/workload_adapter.go index 2610337f..70a5e7e1 100644 --- a/test/e2e/utils/workload_adapter.go +++ b/test/e2e/utils/workload_adapter.go @@ -66,12 +66,27 @@ type WorkloadAdapter interface { // WaitReloaded waits for the workload to have the reload annotation. // Returns true if the annotation was found, false if timeout occurred. + // It captures the baseline annotation value at call time, which races with Reloader + // if the reload trigger happened earlier — prefer WaitReloadedFrom in that case. WaitReloaded(ctx context.Context, namespace, name, annotationKey string, timeout time.Duration) (bool, error) + // WaitReloadedFrom waits for the reload annotation to be present with a value different + // from priorValue. Capture priorValue (via GetPodTemplateAnnotation) BEFORE performing + // the change that triggers the reload; capturing it afterwards can observe the already + // reloaded value and then wait for a further change that never comes. + WaitReloadedFrom(ctx context.Context, namespace, name, annotationKey, priorValue string, timeout time.Duration) (bool, error) + // WaitEnvVar waits for the workload to have a STAKATER_ env var (for envvars strategy). // Returns true if the env var was found, false if timeout occurred. + // It captures the baseline env var value at call time, which races with Reloader + // if the reload trigger happened earlier — prefer WaitEnvVarFrom in that case. WaitEnvVar(ctx context.Context, namespace, name, prefix string, timeout time.Duration) (bool, error) + // WaitEnvVarFrom waits for a STAKATER_ env var whose value differs from priorValue. + // Capture priorValue BEFORE performing the change that triggers the reload + // (an empty priorValue means the env var is expected to appear). + WaitEnvVarFrom(ctx context.Context, namespace, name, prefix, priorValue string, timeout time.Duration) (bool, error) + // SupportsEnvVarStrategy returns true if the workload supports env var reload strategy. // CronJob does not support this as it uses job creation instead. SupportsEnvVarStrategy() bool diff --git a/test/e2e/utils/workload_argo.go b/test/e2e/utils/workload_argo.go index 69d5163e..c976d746 100644 --- a/test/e2e/utils/workload_argo.go +++ b/test/e2e/utils/workload_argo.go @@ -58,6 +58,12 @@ func (a *ArgoRolloutAdapter) WaitReady(ctx context.Context, namespace, name stri // Captures the current annotation value first to avoid false positives from prior reloads. func (a *ArgoRolloutAdapter) WaitReloaded(ctx context.Context, namespace, name, annotationKey string, timeout time.Duration) (bool, error) { priorValue, _ := a.GetPodTemplateAnnotation(ctx, namespace, name, annotationKey) + return a.WaitReloadedFrom(ctx, namespace, name, annotationKey, priorValue, timeout) +} + +// WaitReloadedFrom waits for the reload annotation to be present with a value different from +// priorValue, which the caller captured before triggering the reload. +func (a *ArgoRolloutAdapter) WaitReloadedFrom(ctx context.Context, namespace, name, annotationKey, priorValue string, timeout time.Duration) (bool, error) { watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { return a.rolloutsClient.ArgoprojV1alpha1().Rollouts(namespace).Watch(ctx, opts) } @@ -72,6 +78,12 @@ func (a *ArgoRolloutAdapter) WaitEnvVar(ctx context.Context, namespace, name, pr if r, err := a.rolloutsClient.ArgoprojV1alpha1().Rollouts(namespace).Get(ctx, name, metav1.GetOptions{}); err == nil { priorValue = GetEnvVarValueByPrefix(r.Spec.Template.Spec.Containers, prefix) } + return a.WaitEnvVarFrom(ctx, namespace, name, prefix, priorValue, timeout) +} + +// WaitEnvVarFrom waits for a STAKATER_ env var whose value differs from priorValue, which the +// caller captured before triggering the reload. +func (a *ArgoRolloutAdapter) WaitEnvVarFrom(ctx context.Context, namespace, name, prefix, priorValue string, timeout time.Duration) (bool, error) { watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { return a.rolloutsClient.ArgoprojV1alpha1().Rollouts(namespace).Watch(ctx, opts) } diff --git a/test/e2e/utils/workload_cronjob.go b/test/e2e/utils/workload_cronjob.go index c681cce9..151fe9ae 100644 --- a/test/e2e/utils/workload_cronjob.go +++ b/test/e2e/utils/workload_cronjob.go @@ -50,6 +50,12 @@ func (a *CronJobAdapter) WaitReady(ctx context.Context, namespace, name string, // Captures the current annotation value first to avoid false positives from prior reloads. func (a *CronJobAdapter) WaitReloaded(ctx context.Context, namespace, name, annotationKey string, timeout time.Duration) (bool, error) { priorValue, _ := a.GetPodTemplateAnnotation(ctx, namespace, name, annotationKey) + return a.WaitReloadedFrom(ctx, namespace, name, annotationKey, priorValue, timeout) +} + +// WaitReloadedFrom waits for the reload annotation to be present with a value different from +// priorValue, which the caller captured before triggering the reload. +func (a *CronJobAdapter) WaitReloadedFrom(ctx context.Context, namespace, name, annotationKey, priorValue string, timeout time.Duration) (bool, error) { watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { return a.client.BatchV1().CronJobs(namespace).Watch(ctx, opts) } @@ -62,6 +68,11 @@ func (a *CronJobAdapter) WaitEnvVar(ctx context.Context, namespace, name, prefix return false, ErrUnsupportedOperation } +// WaitEnvVarFrom returns an error because CronJobs don't support env var reload strategy. +func (a *CronJobAdapter) WaitEnvVarFrom(ctx context.Context, namespace, name, prefix, priorValue string, timeout time.Duration) (bool, error) { + return false, ErrUnsupportedOperation +} + // SupportsEnvVarStrategy returns false as CronJobs don't support env var reload strategy. func (a *CronJobAdapter) SupportsEnvVarStrategy() bool { return false diff --git a/test/e2e/utils/workload_daemonset.go b/test/e2e/utils/workload_daemonset.go index 4a7a2b14..74aada0e 100644 --- a/test/e2e/utils/workload_daemonset.go +++ b/test/e2e/utils/workload_daemonset.go @@ -50,6 +50,12 @@ func (a *DaemonSetAdapter) WaitReady(ctx context.Context, namespace, name string // Captures the current annotation value first to avoid false positives from prior reloads. func (a *DaemonSetAdapter) WaitReloaded(ctx context.Context, namespace, name, annotationKey string, timeout time.Duration) (bool, error) { priorValue, _ := a.GetPodTemplateAnnotation(ctx, namespace, name, annotationKey) + return a.WaitReloadedFrom(ctx, namespace, name, annotationKey, priorValue, timeout) +} + +// WaitReloadedFrom waits for the reload annotation to be present with a value different from +// priorValue, which the caller captured before triggering the reload. +func (a *DaemonSetAdapter) WaitReloadedFrom(ctx context.Context, namespace, name, annotationKey, priorValue string, timeout time.Duration) (bool, error) { watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { return a.client.AppsV1().DaemonSets(namespace).Watch(ctx, opts) } @@ -64,6 +70,12 @@ func (a *DaemonSetAdapter) WaitEnvVar(ctx context.Context, namespace, name, pref if ds, err := a.client.AppsV1().DaemonSets(namespace).Get(ctx, name, metav1.GetOptions{}); err == nil { priorValue = GetEnvVarValueByPrefix(ds.Spec.Template.Spec.Containers, prefix) } + return a.WaitEnvVarFrom(ctx, namespace, name, prefix, priorValue, timeout) +} + +// WaitEnvVarFrom waits for a STAKATER_ env var whose value differs from priorValue, which the +// caller captured before triggering the reload. +func (a *DaemonSetAdapter) WaitEnvVarFrom(ctx context.Context, namespace, name, prefix, priorValue string, timeout time.Duration) (bool, error) { watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { return a.client.AppsV1().DaemonSets(namespace).Watch(ctx, opts) } diff --git a/test/e2e/utils/workload_deployment.go b/test/e2e/utils/workload_deployment.go index f7ef5e37..d64e194a 100644 --- a/test/e2e/utils/workload_deployment.go +++ b/test/e2e/utils/workload_deployment.go @@ -51,6 +51,12 @@ func (a *DeploymentAdapter) WaitReady(ctx context.Context, namespace, name strin // does not cause a false positive — the condition triggers only when the value changes. func (a *DeploymentAdapter) WaitReloaded(ctx context.Context, namespace, name, annotationKey string, timeout time.Duration) (bool, error) { priorValue, _ := a.GetPodTemplateAnnotation(ctx, namespace, name, annotationKey) + return a.WaitReloadedFrom(ctx, namespace, name, annotationKey, priorValue, timeout) +} + +// WaitReloadedFrom waits for the reload annotation to be present with a value different from +// priorValue, which the caller captured before triggering the reload. +func (a *DeploymentAdapter) WaitReloadedFrom(ctx context.Context, namespace, name, annotationKey, priorValue string, timeout time.Duration) (bool, error) { watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { return a.client.AppsV1().Deployments(namespace).Watch(ctx, opts) } @@ -66,6 +72,12 @@ func (a *DeploymentAdapter) WaitEnvVar(ctx context.Context, namespace, name, pre if d, err := a.client.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}); err == nil { priorValue = GetEnvVarValueByPrefix(d.Spec.Template.Spec.Containers, prefix) } + return a.WaitEnvVarFrom(ctx, namespace, name, prefix, priorValue, timeout) +} + +// WaitEnvVarFrom waits for a STAKATER_ env var whose value differs from priorValue, which the +// caller captured before triggering the reload. +func (a *DeploymentAdapter) WaitEnvVarFrom(ctx context.Context, namespace, name, prefix, priorValue string, timeout time.Duration) (bool, error) { watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { return a.client.AppsV1().Deployments(namespace).Watch(ctx, opts) } diff --git a/test/e2e/utils/workload_job.go b/test/e2e/utils/workload_job.go index e71c86c2..5f8ad506 100644 --- a/test/e2e/utils/workload_job.go +++ b/test/e2e/utils/workload_job.go @@ -54,11 +54,22 @@ func (a *JobAdapter) WaitReloaded(ctx context.Context, namespace, name, annotati return false, ErrUnsupportedOperation } +// WaitReloadedFrom returns an error because Jobs are recreated, not updated. +// Use the Recreatable interface (GetOriginalUID + WaitRecreated) instead. +func (a *JobAdapter) WaitReloadedFrom(ctx context.Context, namespace, name, annotationKey, priorValue string, timeout time.Duration) (bool, error) { + return false, ErrUnsupportedOperation +} + // WaitEnvVar returns an error because Jobs don't support env var reload strategy. func (a *JobAdapter) WaitEnvVar(ctx context.Context, namespace, name, prefix string, timeout time.Duration) (bool, error) { return false, ErrUnsupportedOperation } +// WaitEnvVarFrom returns an error because Jobs don't support env var reload strategy. +func (a *JobAdapter) WaitEnvVarFrom(ctx context.Context, namespace, name, prefix, priorValue string, timeout time.Duration) (bool, error) { + return false, ErrUnsupportedOperation +} + // WaitRecreated waits for the Job to be recreated with a different UID using watches. func (a *JobAdapter) WaitRecreated(ctx context.Context, namespace, name, originalUID string, timeout time.Duration) (string, bool, error) { watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { diff --git a/test/e2e/utils/workload_openshift.go b/test/e2e/utils/workload_openshift.go index 6f758bf4..2fb11c6c 100644 --- a/test/e2e/utils/workload_openshift.go +++ b/test/e2e/utils/workload_openshift.go @@ -60,6 +60,12 @@ func (a *DeploymentConfigAdapter) WaitReady(ctx context.Context, namespace, name // Captures the current annotation value first to avoid false positives from prior reloads. func (a *DeploymentConfigAdapter) WaitReloaded(ctx context.Context, namespace, name, annotationKey string, timeout time.Duration) (bool, error) { priorValue, _ := a.GetPodTemplateAnnotation(ctx, namespace, name, annotationKey) + return a.WaitReloadedFrom(ctx, namespace, name, annotationKey, priorValue, timeout) +} + +// WaitReloadedFrom waits for the reload annotation to be present with a value different from +// priorValue, which the caller captured before triggering the reload. +func (a *DeploymentConfigAdapter) WaitReloadedFrom(ctx context.Context, namespace, name, annotationKey, priorValue string, timeout time.Duration) (bool, error) { watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { return a.openshiftClient.AppsV1().DeploymentConfigs(namespace).Watch(ctx, opts) } @@ -74,6 +80,12 @@ func (a *DeploymentConfigAdapter) WaitEnvVar(ctx context.Context, namespace, nam if dc, err := a.openshiftClient.AppsV1().DeploymentConfigs(namespace).Get(ctx, name, metav1.GetOptions{}); err == nil && dc.Spec.Template != nil { priorValue = GetEnvVarValueByPrefix(dc.Spec.Template.Spec.Containers, prefix) } + return a.WaitEnvVarFrom(ctx, namespace, name, prefix, priorValue, timeout) +} + +// WaitEnvVarFrom waits for a STAKATER_ env var whose value differs from priorValue, which the +// caller captured before triggering the reload. +func (a *DeploymentConfigAdapter) WaitEnvVarFrom(ctx context.Context, namespace, name, prefix, priorValue string, timeout time.Duration) (bool, error) { watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { return a.openshiftClient.AppsV1().DeploymentConfigs(namespace).Watch(ctx, opts) } diff --git a/test/e2e/utils/workload_statefulset.go b/test/e2e/utils/workload_statefulset.go index d071678a..28052dce 100644 --- a/test/e2e/utils/workload_statefulset.go +++ b/test/e2e/utils/workload_statefulset.go @@ -50,6 +50,12 @@ func (a *StatefulSetAdapter) WaitReady(ctx context.Context, namespace, name stri // Captures the current annotation value first to avoid false positives from prior reloads. func (a *StatefulSetAdapter) WaitReloaded(ctx context.Context, namespace, name, annotationKey string, timeout time.Duration) (bool, error) { priorValue, _ := a.GetPodTemplateAnnotation(ctx, namespace, name, annotationKey) + return a.WaitReloadedFrom(ctx, namespace, name, annotationKey, priorValue, timeout) +} + +// WaitReloadedFrom waits for the reload annotation to be present with a value different from +// priorValue, which the caller captured before triggering the reload. +func (a *StatefulSetAdapter) WaitReloadedFrom(ctx context.Context, namespace, name, annotationKey, priorValue string, timeout time.Duration) (bool, error) { watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { return a.client.AppsV1().StatefulSets(namespace).Watch(ctx, opts) } @@ -64,6 +70,12 @@ func (a *StatefulSetAdapter) WaitEnvVar(ctx context.Context, namespace, name, pr if sts, err := a.client.AppsV1().StatefulSets(namespace).Get(ctx, name, metav1.GetOptions{}); err == nil { priorValue = GetEnvVarValueByPrefix(sts.Spec.Template.Spec.Containers, prefix) } + return a.WaitEnvVarFrom(ctx, namespace, name, prefix, priorValue, timeout) +} + +// WaitEnvVarFrom waits for a STAKATER_ env var whose value differs from priorValue, which the +// caller captured before triggering the reload. +func (a *StatefulSetAdapter) WaitEnvVarFrom(ctx context.Context, namespace, name, prefix, priorValue string, timeout time.Duration) (bool, error) { watchFunc := func(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { return a.client.AppsV1().StatefulSets(namespace).Watch(ctx, opts) }