From 379c4282835a707a56f0e60e1233101e6096058f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Pinkava?= Date: Fri, 6 Jun 2025 17:35:59 +0200 Subject: [PATCH 1/2] Use labels suggested by Kubernetes and Helm best practices This improves the compatibility and makes it easier to work with various tools, which take advantake from the assumption of existence of those labels. Keep the old labels for selectors for backward compatibility. Also note there is slight change in behaviour of PDB which might be backward-incompatible in some rare cases. --- deployments/kubernetes/chart/reloader/Chart.yaml | 2 +- .../chart/reloader/templates/_helpers.tpl | 16 ++++++++++++---- .../chart/reloader/templates/deployment.yaml | 3 +-- .../chart/reloader/templates/networkpolicy.yaml | 3 +-- .../reloader/templates/poddisruptionbudget.yaml | 2 +- .../chart/reloader/templates/podmonitor.yaml | 2 +- .../chart/reloader/templates/servicemonitor.yaml | 2 +- .../kubernetes/chart/reloader/values.yaml | 2 +- 8 files changed, 19 insertions(+), 13 deletions(-) diff --git a/deployments/kubernetes/chart/reloader/Chart.yaml b/deployments/kubernetes/chart/reloader/Chart.yaml index 5668e812..558b7727 100644 --- a/deployments/kubernetes/chart/reloader/Chart.yaml +++ b/deployments/kubernetes/chart/reloader/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 name: reloader description: Reloader chart that runs on kubernetes -version: 2.1.5 +version: 2.2.0 appVersion: v1.4.5 keywords: - Reloader diff --git a/deployments/kubernetes/chart/reloader/templates/_helpers.tpl b/deployments/kubernetes/chart/reloader/templates/_helpers.tpl index 1c276211..26112264 100644 --- a/deployments/kubernetes/chart/reloader/templates/_helpers.tpl +++ b/deployments/kubernetes/chart/reloader/templates/_helpers.tpl @@ -27,12 +27,20 @@ Create chart name and version as used by the chart label. {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} -{{- define "reloader-labels.chart" -}} +{{- define "reloader-match-labels.chart" -}} app: {{ template "reloader-fullname" . }} -chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" release: {{ .Release.Name | quote }} +{{- end -}} + +{{- define "reloader-labels.chart" -}} +{{ include "reloader-match-labels.chart" . }} +app.kubernetes.io/name: {{ template "reloader-name" . }} +app.kubernetes.io/instance: {{ .Release.Name | quote }} +helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" heritage: {{ .Release.Service | quote }} app.kubernetes.io/managed-by: {{ .Release.Service | quote }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end -}} {{/* @@ -45,10 +53,10 @@ podAntiAffinity: podAffinityTerm: labelSelector: matchExpressions: - - key: app + - key: app.kubernetes.io/instance operator: In values: - - {{ template "reloader-fullname" . }} + - {{ .Release.Name | quote }} topologyKey: "kubernetes.io/hostname" {{- end -}} diff --git a/deployments/kubernetes/chart/reloader/templates/deployment.yaml b/deployments/kubernetes/chart/reloader/templates/deployment.yaml index 287567b9..f7359125 100644 --- a/deployments/kubernetes/chart/reloader/templates/deployment.yaml +++ b/deployments/kubernetes/chart/reloader/templates/deployment.yaml @@ -25,8 +25,7 @@ spec: revisionHistoryLimit: {{ .Values.reloader.deployment.revisionHistoryLimit }} selector: matchLabels: - app: {{ template "reloader-fullname" . }} - release: {{ .Release.Name | quote }} +{{ include "reloader-match-labels.chart" . | indent 6 }} {{- if .Values.reloader.matchLabels }} {{ tpl (toYaml .Values.reloader.matchLabels) . | indent 6 }} {{- end }} diff --git a/deployments/kubernetes/chart/reloader/templates/networkpolicy.yaml b/deployments/kubernetes/chart/reloader/templates/networkpolicy.yaml index cf4d0f1a..ecb2c6af 100644 --- a/deployments/kubernetes/chart/reloader/templates/networkpolicy.yaml +++ b/deployments/kubernetes/chart/reloader/templates/networkpolicy.yaml @@ -14,8 +14,7 @@ metadata: spec: podSelector: matchLabels: - app: {{ template "reloader-fullname" . }} - release: {{ .Release.Name | quote }} +{{ include "reloader-match-labels.chart" . | indent 6 }} {{- if .Values.reloader.matchLabels }} {{ tpl (toYaml .Values.reloader.matchLabels) . | indent 6 }} {{- end }} diff --git a/deployments/kubernetes/chart/reloader/templates/poddisruptionbudget.yaml b/deployments/kubernetes/chart/reloader/templates/poddisruptionbudget.yaml index e13c4783..7f877caa 100644 --- a/deployments/kubernetes/chart/reloader/templates/poddisruptionbudget.yaml +++ b/deployments/kubernetes/chart/reloader/templates/poddisruptionbudget.yaml @@ -13,5 +13,5 @@ spec: {{- end }} selector: matchLabels: - app: {{ template "reloader-fullname" . }} + {{ include "reloader-match-labels.chart" . | nindent 6 }} {{- end }} diff --git a/deployments/kubernetes/chart/reloader/templates/podmonitor.yaml b/deployments/kubernetes/chart/reloader/templates/podmonitor.yaml index 7afeba3b..3af66dc3 100644 --- a/deployments/kubernetes/chart/reloader/templates/podmonitor.yaml +++ b/deployments/kubernetes/chart/reloader/templates/podmonitor.yaml @@ -56,5 +56,5 @@ spec: - {{ .Release.Namespace }} selector: matchLabels: - {{ include "reloader-labels.chart" . | nindent 6 }} + {{ include "reloader-match-labels.chart" . | nindent 6 }} {{- end }} diff --git a/deployments/kubernetes/chart/reloader/templates/servicemonitor.yaml b/deployments/kubernetes/chart/reloader/templates/servicemonitor.yaml index c4685fdb..cdd35429 100644 --- a/deployments/kubernetes/chart/reloader/templates/servicemonitor.yaml +++ b/deployments/kubernetes/chart/reloader/templates/servicemonitor.yaml @@ -56,5 +56,5 @@ spec: - {{ .Release.Namespace }} selector: matchLabels: - {{ include "reloader-labels.chart" . | nindent 6 }} + {{ include "reloader-match-labels.chart" . | nindent 6 }} {{- end }} diff --git a/deployments/kubernetes/chart/reloader/values.yaml b/deployments/kubernetes/chart/reloader/values.yaml index 21c11a3e..f6e417dc 100644 --- a/deployments/kubernetes/chart/reloader/values.yaml +++ b/deployments/kubernetes/chart/reloader/values.yaml @@ -99,7 +99,7 @@ reloader: # whenUnsatisfiable: DoNotSchedule # labelSelector: # matchLabels: - # app: my-app + # app.kubernetes.io/instance: my-app topologySpreadConstraints: [] annotations: {} From cafa14f0e7279796f3a858056b10d416c598e7d3 Mon Sep 17 00:00:00 2001 From: Vladislav Gusev Date: Tue, 8 Jul 2025 15:32:33 +0300 Subject: [PATCH 2/2] Add pausing deployments during upgrades --- internal/pkg/handler/pause_deployment.go | 242 +++++++++++ internal/pkg/handler/pause_deployment_test.go | 392 ++++++++++++++++++ internal/pkg/handler/upgrade.go | 14 + internal/pkg/handler/upgrade_test.go | 132 ++++++ internal/pkg/options/flags.go | 6 + internal/pkg/testutil/kube.go | 20 + 6 files changed, 806 insertions(+) create mode 100644 internal/pkg/handler/pause_deployment.go create mode 100644 internal/pkg/handler/pause_deployment_test.go diff --git a/internal/pkg/handler/pause_deployment.go b/internal/pkg/handler/pause_deployment.go new file mode 100644 index 00000000..28d1b9ef --- /dev/null +++ b/internal/pkg/handler/pause_deployment.go @@ -0,0 +1,242 @@ +package handler + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/sirupsen/logrus" + "github.com/stakater/Reloader/internal/pkg/options" + "github.com/stakater/Reloader/pkg/kube" + app "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + patchtypes "k8s.io/apimachinery/pkg/types" +) + +// Keeps track of currently active timers +var activeTimers = make(map[string]*time.Timer) + +// Returns unique key for the activeTimers map +func getTimerKey(namespace, deploymentName string) string { + return fmt.Sprintf("%s/%s", namespace, deploymentName) +} + +// Checks if a deployment is currently paused +func IsPaused(deployment *app.Deployment) bool { + return deployment.Spec.Paused +} + +// Deployment paused by reloader ? +func IsPausedByReloader(deployment *app.Deployment) bool { + if IsPaused(deployment) { + pausedAtAnnotationValue := deployment.Annotations[options.PauseDeploymentTimeAnnotation] + return pausedAtAnnotationValue != "" + } + return false +} + +// Returns the time, the deployment was paused by reloader, nil otherwise +func GetPauseStartTime(deployment *app.Deployment) (*time.Time, error) { + if !IsPausedByReloader(deployment) { + return nil, nil + } + + pausedAtStr := deployment.Annotations[options.PauseDeploymentTimeAnnotation] + parsedTime, err := time.Parse(time.RFC3339, pausedAtStr) + if err != nil { + return nil, err + } + + return &parsedTime, nil +} + +// ParsePauseDuration parses the pause interval value and returns a time.Duration +func ParsePauseDuration(pauseIntervalValue string) (time.Duration, error) { + pauseDuration, err := time.ParseDuration(pauseIntervalValue) + if err != nil { + logrus.Warnf("Failed to parse pause interval value '%s': %v", pauseIntervalValue, err) + return 0, err + } + return pauseDuration, nil +} + +// Pauses a deployment for a specified duration and creates a timer to resume it +// after the specified duration +func PauseDeployment(deployment *app.Deployment, clients kube.Clients, namespace, pauseIntervalValue string) (*app.Deployment, error) { + deploymentName := deployment.Name + pauseDuration, err := ParsePauseDuration(pauseIntervalValue) + + if err != nil { + return nil, err + } + + if !IsPaused(deployment) { + logrus.Infof("Pausing Deployment '%s' in namespace '%s' for %s", deploymentName, namespace, pauseDuration) + + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + + pausePatch, err := CreatePausePatch() + if err != nil { + logrus.Errorf("Failed to create pause patch for deployment '%s': %v", deploymentName, err) + return deployment, err + } + + err = deploymentFuncs.PatchFunc(clients, namespace, deployment, patchtypes.StrategicMergePatchType, pausePatch) + + if err != nil { + logrus.Errorf("Failed to patch deployment '%s' in namespace '%s': %v", deploymentName, namespace, err) + return deployment, err + } + + updatedDeployment, err := clients.KubernetesClient.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) + + CreateResumeTimer(deployment, clients, namespace, pauseDuration) + return updatedDeployment, err + } + + if !IsPausedByReloader(deployment) { + logrus.Infof("Deployment '%s' in namespace '%s' already paused", deploymentName, namespace) + return deployment, nil + } + + // Deployment has already been paused by reloader, check for timer + logrus.Debugf("Deployment '%s' in namespace '%s' is already paused by reloader", deploymentName, namespace) + + timerKey := getTimerKey(namespace, deploymentName) + _, timerExists := activeTimers[timerKey] + + if !timerExists { + logrus.Warnf("Timer does not exist for already paused deployment '%s' in namespace '%s', creating new one", + deploymentName, namespace) + HandleMissingTimer(deployment, pauseDuration, clients, namespace) + } + return deployment, nil +} + +// Handles the case where missing timers for deployments that have been paused by reloader. +// Could occur after new leader election or reloader restart +func HandleMissingTimer(deployment *app.Deployment, pauseDuration time.Duration, clients kube.Clients, namespace string) { + deploymentName := deployment.Name + pauseStartTime, err := GetPauseStartTime(deployment) + if err != nil { + logrus.Errorf("Error parsing pause start time for deployment '%s' in namespace '%s': %v. Resuming deployment immediately", + deploymentName, namespace, err) + ResumeDeployment(deployment, namespace, clients) + return + } + + if pauseStartTime == nil { + return + } + + elapsedPauseTime := time.Since(*pauseStartTime) + remainingPauseTime := pauseDuration - elapsedPauseTime + + if remainingPauseTime <= 0 { + logrus.Infof("Pause period for deployment '%s' in namespace '%s' has expired. Resuming immediately", + deploymentName, namespace) + ResumeDeployment(deployment, namespace, clients) + return + } + + logrus.Infof("Creating missing timer for already paused deployment '%s' in namespace '%s' with remaining time %s", + deploymentName, namespace, remainingPauseTime) + CreateResumeTimer(deployment, clients, namespace, remainingPauseTime) +} + +// CreateResumeTimer creates a timer to resume the deployment after the specified duration +func CreateResumeTimer(deployment *app.Deployment, clients kube.Clients, namespace string, pauseDuration time.Duration) { + deploymentName := deployment.Name + timerKey := getTimerKey(namespace, deployment.Name) + + // Check if there's an existing timer for this deployment + if _, exists := activeTimers[timerKey]; exists { + logrus.Debugf("Timer already exists for deployment '%s' in namespace '%s', Skipping creation", + deploymentName, namespace) + return + } + + // Create and store the new timer + timer := time.AfterFunc(pauseDuration, func() { + ResumeDeployment(deployment, namespace, clients) + }) + + // Add the new timer to the map + activeTimers[timerKey] = timer + + logrus.Debugf("Created pause timer for deployment '%s' in namespace '%s' with duration %s", + deploymentName, namespace, pauseDuration) +} + +// ResumeDeployment resumes a deployment that has been paused by reloader +func ResumeDeployment(deployment *app.Deployment, namespace string, clients kube.Clients) { + deploymentName := deployment.Name + + currentDeployment, err := clients.KubernetesClient.AppsV1().Deployments(namespace).Get(context.TODO(), deploymentName, metav1.GetOptions{}) + + if err != nil { + logrus.Errorf("Failed to get deployment '%s' in namespace '%s': %v", deploymentName, namespace, err) + return + } + + if !IsPausedByReloader(currentDeployment) { + logrus.Infof("Deployment '%s' in namespace '%s' not paused by Reloader. Skipping resume", deploymentName, namespace) + return + } + + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + + resumePatch, err := CreateResumePatch() + if err != nil { + logrus.Errorf("Failed to create resume patch for deployment '%s': %v", deploymentName, err) + return + } + + // Remove the timer + timerKey := getTimerKey(namespace, deploymentName) + if timer, exists := activeTimers[timerKey]; exists { + timer.Stop() + delete(activeTimers, timerKey) + logrus.Debugf("Removed pause timer for deployment '%s' in namespace '%s'", deploymentName, namespace) + } + + err = deploymentFuncs.PatchFunc(clients, namespace, currentDeployment, patchtypes.StrategicMergePatchType, resumePatch) + + if err != nil { + logrus.Errorf("Failed to resume deployment '%s' in namespace '%s': %v", deploymentName, namespace, err) + return + } + + logrus.Infof("Successfully resumed deployment '%s' in namespace '%s'", deploymentName, namespace) +} + +func CreatePausePatch() ([]byte, error) { + patchData := map[string]interface{}{ + "spec": map[string]interface{}{ + "paused": true, + }, + "metadata": map[string]interface{}{ + "annotations": map[string]string{ + options.PauseDeploymentTimeAnnotation: time.Now().Format(time.RFC3339), + }, + }, + } + + return json.Marshal(patchData) +} + +func CreateResumePatch() ([]byte, error) { + patchData := map[string]interface{}{ + "spec": map[string]interface{}{ + "paused": false, + }, + "metadata": map[string]interface{}{ + "annotations": map[string]interface{}{ + options.PauseDeploymentTimeAnnotation: nil, + }, + }, + } + + return json.Marshal(patchData) +} diff --git a/internal/pkg/handler/pause_deployment_test.go b/internal/pkg/handler/pause_deployment_test.go new file mode 100644 index 00000000..076b0b25 --- /dev/null +++ b/internal/pkg/handler/pause_deployment_test.go @@ -0,0 +1,392 @@ +package handler + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stakater/Reloader/internal/pkg/options" + "github.com/stakater/Reloader/pkg/kube" + "github.com/stretchr/testify/assert" + app "k8s.io/api/apps/v1" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + testclient "k8s.io/client-go/kubernetes/fake" +) + +func TestIsPaused(t *testing.T) { + tests := []struct { + name string + deployment *appsv1.Deployment + paused bool + }{ + { + name: "paused deployment", + deployment: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Paused: true, + }, + }, + paused: true, + }, + { + name: "unpaused deployment", + deployment: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Paused: false, + }, + }, + paused: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := IsPaused(test.deployment) + assert.Equal(t, test.paused, result) + }) + } +} + +func TestIsPausedByReloader(t *testing.T) { + tests := []struct { + name string + deployment *appsv1.Deployment + pausedByReloader bool + }{ + { + name: "paused by reloader", + deployment: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Paused: true, + }, + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + options.PauseDeploymentTimeAnnotation: time.Now().Format(time.RFC3339), + }, + }, + }, + pausedByReloader: true, + }, + { + name: "not paused by reloader", + deployment: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Paused: true, + }, + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + }, + pausedByReloader: false, + }, + { + name: "not paused", + deployment: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Paused: false, + }, + }, + pausedByReloader: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + pausedByReloader := IsPausedByReloader(test.deployment) + assert.Equal(t, test.pausedByReloader, pausedByReloader) + }) + } +} + +func TestGetPauseStartTime(t *testing.T) { + now := time.Now() + nowStr := now.Format(time.RFC3339) + + tests := []struct { + name string + deployment *appsv1.Deployment + pausedByReloader bool + expectedStartTime time.Time + }{ + { + name: "valid pause time", + deployment: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Paused: true, + }, + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + options.PauseDeploymentTimeAnnotation: nowStr, + }, + }, + }, + pausedByReloader: true, + expectedStartTime: now, + }, + { + name: "not paused by reloader", + deployment: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Paused: false, + }, + }, + pausedByReloader: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actualStartTime, err := GetPauseStartTime(test.deployment) + + assert.NoError(t, err) + + if !test.pausedByReloader { + assert.Nil(t, actualStartTime) + } else { + assert.NotNil(t, actualStartTime) + assert.WithinDuration(t, test.expectedStartTime, *actualStartTime, time.Second) + } + }) + } +} + +func TestParsePauseDuration(t *testing.T) { + tests := []struct { + name string + pauseIntervalValue string + expectedDuration time.Duration + invalidDuration bool + }{ + { + name: "valid duration", + pauseIntervalValue: "10s", + expectedDuration: 10 * time.Second, + invalidDuration: false, + }, + { + name: "valid minute duration", + pauseIntervalValue: "2m", + expectedDuration: 2 * time.Minute, + invalidDuration: false, + }, + { + name: "invalid duration", + pauseIntervalValue: "invalid", + expectedDuration: 0, + invalidDuration: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actualDuration, err := ParsePauseDuration(test.pauseIntervalValue) + + if test.invalidDuration { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, test.expectedDuration, actualDuration) + } + }) + } +} + +func TestHandleMissingTimerSimple(t *testing.T) { + tests := []struct { + name string + deployment *appsv1.Deployment + shouldBePaused bool // Should be unpaused after HandleMissingTimer ? + }{ + { + name: "deployment paused by reloader, pause period has expired and no timer", + deployment: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment-1", + Annotations: map[string]string{ + options.PauseDeploymentTimeAnnotation: time.Now().Add(-6 * time.Minute).Format(time.RFC3339), + options.PauseDeploymentAnnotation: "5m", + }, + }, + Spec: appsv1.DeploymentSpec{ + Paused: true, + }, + }, + shouldBePaused: false, + }, + { + name: "deployment paused by reloader, pause period expires in the future and no timer", + deployment: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment-2", + Annotations: map[string]string{ + options.PauseDeploymentTimeAnnotation: time.Now().Add(1 * time.Minute).Format(time.RFC3339), + options.PauseDeploymentAnnotation: "5m", + }, + }, + Spec: appsv1.DeploymentSpec{ + Paused: true, + }, + }, + shouldBePaused: true, + }, + } + + for _, test := range tests { + // Clean up any timers at the end of the test + defer func() { + for key, timer := range activeTimers { + timer.Stop() + delete(activeTimers, key) + } + }() + + t.Run(test.name, func(t *testing.T) { + fakeClient := testclient.NewSimpleClientset() + clients := kube.Clients{ + KubernetesClient: fakeClient, + } + + _, err := fakeClient.AppsV1().Deployments("default").Create( + context.TODO(), + test.deployment, + metav1.CreateOptions{}) + assert.NoError(t, err, "Expected no error when creating deployment") + + pauseDuration, _ := ParsePauseDuration(test.deployment.Annotations[options.PauseDeploymentAnnotation]) + HandleMissingTimer(test.deployment, pauseDuration, clients, "default") + + updatedDeployment, _ := fakeClient.AppsV1().Deployments("default").Get(context.TODO(), test.deployment.Name, metav1.GetOptions{}) + + assert.Equal(t, test.shouldBePaused, updatedDeployment.Spec.Paused, + "Deployment should have correct paused state after timer expiration") + + if test.shouldBePaused { + pausedAtAnnotationValue := updatedDeployment.Annotations[options.PauseDeploymentTimeAnnotation] + assert.NotEmpty(t, pausedAtAnnotationValue, + "Pause annotation should be present and contain a value when deployment is paused") + } + }) + } +} + +func TestPauseDeployment(t *testing.T) { + tests := []struct { + name string + deployment *appsv1.Deployment + expectedError bool + expectedPaused bool + expectedAnnotation bool // Should have pause time annotation + pauseInterval string + }{ + { + name: "deployment without pause annotation", + deployment: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + Annotations: map[string]string{}, + }, + Spec: appsv1.DeploymentSpec{ + Paused: false, + }, + }, + expectedError: true, + expectedPaused: false, + expectedAnnotation: false, + pauseInterval: "", + }, + { + name: "deployment already paused but not by reloader", + deployment: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + Annotations: map[string]string{ + options.PauseDeploymentAnnotation: "5m", + }, + }, + Spec: appsv1.DeploymentSpec{ + Paused: true, + }, + }, + expectedError: false, + expectedPaused: true, + expectedAnnotation: false, + pauseInterval: "5m", + }, + { + name: "deployment unpaused that needs to be paused by reloader", + deployment: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment-3", + Annotations: map[string]string{ + options.PauseDeploymentAnnotation: "5m", + }, + }, + Spec: appsv1.DeploymentSpec{ + Paused: false, + }, + }, + expectedError: false, + expectedPaused: true, + expectedAnnotation: true, + pauseInterval: "5m", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeClient := testclient.NewSimpleClientset() + clients := kube.Clients{ + KubernetesClient: fakeClient, + } + + _, err := fakeClient.AppsV1().Deployments("default").Create( + context.TODO(), + test.deployment, + metav1.CreateOptions{}) + assert.NoError(t, err, "Expected no error when creating deployment") + + updatedDeployment, err := PauseDeployment(test.deployment, clients, "default", test.pauseInterval) + if test.expectedError { + assert.Error(t, err, "Expected an error pausing the deployment") + return + } else { + assert.NoError(t, err, "Expected no error pausing the deployment") + } + + assert.Equal(t, test.expectedPaused, updatedDeployment.Spec.Paused, + "Deployment should have correct paused state after pause") + + if test.expectedAnnotation { + pausedAtAnnotationValue := updatedDeployment.Annotations[options.PauseDeploymentTimeAnnotation] + assert.NotEmpty(t, pausedAtAnnotationValue, + "Pause annotation should be present and contain a value when deployment is paused") + } else { + pausedAtAnnotationValue := updatedDeployment.Annotations[options.PauseDeploymentTimeAnnotation] + assert.Empty(t, pausedAtAnnotationValue, + "Pause annotation should not be present when deployment has not been paused by reloader") + } + }) + } +} + +// Simple helper function for test cases +func FindDeploymentByName(deployments []runtime.Object, deploymentName string) (*app.Deployment, error) { + for _, deployment := range deployments { + accessor, err := meta.Accessor(deployment) + if err != nil { + return nil, fmt.Errorf("error getting accessor for item: %v", err) + } + if accessor.GetName() == deploymentName { + deploymentObj, ok := deployment.(*app.Deployment) + if !ok { + return nil, fmt.Errorf("failed to cast to Deployment") + } + return deploymentObj, nil + } + } + return nil, fmt.Errorf("deployment '%s' not found", deploymentName) +} diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index ecb4aae5..67ef778a 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -21,6 +21,7 @@ import ( "github.com/stakater/Reloader/internal/pkg/options" "github.com/stakater/Reloader/internal/pkg/util" "github.com/stakater/Reloader/pkg/kube" + app "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -277,6 +278,7 @@ func upgradeResource(clients kube.Clients, config util.Config, upgradeFuncs call typedAutoAnnotationEnabledValue, foundTypedAuto := annotations[config.TypedAutoAnnotation] excludeConfigmapAnnotationValue, foundExcludeConfigmap := annotations[options.ConfigmapExcludeReloaderAnnotation] excludeSecretAnnotationValue, foundExcludeSecret := annotations[options.SecretExcludeReloaderAnnotation] + pauseInterval, foundPauseInterval := annotations[options.PauseDeploymentAnnotation] if !found && !foundAuto && !foundTypedAuto && !foundSearchAnn { annotations = upgradeFuncs.PodAnnotationsFunc(resource) @@ -331,6 +333,18 @@ func upgradeResource(clients kube.Clients, config util.Config, upgradeFuncs call } } if strategyResult.Result == constants.Updated { + if foundPauseInterval { + deployment, ok := resource.(*app.Deployment) + if !ok { + logrus.Warnf("Annotation '%s' only applicable for deployments", options.PauseDeploymentAnnotation) + } else { + _, err = PauseDeployment(deployment, clients, config.Namespace, pauseInterval) + if err != nil { + logrus.Errorf("Failed to pause deployment '%s' in namespace '%s': %v", resourceName, config.Namespace, err) + return err + } + } + } var err error if upgradeFuncs.SupportsPatch && strategyResult.Patch != nil { err = upgradeFuncs.PatchFunc(clients, config.Namespace, resource, strategyResult.Patch.Type, strategyResult.Patch.Bytes) diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index 5c77cae6..61d8aeb8 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -54,6 +54,7 @@ var ( arsConfigmapWithExcludeConfigMapAnnotation = "testconfigmapwithconfigmapexcludeannotationdeployment-handler-" + testutil.RandSeq(5) arsConfigmapWithIgnoreAnnotation = "testconfigmapWithIgnoreAnnotation-handler-" + testutil.RandSeq(5) arsSecretWithIgnoreAnnotation = "testsecretWithIgnoreAnnotation-handler-" + testutil.RandSeq(5) + arsConfigmapWithPausedDeployment = "testconfigmapWithPausedDeployment-handler-" + testutil.RandSeq(5) ersNamespace = "test-handler-" + testutil.RandSeq(5) ersConfigmapName = "testconfigmap-handler-" + testutil.RandSeq(5) @@ -79,6 +80,7 @@ var ( ersConfigmapWithConfigMapExcludeAnnotation = "testconfigmapwithconfigmapexcludeannotationdeployment-handler-" + testutil.RandSeq(5) ersConfigmapWithIgnoreAnnotation = "testconfigmapWithIgnoreAnnotation-handler-" + testutil.RandSeq(5) ersSecretWithIgnoreAnnotation = "testsecretWithIgnoreAnnotation-handler-" + testutil.RandSeq(5) + ersConfigmapWithPausedDeployment = "testconfigmapWithPausedDeployment-handler-" + testutil.RandSeq(5) ) func TestMain(m *testing.M) { @@ -207,6 +209,12 @@ func setupArs() { logrus.Errorf("Error in configmap creation: %v", err) } + // Creating configmap for testing pausing deployments + _, err = testutil.CreateConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapWithPausedDeployment, "www.google.com") + if err != nil { + logrus.Errorf("Error in configmap creation: %v", err) + } + // Creating secret used with secret auto annotation _, err = testutil.CreateSecret(clients.KubernetesClient, arsNamespace, arsSecretWithExcludeSecretAnnotation, data) if err != nil { @@ -457,6 +465,12 @@ func setupArs() { if err != nil { logrus.Errorf("Error in Deployment with both annotations: %v", err) } + + // Creating Deployment with pause annotation + _, err = testutil.CreateDeploymentWithAnnotations(clients.KubernetesClient, arsConfigmapWithPausedDeployment, arsNamespace, map[string]string{options.PauseDeploymentAnnotation: "10s"}, false) + if err != nil { + logrus.Errorf("Error in Deployment with configmap creation: %v", err) + } } func teardownArs() { @@ -658,6 +672,12 @@ func teardownArs() { logrus.Errorf("Error while deleting statefulSet with secret as env var source %v", statefulSetError) } + // Deleting Deployment with pasuse annotation + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, arsNamespace, arsConfigmapWithPausedDeployment) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with configmap %v", deploymentError) + } + // Deleting Configmap err := testutil.DeleteConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapName) if err != nil { @@ -771,6 +791,12 @@ func teardownArs() { logrus.Errorf("Error while deleting the configmap used with configmap auto annotations: %v", err) } + // Deleting configmap for testing pausing deployments + err = testutil.DeleteConfigMap(clients.KubernetesClient, arsNamespace, arsConfigmapWithPausedDeployment) + if err != nil { + logrus.Errorf("Error while deleting the configmap: %v", err) + } + // Deleting namespace testutil.DeleteNamespace(arsNamespace, clients.KubernetesClient) @@ -830,6 +856,12 @@ func setupErs() { logrus.Errorf("Error in configmap creation: %v", err) } + // Creating configmap for testing pausing deployments + _, err = testutil.CreateConfigMap(clients.KubernetesClient, ersNamespace, ersConfigmapWithPausedDeployment, "www.google.com") + if err != nil { + logrus.Errorf("Error in configmap creation: %v", err) + } + // Creating secret _, err = testutil.CreateSecret(clients.KubernetesClient, ersNamespace, ersSecretWithInitEnv, data) if err != nil { @@ -1034,6 +1066,12 @@ func setupErs() { logrus.Errorf("Error in Deployment with configmap and with configmap exclude annotation: %v", err) } + // Creating Deployment with pause annotation + _, err = testutil.CreateDeploymentWithAnnotations(clients.KubernetesClient, ersConfigmapWithPausedDeployment, ersNamespace, map[string]string{options.PauseDeploymentAnnotation: "10s"}, false) + if err != nil { + logrus.Errorf("Error in Deployment with configmap creation: %v", err) + } + // Creating DaemonSet with configmap _, err = testutil.CreateDaemonSet(clients.KubernetesClient, ersConfigmapName, ersNamespace, true) if err != nil { @@ -1318,6 +1356,12 @@ func teardownErs() { logrus.Errorf("Error while deleting statefulSet with secret as env var source %v", statefulSetError) } + // Deleting Deployment for testing pausing deployments + deploymentError = testutil.DeleteDeployment(clients.KubernetesClient, ersNamespace, ersConfigmapWithPausedDeployment) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with configmap %v", deploymentError) + } + // Deleting Configmap err := testutil.DeleteConfigMap(clients.KubernetesClient, ersNamespace, ersConfigmapName) if err != nil { @@ -1431,6 +1475,12 @@ func teardownErs() { logrus.Errorf("Error while deleting the configmap used with configmap exclude annotation: %v", err) } + // Deleting ConfigMap for testins pausing deployments + err = testutil.DeleteConfigMap(clients.KubernetesClient, ersNamespace, ersConfigmapWithPausedDeployment) + if err != nil { + logrus.Errorf("Error while deleting the configmap: %v", err) + } + // Deleting namespace testutil.DeleteNamespace(ersNamespace, clients.KubernetesClient) @@ -4053,3 +4103,85 @@ func TestFailedRollingUpgradeUsingErs(t *testing.T) { t.Errorf("Counter by namespace was not increased") } } + +func TestPausingDeploymentUsingErs(t *testing.T) { + options.ReloadStrategy = constants.EnvVarsReloadStrategy + testPausingDeployment(t, options.ReloadStrategy, ersConfigmapWithPausedDeployment, ersNamespace) +} + +func TestPausingDeploymentUsingArs(t *testing.T) { + options.ReloadStrategy = constants.AnnotationsReloadStrategy + testPausingDeployment(t, options.ReloadStrategy, arsConfigmapWithPausedDeployment, arsNamespace) +} + +func testPausingDeployment(t *testing.T, reloadStrategy string, testName string, namespace string) { + options.ReloadStrategy = reloadStrategy + envVarPostfix := constants.ConfigmapEnvVarPostfix + + shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, testName, "pause.stakater.com") + config := getConfigWithAnnotations(envVarPostfix, testName, shaData, options.ConfigmapUpdateOnChangeAnnotation, options.ConfigmapReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + collectors := getCollectors() + + _ = PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 1 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": namespace})) != 1 { + t.Errorf("Counter by namespace was not increased") + } + + logrus.Infof("Verifying deployment has been paused") + items := deploymentFuncs.ItemsFunc(clients, config.Namespace) + deploymentPaused, err := isDeploymentPaused(items, testName) + if err != nil { + t.Errorf("%s", err.Error()) + } + if !deploymentPaused { + t.Errorf("Deployment has not been paused") + } + + shaData = testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, testName, "pause-changed.stakater.com") + config = getConfigWithAnnotations(envVarPostfix, testName, shaData, options.ConfigmapUpdateOnChangeAnnotation, options.ConfigmapReloaderAutoAnnotation) + + _ = PerformAction(clients, config, deploymentFuncs, collectors, nil, invokeReloadStrategy) + + if promtestutil.ToFloat64(collectors.Reloaded.With(labelSucceeded)) != 2 { + t.Errorf("Counter was not increased") + } + + if promtestutil.ToFloat64(collectors.ReloadedByNamespace.With(prometheus.Labels{"success": "true", "namespace": namespace})) != 2 { + t.Errorf("Counter by namespace was not increased") + } + + logrus.Infof("Verifying deployment is still paused") + items = deploymentFuncs.ItemsFunc(clients, config.Namespace) + deploymentPaused, err = isDeploymentPaused(items, testName) + if err != nil { + t.Errorf("%s", err.Error()) + } + if !deploymentPaused { + t.Errorf("Deployment should still be paused") + } + + logrus.Infof("Verifying deployment has been resumed after pause interval") + time.Sleep(11 * time.Second) + items = deploymentFuncs.ItemsFunc(clients, config.Namespace) + deploymentPaused, err = isDeploymentPaused(items, testName) + if err != nil { + t.Errorf("%s", err.Error()) + } + if deploymentPaused { + t.Errorf("Deployment should have been resumed after pause interval") + } +} + +func isDeploymentPaused(deployments []runtime.Object, deploymentName string) (bool, error) { + deployment, err := FindDeploymentByName(deployments, deploymentName) + if err != nil { + return false, err + } + return IsPaused(deployment), nil +} diff --git a/internal/pkg/options/flags.go b/internal/pkg/options/flags.go index 7c0e14e1..bfa63fc9 100644 --- a/internal/pkg/options/flags.go +++ b/internal/pkg/options/flags.go @@ -40,6 +40,12 @@ var ( SearchMatchAnnotation = "reloader.stakater.com/match" // RolloutStrategyAnnotation is an annotation to define rollout update strategy RolloutStrategyAnnotation = "reloader.stakater.com/rollout-strategy" + // PauseDeploymentAnnotation is an annotation to define the time period to pause a deployment after + // a configmap/secret change has been detected. Valid values are described here: https://pkg.go.dev/time#ParseDuration + // only positive values are allowed + PauseDeploymentAnnotation = "deployment.reloader.stakater.com/pause-period" + // Annotation set by reloader to indicate that the deployment has been paused + PauseDeploymentTimeAnnotation = "deployment.reloader.stakater.com/paused-at" // LogFormat is the log format to use (json, or empty string for default) LogFormat = "" // LogLevel is the log level to use (trace, debug, info, warning, error, fatal and panic) diff --git a/internal/pkg/testutil/kube.go b/internal/pkg/testutil/kube.go index f2d3bb4a..7a1aae3a 100644 --- a/internal/pkg/testutil/kube.go +++ b/internal/pkg/testutil/kube.go @@ -794,6 +794,26 @@ func CreateDeployment(client kubernetes.Interface, deploymentName string, namesp return deployment, err } +// CreateDeployment creates a deployment in given namespace and returns the Deployment +func CreateDeploymentWithAnnotations(client kubernetes.Interface, deploymentName string, namespace string, additionalAnnotations map[string]string, volumeMount bool) (*appsv1.Deployment, error) { + logrus.Infof("Creating Deployment") + deploymentClient := client.AppsV1().Deployments(namespace) + var deploymentObj *appsv1.Deployment + if volumeMount { + deploymentObj = GetDeployment(namespace, deploymentName) + } else { + deploymentObj = GetDeploymentWithEnvVars(namespace, deploymentName) + } + + for annotationKey, annotationValue := range additionalAnnotations { + deploymentObj.Annotations[annotationKey] = annotationValue + } + + deployment, err := deploymentClient.Create(context.TODO(), deploymentObj, metav1.CreateOptions{}) + time.Sleep(3 * time.Second) + return deployment, err +} + // CreateDeploymentConfig creates a deploymentConfig in given namespace and returns the DeploymentConfig func CreateDeploymentConfig(client appsclient.Interface, deploymentName string, namespace string, volumeMount bool) (*openshiftv1.DeploymentConfig, error) { logrus.Infof("Creating DeploymentConfig")