github/Reloader
1
0
Fork 0
mirror of https://github.com/stakater/Reloader.git synced 2026-02-14 18:09:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #106 from stakater/compatibility-1.16

Browse Source 
Make reloader compatible with kubernetes 1.16
This commit is contained in:
Waseem Hassan
2019-10-16 09:57:50 +02:00
committed by GitHub
parent e14b0c81b0 5fff0c9bb7
commit efa30662ae
3 changed files with 113 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -19,6 +19,10 @@ We would like to watch if some change happens in `ConfigMap` and/or `Secret`; th
Reloader can watch changes in `ConfigMap` and `Secret` and do rolling upgrades on Pods with their associated `DeploymentConfigs`, `Deployments`, `Daemonsets` and `Statefulsets`.
## Compatibility
Reloader is compatible with kubernetes >= 1.9
## How to use Reloader
For a `Deployment` called `foo` have a `ConfigMap` called `foo-configmap` or `Secret` called `foo-secret` or both. Then add your annotation (by default `reloader.stakater.com/auto`) to main metadata of your `Deployment`

39
internal/pkg/callbacks/rolling_upgrade.go
View File

@@ -4,9 +4,8 @@ import (
"github.com/sirupsen/logrus"
"github.com/stakater/Reloader/internal/pkg/util"
"github.com/stakater/Reloader/pkg/kube"
apps_v1beta1 "k8s.io/api/apps/v1beta1"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/api/extensions/v1beta1"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
openshiftv1 "github.com/openshift/api/apps/v1"
@@ -39,7 +38,7 @@ type RollingUpgradeFuncs struct {
// GetDeploymentItems returns the deployments in given namespace
func GetDeploymentItems(clients kube.Clients, namespace string) []interface{} {
deployments, err := clients.KubernetesClient.ExtensionsV1beta1().Deployments(namespace).List(meta_v1.ListOptions{})
deployments, err := clients.KubernetesClient.AppsV1().Deployments(namespace).List(meta_v1.ListOptions{})
if err != nil {
logrus.Errorf("Failed to list deployments %v", err)
}
@@ -48,7 +47,7 @@ func GetDeploymentItems(clients kube.Clients, namespace string) []interface{} {
// GetDaemonSetItems returns the daemonSets in given namespace
func GetDaemonSetItems(clients kube.Clients, namespace string) []interface{} {
daemonSets, err := clients.KubernetesClient.ExtensionsV1beta1().DaemonSets(namespace).List(meta_v1.ListOptions{})
daemonSets, err := clients.KubernetesClient.AppsV1().DaemonSets(namespace).List(meta_v1.ListOptions{})
if err != nil {
logrus.Errorf("Failed to list daemonSets %v", err)
}
@@ -57,7 +56,7 @@ func GetDaemonSetItems(clients kube.Clients, namespace string) []interface{} {
// GetStatefulSetItems returns the statefulSets in given namespace
func GetStatefulSetItems(clients kube.Clients, namespace string) []interface{} {
statefulSets, err := clients.KubernetesClient.AppsV1beta1().StatefulSets(namespace).List(meta_v1.ListOptions{})
statefulSets, err := clients.KubernetesClient.AppsV1().StatefulSets(namespace).List(meta_v1.ListOptions{})
if err != nil {
logrus.Errorf("Failed to list statefulSets %v", err)
}
@@ -75,17 +74,17 @@ func GetDeploymentConfigItems(clients kube.Clients, namespace string) []interfac
// GetDeploymentContainers returns the containers of given deployment
func GetDeploymentContainers(item interface{}) []v1.Container {
return item.(v1beta1.Deployment).Spec.Template.Spec.Containers
return item.(appsv1.Deployment).Spec.Template.Spec.Containers
}
// GetDaemonSetContainers returns the containers of given daemonset
func GetDaemonSetContainers(item interface{}) []v1.Container {
return item.(v1beta1.DaemonSet).Spec.Template.Spec.Containers
return item.(appsv1.DaemonSet).Spec.Template.Spec.Containers
}
// GetStatefulsetContainers returns the containers of given statefulSet
func GetStatefulsetContainers(item interface{}) []v1.Container {
return item.(apps_v1beta1.StatefulSet).Spec.Template.Spec.Containers
return item.(appsv1.StatefulSet).Spec.Template.Spec.Containers
}
// GetDeploymentConfigContainers returns the containers of given deploymentConfig
@@ -95,17 +94,17 @@ func GetDeploymentConfigContainers(item interface{}) []v1.Container {
// GetDeploymentInitContainers returns the containers of given deployment
func GetDeploymentInitContainers(item interface{}) []v1.Container {
return item.(v1beta1.Deployment).Spec.Template.Spec.InitContainers
return item.(appsv1.Deployment).Spec.Template.Spec.InitContainers
}
// GetDaemonSetInitContainers returns the containers of given daemonset
func GetDaemonSetInitContainers(item interface{}) []v1.Container {
return item.(v1beta1.DaemonSet).Spec.Template.Spec.InitContainers
return item.(appsv1.DaemonSet).Spec.Template.Spec.InitContainers
}
// GetStatefulsetInitContainers returns the containers of given statefulSet
func GetStatefulsetInitContainers(item interface{}) []v1.Container {
return item.(apps_v1beta1.StatefulSet).Spec.Template.Spec.InitContainers
return item.(appsv1.StatefulSet).Spec.Template.Spec.InitContainers
}
// GetDeploymentConfigInitContainers returns the containers of given deploymentConfig
@@ -115,22 +114,22 @@ func GetDeploymentConfigInitContainers(item interface{}) []v1.Container {
// UpdateDeployment performs rolling upgrade on deployment
func UpdateDeployment(clients kube.Clients, namespace string, resource interface{}) error {
deployment := resource.(v1beta1.Deployment)
_, err := clients.KubernetesClient.ExtensionsV1beta1().Deployments(namespace).Update(&deployment)
deployment := resource.(appsv1.Deployment)
_, err := clients.KubernetesClient.AppsV1().Deployments(namespace).Update(&deployment)
return err
}
// UpdateDaemonSet performs rolling upgrade on daemonSet
func UpdateDaemonSet(clients kube.Clients, namespace string, resource interface{}) error {
daemonSet := resource.(v1beta1.DaemonSet)
_, err := clients.KubernetesClient.ExtensionsV1beta1().DaemonSets(namespace).Update(&daemonSet)
daemonSet := resource.(appsv1.DaemonSet)
_, err := clients.KubernetesClient.AppsV1().DaemonSets(namespace).Update(&daemonSet)
return err
}
// UpdateStatefulset performs rolling upgrade on statefulSet
func UpdateStatefulset(clients kube.Clients, namespace string, resource interface{}) error {
statefulSet := resource.(apps_v1beta1.StatefulSet)
_, err := clients.KubernetesClient.AppsV1beta1().StatefulSets(namespace).Update(&statefulSet)
statefulSet := resource.(appsv1.StatefulSet)
_, err := clients.KubernetesClient.AppsV1().StatefulSets(namespace).Update(&statefulSet)
return err
}
@@ -143,17 +142,17 @@ func UpdateDeploymentConfig(clients kube.Clients, namespace string, resource int
// GetDeploymentVolumes returns the Volumes of given deployment
func GetDeploymentVolumes(item interface{}) []v1.Volume {
return item.(v1beta1.Deployment).Spec.Template.Spec.Volumes
return item.(appsv1.Deployment).Spec.Template.Spec.Volumes
}
// GetDaemonSetVolumes returns the Volumes of given daemonset
func GetDaemonSetVolumes(item interface{}) []v1.Volume {
return item.(v1beta1.DaemonSet).Spec.Template.Spec.Volumes
return item.(appsv1.DaemonSet).Spec.Template.Spec.Volumes
}
// GetStatefulsetVolumes returns the Volumes of given statefulSet
func GetStatefulsetVolumes(item interface{}) []v1.Volume {
return item.(apps_v1beta1.StatefulSet).Spec.Template.Spec.Volumes
return item.(appsv1.StatefulSet).Spec.Template.Spec.Volumes
}
// GetDeploymentConfigVolumes returns the Volumes of given deploymentConfig

154
internal/pkg/testutil/kube.go
View File

@@ -16,9 +16,8 @@ import (
"github.com/stakater/Reloader/internal/pkg/options"
"github.com/stakater/Reloader/internal/pkg/util"
"github.com/stakater/Reloader/pkg/kube"
v1_beta1 "k8s.io/api/apps/v1beta1"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/api/extensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
core_v1 "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -272,14 +271,17 @@ func getPodTemplateSpecWithInitContainerAndEnv(name string) v1.PodTemplateSpec {
}
// GetDeployment provides deployment for testing
func GetDeployment(namespace string, deploymentName string) *v1beta1.Deployment {
func GetDeployment(namespace string, deploymentName string) *appsv1.Deployment {
replicaset := int32(1)
return &v1beta1.Deployment{
return &appsv1.Deployment{
ObjectMeta: getObjectMeta(namespace, deploymentName, false),
Spec: v1beta1.DeploymentSpec{
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"secondLabel": "temp"},
},
Replicas: &replicaset,
Strategy: v1beta1.DeploymentStrategy{
Type: v1beta1.RollingUpdateDeploymentStrategyType,
Strategy: appsv1.DeploymentStrategy{
Type: appsv1.RollingUpdateDeploymentStrategyType,
},
Template: getPodTemplateSpecWithVolumes(deploymentName),
},
@@ -303,14 +305,17 @@ func GetDeploymentConfig(namespace string, deploymentConfigName string) *openshi
}
// GetDeploymentWithInitContainer provides deployment with init container and volumeMounts
func GetDeploymentWithInitContainer(namespace string, deploymentName string) *v1beta1.Deployment {
func GetDeploymentWithInitContainer(namespace string, deploymentName string) *appsv1.Deployment {
replicaset := int32(1)
return &v1beta1.Deployment{
return &appsv1.Deployment{
ObjectMeta: getObjectMeta(namespace, deploymentName, false),
Spec: v1beta1.DeploymentSpec{
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"secondLabel": "temp"},
},
Replicas: &replicaset,
Strategy: v1beta1.DeploymentStrategy{
Type: v1beta1.RollingUpdateDeploymentStrategyType,
Strategy: appsv1.DeploymentStrategy{
Type: appsv1.RollingUpdateDeploymentStrategyType,
},
Template: getPodTemplateSpecWithInitContainer(deploymentName),
},
@@ -318,28 +323,34 @@ func GetDeploymentWithInitContainer(namespace string, deploymentName string) *v1
}
// GetDeploymentWithInitContainerAndEnv provides deployment with init container and EnvSource
func GetDeploymentWithInitContainerAndEnv(namespace string, deploymentName string) *v1beta1.Deployment {
func GetDeploymentWithInitContainerAndEnv(namespace string, deploymentName string) *appsv1.Deployment {
replicaset := int32(1)
return &v1beta1.Deployment{
return &appsv1.Deployment{
ObjectMeta: getObjectMeta(namespace, deploymentName, true),
Spec: v1beta1.DeploymentSpec{
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"secondLabel": "temp"},
},
Replicas: &replicaset,
Strategy: v1beta1.DeploymentStrategy{
Type: v1beta1.RollingUpdateDeploymentStrategyType,
Strategy: appsv1.DeploymentStrategy{
Type: appsv1.RollingUpdateDeploymentStrategyType,
},
Template: getPodTemplateSpecWithInitContainerAndEnv(deploymentName),
},
}
}
func GetDeploymentWithEnvVars(namespace string, deploymentName string) *v1beta1.Deployment {
func GetDeploymentWithEnvVars(namespace string, deploymentName string) *appsv1.Deployment {
replicaset := int32(1)
return &v1beta1.Deployment{
return &appsv1.Deployment{
ObjectMeta: getObjectMeta(namespace, deploymentName, true),
Spec: v1beta1.DeploymentSpec{
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"secondLabel": "temp"},
},
Replicas: &replicaset,
Strategy: v1beta1.DeploymentStrategy{
Type: v1beta1.RollingUpdateDeploymentStrategyType,
Strategy: appsv1.DeploymentStrategy{
Type: appsv1.RollingUpdateDeploymentStrategyType,
},
Template: getPodTemplateSpecWithEnvVars(deploymentName),
},
@@ -361,14 +372,17 @@ func GetDeploymentConfigWithEnvVars(namespace string, deploymentConfigName strin
}
}
func GetDeploymentWithEnvVarSources(namespace string, deploymentName string) *v1beta1.Deployment {
func GetDeploymentWithEnvVarSources(namespace string, deploymentName string) *appsv1.Deployment {
replicaset := int32(1)
return &v1beta1.Deployment{
return &appsv1.Deployment{
ObjectMeta: getObjectMeta(namespace, deploymentName, true),
Spec: v1beta1.DeploymentSpec{
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"secondLabel": "temp"},
},
Replicas: &replicaset,
Strategy: v1beta1.DeploymentStrategy{
Type: v1beta1.RollingUpdateDeploymentStrategyType,
Strategy: appsv1.DeploymentStrategy{
Type: appsv1.RollingUpdateDeploymentStrategyType,
},
Template: getPodTemplateSpecWithEnvVarSources(deploymentName),
},
@@ -376,24 +390,30 @@ func GetDeploymentWithEnvVarSources(namespace string, deploymentName string) *v1
}
// GetDaemonSet provides daemonset for testing
func GetDaemonSet(namespace string, daemonsetName string) *v1beta1.DaemonSet {
return &v1beta1.DaemonSet{
func GetDaemonSet(namespace string, daemonsetName string) *appsv1.DaemonSet {
return &appsv1.DaemonSet{
ObjectMeta: getObjectMeta(namespace, daemonsetName, false),
Spec: v1beta1.DaemonSetSpec{
UpdateStrategy: v1beta1.DaemonSetUpdateStrategy{
Type: v1beta1.RollingUpdateDaemonSetStrategyType,
Spec: appsv1.DaemonSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"secondLabel": "temp"},
},
UpdateStrategy: appsv1.DaemonSetUpdateStrategy{
Type: appsv1.RollingUpdateDaemonSetStrategyType,
},
Template: getPodTemplateSpecWithVolumes(daemonsetName),
},
}
}
func GetDaemonSetWithEnvVars(namespace string, daemonSetName string) *v1beta1.DaemonSet {
return &v1beta1.DaemonSet{
func GetDaemonSetWithEnvVars(namespace string, daemonSetName string) *appsv1.DaemonSet {
return &appsv1.DaemonSet{
ObjectMeta: getObjectMeta(namespace, daemonSetName, true),
Spec: v1beta1.DaemonSetSpec{
UpdateStrategy: v1beta1.DaemonSetUpdateStrategy{
Type: v1beta1.RollingUpdateDaemonSetStrategyType,
Spec: appsv1.DaemonSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"secondLabel": "temp"},
},
UpdateStrategy: appsv1.DaemonSetUpdateStrategy{
Type: appsv1.RollingUpdateDaemonSetStrategyType,
},
Template: getPodTemplateSpecWithEnvVars(daemonSetName),
},
@@ -401,12 +421,15 @@ func GetDaemonSetWithEnvVars(namespace string, daemonSetName string) *v1beta1.Da
}
// GetStatefulSet provides statefulset for testing
func GetStatefulSet(namespace string, statefulsetName string) *v1_beta1.StatefulSet {
return &v1_beta1.StatefulSet{
func GetStatefulSet(namespace string, statefulsetName string) *appsv1.StatefulSet {
return &appsv1.StatefulSet{
ObjectMeta: getObjectMeta(namespace, statefulsetName, false),
Spec: v1_beta1.StatefulSetSpec{
UpdateStrategy: v1_beta1.StatefulSetUpdateStrategy{
Type: v1_beta1.RollingUpdateStatefulSetStrategyType,
Spec: appsv1.StatefulSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"secondLabel": "temp"},
},
UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
Type: appsv1.RollingUpdateStatefulSetStrategyType,
},
Template: getPodTemplateSpecWithVolumes(statefulsetName),
},
@@ -414,12 +437,15 @@ func GetStatefulSet(namespace string, statefulsetName string) *v1_beta1.Stateful
}
// GetStatefulSet provides statefulset for testing
func GetStatefulSetWithEnvVar(namespace string, statefulsetName string) *v1_beta1.StatefulSet {
return &v1_beta1.StatefulSet{
func GetStatefulSetWithEnvVar(namespace string, statefulsetName string) *appsv1.StatefulSet {
return &appsv1.StatefulSet{
ObjectMeta: getObjectMeta(namespace, statefulsetName, true),
Spec: v1_beta1.StatefulSetSpec{
UpdateStrategy: v1_beta1.StatefulSetUpdateStrategy{
Type: v1_beta1.RollingUpdateStatefulSetStrategyType,
Spec: appsv1.StatefulSetSpec{
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"secondLabel": "temp"},
},
UpdateStrategy: appsv1.StatefulSetUpdateStrategy{
Type: appsv1.RollingUpdateStatefulSetStrategyType,
},
Template: getPodTemplateSpecWithEnvVars(statefulsetName),
},
@@ -524,10 +550,10 @@ func CreateSecret(client kubernetes.Interface, namespace string, secretName stri
}
// CreateDeployment creates a deployment in given namespace and returns the Deployment
func CreateDeployment(client kubernetes.Interface, deploymentName string, namespace string, volumeMount bool) (*v1beta1.Deployment, error) {
func CreateDeployment(client kubernetes.Interface, deploymentName string, namespace string, volumeMount bool) (*appsv1.Deployment, error) {
logrus.Infof("Creating Deployment")
deploymentClient := client.ExtensionsV1beta1().Deployments(namespace)
var deploymentObj *v1beta1.Deployment
deploymentClient := client.AppsV1().Deployments(namespace)
var deploymentObj *appsv1.Deployment
if volumeMount {
deploymentObj = GetDeployment(namespace, deploymentName)
} else {
@@ -554,10 +580,10 @@ func CreateDeploymentConfig(client appsclient.Interface, deploymentName string,
}
// CreateDeploymentWithInitContainer creates a deployment in given namespace with init container and returns the Deployment
func CreateDeploymentWithInitContainer(client kubernetes.Interface, deploymentName string, namespace string, volumeMount bool) (*v1beta1.Deployment, error) {
func CreateDeploymentWithInitContainer(client kubernetes.Interface, deploymentName string, namespace string, volumeMount bool) (*appsv1.Deployment, error) {
logrus.Infof("Creating Deployment")
deploymentClient := client.ExtensionsV1beta1().Deployments(namespace)
var deploymentObj *v1beta1.Deployment
deploymentClient := client.AppsV1().Deployments(namespace)
var deploymentObj *appsv1.Deployment
if volumeMount {
deploymentObj = GetDeploymentWithInitContainer(namespace, deploymentName)
} else {
@@ -569,9 +595,9 @@ func CreateDeploymentWithInitContainer(client kubernetes.Interface, deploymentNa
}
// CreateDeploymentWithEnvVarSource creates a deployment in given namespace and returns the Deployment
func CreateDeploymentWithEnvVarSource(client kubernetes.Interface, deploymentName string, namespace string) (*v1beta1.Deployment, error) {
func CreateDeploymentWithEnvVarSource(client kubernetes.Interface, deploymentName string, namespace string) (*appsv1.Deployment, error) {
logrus.Infof("Creating Deployment")
deploymentClient := client.ExtensionsV1beta1().Deployments(namespace)
deploymentClient := client.AppsV1().Deployments(namespace)
deploymentObj := GetDeploymentWithEnvVarSources(namespace, deploymentName)
deployment, err := deploymentClient.Create(deploymentObj)
time.Sleep(3 * time.Second)
@@ -579,10 +605,10 @@ func CreateDeploymentWithEnvVarSource(client kubernetes.Interface, deploymentNam
}
// CreateDaemonSet creates a deployment in given namespace and returns the DaemonSet
func CreateDaemonSet(client kubernetes.Interface, daemonsetName string, namespace string, volumeMount bool) (*v1beta1.DaemonSet, error) {
func CreateDaemonSet(client kubernetes.Interface, daemonsetName string, namespace string, volumeMount bool) (*appsv1.DaemonSet, error) {
logrus.Infof("Creating DaemonSet")
daemonsetClient := client.ExtensionsV1beta1().DaemonSets(namespace)
var daemonsetObj *v1beta1.DaemonSet
daemonsetClient := client.AppsV1().DaemonSets(namespace)
var daemonsetObj *appsv1.DaemonSet
if volumeMount {
daemonsetObj = GetDaemonSet(namespace, daemonsetName)
} else {
@@ -594,10 +620,10 @@ func CreateDaemonSet(client kubernetes.Interface, daemonsetName string, namespac
}
// CreateStatefulSet creates a deployment in given namespace and returns the StatefulSet
func CreateStatefulSet(client kubernetes.Interface, statefulsetName string, namespace string, volumeMount bool) (*v1_beta1.StatefulSet, error) {
func CreateStatefulSet(client kubernetes.Interface, statefulsetName string, namespace string, volumeMount bool) (*appsv1.StatefulSet, error) {
logrus.Infof("Creating StatefulSet")
statefulsetClient := client.AppsV1beta1().StatefulSets(namespace)
var statefulsetObj *v1_beta1.StatefulSet
statefulsetClient := client.AppsV1().StatefulSets(namespace)
var statefulsetObj *appsv1.StatefulSet
if volumeMount {
statefulsetObj = GetStatefulSet(namespace, statefulsetName)
} else {
@@ -611,7 +637,7 @@ func CreateStatefulSet(client kubernetes.Interface, statefulsetName string, name
// DeleteDeployment creates a deployment in given namespace and returns the error if any
func DeleteDeployment(client kubernetes.Interface, namespace string, deploymentName string) error {
logrus.Infof("Deleting Deployment")
deploymentError := client.ExtensionsV1beta1().Deployments(namespace).Delete(deploymentName, &metav1.DeleteOptions{})
deploymentError := client.AppsV1().Deployments(namespace).Delete(deploymentName, &metav1.DeleteOptions{})
time.Sleep(3 * time.Second)
return deploymentError
}
@@ -627,7 +653,7 @@ func DeleteDeploymentConfig(client appsclient.Interface, namespace string, deplo
// DeleteDaemonSet creates a daemonset in given namespace and returns the error if any
func DeleteDaemonSet(client kubernetes.Interface, namespace string, daemonsetName string) error {
logrus.Infof("Deleting DaemonSet %s", daemonsetName)
daemonsetError := client.ExtensionsV1beta1().DaemonSets(namespace).Delete(daemonsetName, &metav1.DeleteOptions{})
daemonsetError := client.AppsV1().DaemonSets(namespace).Delete(daemonsetName, &metav1.DeleteOptions{})
time.Sleep(3 * time.Second)
return daemonsetError
}
@@ -635,7 +661,7 @@ func DeleteDaemonSet(client kubernetes.Interface, namespace string, daemonsetNam
// DeleteStatefulSet creates a statefulset in given namespace and returns the error if any
func DeleteStatefulSet(client kubernetes.Interface, namespace string, statefulsetName string) error {
logrus.Infof("Deleting StatefulSet %s", statefulsetName)
statefulsetError := client.AppsV1beta1().StatefulSets(namespace).Delete(statefulsetName, &metav1.DeleteOptions{})
statefulsetError := client.AppsV1().StatefulSets(namespace).Delete(statefulsetName, &metav1.DeleteOptions{})
time.Sleep(3 * time.Second)
return statefulsetError
}