From 6fb1266637220809bfaf6e4505b77373e929c12b Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Sat, 16 Feb 2019 15:43:56 +0100 Subject: [PATCH 1/5] Initial implementation for init container support Signed-off-by: faizanahmad055 --- internal/pkg/callbacks/rolling_upgrade.go | 31 +++- internal/pkg/handler/upgrade.go | 181 +++++++++++++--------- internal/pkg/handler/upgrade_test.go | 102 ++++++++++-- internal/pkg/testutil/kube.go | 84 ++++++++++ 4 files changed, 306 insertions(+), 92 deletions(-) diff --git a/internal/pkg/callbacks/rolling_upgrade.go b/internal/pkg/callbacks/rolling_upgrade.go index 4d487546..803ce78e 100644 --- a/internal/pkg/callbacks/rolling_upgrade.go +++ b/internal/pkg/callbacks/rolling_upgrade.go @@ -4,7 +4,7 @@ import ( "github.com/sirupsen/logrus" "github.com/stakater/Reloader/internal/pkg/util" apps_v1beta1 "k8s.io/api/apps/v1beta1" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/api/extensions/v1beta1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -16,6 +16,9 @@ type ItemsFunc func(kubernetes.Interface, string) []interface{} //ContainersFunc is a generic func to return containers type ContainersFunc func(interface{}) []v1.Container +//InitContainersFunc is a generic func to return containers +type InitContainersFunc func(interface{}) []v1.Container + //VolumesFunc is a generic func to return volumes type VolumesFunc func(interface{}) []v1.Volume @@ -24,11 +27,12 @@ type UpdateFunc func(kubernetes.Interface, string, interface{}) error //RollingUpgradeFuncs contains generic functions to perform rolling upgrade type RollingUpgradeFuncs struct { - ItemsFunc ItemsFunc - ContainersFunc ContainersFunc - UpdateFunc UpdateFunc - VolumesFunc VolumesFunc - ResourceType string + ItemsFunc ItemsFunc + ContainersFunc ContainersFunc + InitContainersFunc InitContainersFunc + UpdateFunc UpdateFunc + VolumesFunc VolumesFunc + ResourceType string } // GetDeploymentItems returns the deployments in given namespace @@ -73,6 +77,21 @@ func GetStatefulsetContainers(item interface{}) []v1.Container { return item.(apps_v1beta1.StatefulSet).Spec.Template.Spec.Containers } +// GetDeploymentInitContainers returns the containers of given deployment +func GetDeploymentInitContainers(item interface{}) []v1.Container { + return item.(v1beta1.Deployment).Spec.Template.Spec.InitContainers +} + +// GetDaemonSetInitContainers returns the containers of given daemonset +func GetDaemonSetInitContainers(item interface{}) []v1.Container { + return item.(v1beta1.DaemonSet).Spec.Template.Spec.InitContainers +} + +// GetStatefulsetInitContainers returns the containers of given statefulSet +func GetStatefulsetInitContainers(item interface{}) []v1.Container { + return item.(apps_v1beta1.StatefulSet).Spec.Template.Spec.InitContainers +} + // UpdateDeployment performs rolling upgrade on deployment func UpdateDeployment(client kubernetes.Interface, namespace string, resource interface{}) error { deployment := resource.(v1beta1.Deployment) diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index 5ade0792..c7d6ae48 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -17,33 +17,36 @@ import ( // GetDeploymentRollingUpgradeFuncs returns all callback funcs for a deployment func GetDeploymentRollingUpgradeFuncs() callbacks.RollingUpgradeFuncs { return callbacks.RollingUpgradeFuncs{ - ItemsFunc: callbacks.GetDeploymentItems, - ContainersFunc: callbacks.GetDeploymentContainers, - UpdateFunc: callbacks.UpdateDeployment, - VolumesFunc: callbacks.GetDeploymentVolumes, - ResourceType: "Deployment", + ItemsFunc: callbacks.GetDeploymentItems, + ContainersFunc: callbacks.GetDeploymentContainers, + InitContainersFunc: callbacks.GetDeploymentInitContainers, + UpdateFunc: callbacks.UpdateDeployment, + VolumesFunc: callbacks.GetDeploymentVolumes, + ResourceType: "Deployment", } } // GetDaemonSetRollingUpgradeFuncs returns all callback funcs for a daemonset func GetDaemonSetRollingUpgradeFuncs() callbacks.RollingUpgradeFuncs { return callbacks.RollingUpgradeFuncs{ - ItemsFunc: callbacks.GetDaemonSetItems, - ContainersFunc: callbacks.GetDaemonSetContainers, - UpdateFunc: callbacks.UpdateDaemonSet, - VolumesFunc: callbacks.GetDaemonSetVolumes, - ResourceType: "DaemonSet", + ItemsFunc: callbacks.GetDaemonSetItems, + ContainersFunc: callbacks.GetDaemonSetContainers, + InitContainersFunc: callbacks.GetDaemonSetInitContainers, + UpdateFunc: callbacks.UpdateDaemonSet, + VolumesFunc: callbacks.GetDaemonSetVolumes, + ResourceType: "DaemonSet", } } // GetStatefulSetRollingUpgradeFuncs returns all callback funcs for a statefulSet func GetStatefulSetRollingUpgradeFuncs() callbacks.RollingUpgradeFuncs { return callbacks.RollingUpgradeFuncs{ - ItemsFunc: callbacks.GetStatefulSetItems, - ContainersFunc: callbacks.GetStatefulsetContainers, - UpdateFunc: callbacks.UpdateStatefulset, - VolumesFunc: callbacks.GetStatefulsetVolumes, - ResourceType: "StatefulSet", + ItemsFunc: callbacks.GetStatefulSetItems, + ContainersFunc: callbacks.GetStatefulsetContainers, + InitContainersFunc: callbacks.GetStatefulsetInitContainers, + UpdateFunc: callbacks.UpdateStatefulset, + VolumesFunc: callbacks.GetStatefulsetVolumes, + ResourceType: "StatefulSet", } } @@ -70,35 +73,31 @@ func PerformRollingUpgrade(client kubernetes.Interface, config util.Config, upgr items := upgradeFuncs.ItemsFunc(client, config.Namespace) var err error for _, i := range items { - containers := upgradeFuncs.ContainersFunc(i) - volumes := upgradeFuncs.VolumesFunc(i) // find correct annotation and update the resource annotationValue := util.ToObjectMeta(i).Annotations[config.Annotation] reloaderEnabledValue := util.ToObjectMeta(i).Annotations[options.ReloaderAutoAnnotation] - if len(containers) > 0 { - resourceName := util.ToObjectMeta(i).Name - result := constants.NotUpdated - reloaderEnabled, err := strconv.ParseBool(reloaderEnabledValue) - if err == nil && reloaderEnabled { - result = updateContainers(volumes, containers, config.ResourceName, config) - } else if annotationValue != "" { - values := strings.Split(annotationValue, ",") - for _, value := range values { - if value == config.ResourceName { - result = updateContainers(volumes, containers, value, config) - if result == constants.Updated { - break - } + result := constants.NotUpdated + reloaderEnabled, err := strconv.ParseBool(reloaderEnabledValue) + if err == nil && reloaderEnabled { + result = updateContainers(upgradeFuncs, i, config) + } else if annotationValue != "" { + values := strings.Split(annotationValue, ",") + for _, value := range values { + if value == config.ResourceName { + result = updateContainers(upgradeFuncs, i, config) + if result == constants.Updated { + break } } } - if result == constants.Updated { - err = upgradeFuncs.UpdateFunc(client, config.Namespace, i) - if err != nil { - logrus.Errorf("Update for '%s' of type '%s' in namespace '%s' failed with error %v", resourceName, upgradeFuncs.ResourceType, config.Namespace, err) - } else { - logrus.Infof("Updated '%s' of type '%s' in namespace '%s'", resourceName, upgradeFuncs.ResourceType, config.Namespace) - } + } + if result == constants.Updated { + err = upgradeFuncs.UpdateFunc(client, config.Namespace, i) + resourceName := util.ToObjectMeta(i).Name + if err != nil { + logrus.Errorf("Update for '%s' of type '%s' in namespace '%s' failed with error %v", resourceName, upgradeFuncs.ResourceType, config.Namespace, err) + } else { + logrus.Infof("Updated '%s' of type '%s' in namespace '%s'", resourceName, upgradeFuncs.ResourceType, config.Namespace) } } } @@ -116,42 +115,11 @@ func getVolumeMountName(volumes []v1.Volume, mountType string, volumeName string return "" } -func getContainerToUpdate(volumes []v1.Volume, containers []v1.Container, envarPostfix string, volumeName string) *v1.Container { - // Get the volumeMountName to find volumeMount in container - if len(volumes) > 0 { - volumeMountName := getVolumeMountName(volumes, envarPostfix, volumeName) - // Get the container with mounted configmap/secret - if volumeMountName != "" { - for i := range containers { - volumeMounts := containers[i].VolumeMounts - for j := range volumeMounts { - if volumeMounts[j].Name == volumeMountName { - return &containers[i] - } - } - } - } - } - - // Get the container with referenced secret or configmap as env var +func getContainerWithVolumeMount(volumes []v1.Volume, containers []v1.Container, volumeMountName string) *v1.Container { for i := range containers { - envs := containers[i].Env - for j := range envs { - envVarSource := envs[j].ValueFrom - if envVarSource != nil { - if envVarSource.SecretKeyRef != nil && envVarSource.SecretKeyRef.LocalObjectReference.Name == volumeName { - return &containers[i] - } else if envVarSource.ConfigMapKeyRef != nil && envVarSource.ConfigMapKeyRef.LocalObjectReference.Name == volumeName { - return &containers[i] - } - } - } - - envsFrom := containers[i].EnvFrom - for j := range envsFrom { - if envsFrom[j].SecretRef != nil && envsFrom[j].SecretRef.LocalObjectReference.Name == volumeName { - return &containers[i] - } else if envsFrom[j].ConfigMapRef != nil && envsFrom[j].ConfigMapRef.LocalObjectReference.Name == volumeName { + volumeMounts := containers[i].VolumeMounts + for j := range volumeMounts { + if volumeMounts[j].Name == volumeMountName { return &containers[i] } } @@ -160,17 +128,76 @@ func getContainerToUpdate(volumes []v1.Volume, containers []v1.Container, envarP return nil } -func updateContainers(volumes []v1.Volume, containers []v1.Container, annotationValue string, config util.Config) constants.Result { +func getContainerWithEnvReference(containers []v1.Container, resourceName string) *v1.Container { + for i := range containers { + envs := containers[i].Env + for j := range envs { + envVarSource := envs[j].ValueFrom + if envVarSource != nil { + if envVarSource.SecretKeyRef != nil && envVarSource.SecretKeyRef.LocalObjectReference.Name == resourceName{ + return &containers[i] + } else if envVarSource.ConfigMapKeyRef != nil && envVarSource.ConfigMapKeyRef.LocalObjectReference.Name == resourceName { + return &containers[i] + } + } + } + + envsFrom := containers[i].EnvFrom + for j := range envsFrom { + if envsFrom[j].SecretRef != nil && envsFrom[j].SecretRef.LocalObjectReference.Name == resourceName { + return &containers[i] + } else if envsFrom[j].ConfigMapRef != nil && envsFrom[j].ConfigMapRef.LocalObjectReference.Name == resourceName { + return &containers[i] + } + } + } + return nil +} + +func getContainerToUpdate(upgradeFuncs callbacks.RollingUpgradeFuncs, item interface{}, config util.Config) *v1.Container { + volumes := upgradeFuncs.VolumesFunc(item) + containers := upgradeFuncs.ContainersFunc(item) + initContainers := upgradeFuncs.InitContainersFunc(item) + var container *v1.Container + // Get the volumeMountName to find volumeMount in container + volumeMountName := getVolumeMountName(volumes, config.Type, config.ResourceName) + // Get the container with mounted configmap/secret + if volumeMountName != "" { + container = getContainerWithVolumeMount(volumes, containers, volumeMountName) + if container == nil && len(initContainers) > 0 { + container = getContainerWithVolumeMount(volumes, initContainers, volumeMountName) + if container != nil { + // if configmap/secret is being used in init container then return the first Pod container to save reloader env + return &containers[0] + } + } else if container != nil { + return container + } + } + + // Get the container with referenced secret or configmap as env var + container = getContainerWithEnvReference(containers, config.ResourceName) + if container == nil && len(initContainers) > 0 { + container = getContainerWithEnvReference(initContainers, config.ResourceName) + if container == nil { + // if configmap/secret is being used in init container then return the first Pod container to save reloader env + return &containers[0] + } + } + return container +} + +func updateContainers(upgradeFuncs callbacks.RollingUpgradeFuncs, item interface{}, config util.Config) constants.Result { var result constants.Result - envar := constants.EnvVarPrefix + util.ConvertToEnvVarName(annotationValue) + "_" + config.Type - container := getContainerToUpdate(volumes, containers, config.Type, config.ResourceName) + envar := constants.EnvVarPrefix + util.ConvertToEnvVarName(config.ResourceName) + "_" + config.Type + container := getContainerToUpdate(upgradeFuncs, item, config) if container == nil { return constants.NoContainerFound } //update if env var exists - result = updateEnvVar(containers, envar, config.SHAValue) + result = updateEnvVar(upgradeFuncs.ContainersFunc(item), envar, config.SHAValue) // if no existing env var exists lets create one if result == constants.NoEnvVarFound { diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index 6e2439c6..38d5a850 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -14,14 +14,16 @@ import ( ) var ( - client = testclient.NewSimpleClientset() - namespace = "test-handler-" + testutil.RandSeq(5) - configmapName = "testconfigmap-handler-" + testutil.RandSeq(5) - secretName = "testsecret-handler-" + testutil.RandSeq(5) - configmapWithEnvName = "testconfigmapWithEnv-handler-" + testutil.RandSeq(3) - configmapWithEnvFromName = "testconfigmapWithEnvFrom-handler-" + testutil.RandSeq(3) - secretWithEnvName = "testsecretWithEnv-handler-" + testutil.RandSeq(5) - secretWithEnvFromName = "testsecretWithEnvFrom-handler-" + testutil.RandSeq(5) + client = testclient.NewSimpleClientset() + namespace = "test-handler-" + testutil.RandSeq(5) + configmapName = "testconfigmap-handler-" + testutil.RandSeq(5) + secretName = "testsecret-handler-" + testutil.RandSeq(5) + configmapWithInitContainer = "testconfigmapInitContainerhandler-" + testutil.RandSeq(3) + secretWithInitContainer = "testsecretWithInitContainer-handler-" + testutil.RandSeq(3) + configmapWithEnvName = "testconfigmapWithEnv-handler-" + testutil.RandSeq(3) + configmapWithEnvFromName = "testconfigmapWithEnvFrom-handler-" + testutil.RandSeq(3) + secretWithEnvName = "testsecretWithEnv-handler-" + testutil.RandSeq(5) + secretWithEnvFromName = "testsecretWithEnvFrom-handler-" + testutil.RandSeq(5) ) func TestMain(m *testing.M) { @@ -77,12 +79,35 @@ func setup() { logrus.Errorf("Error in secret creation: %v", err) } + _, err = testutil.CreateConfigMap(client, namespace, configmapWithInitContainer, "www.google.com") + if err != nil { + logrus.Errorf("Error in configmap creation: %v", err) + } + + // Creating secret + _, err = testutil.CreateSecret(client, namespace, secretWithInitContainer, data) + if err != nil { + logrus.Errorf("Error in secret creation: %v", err) + } + // Creating Deployment with configmap _, err = testutil.CreateDeployment(client, configmapName, namespace, true) if err != nil { logrus.Errorf("Error in Deployment with configmap creation: %v", err) } + // Creating Deployment with configmap mounted in init container + _, err = testutil.CreateDeploymentWithInitContainer(client, configmapWithInitContainer, namespace) + if err != nil { + logrus.Errorf("Error in Deployment with configmap creation: %v", err) + } + + // Creating Deployment with secret mounted in init container + _, err = testutil.CreateDeploymentWithInitContainer(client, secretWithInitContainer, namespace) + if err != nil { + logrus.Errorf("Error in Deployment with secret creation: %v", err) + } + // Creating Deployment with secret _, err = testutil.CreateDeployment(client, secretName, namespace, true) if err != nil { @@ -188,6 +213,18 @@ func teardown() { logrus.Errorf("Error while deleting deployment with secret as env var source %v", deploymentError) } + // Deleting Deployment with configmap mounted in init container + deploymentError = testutil.DeleteDeployment(client, namespace, configmapWithInitContainer) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with configmap mounted in init container %v", deploymentError) + } + + // Deleting Deployment with secret mounted in init container + deploymentError = testutil.DeleteDeployment(client, namespace, secretWithInitContainer) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secret mounted in init container %v", deploymentError) + } + // Deleting Deployment with configmap as envFrom source deploymentError = testutil.DeleteDeployment(client, namespace, configmapWithEnvFromName) if deploymentError != nil { @@ -272,6 +309,18 @@ func teardown() { logrus.Errorf("Error while deleting the secret used as env var source %v", err) } + // Deleting Configmap used in init container + err = testutil.DeleteConfigMap(client, namespace, configmapWithInitContainer) + if err != nil { + logrus.Errorf("Error while deleting the configmap used in init container %v", err) + } + + // Deleting Secret used in init container + err = testutil.DeleteSecret(client, namespace, secretWithInitContainer) + if err != nil { + logrus.Errorf("Error while deleting the secret used in init container %v", err) + } + // Deleting Configmap used as env var source err = testutil.DeleteConfigMap(client, namespace, configmapWithEnvFromName) if err != nil { @@ -297,7 +346,6 @@ func getConfigWithAnnotations(resourceType string, name string, shaData string, Annotation: annotation, Type: resourceType, } - } func TestRollingUpgradeForDeploymentWithConfigmap(t *testing.T) { @@ -318,6 +366,24 @@ func TestRollingUpgradeForDeploymentWithConfigmap(t *testing.T) { } } +func TestRollingUpgradeForDeploymentWithConfigmapInInitContainer(t *testing.T) { + shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, configmapWithInitContainer, "www.stakater.com") + config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, configmapWithInitContainer, shaData, options.ConfigmapUpdateOnChangeAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + + err := PerformRollingUpgrade(client, config, deploymentFuncs) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with Configmap") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceUpdate(client, config, constants.ConfigmapEnvVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } +} + func TestRollingUpgradeForDeploymentWithConfigmapAsEnvVar(t *testing.T) { shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, configmapWithEnvName, "www.stakater.com") config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, configmapWithEnvName, shaData, options.ReloaderAutoAnnotation) @@ -372,6 +438,24 @@ func TestRollingUpgradeForDeploymentWithSecret(t *testing.T) { } } +func TestRollingUpgradeForDeploymentWithSecretinInitContainer(t *testing.T) { + shaData := testutil.ConvertResourceToSHA(testutil.SecretResourceType, namespace, secretWithInitContainer, "dGVzdFVwZGF0ZWRTZWNyZXRFbmNvZGluZ0ZvclJlbG9hZGVy") + config := getConfigWithAnnotations(constants.SecretEnvVarPostfix, secretWithInitContainer, shaData, options.SecretUpdateOnChangeAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + + err := PerformRollingUpgrade(client, config, deploymentFuncs) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with Secret") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceUpdate(client, config, constants.SecretEnvVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } +} + func TestRollingUpgradeForDeploymentWithSecretAsEnvVar(t *testing.T) { shaData := testutil.ConvertResourceToSHA(testutil.SecretResourceType, namespace, secretWithEnvName, "dGVzdFVwZGF0ZWRTZWNyZXRFbmNvZGluZ0ZvclJlbG9hZGVy") config := getConfigWithAnnotations(constants.SecretEnvVarPostfix, secretWithEnvName, shaData, options.ReloaderAutoAnnotation) diff --git a/internal/pkg/testutil/kube.go b/internal/pkg/testutil/kube.go index b48a18c7..6f7ce8e0 100644 --- a/internal/pkg/testutil/kube.go +++ b/internal/pkg/testutil/kube.go @@ -206,6 +206,64 @@ func getPodTemplateSpecWithVolumes(name string) v1.PodTemplateSpec { } } +func getPodTemplateSpecWithInitContainer(name string) v1.PodTemplateSpec { + return v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"secondLabel": "temp"}, + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ + { + Image: "busybox", + Name: "busyBox", + VolumeMounts: []v1.VolumeMount{ + { + MountPath: "etc/config", + Name: "configmap", + }, + { + MountPath: "etc/sec", + Name: "secret", + }, + }, + }, + }, + Containers: []v1.Container{ + { + Image: "tutum/hello-world", + Name: name, + Env: []v1.EnvVar{ + { + Name: "BUCKET_NAME", + Value: "test", + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "configmap", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: name, + }, + }, + }, + }, + { + Name: "secret", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: name, + }, + }, + }, + }, + }, + } +} + // GetDeployment provides deployment for testing func GetDeployment(namespace string, deploymentName string) *v1beta1.Deployment { replicaset := int32(1) @@ -221,6 +279,21 @@ func GetDeployment(namespace string, deploymentName string) *v1beta1.Deployment } } +// GetDeploymentWithInitContainer provides deployment with init container +func GetDeploymentWithInitContainer(namespace string, deploymentName string) *v1beta1.Deployment { + replicaset := int32(1) + return &v1beta1.Deployment{ + ObjectMeta: getObjectMeta(namespace, deploymentName, false), + Spec: v1beta1.DeploymentSpec{ + Replicas: &replicaset, + Strategy: v1beta1.DeploymentStrategy{ + Type: v1beta1.RollingUpdateDeploymentStrategyType, + }, + Template: getPodTemplateSpecWithInitContainer(deploymentName), + }, + } +} + func GetDeploymentWithEnvVars(namespace string, deploymentName string) *v1beta1.Deployment { replicaset := int32(1) return &v1beta1.Deployment{ @@ -412,6 +485,17 @@ func CreateDeployment(client kubernetes.Interface, deploymentName string, namesp return deployment, err } +// CreateDeploymentWithInitContainer creates a deployment in given namespace with init container and returns the Deployment +func CreateDeploymentWithInitContainer(client kubernetes.Interface, deploymentName string, namespace string) (*v1beta1.Deployment, error) { + logrus.Infof("Creating Deployment") + deploymentClient := client.ExtensionsV1beta1().Deployments(namespace) + var deploymentObj *v1beta1.Deployment + deploymentObj = GetDeploymentWithInitContainer(namespace, deploymentName) + deployment, err := deploymentClient.Create(deploymentObj) + time.Sleep(10 * time.Second) + return deployment, err +} + // CreateDeploymentWithEnvVarSource creates a deployment in given namespace and returns the Deployment func CreateDeploymentWithEnvVarSource(client kubernetes.Interface, deploymentName string, namespace string) (*v1beta1.Deployment, error) { logrus.Infof("Creating Deployment") From 39944497f31bfe8f0eda135dcba9b33d7cb8adf1 Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Sat, 16 Feb 2019 15:50:47 +0100 Subject: [PATCH 2/5] Implement golang-ci comment Signed-off-by: faizanahmad055 --- internal/pkg/testutil/kube.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/pkg/testutil/kube.go b/internal/pkg/testutil/kube.go index 6f7ce8e0..296eec04 100644 --- a/internal/pkg/testutil/kube.go +++ b/internal/pkg/testutil/kube.go @@ -489,8 +489,7 @@ func CreateDeployment(client kubernetes.Interface, deploymentName string, namesp func CreateDeploymentWithInitContainer(client kubernetes.Interface, deploymentName string, namespace string) (*v1beta1.Deployment, error) { logrus.Infof("Creating Deployment") deploymentClient := client.ExtensionsV1beta1().Deployments(namespace) - var deploymentObj *v1beta1.Deployment - deploymentObj = GetDeploymentWithInitContainer(namespace, deploymentName) + deploymentObj := GetDeploymentWithInitContainer(namespace, deploymentName) deployment, err := deploymentClient.Create(deploymentObj) time.Sleep(10 * time.Second) return deployment, err From 512cbd8c85faf4f5b0c548b47009f1e3f74c80c6 Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Sat, 16 Feb 2019 16:52:12 +0100 Subject: [PATCH 3/5] Fix incorrect container update Signed-off-by: faizanahmad055 --- internal/pkg/handler/upgrade.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index c7d6ae48..26e69831 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -115,7 +115,7 @@ func getVolumeMountName(volumes []v1.Volume, mountType string, volumeName string return "" } -func getContainerWithVolumeMount(volumes []v1.Volume, containers []v1.Container, volumeMountName string) *v1.Container { +func getContainerWithVolumeMount(containers []v1.Container, volumeMountName string) *v1.Container { for i := range containers { volumeMounts := containers[i].VolumeMounts for j := range volumeMounts { @@ -163,9 +163,9 @@ func getContainerToUpdate(upgradeFuncs callbacks.RollingUpgradeFuncs, item inter volumeMountName := getVolumeMountName(volumes, config.Type, config.ResourceName) // Get the container with mounted configmap/secret if volumeMountName != "" { - container = getContainerWithVolumeMount(volumes, containers, volumeMountName) + container = getContainerWithVolumeMount(containers, volumeMountName) if container == nil && len(initContainers) > 0 { - container = getContainerWithVolumeMount(volumes, initContainers, volumeMountName) + container = getContainerWithVolumeMount(initContainers, volumeMountName) if container != nil { // if configmap/secret is being used in init container then return the first Pod container to save reloader env return &containers[0] @@ -179,7 +179,7 @@ func getContainerToUpdate(upgradeFuncs callbacks.RollingUpgradeFuncs, item inter container = getContainerWithEnvReference(containers, config.ResourceName) if container == nil && len(initContainers) > 0 { container = getContainerWithEnvReference(initContainers, config.ResourceName) - if container == nil { + if container != nil { // if configmap/secret is being used in init container then return the first Pod container to save reloader env return &containers[0] } From 889b16718a777a8a84c7ffb14cf51f025c6ef9bf Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Mon, 18 Feb 2019 10:20:52 +0100 Subject: [PATCH 4/5] Add test cases Signed-off-by: faizanahmad055 --- .../chart/reloader/templates/deployment.yaml | 2 +- internal/pkg/handler/upgrade_test.go | 89 ++++++++- internal/pkg/testutil/kube.go | 177 +++++++++++------- 3 files changed, 193 insertions(+), 75 deletions(-) diff --git a/deployments/kubernetes/chart/reloader/templates/deployment.yaml b/deployments/kubernetes/chart/reloader/templates/deployment.yaml index 53b999ce..fb40b3f0 100644 --- a/deployments/kubernetes/chart/reloader/templates/deployment.yaml +++ b/deployments/kubernetes/chart/reloader/templates/deployment.yaml @@ -1,4 +1,4 @@ -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: Deployment metadata: {{- if .Values.reloader.deployment.annotations }} diff --git a/internal/pkg/handler/upgrade_test.go b/internal/pkg/handler/upgrade_test.go index 38d5a850..9b40806a 100644 --- a/internal/pkg/handler/upgrade_test.go +++ b/internal/pkg/handler/upgrade_test.go @@ -20,6 +20,8 @@ var ( secretName = "testsecret-handler-" + testutil.RandSeq(5) configmapWithInitContainer = "testconfigmapInitContainerhandler-" + testutil.RandSeq(3) secretWithInitContainer = "testsecretWithInitContainer-handler-" + testutil.RandSeq(3) + configmapWithInitEnv = "configmapWithInitEnv-" + testutil.RandSeq(3) + secretWithInitEnv = "secretWithInitEnv-handler-" + testutil.RandSeq(3) configmapWithEnvName = "testconfigmapWithEnv-handler-" + testutil.RandSeq(3) configmapWithEnvFromName = "testconfigmapWithEnvFrom-handler-" + testutil.RandSeq(3) secretWithEnvName = "testsecretWithEnv-handler-" + testutil.RandSeq(5) @@ -73,6 +75,17 @@ func setup() { logrus.Errorf("Error in configmap creation: %v", err) } + // Creating secret + _, err = testutil.CreateSecret(client, namespace, secretWithInitEnv, data) + if err != nil { + logrus.Errorf("Error in secret creation: %v", err) + } + + _, err = testutil.CreateConfigMap(client, namespace, configmapWithInitContainer, "www.google.com") + if err != nil { + logrus.Errorf("Error in configmap creation: %v", err) + } + // Creating secret _, err = testutil.CreateSecret(client, namespace, secretWithEnvFromName, data) if err != nil { @@ -97,13 +110,25 @@ func setup() { } // Creating Deployment with configmap mounted in init container - _, err = testutil.CreateDeploymentWithInitContainer(client, configmapWithInitContainer, namespace) + _, err = testutil.CreateDeploymentWithInitContainer(client, configmapWithInitContainer, namespace, true) if err != nil { logrus.Errorf("Error in Deployment with configmap creation: %v", err) } // Creating Deployment with secret mounted in init container - _, err = testutil.CreateDeploymentWithInitContainer(client, secretWithInitContainer, namespace) + _, err = testutil.CreateDeploymentWithInitContainer(client, secretWithInitContainer, namespace, true) + if err != nil { + logrus.Errorf("Error in Deployment with secret creation: %v", err) + } + + // Creating Deployment with configmap mounted as Env in init container + _, err = testutil.CreateDeploymentWithInitContainer(client, configmapWithInitEnv, namespace, false) + if err != nil { + logrus.Errorf("Error in Deployment with configmap creation: %v", err) + } + + // Creating Deployment with secret mounted as Env in init container + _, err = testutil.CreateDeploymentWithInitContainer(client, secretWithInitEnv, namespace, false) if err != nil { logrus.Errorf("Error in Deployment with secret creation: %v", err) } @@ -225,6 +250,18 @@ func teardown() { logrus.Errorf("Error while deleting deployment with secret mounted in init container %v", deploymentError) } + // Deleting Deployment with configmap mounted as env in init container + deploymentError = testutil.DeleteDeployment(client, namespace, configmapWithInitEnv) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with configmap mounted as env in init container %v", deploymentError) + } + + // Deleting Deployment with secret mounted as env in init container + deploymentError = testutil.DeleteDeployment(client, namespace, secretWithInitEnv) + if deploymentError != nil { + logrus.Errorf("Error while deleting deployment with secret mounted as env in init container %v", deploymentError) + } + // Deleting Deployment with configmap as envFrom source deploymentError = testutil.DeleteDeployment(client, namespace, configmapWithEnvFromName) if deploymentError != nil { @@ -333,6 +370,18 @@ func teardown() { logrus.Errorf("Error while deleting the secret used as env var source %v", err) } + // Deleting Configmap used as env var source + err = testutil.DeleteConfigMap(client, namespace, configmapWithInitEnv) + if err != nil { + logrus.Errorf("Error while deleting the configmap used as env var source in init container %v", err) + } + + // Deleting Secret used as env var source + err = testutil.DeleteSecret(client, namespace, secretWithInitEnv) + if err != nil { + logrus.Errorf("Error while deleting the secret used as env var source in init container %v", err) + } + // Deleting namespace testutil.DeleteNamespace(namespace, client) @@ -402,6 +451,24 @@ func TestRollingUpgradeForDeploymentWithConfigmapAsEnvVar(t *testing.T) { } } +func TestRollingUpgradeForDeploymentWithConfigmapAsEnvVarInInitContainer(t *testing.T) { + shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, configmapWithInitEnv, "www.stakater.com") + config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, configmapWithInitEnv, shaData, options.ReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + + err := PerformRollingUpgrade(client, config, deploymentFuncs) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with Configmap used as env var") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceUpdate(client, config, constants.ConfigmapEnvVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } +} + func TestRollingUpgradeForDeploymentWithConfigmapAsEnvVarFrom(t *testing.T) { shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, configmapWithEnvFromName, "www.stakater.com") config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, configmapWithEnvFromName, shaData, options.ReloaderAutoAnnotation) @@ -492,6 +559,24 @@ func TestRollingUpgradeForDeploymentWithSecretAsEnvVarFrom(t *testing.T) { } } +func TestRollingUpgradeForDeploymentWithSecretAsEnvVarInInitContainer(t *testing.T) { + shaData := testutil.ConvertResourceToSHA(testutil.SecretResourceType, namespace, secretWithInitEnv, "dGVzdFVwZGF0ZWRTZWNyZXRFbmNvZGluZ0ZvclJlbG9hZGVy") + config := getConfigWithAnnotations(constants.SecretEnvVarPostfix, secretWithInitEnv, shaData, options.ReloaderAutoAnnotation) + deploymentFuncs := GetDeploymentRollingUpgradeFuncs() + + err := PerformRollingUpgrade(client, config, deploymentFuncs) + time.Sleep(5 * time.Second) + if err != nil { + t.Errorf("Rolling upgrade failed for Deployment with Secret") + } + + logrus.Infof("Verifying deployment update") + updated := testutil.VerifyResourceUpdate(client, config, constants.SecretEnvVarPostfix, deploymentFuncs) + if !updated { + t.Errorf("Deployment was not updated") + } +} + func TestRollingUpgradeForDaemonSetWithConfigmap(t *testing.T) { shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, configmapName, "www.facebook.com") config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, configmapName, shaData, options.ConfigmapUpdateOnChangeAnnotation) diff --git a/internal/pkg/testutil/kube.go b/internal/pkg/testutil/kube.go index 296eec04..feaec282 100644 --- a/internal/pkg/testutil/kube.go +++ b/internal/pkg/testutil/kube.go @@ -78,6 +78,61 @@ func getAnnotations(name string, autoReload bool) map[string]string { options.SecretUpdateOnChangeAnnotation: name} } +func getEnvVarSources(name string) []v1.EnvFromSource { + return []v1.EnvFromSource{ + { + ConfigMapRef: &v1.ConfigMapEnvSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: name, + }, + }, + }, + { + SecretRef: &v1.SecretEnvSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: name, + }, + }, + }, + } +} + +func getVolumes(name string) []v1.Volume { + return []v1.Volume{ + { + Name: "configmap", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: name, + }, + }, + }, + }, + { + Name: "secret", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: name, + }, + }, + }, + } +} + +func getVolumeMounts(name string) []v1.VolumeMount { + return []v1.VolumeMount{ + { + MountPath: "etc/config", + Name: "configmap", + }, + { + MountPath: "etc/sec", + Name: "secret", + }, + } +} + func getPodTemplateSpecWithEnvVars(name string) v1.PodTemplateSpec { return v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -132,22 +187,7 @@ func getPodTemplateSpecWithEnvVarSources(name string) v1.PodTemplateSpec { { Image: "tutum/hello-world", Name: name, - EnvFrom: []v1.EnvFromSource{ - { - ConfigMapRef: &v1.ConfigMapEnvSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: name, - }, - }, - }, - { - SecretRef: &v1.SecretEnvSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: name, - }, - }, - }, - }, + EnvFrom: getEnvVarSources(name), }, }, }, @@ -170,38 +210,10 @@ func getPodTemplateSpecWithVolumes(name string) v1.PodTemplateSpec { Value: "test", }, }, - VolumeMounts: []v1.VolumeMount{ - { - MountPath: "etc/config", - Name: "configmap", - }, - { - MountPath: "etc/sec", - Name: "secret", - }, - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "configmap", - VolumeSource: v1.VolumeSource{ - ConfigMap: &v1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: name, - }, - }, - }, - }, - { - Name: "secret", - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: name, - }, - }, + VolumeMounts: getVolumeMounts(name), }, }, + Volumes: getVolumes(name), }, } } @@ -216,16 +228,7 @@ func getPodTemplateSpecWithInitContainer(name string) v1.PodTemplateSpec { { Image: "busybox", Name: "busyBox", - VolumeMounts: []v1.VolumeMount{ - { - MountPath: "etc/config", - Name: "configmap", - }, - { - MountPath: "etc/sec", - Name: "secret", - }, - }, + VolumeMounts: getVolumeMounts(name), }, }, Containers: []v1.Container{ @@ -240,22 +243,32 @@ func getPodTemplateSpecWithInitContainer(name string) v1.PodTemplateSpec { }, }, }, - Volumes: []v1.Volume{ + Volumes: getVolumes(name), + }, + } +} + +func getPodTemplateSpecWithInitContainerAndEnv(name string) v1.PodTemplateSpec { + return v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"secondLabel": "temp"}, + }, + Spec: v1.PodSpec{ + InitContainers: []v1.Container{ { - Name: "configmap", - VolumeSource: v1.VolumeSource{ - ConfigMap: &v1.ConfigMapVolumeSource{ - LocalObjectReference: v1.LocalObjectReference{ - Name: name, - }, - }, - }, + Image: "busybox", + Name: "busyBox", + EnvFrom: getEnvVarSources(name), }, + }, + Containers: []v1.Container{ { - Name: "secret", - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: name, + Image: "tutum/hello-world", + Name: name, + Env: []v1.EnvVar{ + { + Name: "BUCKET_NAME", + Value: "test", }, }, }, @@ -279,7 +292,7 @@ func GetDeployment(namespace string, deploymentName string) *v1beta1.Deployment } } -// GetDeploymentWithInitContainer provides deployment with init container +// GetDeploymentWithInitContainer provides deployment with init container and volumeMounts func GetDeploymentWithInitContainer(namespace string, deploymentName string) *v1beta1.Deployment { replicaset := int32(1) return &v1beta1.Deployment{ @@ -294,6 +307,21 @@ func GetDeploymentWithInitContainer(namespace string, deploymentName string) *v1 } } +// GetDeploymentWithInitContainerAndEnv provides deployment with init container and EnvSource +func GetDeploymentWithInitContainerAndEnv(namespace string, deploymentName string) *v1beta1.Deployment { + replicaset := int32(1) + return &v1beta1.Deployment{ + ObjectMeta: getObjectMeta(namespace, deploymentName, true), + Spec: v1beta1.DeploymentSpec{ + Replicas: &replicaset, + Strategy: v1beta1.DeploymentStrategy{ + Type: v1beta1.RollingUpdateDeploymentStrategyType, + }, + Template: getPodTemplateSpecWithInitContainerAndEnv(deploymentName), + }, + } +} + func GetDeploymentWithEnvVars(namespace string, deploymentName string) *v1beta1.Deployment { replicaset := int32(1) return &v1beta1.Deployment{ @@ -486,10 +514,15 @@ func CreateDeployment(client kubernetes.Interface, deploymentName string, namesp } // CreateDeploymentWithInitContainer creates a deployment in given namespace with init container and returns the Deployment -func CreateDeploymentWithInitContainer(client kubernetes.Interface, deploymentName string, namespace string) (*v1beta1.Deployment, error) { +func CreateDeploymentWithInitContainer(client kubernetes.Interface, deploymentName string, namespace string, volumeMount bool) (*v1beta1.Deployment, error) { logrus.Infof("Creating Deployment") deploymentClient := client.ExtensionsV1beta1().Deployments(namespace) - deploymentObj := GetDeploymentWithInitContainer(namespace, deploymentName) + var deploymentObj *v1beta1.Deployment + if volumeMount { + deploymentObj = GetDeploymentWithInitContainer(namespace, deploymentName) + } else { + deploymentObj = GetDeploymentWithInitContainerAndEnv(namespace, deploymentName) + } deployment, err := deploymentClient.Create(deploymentObj) time.Sleep(10 * time.Second) return deployment, err From 0c340fcb482b1a7591f1a7a81888d0ebc2c81858 Mon Sep 17 00:00:00 2001 From: faizanahmad055 Date: Mon, 18 Feb 2019 12:09:53 +0100 Subject: [PATCH 5/5] Fix same name configmap and secret conflict Signed-off-by: faizanahmad055 --- internal/pkg/handler/upgrade.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/pkg/handler/upgrade.go b/internal/pkg/handler/upgrade.go index 26e69831..8c3fb9fa 100644 --- a/internal/pkg/handler/upgrade.go +++ b/internal/pkg/handler/upgrade.go @@ -128,15 +128,15 @@ func getContainerWithVolumeMount(containers []v1.Container, volumeMountName stri return nil } -func getContainerWithEnvReference(containers []v1.Container, resourceName string) *v1.Container { +func getContainerWithEnvReference(containers []v1.Container, resourceName string, resourceType string) *v1.Container { for i := range containers { envs := containers[i].Env for j := range envs { envVarSource := envs[j].ValueFrom if envVarSource != nil { - if envVarSource.SecretKeyRef != nil && envVarSource.SecretKeyRef.LocalObjectReference.Name == resourceName{ + if resourceType == constants.SecretEnvVarPostfix && envVarSource.SecretKeyRef != nil && envVarSource.SecretKeyRef.LocalObjectReference.Name == resourceName { return &containers[i] - } else if envVarSource.ConfigMapKeyRef != nil && envVarSource.ConfigMapKeyRef.LocalObjectReference.Name == resourceName { + } else if resourceType == constants.ConfigmapEnvVarPostfix && envVarSource.ConfigMapKeyRef != nil && envVarSource.ConfigMapKeyRef.LocalObjectReference.Name == resourceName { return &containers[i] } } @@ -144,9 +144,9 @@ func getContainerWithEnvReference(containers []v1.Container, resourceName string envsFrom := containers[i].EnvFrom for j := range envsFrom { - if envsFrom[j].SecretRef != nil && envsFrom[j].SecretRef.LocalObjectReference.Name == resourceName { + if resourceType == constants.SecretEnvVarPostfix && envsFrom[j].SecretRef != nil && envsFrom[j].SecretRef.LocalObjectReference.Name == resourceName { return &containers[i] - } else if envsFrom[j].ConfigMapRef != nil && envsFrom[j].ConfigMapRef.LocalObjectReference.Name == resourceName { + } else if resourceType == constants.ConfigmapEnvVarPostfix && envsFrom[j].ConfigMapRef != nil && envsFrom[j].ConfigMapRef.LocalObjectReference.Name == resourceName { return &containers[i] } } @@ -176,9 +176,9 @@ func getContainerToUpdate(upgradeFuncs callbacks.RollingUpgradeFuncs, item inter } // Get the container with referenced secret or configmap as env var - container = getContainerWithEnvReference(containers, config.ResourceName) + container = getContainerWithEnvReference(containers, config.ResourceName, config.Type) if container == nil && len(initContainers) > 0 { - container = getContainerWithEnvReference(initContainers, config.ResourceName) + container = getContainerWithEnvReference(initContainers, config.ResourceName, config.Type) if container != nil { // if configmap/secret is being used in init container then return the first Pod container to save reloader env return &containers[0]