Initial implementation for init container support

Signed-off-by: faizanahmad055 <faizan.ahmad55@outlook.com>
This commit is contained in:
faizanahmad055
2019-02-16 15:43:56 +01:00
parent 724cda887e
commit 6fb1266637
4 changed files with 306 additions and 92 deletions
+25 -6
View File
@@ -4,7 +4,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/stakater/Reloader/internal/pkg/util"
apps_v1beta1 "k8s.io/api/apps/v1beta1"
"k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/api/extensions/v1beta1"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
@@ -16,6 +16,9 @@ type ItemsFunc func(kubernetes.Interface, string) []interface{}
//ContainersFunc is a generic func to return containers
type ContainersFunc func(interface{}) []v1.Container
//InitContainersFunc is a generic func to return containers
type InitContainersFunc func(interface{}) []v1.Container
//VolumesFunc is a generic func to return volumes
type VolumesFunc func(interface{}) []v1.Volume
@@ -24,11 +27,12 @@ type UpdateFunc func(kubernetes.Interface, string, interface{}) error
//RollingUpgradeFuncs contains generic functions to perform rolling upgrade
type RollingUpgradeFuncs struct {
ItemsFunc ItemsFunc
ContainersFunc ContainersFunc
UpdateFunc UpdateFunc
VolumesFunc VolumesFunc
ResourceType string
ItemsFunc ItemsFunc
ContainersFunc ContainersFunc
InitContainersFunc InitContainersFunc
UpdateFunc UpdateFunc
VolumesFunc VolumesFunc
ResourceType string
}
// GetDeploymentItems returns the deployments in given namespace
@@ -73,6 +77,21 @@ func GetStatefulsetContainers(item interface{}) []v1.Container {
return item.(apps_v1beta1.StatefulSet).Spec.Template.Spec.Containers
}
// GetDeploymentInitContainers returns the containers of given deployment
func GetDeploymentInitContainers(item interface{}) []v1.Container {
return item.(v1beta1.Deployment).Spec.Template.Spec.InitContainers
}
// GetDaemonSetInitContainers returns the containers of given daemonset
func GetDaemonSetInitContainers(item interface{}) []v1.Container {
return item.(v1beta1.DaemonSet).Spec.Template.Spec.InitContainers
}
// GetStatefulsetInitContainers returns the containers of given statefulSet
func GetStatefulsetInitContainers(item interface{}) []v1.Container {
return item.(apps_v1beta1.StatefulSet).Spec.Template.Spec.InitContainers
}
// UpdateDeployment performs rolling upgrade on deployment
func UpdateDeployment(client kubernetes.Interface, namespace string, resource interface{}) error {
deployment := resource.(v1beta1.Deployment)
+104 -77
View File
@@ -17,33 +17,36 @@ import (
// GetDeploymentRollingUpgradeFuncs returns all callback funcs for a deployment
func GetDeploymentRollingUpgradeFuncs() callbacks.RollingUpgradeFuncs {
return callbacks.RollingUpgradeFuncs{
ItemsFunc: callbacks.GetDeploymentItems,
ContainersFunc: callbacks.GetDeploymentContainers,
UpdateFunc: callbacks.UpdateDeployment,
VolumesFunc: callbacks.GetDeploymentVolumes,
ResourceType: "Deployment",
ItemsFunc: callbacks.GetDeploymentItems,
ContainersFunc: callbacks.GetDeploymentContainers,
InitContainersFunc: callbacks.GetDeploymentInitContainers,
UpdateFunc: callbacks.UpdateDeployment,
VolumesFunc: callbacks.GetDeploymentVolumes,
ResourceType: "Deployment",
}
}
// GetDaemonSetRollingUpgradeFuncs returns all callback funcs for a daemonset
func GetDaemonSetRollingUpgradeFuncs() callbacks.RollingUpgradeFuncs {
return callbacks.RollingUpgradeFuncs{
ItemsFunc: callbacks.GetDaemonSetItems,
ContainersFunc: callbacks.GetDaemonSetContainers,
UpdateFunc: callbacks.UpdateDaemonSet,
VolumesFunc: callbacks.GetDaemonSetVolumes,
ResourceType: "DaemonSet",
ItemsFunc: callbacks.GetDaemonSetItems,
ContainersFunc: callbacks.GetDaemonSetContainers,
InitContainersFunc: callbacks.GetDaemonSetInitContainers,
UpdateFunc: callbacks.UpdateDaemonSet,
VolumesFunc: callbacks.GetDaemonSetVolumes,
ResourceType: "DaemonSet",
}
}
// GetStatefulSetRollingUpgradeFuncs returns all callback funcs for a statefulSet
func GetStatefulSetRollingUpgradeFuncs() callbacks.RollingUpgradeFuncs {
return callbacks.RollingUpgradeFuncs{
ItemsFunc: callbacks.GetStatefulSetItems,
ContainersFunc: callbacks.GetStatefulsetContainers,
UpdateFunc: callbacks.UpdateStatefulset,
VolumesFunc: callbacks.GetStatefulsetVolumes,
ResourceType: "StatefulSet",
ItemsFunc: callbacks.GetStatefulSetItems,
ContainersFunc: callbacks.GetStatefulsetContainers,
InitContainersFunc: callbacks.GetStatefulsetInitContainers,
UpdateFunc: callbacks.UpdateStatefulset,
VolumesFunc: callbacks.GetStatefulsetVolumes,
ResourceType: "StatefulSet",
}
}
@@ -70,35 +73,31 @@ func PerformRollingUpgrade(client kubernetes.Interface, config util.Config, upgr
items := upgradeFuncs.ItemsFunc(client, config.Namespace)
var err error
for _, i := range items {
containers := upgradeFuncs.ContainersFunc(i)
volumes := upgradeFuncs.VolumesFunc(i)
// find correct annotation and update the resource
annotationValue := util.ToObjectMeta(i).Annotations[config.Annotation]
reloaderEnabledValue := util.ToObjectMeta(i).Annotations[options.ReloaderAutoAnnotation]
if len(containers) > 0 {
resourceName := util.ToObjectMeta(i).Name
result := constants.NotUpdated
reloaderEnabled, err := strconv.ParseBool(reloaderEnabledValue)
if err == nil && reloaderEnabled {
result = updateContainers(volumes, containers, config.ResourceName, config)
} else if annotationValue != "" {
values := strings.Split(annotationValue, ",")
for _, value := range values {
if value == config.ResourceName {
result = updateContainers(volumes, containers, value, config)
if result == constants.Updated {
break
}
result := constants.NotUpdated
reloaderEnabled, err := strconv.ParseBool(reloaderEnabledValue)
if err == nil && reloaderEnabled {
result = updateContainers(upgradeFuncs, i, config)
} else if annotationValue != "" {
values := strings.Split(annotationValue, ",")
for _, value := range values {
if value == config.ResourceName {
result = updateContainers(upgradeFuncs, i, config)
if result == constants.Updated {
break
}
}
}
if result == constants.Updated {
err = upgradeFuncs.UpdateFunc(client, config.Namespace, i)
if err != nil {
logrus.Errorf("Update for '%s' of type '%s' in namespace '%s' failed with error %v", resourceName, upgradeFuncs.ResourceType, config.Namespace, err)
} else {
logrus.Infof("Updated '%s' of type '%s' in namespace '%s'", resourceName, upgradeFuncs.ResourceType, config.Namespace)
}
}
if result == constants.Updated {
err = upgradeFuncs.UpdateFunc(client, config.Namespace, i)
resourceName := util.ToObjectMeta(i).Name
if err != nil {
logrus.Errorf("Update for '%s' of type '%s' in namespace '%s' failed with error %v", resourceName, upgradeFuncs.ResourceType, config.Namespace, err)
} else {
logrus.Infof("Updated '%s' of type '%s' in namespace '%s'", resourceName, upgradeFuncs.ResourceType, config.Namespace)
}
}
}
@@ -116,42 +115,11 @@ func getVolumeMountName(volumes []v1.Volume, mountType string, volumeName string
return ""
}
func getContainerToUpdate(volumes []v1.Volume, containers []v1.Container, envarPostfix string, volumeName string) *v1.Container {
// Get the volumeMountName to find volumeMount in container
if len(volumes) > 0 {
volumeMountName := getVolumeMountName(volumes, envarPostfix, volumeName)
// Get the container with mounted configmap/secret
if volumeMountName != "" {
for i := range containers {
volumeMounts := containers[i].VolumeMounts
for j := range volumeMounts {
if volumeMounts[j].Name == volumeMountName {
return &containers[i]
}
}
}
}
}
// Get the container with referenced secret or configmap as env var
func getContainerWithVolumeMount(volumes []v1.Volume, containers []v1.Container, volumeMountName string) *v1.Container {
for i := range containers {
envs := containers[i].Env
for j := range envs {
envVarSource := envs[j].ValueFrom
if envVarSource != nil {
if envVarSource.SecretKeyRef != nil && envVarSource.SecretKeyRef.LocalObjectReference.Name == volumeName {
return &containers[i]
} else if envVarSource.ConfigMapKeyRef != nil && envVarSource.ConfigMapKeyRef.LocalObjectReference.Name == volumeName {
return &containers[i]
}
}
}
envsFrom := containers[i].EnvFrom
for j := range envsFrom {
if envsFrom[j].SecretRef != nil && envsFrom[j].SecretRef.LocalObjectReference.Name == volumeName {
return &containers[i]
} else if envsFrom[j].ConfigMapRef != nil && envsFrom[j].ConfigMapRef.LocalObjectReference.Name == volumeName {
volumeMounts := containers[i].VolumeMounts
for j := range volumeMounts {
if volumeMounts[j].Name == volumeMountName {
return &containers[i]
}
}
@@ -160,17 +128,76 @@ func getContainerToUpdate(volumes []v1.Volume, containers []v1.Container, envarP
return nil
}
func updateContainers(volumes []v1.Volume, containers []v1.Container, annotationValue string, config util.Config) constants.Result {
func getContainerWithEnvReference(containers []v1.Container, resourceName string) *v1.Container {
for i := range containers {
envs := containers[i].Env
for j := range envs {
envVarSource := envs[j].ValueFrom
if envVarSource != nil {
if envVarSource.SecretKeyRef != nil && envVarSource.SecretKeyRef.LocalObjectReference.Name == resourceName{
return &containers[i]
} else if envVarSource.ConfigMapKeyRef != nil && envVarSource.ConfigMapKeyRef.LocalObjectReference.Name == resourceName {
return &containers[i]
}
}
}
envsFrom := containers[i].EnvFrom
for j := range envsFrom {
if envsFrom[j].SecretRef != nil && envsFrom[j].SecretRef.LocalObjectReference.Name == resourceName {
return &containers[i]
} else if envsFrom[j].ConfigMapRef != nil && envsFrom[j].ConfigMapRef.LocalObjectReference.Name == resourceName {
return &containers[i]
}
}
}
return nil
}
func getContainerToUpdate(upgradeFuncs callbacks.RollingUpgradeFuncs, item interface{}, config util.Config) *v1.Container {
volumes := upgradeFuncs.VolumesFunc(item)
containers := upgradeFuncs.ContainersFunc(item)
initContainers := upgradeFuncs.InitContainersFunc(item)
var container *v1.Container
// Get the volumeMountName to find volumeMount in container
volumeMountName := getVolumeMountName(volumes, config.Type, config.ResourceName)
// Get the container with mounted configmap/secret
if volumeMountName != "" {
container = getContainerWithVolumeMount(volumes, containers, volumeMountName)
if container == nil && len(initContainers) > 0 {
container = getContainerWithVolumeMount(volumes, initContainers, volumeMountName)
if container != nil {
// if configmap/secret is being used in init container then return the first Pod container to save reloader env
return &containers[0]
}
} else if container != nil {
return container
}
}
// Get the container with referenced secret or configmap as env var
container = getContainerWithEnvReference(containers, config.ResourceName)
if container == nil && len(initContainers) > 0 {
container = getContainerWithEnvReference(initContainers, config.ResourceName)
if container == nil {
// if configmap/secret is being used in init container then return the first Pod container to save reloader env
return &containers[0]
}
}
return container
}
func updateContainers(upgradeFuncs callbacks.RollingUpgradeFuncs, item interface{}, config util.Config) constants.Result {
var result constants.Result
envar := constants.EnvVarPrefix + util.ConvertToEnvVarName(annotationValue) + "_" + config.Type
container := getContainerToUpdate(volumes, containers, config.Type, config.ResourceName)
envar := constants.EnvVarPrefix + util.ConvertToEnvVarName(config.ResourceName) + "_" + config.Type
container := getContainerToUpdate(upgradeFuncs, item, config)
if container == nil {
return constants.NoContainerFound
}
//update if env var exists
result = updateEnvVar(containers, envar, config.SHAValue)
result = updateEnvVar(upgradeFuncs.ContainersFunc(item), envar, config.SHAValue)
// if no existing env var exists lets create one
if result == constants.NoEnvVarFound {
+93 -9
View File
@@ -14,14 +14,16 @@ import (
)
var (
client = testclient.NewSimpleClientset()
namespace = "test-handler-" + testutil.RandSeq(5)
configmapName = "testconfigmap-handler-" + testutil.RandSeq(5)
secretName = "testsecret-handler-" + testutil.RandSeq(5)
configmapWithEnvName = "testconfigmapWithEnv-handler-" + testutil.RandSeq(3)
configmapWithEnvFromName = "testconfigmapWithEnvFrom-handler-" + testutil.RandSeq(3)
secretWithEnvName = "testsecretWithEnv-handler-" + testutil.RandSeq(5)
secretWithEnvFromName = "testsecretWithEnvFrom-handler-" + testutil.RandSeq(5)
client = testclient.NewSimpleClientset()
namespace = "test-handler-" + testutil.RandSeq(5)
configmapName = "testconfigmap-handler-" + testutil.RandSeq(5)
secretName = "testsecret-handler-" + testutil.RandSeq(5)
configmapWithInitContainer = "testconfigmapInitContainerhandler-" + testutil.RandSeq(3)
secretWithInitContainer = "testsecretWithInitContainer-handler-" + testutil.RandSeq(3)
configmapWithEnvName = "testconfigmapWithEnv-handler-" + testutil.RandSeq(3)
configmapWithEnvFromName = "testconfigmapWithEnvFrom-handler-" + testutil.RandSeq(3)
secretWithEnvName = "testsecretWithEnv-handler-" + testutil.RandSeq(5)
secretWithEnvFromName = "testsecretWithEnvFrom-handler-" + testutil.RandSeq(5)
)
func TestMain(m *testing.M) {
@@ -77,12 +79,35 @@ func setup() {
logrus.Errorf("Error in secret creation: %v", err)
}
_, err = testutil.CreateConfigMap(client, namespace, configmapWithInitContainer, "www.google.com")
if err != nil {
logrus.Errorf("Error in configmap creation: %v", err)
}
// Creating secret
_, err = testutil.CreateSecret(client, namespace, secretWithInitContainer, data)
if err != nil {
logrus.Errorf("Error in secret creation: %v", err)
}
// Creating Deployment with configmap
_, err = testutil.CreateDeployment(client, configmapName, namespace, true)
if err != nil {
logrus.Errorf("Error in Deployment with configmap creation: %v", err)
}
// Creating Deployment with configmap mounted in init container
_, err = testutil.CreateDeploymentWithInitContainer(client, configmapWithInitContainer, namespace)
if err != nil {
logrus.Errorf("Error in Deployment with configmap creation: %v", err)
}
// Creating Deployment with secret mounted in init container
_, err = testutil.CreateDeploymentWithInitContainer(client, secretWithInitContainer, namespace)
if err != nil {
logrus.Errorf("Error in Deployment with secret creation: %v", err)
}
// Creating Deployment with secret
_, err = testutil.CreateDeployment(client, secretName, namespace, true)
if err != nil {
@@ -188,6 +213,18 @@ func teardown() {
logrus.Errorf("Error while deleting deployment with secret as env var source %v", deploymentError)
}
// Deleting Deployment with configmap mounted in init container
deploymentError = testutil.DeleteDeployment(client, namespace, configmapWithInitContainer)
if deploymentError != nil {
logrus.Errorf("Error while deleting deployment with configmap mounted in init container %v", deploymentError)
}
// Deleting Deployment with secret mounted in init container
deploymentError = testutil.DeleteDeployment(client, namespace, secretWithInitContainer)
if deploymentError != nil {
logrus.Errorf("Error while deleting deployment with secret mounted in init container %v", deploymentError)
}
// Deleting Deployment with configmap as envFrom source
deploymentError = testutil.DeleteDeployment(client, namespace, configmapWithEnvFromName)
if deploymentError != nil {
@@ -272,6 +309,18 @@ func teardown() {
logrus.Errorf("Error while deleting the secret used as env var source %v", err)
}
// Deleting Configmap used in init container
err = testutil.DeleteConfigMap(client, namespace, configmapWithInitContainer)
if err != nil {
logrus.Errorf("Error while deleting the configmap used in init container %v", err)
}
// Deleting Secret used in init container
err = testutil.DeleteSecret(client, namespace, secretWithInitContainer)
if err != nil {
logrus.Errorf("Error while deleting the secret used in init container %v", err)
}
// Deleting Configmap used as env var source
err = testutil.DeleteConfigMap(client, namespace, configmapWithEnvFromName)
if err != nil {
@@ -297,7 +346,6 @@ func getConfigWithAnnotations(resourceType string, name string, shaData string,
Annotation: annotation,
Type: resourceType,
}
}
func TestRollingUpgradeForDeploymentWithConfigmap(t *testing.T) {
@@ -318,6 +366,24 @@ func TestRollingUpgradeForDeploymentWithConfigmap(t *testing.T) {
}
}
func TestRollingUpgradeForDeploymentWithConfigmapInInitContainer(t *testing.T) {
shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, configmapWithInitContainer, "www.stakater.com")
config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, configmapWithInitContainer, shaData, options.ConfigmapUpdateOnChangeAnnotation)
deploymentFuncs := GetDeploymentRollingUpgradeFuncs()
err := PerformRollingUpgrade(client, config, deploymentFuncs)
time.Sleep(5 * time.Second)
if err != nil {
t.Errorf("Rolling upgrade failed for Deployment with Configmap")
}
logrus.Infof("Verifying deployment update")
updated := testutil.VerifyResourceUpdate(client, config, constants.ConfigmapEnvVarPostfix, deploymentFuncs)
if !updated {
t.Errorf("Deployment was not updated")
}
}
func TestRollingUpgradeForDeploymentWithConfigmapAsEnvVar(t *testing.T) {
shaData := testutil.ConvertResourceToSHA(testutil.ConfigmapResourceType, namespace, configmapWithEnvName, "www.stakater.com")
config := getConfigWithAnnotations(constants.ConfigmapEnvVarPostfix, configmapWithEnvName, shaData, options.ReloaderAutoAnnotation)
@@ -372,6 +438,24 @@ func TestRollingUpgradeForDeploymentWithSecret(t *testing.T) {
}
}
func TestRollingUpgradeForDeploymentWithSecretinInitContainer(t *testing.T) {
shaData := testutil.ConvertResourceToSHA(testutil.SecretResourceType, namespace, secretWithInitContainer, "dGVzdFVwZGF0ZWRTZWNyZXRFbmNvZGluZ0ZvclJlbG9hZGVy")
config := getConfigWithAnnotations(constants.SecretEnvVarPostfix, secretWithInitContainer, shaData, options.SecretUpdateOnChangeAnnotation)
deploymentFuncs := GetDeploymentRollingUpgradeFuncs()
err := PerformRollingUpgrade(client, config, deploymentFuncs)
time.Sleep(5 * time.Second)
if err != nil {
t.Errorf("Rolling upgrade failed for Deployment with Secret")
}
logrus.Infof("Verifying deployment update")
updated := testutil.VerifyResourceUpdate(client, config, constants.SecretEnvVarPostfix, deploymentFuncs)
if !updated {
t.Errorf("Deployment was not updated")
}
}
func TestRollingUpgradeForDeploymentWithSecretAsEnvVar(t *testing.T) {
shaData := testutil.ConvertResourceToSHA(testutil.SecretResourceType, namespace, secretWithEnvName, "dGVzdFVwZGF0ZWRTZWNyZXRFbmNvZGluZ0ZvclJlbG9hZGVy")
config := getConfigWithAnnotations(constants.SecretEnvVarPostfix, secretWithEnvName, shaData, options.ReloaderAutoAnnotation)
+84
View File
@@ -206,6 +206,64 @@ func getPodTemplateSpecWithVolumes(name string) v1.PodTemplateSpec {
}
}
func getPodTemplateSpecWithInitContainer(name string) v1.PodTemplateSpec {
return v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{"secondLabel": "temp"},
},
Spec: v1.PodSpec{
InitContainers: []v1.Container{
{
Image: "busybox",
Name: "busyBox",
VolumeMounts: []v1.VolumeMount{
{
MountPath: "etc/config",
Name: "configmap",
},
{
MountPath: "etc/sec",
Name: "secret",
},
},
},
},
Containers: []v1.Container{
{
Image: "tutum/hello-world",
Name: name,
Env: []v1.EnvVar{
{
Name: "BUCKET_NAME",
Value: "test",
},
},
},
},
Volumes: []v1.Volume{
{
Name: "configmap",
VolumeSource: v1.VolumeSource{
ConfigMap: &v1.ConfigMapVolumeSource{
LocalObjectReference: v1.LocalObjectReference{
Name: name,
},
},
},
},
{
Name: "secret",
VolumeSource: v1.VolumeSource{
Secret: &v1.SecretVolumeSource{
SecretName: name,
},
},
},
},
},
}
}
// GetDeployment provides deployment for testing
func GetDeployment(namespace string, deploymentName string) *v1beta1.Deployment {
replicaset := int32(1)
@@ -221,6 +279,21 @@ func GetDeployment(namespace string, deploymentName string) *v1beta1.Deployment
}
}
// GetDeploymentWithInitContainer provides deployment with init container
func GetDeploymentWithInitContainer(namespace string, deploymentName string) *v1beta1.Deployment {
replicaset := int32(1)
return &v1beta1.Deployment{
ObjectMeta: getObjectMeta(namespace, deploymentName, false),
Spec: v1beta1.DeploymentSpec{
Replicas: &replicaset,
Strategy: v1beta1.DeploymentStrategy{
Type: v1beta1.RollingUpdateDeploymentStrategyType,
},
Template: getPodTemplateSpecWithInitContainer(deploymentName),
},
}
}
func GetDeploymentWithEnvVars(namespace string, deploymentName string) *v1beta1.Deployment {
replicaset := int32(1)
return &v1beta1.Deployment{
@@ -412,6 +485,17 @@ func CreateDeployment(client kubernetes.Interface, deploymentName string, namesp
return deployment, err
}
// CreateDeploymentWithInitContainer creates a deployment in given namespace with init container and returns the Deployment
func CreateDeploymentWithInitContainer(client kubernetes.Interface, deploymentName string, namespace string) (*v1beta1.Deployment, error) {
logrus.Infof("Creating Deployment")
deploymentClient := client.ExtensionsV1beta1().Deployments(namespace)
var deploymentObj *v1beta1.Deployment
deploymentObj = GetDeploymentWithInitContainer(namespace, deploymentName)
deployment, err := deploymentClient.Create(deploymentObj)
time.Sleep(10 * time.Second)
return deployment, err
}
// CreateDeploymentWithEnvVarSource creates a deployment in given namespace and returns the Deployment
func CreateDeploymentWithEnvVarSource(client kubernetes.Interface, deploymentName string, namespace string) (*v1beta1.Deployment, error) {
logrus.Infof("Creating Deployment")