diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index b6990df9..2a68db90 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -74,6 +74,11 @@ func NewController( kubeClient: kubeClient, istioClient: istioClient, flaggerClient: flaggerClient, + configTracker: ConfigTracker{ + logger: logger, + kubeClient: kubeClient, + flaggerClient: flaggerClient, + }, } router := CanaryRouter{ diff --git a/pkg/controller/deployer.go b/pkg/controller/deployer.go index 3eed041a..0019fd8b 100644 --- a/pkg/controller/deployer.go +++ b/pkg/controller/deployer.go @@ -28,9 +28,10 @@ type CanaryDeployer struct { istioClient istioclientset.Interface flaggerClient clientset.Interface logger *zap.SugaredLogger + configTracker ConfigTracker } -// Promote copies the pod spec from canary to primary +// Promote copies the pod spec, secrets and config maps from canary to primary func (c *CanaryDeployer) Promote(cd *flaggerv1.Canary) error { targetName := cd.Spec.TargetRef.Name primaryName := fmt.Sprintf("%s-primary", targetName) @@ -51,12 +52,23 @@ func (c *CanaryDeployer) Promote(cd *flaggerv1.Canary) error { return fmt.Errorf("deployment %s.%s query error %v", primaryName, cd.Namespace, err) } + // promote secrets and config maps + configRefs, err := c.configTracker.GetTargetConfigs(cd) + if err != nil { + return err + } + if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs); err != nil { + return err + } + primaryCopy := primary.DeepCopy() primaryCopy.Spec.ProgressDeadlineSeconds = canary.Spec.ProgressDeadlineSeconds primaryCopy.Spec.MinReadySeconds = canary.Spec.MinReadySeconds primaryCopy.Spec.RevisionHistoryLimit = canary.Spec.RevisionHistoryLimit primaryCopy.Spec.Strategy = canary.Spec.Strategy - primaryCopy.Spec.Template.Spec = canary.Spec.Template.Spec + + // update spec with primary secrets and config maps + primaryCopy.Spec.Template.Spec = c.configTracker.ApplyPrimaryConfigs(canary.Spec.Template.Spec, configRefs) _, err = c.kubeClient.AppsV1().Deployments(cd.Namespace).Update(primaryCopy) if err != nil { @@ -290,6 +302,15 @@ func (c *CanaryDeployer) createPrimaryDeployment(cd *flaggerv1.Canary) error { primaryDep, err := c.kubeClient.AppsV1().Deployments(cd.Namespace).Get(primaryName, metav1.GetOptions{}) if errors.IsNotFound(err) { + // create primary secrets and config maps + configRefs, err := c.configTracker.GetTargetConfigs(cd) + if err != nil { + return err + } + if err := c.configTracker.CreatePrimaryConfigs(cd, configRefs); err != nil { + return err + } + // create primary deployment primaryDep = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: primaryName, @@ -319,7 +340,8 @@ func (c *CanaryDeployer) createPrimaryDeployment(cd *flaggerv1.Canary) error { Labels: map[string]string{"app": primaryName}, Annotations: canaryDep.Spec.Template.Annotations, }, - Spec: canaryDep.Spec.Template.Spec, + // update spec with the primary secrets and config maps + Spec: c.configTracker.ApplyPrimaryConfigs(canaryDep.Spec.Template.Spec, configRefs), }, }, } diff --git a/pkg/controller/tracker.go b/pkg/controller/tracker.go index c24fdf64..f7eb767a 100644 --- a/pkg/controller/tracker.go +++ b/pkg/controller/tracker.go @@ -10,10 +10,11 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" ) -// ConfigTracker is managing the operations for Kubernetes configmaps and secrets +// ConfigTracker is managing the operations for Kubernetes ConfigMaps and Secrets type ConfigTracker struct { kubeClient kubernetes.Interface flaggerClient clientset.Interface @@ -27,13 +28,14 @@ const ( ConfigRefSecret ConfigRefType = "secret" ) -// ConfigRef holds the reference to a tracked configmap or secrets +// ConfigRef holds the reference to a tracked Kubernetes ConfigMap or Secret type ConfigRef struct { Name string Type ConfigRefType Checksum string } +// GetName returns the config ref type and name func (c *ConfigRef) GetName() string { return fmt.Sprintf("%s/%s", c.Type, c.Name) } @@ -41,10 +43,52 @@ func (c *ConfigRef) GetName() string { func checksum(data interface{}) string { jsonBytes, _ := json.Marshal(data) hashBytes := sha256.Sum256(jsonBytes) + return fmt.Sprintf("%x", hashBytes[:8]) } -func (ct *ConfigTracker) getTargetConfigs(cd *flaggerv1.Canary) (map[string]ConfigRef, error) { +// getRefFromConfigMap transforms a Kubernetes ConfigMap into a ConfigRef +// and computes the checksum of the ConfigMap data +func (ct *ConfigTracker) getRefFromConfigMap(name string, namespace string) (*ConfigRef, error) { + config, err := ct.kubeClient.CoreV1().ConfigMaps(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + return &ConfigRef{ + Name: config.Name, + Type: ConfigRefMap, + Checksum: checksum(config.Data), + }, nil +} + +// getRefFromConfigMap transforms a Kubernetes Secret into a ConfigRef +// and computes the checksum of the Secret data +func (ct *ConfigTracker) getRefFromSecret(name string, namespace string) (*ConfigRef, error) { + secret, err := ct.kubeClient.CoreV1().Secrets(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + // ignore registry secrets (those should be set via service account) + if secret.Type != corev1.SecretTypeOpaque && + secret.Type != corev1.SecretTypeBasicAuth && + secret.Type != corev1.SecretTypeSSHAuth && + secret.Type != corev1.SecretTypeTLS { + ct.logger.Debugf("ignoring secret %s.%s type not supported %v", name, namespace, secret.Type) + return nil, nil + } + + return &ConfigRef{ + Name: secret.Name, + Type: ConfigRefSecret, + Checksum: checksum(secret.Data), + }, nil +} + +// GetTargetConfigs scans the target deployment for Kubernetes ConfigMaps and Secretes +// and returns a list of config references +func (ct *ConfigTracker) GetTargetConfigs(cd *flaggerv1.Canary) (map[string]ConfigRef, error) { res := make(map[string]ConfigRef) targetName := cd.Spec.TargetRef.Name targetDep, err := ct.kubeClient.AppsV1().Deployments(cd.Namespace).Get(targetName, metav1.GetOptions{}) @@ -79,7 +123,6 @@ func (ct *ConfigTracker) getTargetConfigs(cd *flaggerv1.Canary) (map[string]Conf } } } - // scan containers for _, container := range targetDep.Spec.Template.Spec.Containers { // scan env @@ -139,36 +182,145 @@ func (ct *ConfigTracker) getTargetConfigs(cd *flaggerv1.Canary) (map[string]Conf return res, nil } -func (ct *ConfigTracker) getRefFromConfigMap(name string, namespace string) (*ConfigRef, error) { - config, err := ct.kubeClient.CoreV1().ConfigMaps(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - return nil, err +// CreatePrimaryConfigs syncs the primary Kubernetes ConfigMaps and Secretes +// with those found in the target deployment +func (ct *ConfigTracker) CreatePrimaryConfigs(cd *flaggerv1.Canary, refs map[string]ConfigRef) error { + for _, ref := range refs { + switch ref.Type { + case ConfigRefMap: + config, err := ct.kubeClient.CoreV1().ConfigMaps(cd.Namespace).Get(ref.Name, metav1.GetOptions{}) + if err != nil { + return err + } + primaryName := fmt.Sprintf("%s-primary", config.GetName()) + primaryConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: primaryName, + Namespace: cd.Namespace, + Annotations: config.Annotations, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(cd, schema.GroupVersionKind{ + Group: flaggerv1.SchemeGroupVersion.Group, + Version: flaggerv1.SchemeGroupVersion.Version, + Kind: flaggerv1.CanaryKind, + }), + }, + }, + Data: config.Data, + } + + // update or insert primary ConfigMap + _, err = ct.kubeClient.CoreV1().ConfigMaps(cd.Namespace).Update(primaryConfigMap) + if err != nil { + if errors.IsNotFound(err) { + _, err = ct.kubeClient.CoreV1().ConfigMaps(cd.Namespace).Create(primaryConfigMap) + if err != nil { + return err + } + } else { + return err + } + } + + ct.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)). + Infof("ConfigMap %s synced", primaryConfigMap.GetName()) + case ConfigRefSecret: + secret, err := ct.kubeClient.CoreV1().Secrets(cd.Namespace).Get(ref.Name, metav1.GetOptions{}) + if err != nil { + return err + } + primaryName := fmt.Sprintf("%s-primary", secret.GetName()) + primarySecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: primaryName, + Namespace: cd.Namespace, + Annotations: secret.Annotations, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(cd, schema.GroupVersionKind{ + Group: flaggerv1.SchemeGroupVersion.Group, + Version: flaggerv1.SchemeGroupVersion.Version, + Kind: flaggerv1.CanaryKind, + }), + }, + }, + Type: secret.Type, + Data: secret.Data, + } + + // update or insert primary Secret + _, err = ct.kubeClient.CoreV1().Secrets(cd.Namespace).Update(primarySecret) + if err != nil { + if errors.IsNotFound(err) { + _, err = ct.kubeClient.CoreV1().Secrets(cd.Namespace).Create(primarySecret) + if err != nil { + return err + } + } else { + return err + } + } + + ct.logger.With("canary", fmt.Sprintf("%s.%s", cd.Name, cd.Namespace)). + Infof("Secret %s synced", primarySecret.GetName()) + } } - return &ConfigRef{ - Name: config.Name, - Type: ConfigRefMap, - Checksum: checksum(config.Data), - }, nil + return nil } -func (ct *ConfigTracker) getRefFromSecret(name string, namespace string) (*ConfigRef, error) { - secret, err := ct.kubeClient.CoreV1().Secrets(namespace).Get(name, metav1.GetOptions{}) - if err != nil { - return nil, err +// ApplyPrimaryConfigs appends the primary suffix to all ConfigMaps and Secretes found in the PodSpec +func (ct *ConfigTracker) ApplyPrimaryConfigs(spec corev1.PodSpec, refs map[string]ConfigRef) corev1.PodSpec { + // update volumes + for i, volume := range spec.Volumes { + if cmv := volume.ConfigMap; cmv != nil { + name := fmt.Sprintf("%s/%s", ConfigRefMap, cmv.Name) + if _, exists := refs[name]; exists { + spec.Volumes[i].ConfigMap.Name += "-primary" + } + } + + if sv := volume.Secret; sv != nil { + name := fmt.Sprintf("%s/%s", ConfigRefSecret, sv.SecretName) + if _, exists := refs[name]; exists { + spec.Volumes[i].Secret.SecretName += "-primary" + } + } + } + // update containers + for _, container := range spec.Containers { + // update env + for i, env := range container.Env { + if env.ValueFrom != nil { + switch { + case env.ValueFrom.ConfigMapKeyRef != nil: + name := fmt.Sprintf("%s/%s", ConfigRefMap, env.ValueFrom.ConfigMapKeyRef.Name) + if _, exists := refs[name]; exists { + container.Env[i].ValueFrom.ConfigMapKeyRef.Name += "-primary" + } + case env.ValueFrom.SecretKeyRef != nil: + name := fmt.Sprintf("%s/%s", ConfigRefSecret, env.ValueFrom.SecretKeyRef.Name) + if _, exists := refs[name]; exists { + container.Env[i].ValueFrom.SecretKeyRef.Name += "-primary" + } + } + } + } + // update envFrom + for i, envFrom := range container.EnvFrom { + switch { + case envFrom.ConfigMapRef != nil: + name := fmt.Sprintf("%s/%s", ConfigRefMap, envFrom.ConfigMapRef.Name) + if _, exists := refs[name]; exists { + container.EnvFrom[i].ConfigMapRef.Name += "-primary" + } + case envFrom.SecretRef != nil: + name := fmt.Sprintf("%s/%s", ConfigRefSecret, envFrom.SecretRef.Name) + if _, exists := refs[name]; exists { + container.EnvFrom[i].SecretRef.Name += "-primary" + } + } + } } - if secret.Type != corev1.SecretTypeOpaque && - secret.Type != corev1.SecretTypeBasicAuth && - secret.Type != corev1.SecretTypeSSHAuth && - secret.Type != corev1.SecretTypeTLS { - ct.logger.Debugf("ignoring secret %s.%s type not supported %v", name, namespace, secret.Type) - return nil, nil - } - - return &ConfigRef{ - Name: secret.Name, - Type: ConfigRefSecret, - Checksum: checksum(secret.Data), - }, nil + return spec }