package templateagent import ( "fmt" "path" "regexp" "strings" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2" "open-cluster-management.io/addon-framework/pkg/addonfactory" "open-cluster-management.io/addon-framework/pkg/utils" addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" addonapiv1beta1 "open-cluster-management.io/api/addon/v1beta1" ) // decorator mutate the unstructured and returns an unstructured. type decorator interface { decorate(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) } type namespaceDecorator struct { installNamespace string // paths is the paths of a resource kind that the decorator needs to set namespace field in it. // if the returned object is a list, decorator will set namespace for each item. paths map[string][]string } func newNamespaceDecorator(privateValues addonfactory.Values) *namespaceDecorator { decorator := &namespaceDecorator{ paths: map[string][]string{ "ClusterRoleBinding": {"subjects", "namespace"}, "RoleBinding": {"subjects", "namespace"}, "Namespace": {"metadata", "name"}, }, } namespace, ok := privateValues[InstallNamespacePrivateValueKey] if ok { decorator.installNamespace = namespace.(string) } return decorator } func (d *namespaceDecorator) decorate(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { if len(d.installNamespace) == 0 { return obj, nil } // If obj has no namespace set, we do not mutate namespace assuming it is cluster scoped. if len(obj.GetNamespace()) > 0 { obj.SetNamespace(d.installNamespace) } paths, ok := d.paths[obj.GetKind()] if !ok { return obj, nil } err := setUnstructuredNestedField(obj.Object, d.installNamespace, paths) return obj, err } // search the object to set the val, if an array is found, find every item in the array. func setUnstructuredNestedField(obj interface{}, val string, paths []string) error { switch f := obj.(type) { case []interface{}: for _, item := range f { if err := setUnstructuredNestedField(item, val, paths); err != nil { return err } } case map[string]interface{}: if len(paths) == 1 { f[paths[0]] = val return nil } field, ok := f[paths[0]] if !ok { return fmt.Errorf("failed to find field %s", paths[0]) } return setUnstructuredNestedField(field, val, paths[1:]) } return nil } type deploymentDecorator struct { logger klog.Logger decorators []podTemplateSpecDecorator } func newDeploymentDecorator( logger klog.Logger, addonName string, template *addonapiv1alpha1.AddOnTemplate, orderedValues orderedValues, privateValues addonfactory.Values, ) decorator { return &deploymentDecorator{ logger: logger, decorators: []podTemplateSpecDecorator{ newEnvironmentDecorator(orderedValues), newVolumeDecorator(addonName, template), newNodePlacementDecorator(privateValues), newImageDecorator(privateValues), newProxyHandler(logger, addonName, privateValues), newResourceRequirementsDecorator(logger, supportResourceDeployment, privateValues), }, } } func (d *deploymentDecorator) decorate(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { deployment, err := utils.ConvertToDeployment(obj) // not a deployment, directly return if err != nil { return obj, nil } for _, decorator := range d.decorators { err = decorator.decorate(deployment.Name, &deployment.Spec.Template) if err != nil { return obj, err } } result, err := runtime.DefaultUnstructuredConverter.ToUnstructured(deployment) if err != nil { return obj, err } return &unstructured.Unstructured{Object: result}, nil } type daemonSetDecorator struct { logger klog.Logger decorators []podTemplateSpecDecorator } func newDaemonSetDecorator( logger klog.Logger, addonName string, template *addonapiv1alpha1.AddOnTemplate, orderedValues orderedValues, privateValues addonfactory.Values, ) decorator { return &daemonSetDecorator{ logger: logger, decorators: []podTemplateSpecDecorator{ newEnvironmentDecorator(orderedValues), newVolumeDecorator(addonName, template), newNodePlacementDecorator(privateValues), newImageDecorator(privateValues), newProxyHandler(logger, addonName, privateValues), newResourceRequirementsDecorator(logger, supportResourceDaemonset, privateValues), }, } } func (d *daemonSetDecorator) decorate(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { daemonSet, err := utils.ConvertToDaemonSet(obj) // not a daemonset, directly return if err != nil { return obj, nil } for _, decorator := range d.decorators { err = decorator.decorate(daemonSet.Name, &daemonSet.Spec.Template) if err != nil { return obj, err } } result, err := runtime.DefaultUnstructuredConverter.ToUnstructured(daemonSet) if err != nil { return obj, err } return &unstructured.Unstructured{Object: result}, nil } type podTemplateSpecDecorator interface { // decorate modifies the pod template in place // resourceName is the name of the resource, could be a deployment name or a daemonset name decorate(resourceName string, pod *corev1.PodTemplateSpec) error } type environmentDecorator struct { orderedValues orderedValues } func newEnvironmentDecorator(orderedValues orderedValues) podTemplateSpecDecorator { return &environmentDecorator{ orderedValues: orderedValues, } } func (d *environmentDecorator) decorate(_ string, pod *corev1.PodTemplateSpec) error { envVars := make([]corev1.EnvVar, len(d.orderedValues)) for index, value := range d.orderedValues { envVars[index] = corev1.EnvVar{ Name: value.name, Value: value.value, } } for j := range pod.Spec.Containers { pod.Spec.Containers[j].Env = append( pod.Spec.Containers[j].Env, envVars...) } return nil } type volumeDecorator struct { template *addonapiv1alpha1.AddOnTemplate addonName string } func newVolumeDecorator(addonName string, template *addonapiv1alpha1.AddOnTemplate) podTemplateSpecDecorator { return &volumeDecorator{ addonName: addonName, template: template, } } func (d *volumeDecorator) decorate(_ string, pod *corev1.PodTemplateSpec) error { volumeMounts := []corev1.VolumeMount{} volumes := []corev1.Volume{} for _, registration := range d.template.Spec.Registration { if registration.Type == addonapiv1alpha1.RegistrationTypeKubeClient { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: "hub-kubeconfig", MountPath: hubKubeconfigSecretMountPath(), }) volumes = append(volumes, corev1.Volume{ Name: "hub-kubeconfig", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: HubKubeconfigSecretName(d.addonName), }, }, }) } if registration.Type == addonapiv1alpha1.RegistrationTypeCustomSigner { if registration.CustomSigner == nil { return fmt.Errorf("custom signer is nil") } name := fmt.Sprintf("cert-%s", strings.ReplaceAll( strings.ReplaceAll(registration.CustomSigner.SignerName, "/", "-"), ".", "-")) volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: name, MountPath: customSignedSecretMountPath(registration.CustomSigner.SignerName), }) volumes = append(volumes, corev1.Volume{ Name: name, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: CustomSignedSecretName(d.addonName, registration.CustomSigner.SignerName), }, }, }) } } if len(volumeMounts) == 0 || len(volumes) == 0 { return nil } for j := range pod.Spec.Containers { pod.Spec.Containers[j].VolumeMounts = append( pod.Spec.Containers[j].VolumeMounts, volumeMounts...) } pod.Spec.Volumes = append(pod.Spec.Volumes, volumes...) return nil } type nodePlacementDecorator struct { privateValues addonfactory.Values } func newNodePlacementDecorator(privateValues addonfactory.Values) podTemplateSpecDecorator { return &nodePlacementDecorator{ privateValues: privateValues, } } func (d *nodePlacementDecorator) decorate(_ string, pod *corev1.PodTemplateSpec) error { nodePlacement, ok := d.privateValues[NodePlacementPrivateValueKey] if !ok { return nil } np, ok := nodePlacement.(*addonapiv1beta1.NodePlacement) if !ok { return fmt.Errorf("node placement value is invalid") } if np.NodeSelector != nil { pod.Spec.NodeSelector = np.NodeSelector } if np.NodeSelector != nil { pod.Spec.Tolerations = np.Tolerations } return nil } type imageDecorator struct { privateValues addonfactory.Values } func newImageDecorator(privateValues addonfactory.Values) podTemplateSpecDecorator { return &imageDecorator{ privateValues: privateValues, } } func (d *imageDecorator) decorate(_ string, pod *corev1.PodTemplateSpec) error { registries, ok := d.privateValues[RegistriesPrivateValueKey] if !ok { return nil } ims, ok := registries.([]addonapiv1beta1.ImageMirror) if !ok { return fmt.Errorf("registries value is invalid") } for i := range pod.Spec.Containers { pod.Spec.Containers[i].Image = addonfactory.OverrideImage( ims, pod.Spec.Containers[i].Image) } return nil } // objectsInjector injects additional runtime objects to the manifests, these objects will be created // in the managed clusters type objectsInjector interface { // inject returns a list of runtime objects to be created in the managed cluster inject() ([]runtime.Object, error) } // podTemplateHandler is a combination of podTemplateSpecDecorator and objectsInjector, it can decorate // the pod in the deployments/daemonsets and inject additional runtime objects into the manifests type podTemplateHandler interface { podTemplateSpecDecorator objectsInjector } type proxyHandler struct { logger klog.Logger addonName string privateValues addonfactory.Values } func newProxyHandler(logger klog.Logger, addonName string, privateValues addonfactory.Values) podTemplateHandler { return &proxyHandler{ logger: logger, addonName: addonName, privateValues: privateValues, } } func (d *proxyHandler) decorate(name string, pod *corev1.PodTemplateSpec) error { pc, ok := d.getProxyConfig() if !ok { return nil } keyValues := []keyValuePair{} if len(pc.HTTPProxy) > 0 { keyValues = append(keyValues, keyValuePair{name: "HTTP_PROXY", value: pc.HTTPProxy}, keyValuePair{name: "http_proxy", value: pc.HTTPProxy}, ) } if len(pc.HTTPSProxy) > 0 { keyValues = append(keyValues, keyValuePair{name: "HTTPS_PROXY", value: pc.HTTPSProxy}, keyValuePair{name: "https_proxy", value: pc.HTTPSProxy}, ) } if len(pc.NoProxy) > 0 { keyValues = append(keyValues, keyValuePair{name: "NO_PROXY", value: pc.NoProxy}, keyValuePair{name: "no_proxy", value: pc.NoProxy}, ) } if len(keyValues) == 0 { return nil } err := newEnvironmentDecorator(keyValues).decorate(name, pod) if err != nil { return err } if len(pc.CABundle) == 0 { return nil } return newCABundleDecorator(d.addonName, pc.CABundle).decorate(name, pod) } func (d *proxyHandler) getProxyConfig() (addonapiv1beta1.ProxyConfig, bool) { proxyConfig, ok := d.privateValues[ProxyPrivateValueKey] if !ok { return addonapiv1beta1.ProxyConfig{}, false } pc, ok := proxyConfig.(addonapiv1beta1.ProxyConfig) if !ok { d.logger.Error(nil, "proxy config value is invalid", "value", proxyConfig) return addonapiv1beta1.ProxyConfig{}, false } return pc, true } func (d *proxyHandler) inject() ([]runtime.Object, error) { pc, ok := d.getProxyConfig() if !ok { return nil, nil } if len(pc.CABundle) == 0 { return nil, nil } return []runtime.Object{ &corev1.ConfigMap{ // add TypeMeta to prevent error: // "failed to generate required mapper.err got empty kind/version from object" TypeMeta: metav1.TypeMeta{ Kind: "ConfigMap", APIVersion: "v1", }, ObjectMeta: metav1.ObjectMeta{ Name: proxyCABundleConfigMapName(d.addonName), // use the default namespace, will be decorated by the namespaceDecorator Namespace: "open-cluster-management-agent-addon", }, Data: map[string]string{ proxyCABundleConfigMapDataKey(): string(pc.CABundle), }, }, }, nil } type caBundleDecorator struct { addonName string caBundle []byte envDecorator podTemplateSpecDecorator } func newCABundleDecorator(addonName string, caBundle []byte) podTemplateSpecDecorator { keyValues := []keyValuePair{} keyValues = append(keyValues, keyValuePair{name: "CA_BUNDLE_FILE_PATH", value: proxyCABundleFilePath()}, ) return &caBundleDecorator{ addonName: addonName, caBundle: caBundle, envDecorator: newEnvironmentDecorator(keyValues), } } func (d *caBundleDecorator) decorate(name string, pod *corev1.PodTemplateSpec) error { err := d.envDecorator.decorate(name, pod) if err != nil { return err } volumeMounts := []corev1.VolumeMount{ { Name: "proxy-ca-bundle", MountPath: proxyCABundleConfigMapMountPath(), }, } volumes := []corev1.Volume{ { Name: "proxy-ca-bundle", VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ Name: proxyCABundleConfigMapName(d.addonName), }, }, }, }, } for j := range pod.Spec.Containers { pod.Spec.Containers[j].VolumeMounts = append( pod.Spec.Containers[j].VolumeMounts, volumeMounts...) } pod.Spec.Volumes = append(pod.Spec.Volumes, volumes...) return nil } type supportResource string const ( supportResourceDeployment supportResource = "deployments" supportResourceDaemonset supportResource = "daemonsets" ) type resourceRequirementsDecorator struct { privateValues addonfactory.Values resource supportResource // only support daemonsets, deployments for now logger klog.Logger } func newResourceRequirementsDecorator(logger klog.Logger, resource supportResource, privateValues addonfactory.Values) podTemplateSpecDecorator { return &resourceRequirementsDecorator{ resource: resource, privateValues: privateValues, } } func (d *resourceRequirementsDecorator) decorate(name string, pod *corev1.PodTemplateSpec) error { requirements, ok := d.privateValues[ResourceRequirementsPrivateValueKey] if !ok { return nil } regexRequirements, ok := requirements.([]addonfactory.RegexResourceRequirements) if !ok { return fmt.Errorf("resource requirements value is invalid") } for i := range pod.Spec.Containers { containerID := fmt.Sprintf("%s:%s:%s", d.resource, name, pod.Spec.Containers[i].Name) // revese the requirements array to make the later elements in the array have higher priority for j := len(regexRequirements) - 1; j >= 0; j-- { matched, err := regexp.MatchString(regexRequirements[j].ContainerIDRegex, containerID) if err != nil { d.logger.Info("regex match container id failed", "pattern", regexRequirements[j].ContainerIDRegex, "containerID", containerID) continue } if !matched { continue } pod.Spec.Containers[i].Resources = regexRequirements[j].ResourcesRaw break } } return nil } func hubKubeconfigSecretMountPath() string { return "/managed/hub-kubeconfig" } func HubKubeconfigSecretName(addonName string) string { return fmt.Sprintf("%s-hub-kubeconfig", addonName) } func CustomSignedSecretName(addonName, signerName string) string { return fmt.Sprintf("%s-%s-client-cert", addonName, strings.ReplaceAll(signerName, "/", "-")) } func customSignedSecretMountPath(signerName string) string { return fmt.Sprintf("/managed/%s", strings.ReplaceAll(signerName, "/", "-")) } func proxyCABundleConfigMapMountPath() string { return "/managed/proxy-ca" } func proxyCABundleConfigMapName(addonName string) string { return fmt.Sprintf("%s-proxy-ca", addonName) } func proxyCABundleConfigMapDataKey() string { return "ca-bundle.crt" } func proxyCABundleFilePath() string { return path.Join(proxyCABundleConfigMapMountPath(), proxyCABundleConfigMapDataKey()) }