package templateagent import ( "context" "crypto/x509" "encoding/pem" "fmt" "os" "strings" "time" openshiftcrypto "github.com/openshift/library-go/pkg/crypto" "github.com/openshift/library-go/pkg/operator/resource/resourceapply" "github.com/pkg/errors" certificatesv1 "k8s.io/api/certificates/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" "open-cluster-management.io/addon-framework/pkg/agent" "open-cluster-management.io/addon-framework/pkg/utils" addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" clusterv1 "open-cluster-management.io/api/cluster/v1" ) const ( // AddonTemplateLabelKey is the label key to set addon template name. It is to set the resources on the hub relating // to an addon template AddonTemplateLabelKey = "open-cluster-management.io/addon-template-name" ) var ( podNamespace = "" ) func AddonManagerNamespace() string { if len(podNamespace) != 0 { return podNamespace } namespace := os.Getenv("POD_NAMESPACE") if len(namespace) != 0 { podNamespace = namespace } else { podNamespace = "open-cluster-management-hub" } return podNamespace } // GetDesiredAddOnTemplate returns the desired AddOnTemplate for the given ManagedClusterAddOn. // If the desired AddOnTemplate is not found in the ManagedClusterAddOn Status ConfigReferences, // it will return a nil AddOnTemplate with no error. the caller should handle the nil // AddOnTemplate case. func (a *CRDTemplateAgentAddon) GetDesiredAddOnTemplate(addon *addonapiv1alpha1.ManagedClusterAddOn, clusterName, addonName string) (*addonapiv1alpha1.AddOnTemplate, error) { if addon != nil { return a.getDesiredAddOnTemplateInner(addon.Name, addon.Status.ConfigReferences) } if len(clusterName) != 0 { addon, err := a.addonLister.ManagedClusterAddOns(clusterName).Get(addonName) if err != nil { return nil, err } return a.getDesiredAddOnTemplateInner(addon.Name, addon.Status.ConfigReferences) } // clusterName and addon are both empty, backoff to get the template from the clusterManagementAddOn cma, err := a.cmaLister.Get(addonName) if err != nil { return nil, err } // convert the DefaultConfigReference to ConfigReference var configReferences []addonapiv1alpha1.ConfigReference for _, configReference := range cma.Status.DefaultConfigReferences { configReferences = append(configReferences, addonapiv1alpha1.ConfigReference{ ConfigGroupResource: configReference.ConfigGroupResource, DesiredConfig: configReference.DesiredConfig, }) } return a.getDesiredAddOnTemplateInner(cma.Name, configReferences) } func (a *CRDTemplateAgentAddon) TemplateCSRConfigurationsFunc() func(cluster *clusterv1.ManagedCluster) []addonapiv1alpha1.RegistrationConfig { return func(cluster *clusterv1.ManagedCluster) []addonapiv1alpha1.RegistrationConfig { template, err := a.GetDesiredAddOnTemplate(nil, cluster.Name, a.addonName) if err != nil { utilruntime.HandleError(fmt.Errorf("failed to get addon %s template: %v", a.addonName, err)) return nil } if template == nil { return nil } contain := func(rcs []addonapiv1alpha1.RegistrationConfig, signerName string) bool { for _, rc := range rcs { if rc.SignerName == signerName { return true } } return false } registrationConfigs := make([]addonapiv1alpha1.RegistrationConfig, 0) for _, registration := range template.Spec.Registration { switch registration.Type { case addonapiv1alpha1.RegistrationTypeKubeClient: if !contain(registrationConfigs, certificatesv1.KubeAPIServerClientSignerName) { configs := agent.KubeClientSignerConfigurations(a.addonName, a.agentName)(cluster) registrationConfigs = append(registrationConfigs, configs...) } case addonapiv1alpha1.RegistrationTypeCustomSigner: if registration.CustomSigner == nil { continue } if !contain(registrationConfigs, registration.CustomSigner.SignerName) { configs := CustomSignerConfigurations( a.addonName, a.agentName, registration.CustomSigner)(cluster) registrationConfigs = append(registrationConfigs, configs...) } default: utilruntime.HandleError(fmt.Errorf("unsupported registration type %s", registration.Type)) } } return registrationConfigs } } // CustomSignerConfigurations returns a func that can generate RegistrationConfig // for CustomSigner type registration addon func CustomSignerConfigurations(addonName, agentName string, customSignerConfig *addonapiv1alpha1.CustomSignerRegistrationConfig, ) func(cluster *clusterv1.ManagedCluster) []addonapiv1alpha1.RegistrationConfig { return func(cluster *clusterv1.ManagedCluster) []addonapiv1alpha1.RegistrationConfig { if customSignerConfig == nil { utilruntime.HandleError(fmt.Errorf("custome signer is nil")) } config := addonapiv1alpha1.RegistrationConfig{ SignerName: customSignerConfig.SignerName, // TODO: confirm the subject Subject: addonapiv1alpha1.Subject{ User: agent.DefaultUser(cluster.Name, addonName, agentName), Groups: agent.DefaultGroups(cluster.Name, addonName), }, } if customSignerConfig.Subject != nil { config.Subject = *customSignerConfig.Subject } return []addonapiv1alpha1.RegistrationConfig{config} } } func (a *CRDTemplateAgentAddon) TemplateCSRApproveCheckFunc() agent.CSRApproveFunc { return func(cluster *clusterv1.ManagedCluster, addon *addonapiv1alpha1.ManagedClusterAddOn, csr *certificatesv1.CertificateSigningRequest) bool { template, err := a.GetDesiredAddOnTemplate(addon, cluster.Name, a.addonName) if err != nil { utilruntime.HandleError(fmt.Errorf("failed to get addon %s template: %v", a.addonName, err)) return false } if template == nil { return false } for _, registration := range template.Spec.Registration { switch registration.Type { case addonapiv1alpha1.RegistrationTypeKubeClient: if csr.Spec.SignerName == certificatesv1.KubeAPIServerClientSignerName { return KubeClientCSRApprover(a.agentName)(cluster, addon, csr) } case addonapiv1alpha1.RegistrationTypeCustomSigner: if registration.CustomSigner == nil { continue } if csr.Spec.SignerName == registration.CustomSigner.SignerName { return CustomerSignerCSRApprover(a.logger, a.addonName)(cluster, addon, csr) } default: utilruntime.HandleError(fmt.Errorf("unsupported registration type %s", registration.Type)) } } return false } } // KubeClientCSRApprover approve the csr when addon agent uses default group, default user and // "kubernetes.io/kube-apiserver-client" signer to sign csr. func KubeClientCSRApprover(agentName string) agent.CSRApproveFunc { return func( cluster *clusterv1.ManagedCluster, addon *addonapiv1alpha1.ManagedClusterAddOn, csr *certificatesv1.CertificateSigningRequest) bool { if csr.Spec.SignerName != certificatesv1.KubeAPIServerClientSignerName { return false } return utils.DefaultCSRApprover(agentName)(cluster, addon, csr) } } // CustomerSignerCSRApprover approve the csr when addon agent uses custom signer to sign csr. func CustomerSignerCSRApprover(logger klog.Logger, agentName string) agent.CSRApproveFunc { return func( cluster *clusterv1.ManagedCluster, addon *addonapiv1alpha1.ManagedClusterAddOn, csr *certificatesv1.CertificateSigningRequest) bool { logger.Info("Customer signer CSR is approved", "clusterName", cluster.Name, "addonName", addon.Name, "requester", csr.Spec.Username) return true } } func (a *CRDTemplateAgentAddon) TemplateCSRSignFunc() agent.CSRSignerFunc { return func(csr *certificatesv1.CertificateSigningRequest) []byte { // TODO: consider to change the agent.CSRSignerFun to accept parameter addon getClusterName := func(userName string) string { return csr.Labels[clusterv1.ClusterNameLabelKey] } clusterName := getClusterName(csr.Spec.Username) template, err := a.GetDesiredAddOnTemplate(nil, clusterName, a.addonName) if err != nil { utilruntime.HandleError(fmt.Errorf("failed to get template for addon %s in cluster %s: %v", a.addonName, clusterName, err)) return nil } if template == nil { return nil } for _, registration := range template.Spec.Registration { switch registration.Type { case addonapiv1alpha1.RegistrationTypeKubeClient: continue case addonapiv1alpha1.RegistrationTypeCustomSigner: if registration.CustomSigner == nil { continue } if csr.Spec.SignerName == registration.CustomSigner.SignerName { return CustomSignerWithExpiry(a.hubKubeClient, registration.CustomSigner, 24*time.Hour)(csr) } default: utilruntime.HandleError(fmt.Errorf("unsupported registration type %s", registration.Type)) } } return nil } } func CustomSignerWithExpiry( kubeclient kubernetes.Interface, customSignerConfig *addonapiv1alpha1.CustomSignerRegistrationConfig, duration time.Duration) agent.CSRSignerFunc { return func(csr *certificatesv1.CertificateSigningRequest) []byte { if customSignerConfig == nil { utilruntime.HandleError(fmt.Errorf("custome signer is nil")) return nil } if csr.Spec.SignerName != customSignerConfig.SignerName { return nil } secretNamespace := AddonManagerNamespace() if len(customSignerConfig.SigningCA.Namespace) != 0 { secretNamespace = customSignerConfig.SigningCA.Namespace } caSecret, err := kubeclient.CoreV1().Secrets(secretNamespace).Get( context.TODO(), customSignerConfig.SigningCA.Name, metav1.GetOptions{}) if err != nil { utilruntime.HandleError(fmt.Errorf("get custome signer ca %s/%s failed, %v", secretNamespace, customSignerConfig.SigningCA.Name, err)) return nil } caData, caKey, err := extractCAdata(caSecret.Data[corev1.TLSCertKey], caSecret.Data[corev1.TLSPrivateKeyKey]) if err != nil { utilruntime.HandleError(fmt.Errorf("get ca %s/%s data failed, %v", secretNamespace, customSignerConfig.SigningCA.Name, err)) return nil } return utils.DefaultSignerWithExpiry(caKey, caData, duration)(csr) } } func extractCAdata(caCertData, caKeyData []byte) ([]byte, []byte, error) { certBlock, _ := pem.Decode(caCertData) if certBlock == nil { return nil, nil, errors.New("failed to decode ca cert") } caCert, err := x509.ParseCertificate(certBlock.Bytes) if err != nil { return nil, nil, errors.Wrapf(err, "failed to parse ca certificate") } keyBlock, _ := pem.Decode(caKeyData) if keyBlock == nil { return nil, nil, errors.New("failed to decode ca key") } var errPkcs8, errPkcs1 error var caKey any caKey, errPkcs8 = x509.ParsePKCS8PrivateKey(keyBlock.Bytes) if errPkcs8 != nil { caKey, errPkcs1 = x509.ParsePKCS1PrivateKey(keyBlock.Bytes) if err != nil { return nil, nil, fmt.Errorf("failed to parse ca key with pkcs8: %v and pkcs1: %v", errPkcs8, errPkcs1) } } caConfig := &openshiftcrypto.TLSCertificateConfig{ Certs: []*x509.Certificate{caCert}, Key: caKey, } return caConfig.GetPEMBytes() } // TemplatePermissionConfigFunc returns a func that can grant permission for addon agent // that is deployed by addon template. // the returned func will create a rolebinding to bind the clusterRole/role which is // specified by the user, so the user is required to make sure the existence of the // clusterRole/role func (a *CRDTemplateAgentAddon) TemplatePermissionConfigFunc() agent.PermissionConfigFunc { return func(cluster *clusterv1.ManagedCluster, addon *addonapiv1alpha1.ManagedClusterAddOn) error { template, err := a.GetDesiredAddOnTemplate(addon, cluster.Name, a.addonName) if err != nil { return err } if template == nil { return nil } for _, registration := range template.Spec.Registration { switch registration.Type { case addonapiv1alpha1.RegistrationTypeKubeClient: kcrc := registration.KubeClient if kcrc == nil { continue } err := a.createKubeClientPermissions(kcrc, cluster, addon) if err != nil { return err } case addonapiv1alpha1.RegistrationTypeCustomSigner: continue default: utilruntime.HandleError(fmt.Errorf("unsupported registration type %s", registration.Type)) } } return nil } } func (a *CRDTemplateAgentAddon) createKubeClientPermissions( kcrc *addonapiv1alpha1.KubeClientRegistrationConfig, cluster *clusterv1.ManagedCluster, addon *addonapiv1alpha1.ManagedClusterAddOn, ) error { for _, pc := range kcrc.HubPermissions { switch pc.Type { case addonapiv1alpha1.HubPermissionsBindingCurrentCluster: if pc.CurrentCluster == nil { return fmt.Errorf("current cluster is required when the HubPermission type is CurrentCluster") } a.logger.V(5).Info("Set hub permission for addon", "addonNamespace", addon.Namespace, "addonName", addon.Name, "UID", addon.UID, "APIVersion", addon.APIVersion, "Kind", addon.Kind) owner := metav1.OwnerReference{ // TODO: use apiVersion and kind in addon object, but now they could be empty at some unknown reason APIVersion: "addon.open-cluster-management.io/v1alpha1", Kind: "ManagedClusterAddOn", Name: addon.Name, UID: addon.UID, } roleRef := rbacv1.RoleRef{ Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: pc.CurrentCluster.ClusterRoleName, } err := a.createPermissionBinding(cluster.Name, addon.Name, cluster.Name, roleRef, &owner) if err != nil { return err } case addonapiv1alpha1.HubPermissionsBindingSingleNamespace: if pc.SingleNamespace == nil { return fmt.Errorf("single namespace is required when the HubPermission type is SingleNamespace") } // set owner reference nil since the rolebinding has different namespace with the ManagedClusterAddon // TODO: cleanup the rolebinding when the addon is deleted err := a.createPermissionBinding(cluster.Name, addon.Name, pc.SingleNamespace.Namespace, pc.SingleNamespace.RoleRef, nil) if err != nil { return err } } } return nil } func (a *CRDTemplateAgentAddon) createPermissionBinding(clusterName, addonName, namespace string, roleRef rbacv1.RoleRef, owner *metav1.OwnerReference) error { binding := &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("open-cluster-management:%s:%s:agent", addonName, strings.ToLower(roleRef.Kind)), Namespace: namespace, Labels: map[string]string{ addonapiv1alpha1.AddonLabelKey: addonName, AddonTemplateLabelKey: "", }, }, RoleRef: roleRef, Subjects: []rbacv1.Subject{ { Kind: rbacv1.GroupKind, APIGroup: rbacv1.GroupName, Name: clusterAddonGroup(clusterName, addonName), }, }, } if owner != nil { binding.OwnerReferences = []metav1.OwnerReference{*owner} } _, modified, err := resourceapply.ApplyRoleBinding(context.TODO(), a.hubKubeClient.RbacV1(), a.eventRecorder, binding) if err == nil && modified { a.logger.Info("Rolebinding for addon updated", "namespace", binding.Namespace, "name", binding.Name, "clusterName", clusterName, "addonName", addonName) } return err } // clusterAddonGroup returns the group that represents the addon for the cluster func clusterAddonGroup(clusterName, addonName string) string { return fmt.Sprintf("system:open-cluster-management:cluster:%s:addon:%s", clusterName, addonName) }