// Copyright 2020-2026 Project Capsule Authors // SPDX-License-Identifier: Apache-2.0 package main import ( "context" "crypto/tls" goflag "flag" "fmt" "os" "path/filepath" goRuntime "runtime" "time" flag "github.com/spf13/pflag" _ "go.uber.org/automaxprocs" "go.uber.org/zap/zapcore" admissionv1 "k8s.io/api/admissionregistration/v1" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" utilVersion "k8s.io/apimachinery/pkg/util/version" "k8s.io/client-go/discovery" "k8s.io/client-go/dynamic" clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ctrlwebhook "sigs.k8s.io/controller-runtime/pkg/webhook" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" capsulev1beta1 "github.com/projectcapsule/capsule/api/v1beta1" capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2" "github.com/projectcapsule/capsule/internal/cache" "github.com/projectcapsule/capsule/internal/controllers/admission" cacheinvalidator "github.com/projectcapsule/capsule/internal/controllers/cfg/invalidator" configcontroller "github.com/projectcapsule/capsule/internal/controllers/cfg/status" customquotacontroller "github.com/projectcapsule/capsule/internal/controllers/customquotas" globalresourcequotacontroller "github.com/projectcapsule/capsule/internal/controllers/globalresourcequotas" podlabelscontroller "github.com/projectcapsule/capsule/internal/controllers/pod" "github.com/projectcapsule/capsule/internal/controllers/pv" rbaccontroller "github.com/projectcapsule/capsule/internal/controllers/rbac" "github.com/projectcapsule/capsule/internal/controllers/resourcepools" "github.com/projectcapsule/capsule/internal/controllers/resources" rulestatuscontroller "github.com/projectcapsule/capsule/internal/controllers/rulestatus" servicelabelscontroller "github.com/projectcapsule/capsule/internal/controllers/servicelabels" tenantcontroller "github.com/projectcapsule/capsule/internal/controllers/tenant" tenantownercontroller "github.com/projectcapsule/capsule/internal/controllers/tenantowner" tlscontroller "github.com/projectcapsule/capsule/internal/controllers/tls" utilscontroller "github.com/projectcapsule/capsule/internal/controllers/utils" "github.com/projectcapsule/capsule/internal/metrics" capsuleversion "github.com/projectcapsule/capsule/internal/version" "github.com/projectcapsule/capsule/internal/webhook" cfgvalidation "github.com/projectcapsule/capsule/internal/webhook/cfg" customquotavalidation "github.com/projectcapsule/capsule/internal/webhook/customquota" "github.com/projectcapsule/capsule/internal/webhook/defaults" "github.com/projectcapsule/capsule/internal/webhook/dra" "github.com/projectcapsule/capsule/internal/webhook/gateway" "github.com/projectcapsule/capsule/internal/webhook/generic" globalresourcequotavalidation "github.com/projectcapsule/capsule/internal/webhook/globalresourcequota" "github.com/projectcapsule/capsule/internal/webhook/ingress" namespacemutation "github.com/projectcapsule/capsule/internal/webhook/namespace/mutation" namespacevalidation "github.com/projectcapsule/capsule/internal/webhook/namespace/validation" "github.com/projectcapsule/capsule/internal/webhook/node" "github.com/projectcapsule/capsule/internal/webhook/owners" "github.com/projectcapsule/capsule/internal/webhook/pod" "github.com/projectcapsule/capsule/internal/webhook/pvc" "github.com/projectcapsule/capsule/internal/webhook/resourcepool" "github.com/projectcapsule/capsule/internal/webhook/route" rulesgenericmutation "github.com/projectcapsule/capsule/internal/webhook/rules/generic/mutation" rulesgenericvalidation "github.com/projectcapsule/capsule/internal/webhook/rules/generic/validation" podrules "github.com/projectcapsule/capsule/internal/webhook/rules/pods/validation" servicerules "github.com/projectcapsule/capsule/internal/webhook/rules/services/validation" "github.com/projectcapsule/capsule/internal/webhook/service" "github.com/projectcapsule/capsule/internal/webhook/serviceaccounts" tenantmutation "github.com/projectcapsule/capsule/internal/webhook/tenant/mutation" tenantvalidation "github.com/projectcapsule/capsule/internal/webhook/tenant/validation" "github.com/projectcapsule/capsule/pkg/runtime/configuration" evt "github.com/projectcapsule/capsule/pkg/runtime/events" "github.com/projectcapsule/capsule/pkg/runtime/handlers" "github.com/projectcapsule/capsule/pkg/runtime/indexers" "github.com/projectcapsule/capsule/pkg/utils" ) var ( scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") ) func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(capsulev1beta1.AddToScheme(scheme)) utilruntime.Must(capsulev1beta2.AddToScheme(scheme)) utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) utilruntime.Must(gatewayv1.Install(scheme)) utilruntime.Must(admissionv1.AddToScheme(scheme)) } func printVersion() { setupLog.Info(fmt.Sprintf("Capsule Version %s %s%s", capsuleversion.GitTag, capsuleversion.GitCommit, capsuleversion.GitDirty)) setupLog.Info(fmt.Sprintf("Build from: %s", capsuleversion.GitRepo)) setupLog.Info(fmt.Sprintf("Build date: %s", capsuleversion.BuildTime)) setupLog.Info(fmt.Sprintf("Go Version: %s", goRuntime.Version())) setupLog.Info(fmt.Sprintf("Go OS/Arch: %s/%s", goRuntime.GOOS, goRuntime.GOARCH)) } //nolint:maintidx,gocyclo,cyclop,gocognit func main() { controllerConfig := utilscontroller.ControllerOptions{} var ( metricsAddr, metricsCertPath, metricsCertName, metricsCertKey string webhookCertPath, webhookCertName, webhookCertKey string enableLeaderElection bool enablePprof bool enableTracing bool version bool secureMetrics bool enableHTTP2 bool tracingEndpoint string tracingSampleRatio float64 tracingInsecure bool tracingHeaders = tracingHeadersFlag{} tracingBasicAuthUsername string tracingBasicAuthPassword string tracingTimeout time.Duration tracingCompression string tracingTLSServerName string tracingTLSInsecureSkipVerify bool clientConnectionQPS float32 clientConnectionBurst int32 webhookPort int cacheSyncTimeout time.Duration leaderElectionLeaseDuration time.Duration leaderElectionRenewDeadline time.Duration leaderElectionRetryPeriod time.Duration ) var goFlagSet goflag.FlagSet var tlsOpts []func(*tls.Config) flag.StringVar( &controllerConfig.ConfigurationName, "configuration-name", "default", "The CapsuleConfiguration resource name to use", ) flag.BoolVar( &enableLeaderElection, "enable-leader-election", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.", ) flag.BoolVar( &enableTracing, "enable-tracing", false, "Enable OpenTelemetry tracing for admission webhook requests.", ) flag.StringVar( &tracingEndpoint, "tracing-otlp-endpoint", "", "OTLP gRPC endpoint for exporting traces, for example otel-collector.observability.svc:4317. If unset, OpenTelemetry environment variables are used.", ) flag.BoolVar( &tracingInsecure, "tracing-otlp-insecure", true, "Disable transport security for the OTLP gRPC trace exporter.", ) flag.Float64Var( &tracingSampleRatio, "tracing-sample-ratio", 1.0, "Trace sampling ratio for admission webhook requests. Must be between 0 and 1.", ) flag.Var( tracingHeaders, "tracing-otlp-header", "OTLP gRPC metadata header in key=value format. Can be set multiple times.", ) flag.StringVar( &tracingBasicAuthUsername, "tracing-otlp-basic-auth-username", "", "Basic auth username for the OTLP gRPC trace exporter. Can also be set with CAPSULE_TRACING_OTLP_BASIC_AUTH_USERNAME.", ) flag.StringVar( &tracingBasicAuthPassword, "tracing-otlp-basic-auth-password", "", "Basic auth password for the OTLP gRPC trace exporter. Can also be set with CAPSULE_TRACING_OTLP_BASIC_AUTH_PASSWORD.", ) flag.DurationVar( &tracingTimeout, "tracing-otlp-timeout", 0, "Timeout for exporting a batch of spans. Empty or 0 uses OpenTelemetry's default.", ) flag.StringVar( &tracingCompression, "tracing-otlp-compression", "", "Compression for OTLP gRPC trace exports. Supported value: gzip. Empty disables compression.", ) flag.StringVar( &tracingTLSServerName, "tracing-otlp-tls-server-name", "", "TLS server name override for the OTLP gRPC trace exporter.", ) flag.BoolVar( &tracingTLSInsecureSkipVerify, "tracing-otlp-tls-insecure-skip-verify", false, "Skip OTLP gRPC trace exporter TLS certificate verification. Not recommended for production.", ) flag.IntVar( &controllerConfig.Runtime.MaxConcurrentReconciles, "workers", 1, "MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run.", ) flag.DurationVar( &cacheSyncTimeout, "cache-sync-timeout", 0, "The timeout used when waiting for controller cache synchronization. If unset or 0, the controller-runtime default is used.", ) flag.DurationVar( &leaderElectionLeaseDuration, "leader-election-lease-duration", 0, "The duration that non-leader candidates wait to force acquire leadership. If unset or 0, the controller-runtime default is used.", ) flag.DurationVar( &leaderElectionRenewDeadline, "leader-election-renew-deadline", 0, "The duration that the acting leader retries refreshing leadership before giving up. If unset or 0, the controller-runtime default is used.", ) flag.DurationVar( &leaderElectionRetryPeriod, "leader-election-retry-period", 0, "The duration that leader election clients wait between retries. If unset or 0, the controller-runtime default is used.", ) flag.StringVar( &metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.", ) flag.BoolVar( &secureMetrics, "metrics-secure", false, "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.", ) flag.StringVar( &metricsCertPath, "metrics-cert-path", "", "The directory that contains the metrics server certificate.", ) flag.StringVar( &metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.", ) flag.StringVar( &metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.", ) flag.IntVar( &webhookPort, "webhook-port", 9443, "The port the webhook server binds to.", ) flag.StringVar( &webhookCertPath, "webhook-cert-path", "/tmp/k8s-webhook-server/serving-certs", "The directory that contains the webhook certificate.", ) flag.StringVar( &webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.", ) flag.StringVar( &webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.", ) flag.BoolVar( &enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers", ) flag.Float32Var( &clientConnectionQPS, "client-connection-qps", 20.0, "QPS to use for interacting with kubernetes apiserver.", ) flag.Int32Var( &clientConnectionBurst, "client-connection-burst", 30, "Burst to use for interacting with kubernetes apiserver.", ) flag.BoolVar( &enablePprof, "enable-pprof", false, "Enables Pprof endpoint for profiling (not recommend in production)", ) flag.BoolVar( &version, "version", false, "Print the Capsule version and exit", ) opts := zap.Options{ EncoderConfigOptions: append([]zap.EncoderConfigOption{}, func(config *zapcore.EncoderConfig) { config.EncodeTime = zapcore.ISO8601TimeEncoder }), } opts.BindFlags(&goFlagSet) flag.CommandLine.AddGoFlagSet(&goFlagSet) flag.Parse() ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) printVersion() if version { os.Exit(0) } ctx := ctrl.SetupSignalHandler() setupLog.V(5).Info("Controller", "Options", controllerConfig) var ns string if ns = os.Getenv(configuration.EnvironmentControllerNamespace); len(ns) == 0 { setupLog.Error(fmt.Errorf("unable to determinate the Namespace Capsule is running on. Please export %s", configuration.EnvironmentControllerNamespace), "unable to start manager") os.Exit(1) } if serviceAccountName := os.Getenv(configuration.EnvironmentServiceaccountName); len(serviceAccountName) == 0 { setupLog.Error(fmt.Errorf("unable to determinate the ServiceAccount Capsule is running with. Please export %s", configuration.EnvironmentServiceaccountName), "unable to start manager") os.Exit(1) } if cacheSyncTimeout > 0 { controllerConfig.Runtime.CacheSyncTimeout = cacheSyncTimeout } if len(controllerConfig.ConfigurationName) == 0 { setupLog.Error(fmt.Errorf("missing CapsuleConfiguration resource name"), "unable to start manager") os.Exit(1) } restConfig, err := ctrl.GetConfig() if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) } // Suppress Warnings restConfig.WarningHandler = rest.NoWarnings{} restConfig.QPS = clientConnectionQPS restConfig.Burst = int(clientConnectionBurst) directClient, err := client.New(ctrl.GetConfigOrDie(), client.Options{ Scheme: scheme, }) if err != nil { setupLog.Error(err, "unable to create the direct client") os.Exit(1) } directCfg := configuration.NewCapsuleConfiguration(ctx, directClient, directClient, restConfig, controllerConfig.ConfigurationName) tlsReconciler := &tlscontroller.Reconciler{} if directCfg.EnableTLSConfiguration() { tlsReconciler = &tlscontroller.Reconciler{ Client: directClient, Log: ctrl.Log.WithName("capsule.ctrl").WithName("tls"), Namespace: ns, Configuration: directCfg, } tlsCert := &corev1.Secret{} if err = directClient.Get(ctx, types.NamespacedName{ Namespace: ns, Name: directCfg.TLSSecretName(), }, tlsCert); err != nil { if !apierrors.IsNotFound(err) { setupLog.Error(err, "unable to get Capsule TLS secret") os.Exit(1) } tlsCert = &corev1.Secret{} tlsCert.Name = directCfg.TLSSecretName() tlsCert.Namespace = ns tlsCert.Data = map[string][]byte{} } // Reconcile TLS certificates before starting controllers and webhooks if err = tlsReconciler.ReconcileCertificates(ctx, ctrl.Log.WithName("capsule.setup").WithName("tls"), tlsCert); err != nil { setupLog.Error(err, "unable to reconcile Capsule TLS secret") os.Exit(1) } } // if the enable-http2 flag is false (the default), http/2 should be disabled // due to its vulnerabilities. More specifically, disabling http/2 will // prevent from being vulnerable to the HTTP/2 Stream Cancellation and // Rapid Reset CVEs. For more information see: // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 // - https://github.com/advisories/GHSA-4374-p667-p6c8 disableHTTP2 := func(c *tls.Config) { setupLog.Info("disabling http/2") c.NextProtos = []string{"http/1.1"} } if !enableHTTP2 { tlsOpts = append(tlsOpts, disableHTTP2) } // Create watchers for metrics and webhooks certificates var metricsCertWatcher, webhookCertWatcher *certwatcher.CertWatcher // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. // More info: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.20.4/pkg/metrics/server // - https://book.kubebuilder.io/reference/metrics.html metricsServerOptions := metricsserver.Options{ BindAddress: metricsAddr, SecureServing: secureMetrics, TLSOpts: tlsOpts, } if secureMetrics { // FilterProvider is used to protect the metrics endpoint with authn/authz. // These configurations ensure that only authorized users and service accounts // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.20.4/pkg/metrics/filters#WithAuthenticationAndAuthorization metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization } // If the certificate is not specified, controller-runtime will automatically // generate self-signed certificates for the metrics server. While convenient for development and testing, // this setup is not recommended for production. // if len(metricsCertPath) > 0 { setupLog.Info( "Initializing metrics certificate watcher using provided certificates", "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey, ) var err error metricsCertWatcher, err = certwatcher.New( filepath.Join(metricsCertPath, metricsCertName), filepath.Join(metricsCertPath, metricsCertKey), ) if err != nil { setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) os.Exit(1) } metricsServerOptions.TLSOpts = append( metricsServerOptions.TLSOpts, func(config *tls.Config) { config.GetCertificate = metricsCertWatcher.GetCertificate }, ) } // Initial webhook TLS options webhookTLSOpts := tlsOpts if len(webhookCertPath) > 0 { setupLog.Info( "Initializing webhook certificate watcher using provided certificates", "webhook-cert-path", webhookCertPath, "webhook-cert-name", webhookCertName, "webhook-cert-key", webhookCertKey, ) var err error webhookCertWatcher, err = certwatcher.New( filepath.Join(webhookCertPath, webhookCertName), filepath.Join(webhookCertPath, webhookCertKey), ) if err != nil { setupLog.Error(err, "Failed to initialize webhook certificate watcher") os.Exit(1) } webhookTLSOpts = append(webhookTLSOpts, func(config *tls.Config) { config.GetCertificate = webhookCertWatcher.GetCertificate }) } ctrlOpts := ctrl.Options{ Scheme: scheme, Metrics: metricsServerOptions, WebhookServer: ctrlwebhook.NewServer(ctrlwebhook.Options{ Port: webhookPort, TLSOpts: webhookTLSOpts, }), LeaderElection: enableLeaderElection, LeaderElectionID: "42c733ea.clastix.capsule.io", LeaderElectionNamespace: ns, HealthProbeBindAddress: ":10080", NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { options.Cache.Unstructured = true return client.New(config, options) }, } if leaderElectionLeaseDuration > 0 { ctrlOpts.LeaseDuration = &leaderElectionLeaseDuration } if leaderElectionRenewDeadline > 0 { ctrlOpts.RenewDeadline = &leaderElectionRenewDeadline } if leaderElectionRetryPeriod > 0 { ctrlOpts.RetryPeriod = &leaderElectionRetryPeriod } if enablePprof { ctrlOpts.PprofBindAddress = ":8082" } setupLog.Info("initializing manager") // Suppress Warnings restConfig.WarningHandler = rest.NoWarnings{} restConfig.QPS = clientConnectionQPS restConfig.Burst = int(clientConnectionBurst) manager, err := ctrl.NewManager(restConfig, ctrlOpts) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) } if metricsCertWatcher != nil { if err := manager.Add(metricsCertWatcher); err != nil { setupLog.Error(err, "unable to add metrics certificate watcher") os.Exit(1) } } if webhookCertWatcher != nil { if err := manager.Add(webhookCertWatcher); err != nil { setupLog.Error(err, "unable to add webhook certificate watcher") os.Exit(1) } } _ = manager.AddReadyzCheck("ping", healthz.Ping) _ = manager.AddHealthzCheck("ping", healthz.Ping) dc, err := discovery.NewDiscoveryClientForConfig(manager.GetConfig()) if err != nil { setupLog.Error(err, "unable to create discovery client") os.Exit(1) } dynamicClient, err := dynamic.NewForConfig(manager.GetConfig()) if err != nil { setupLog.Error(err, "unable to create dynamic client") os.Exit(1) } setupLog.Info("initializing capsule configuration") cfg := configuration.NewCapsuleConfiguration(ctx, manager.GetClient(), manager.GetAPIReader(), manager.GetConfig(), controllerConfig.ConfigurationName) setupLog.Info("initializing caches") // Initialize Caches impersonationCache := cache.NewImpersonationCache() regexCache := cache.NewRegexCache() registryCache := cache.NewRegistryRuleSetCache(regexCache) jsonPathCache := cache.NewJSONPathCache() celCache, err := cache.NewCELCache() if err != nil { setupLog.Error(err, "unable to initialize Kubernetes CEL cache") os.Exit(1) } targetsCache := cache.NewCompiledTargetsCache[string]() if directCfg.EnableTLSConfiguration() { if err = tlsReconciler.SetupWithManager(manager, controllerConfig); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Namespace") os.Exit(1) } } if err = (&tenantcontroller.Manager{ RESTConfig: manager.GetConfig(), Client: manager.GetClient(), DynamicClient: dynamicClient, DiscoveryClient: dc, Metrics: metrics.MustMakeTenantRecorder(), Log: ctrl.Log.WithName("capsule.ctrl").WithName("tenant"), Recorder: manager.GetEventRecorder("tenant-controller"), Configuration: cfg, }).SetupWithManager(manager, controllerConfig); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Tenant") os.Exit(1) } if err = (&capsulev1beta1.Tenant{}).SetupWebhookWithManager(manager); err != nil { setupLog.Error(err, "unable to create conversion webhook", "webhook", "capsulev1beta1.Tenant") os.Exit(1) } setupLog.Info("registering indexers") if err = indexers.AddToManager(ctx, setupLog, manager); err != nil { setupLog.Error(err, "unable to setup indexers") os.Exit(1) } var kubeVersion *utilVersion.Version if kubeVersion, err = utils.GetK8sVersionFromConfig(dc); err != nil { setupLog.Error(err, "unable to get kubernetes version") os.Exit(1) } setupLog.Info("registering webhooks") // webhooks: the order matters, don't change it and just append webhooksList := append( make([]handlers.Webhook, 0), rulesgenericmutation.Register(cfg), rulesgenericvalidation.Register( regexCache, cfg, rulesgenericvalidation.ForKind( corev1.SchemeGroupVersion.WithKind("Pod").GroupKind(), pod.Handler(cfg, podrules.PodRules(regexCache, registryCache), ), "ephemeralcontainers", ), rulesgenericvalidation.ForKind( corev1.SchemeGroupVersion.WithKind("Service").GroupKind(), service.Handler(cfg, servicerules.ServiceRules(regexCache), ), ), ), route.GenericReplicasHandler(), route.GenericManagedHandler(cfg), route.Pod( pod.Handler(cfg, pod.ImagePullPolicy(), pod.ContainerRegistryLegacy(cfg), pod.PriorityClass(), pod.RuntimeClass(), ), ), route.Ingress(ingress.Class(cfg, kubeVersion), ingress.Hostnames(cfg), ingress.Collision(cfg), ingress.Wildcard()), route.PVCValidating( pvc.Handler( pvc.PersistentVolumeValidatingVolume(), pvc.PersistentVolumeValidatingClass(), ), ), route.PVCMutating( pvc.MutatingHandler( pvc.PersistentVolumeMutatingVolume(), ), ), route.Service( service.Handler(cfg, service.Validating(), ), ), route.Node(handlers.InCapsuleGroups(cfg, node.UserMetadataHandler(cfg, kubeVersion))), route.Cordoning(handlers.InCapsuleGroups(cfg, generic.CordoningHandler(cfg))), route.ServiceAccounts( serviceaccounts.Handler( cfg, serviceaccounts.Promotion(cfg), serviceaccounts.OwnerPromotion(cfg), ), ), route.GenericCustomResources(generic.ResourceCounterHandler(manager.GetClient())), route.Gateway(gateway.Class(cfg)), route.DeviceClass(dra.DeviceClass()), route.Defaults(defaults.Handler(cfg, kubeVersion)), route.TenantMutation( tenantmutation.MetaHandler(), ), route.TenantValidation( tenantvalidation.Handler(cfg, tenantvalidation.NameHandler(), tenantvalidation.NamespaceMetadataHandler(), tenantvalidation.RoleBindingRegexHandler(), tenantvalidation.IngressClassRegexHandler(), tenantvalidation.StorageClassRegexHandler(), tenantvalidation.ContainerRegistryRegexHandler(), tenantvalidation.RuleHandler(manager.GetRESTMapper()), tenantvalidation.HostnameRegexHandler(), tenantvalidation.FreezedEmitter(), tenantvalidation.OwnersHandler(), tenantvalidation.ForbiddenAnnotationsRegexHandler(), tenantvalidation.ProtectedHandler(), tenantvalidation.RequiredMetadataHandler(), // Must run last, because always returns response tenantvalidation.WarningHandler(cfg), ), ), route.TenantOwnersValidation( owners.UserMetadataHandler(), ), route.NamespaceValidation( namespacevalidation.NamespaceHandler( cfg, namespacevalidation.CordoningHandler(cfg), namespacevalidation.QuotaHandler(), namespacevalidation.PrefixHandler(cfg), namespacevalidation.RulesMetadataHandler(regexCache, cfg), namespacevalidation.UserMetadataHandler(), namespacevalidation.RequiredMetadataHandler(), ), ), route.NamespaceMutation( namespacemutation.NamespaceHandler( cfg, namespacemutation.OwnerReferenceHandler(cfg), namespacemutation.MetadataHandler(cfg), // Tenant metadata must be resolved before applying namespace rules. namespacemutation.RulesMetadataHandler(cfg), ), ), route.ResourcePoolMutation(resourcepool.PoolMutationHandler(ctrl.Log.WithName("webhooks").WithName("resourcepool"))), route.ResourcePoolValidation(resourcepool.PoolValidationHandler(ctrl.Log.WithName("webhooks").WithName("resourcepool"))), route.ResourcePoolClaimMutation(resourcepool.ClaimMutationHandler(ctrl.Log.WithName("webhooks").WithName("resourcepoolclaims"))), route.ResourcePoolClaimValidation(resourcepool.ClaimValidationHandler(ctrl.Log.WithName("webhooks").WithName("resourcepoolclaims"))), route.CustomQuotaValidation(customquotavalidation.CustomQuotaValidationHandler( targetsCache, jsonPathCache, celCache, )), route.GlobalCustomQuotaValidation(customquotavalidation.GlobalCustomQuotaValidationHandler( targetsCache, jsonPathCache, celCache, )), route.GlobalResourceQuotaCalculation(globalresourcequotavalidation.Handler()), route.CalculationCustomQuotas( customquotavalidation.ObjectCalculationHandler( targetsCache, jsonPathCache, celCache, ), ), route.GenericTenantAssignment( generic.TenantAssignmentHandler(), ), route.ConfigValidation( cfgvalidation.Handler(cfg, cfgvalidation.ValidationHandler(regexCache), cfgvalidation.ServiceAccountHandler(), cfgvalidation.OwnerHandler(), // Must run last, because always returns response cfgvalidation.WarningHandler(), ), ), route.RulesValidating(manager.GetRESTMapper(), cfg), ) nodeWebhookSupported, _ := utils.NodeWebhookSupported(kubeVersion) if !nodeWebhookSupported { setupLog.Info("disabling node labels verification webhook as current Kubernetes version doesn't have fix for CVE-2021-25735") } if err = webhook.Register( manager, evt.NewEventRecorder( manager.GetClient(), ctrl.Log.WithName("capsule.ctrl").WithName("events"), manager.GetEventRecorder("tenant-controller"), cfg, ), webhook.RegistrationOptions{ EnableTracing: enableTracing, }, webhooksList...); err != nil { setupLog.Error(err, "unable to setup webhooks") os.Exit(1) } rbacManager := &rbaccontroller.Manager{ Log: ctrl.Log.WithName("capsule.ctrl").WithName("rbac"), Client: manager.GetClient(), Configuration: cfg, } if err = manager.Add(rbacManager); err != nil { setupLog.Error(err, "unable to create cluster roles") os.Exit(1) } if err = rbacManager.SetupWithManager(ctx, manager, controllerConfig); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Rbac") os.Exit(1) } if err = (&tenantownercontroller.TenantOwnerManager{ Log: ctrl.Log.WithName("capsule.ctrl").WithName("tenantowners"), Client: manager.GetClient(), }).SetupWithManager(manager, controllerConfig, metrics.MustMakeTenantOwnerRecorder()); err != nil { setupLog.Error(err, "unable to create controller", "controller", "TenantOwners") os.Exit(1) } if err = (&servicelabelscontroller.ServicesLabelsReconciler{ Log: ctrl.Log.WithName("capsule.ctrl").WithName("services"), }).SetupWithManager(ctx, manager, controllerConfig); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ServiceLabels") os.Exit(1) } if err = (&servicelabelscontroller.EndpointSlicesLabelsReconciler{ Log: ctrl.Log.WithName("capsule.ctrl").WithName("endpointslices"), }).SetupWithManager(ctx, manager, controllerConfig); err != nil { setupLog.Error(err, "unable to create controller", "controller", "EndpointSliceLabels") } if err = (&podlabelscontroller.MetadataReconciler{ Client: manager.GetClient(), Log: ctrl.Log.WithName("capsule.ctrl").WithName("pods"), }).SetupWithManager(ctx, manager, controllerConfig); err != nil { setupLog.Error(err, "unable to create controller", "controller", "PodLabels") os.Exit(1) } if err = (&pv.Controller{}).SetupWithManager(manager, controllerConfig); err != nil { setupLog.Error(err, "unable to create controller", "controller", "PersistentVolume") os.Exit(1) } configrecorder := metrics.MustMakeConfigRecorder() if err = (&configcontroller.Manager{ Rest: manager.GetConfig(), Client: manager.GetClient(), Log: ctrl.Log.WithName("capsule.ctrl").WithName("configuration"), }).SetupWithManager(manager, controllerConfig, configrecorder); err != nil { setupLog.Error(err, "unable to create controller", "controller", "CapsuleConfiguration") os.Exit(1) } if err = (&rulestatuscontroller.Manager{ Client: manager.GetClient(), RESTConfig: manager.GetConfig(), Log: ctrl.Log.WithName("capsule.ctrl").WithName("ruleset"), Metrics: metrics.MustMakeRuleStatusRecorder(), }).SetupWithManager(manager, controllerConfig); err != nil { setupLog.Error(err, "unable to create controller", "controller", "RuleSet") os.Exit(1) } setupLog.Info("initializing controllers") localInvalidator := &cacheinvalidator.CacheInvalidator{ Log: ctrl.Log.WithName("capsule.ctrl").WithName("invalidator"), Client: manager.GetClient(), Configuration: directCfg, ImpersonationCache: impersonationCache, RegistryCache: registryCache, JSONPathCache: jsonPathCache, CELCache: celCache, TargetsCache: targetsCache, RegexCache: regexCache, } if err := localInvalidator.SetupWithManager(manager, controllerConfig, configrecorder); err != nil { setupLog.Error(err, "unable to create controller", "controller", "invalidator") os.Exit(1) } if err := resources.Add( ctrl.Log.WithName("controllers").WithName("TenantResources"), manager, cfg, controllerConfig, impersonationCache, ); err != nil { setupLog.Error(err, "unable to create controller", "controller", "tenantresources") os.Exit(1) } if err := admission.Add( ctrl.Log.WithName("capsule.ctrl").WithName("admission"), manager, manager.GetEventRecorder("admission-ctrl"), controllerConfig, directCfg, ); err != nil { setupLog.Error(err, "unable to create controller", "controller", "admission") os.Exit(1) } if err := resourcepools.Add( ctrl.Log.WithName("capsule.ctrl").WithName("resourcepools"), manager, manager.GetEventRecorder("pools-ctrl"), controllerConfig, ); err != nil { setupLog.Error(err, "unable to create controller", "controller", "resourcepools") os.Exit(1) } if err := globalresourcequotacontroller.Add( ctrl.Log.WithName("capsule.ctrl").WithName("globalresourcequotas"), manager, manager.GetEventRecorder("globalresourcequotas-ctrl"), controllerConfig, ); err != nil { setupLog.Error(err, "unable to create controller", "controller", "globalresourcequotas") os.Exit(1) } if err = customquotacontroller.Add(ctrl.Log.WithName("controllers").WithName("CustomQuotas"), manager, manager.GetEventRecorder("customquotas-ctrl"), controllerConfig, jsonPathCache, celCache, targetsCache, ); err != nil { setupLog.Error(err, "unable to create controller", "controller", "customquotas") os.Exit(1) } setupLog.Info("starting manager") if tracingBasicAuthUsername == "" { tracingBasicAuthUsername = os.Getenv("CAPSULE_TRACING_OTLP_BASIC_AUTH_USERNAME") } if tracingBasicAuthPassword == "" { tracingBasicAuthPassword = os.Getenv("CAPSULE_TRACING_OTLP_BASIC_AUTH_PASSWORD") } tracingShutdown, err := setupTracing(ctx, tracingOptions{ enabled: enableTracing, endpoint: tracingEndpoint, insecure: tracingInsecure, sampleRatio: tracingSampleRatio, headers: tracingHeaders, basicAuthUsername: tracingBasicAuthUsername, basicAuthPassword: tracingBasicAuthPassword, timeout: tracingTimeout, compression: tracingCompression, tlsServerName: tracingTLSServerName, tlsInsecureSkipVerify: tracingTLSInsecureSkipVerify, }) if err != nil { setupLog.Error(err, "unable to initialize tracing") os.Exit(1) } if err = manager.Start(ctx); err != nil { setupLog.Error(err, "problem running manager") shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) if shutdownErr := tracingShutdown(shutdownCtx); shutdownErr != nil { setupLog.Error(shutdownErr, "unable to shutdown tracing") } cancel() os.Exit(1) } shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) if err := tracingShutdown(shutdownCtx); err != nil { setupLog.Error(err, "unable to shutdown tracing") } cancel() }