feat: requests and limit policies (#2095)

* chore: save progress

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: requests and limit policies

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: requests and limit policies

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: requests and limit policies

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: requests and limit policies

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: requests and limit policies

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: requests and limit policies

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: requests and limit policies

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: requests and limit policies

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: requests and limit policies

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

---------

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>
This commit is contained in:
Oliver Bähler
2026-08-21 14:31:06 +02:00
committed by GitHub
parent 88217e9562
commit 65a4e19e02
57 changed files with 5627 additions and 265 deletions
+14 -1
View File
@@ -42,10 +42,14 @@ type Manager struct {
Log logr.Logger
Client client.Client
Configuration configuration.Configuration
reader client.Reader
}
//nolint:revive
func (r *Manager) SetupWithManager(ctx context.Context, mgr ctrl.Manager, ctrlConfig utils.ControllerOptions) (err error) {
r.reader = mgr.GetAPIReader()
namesPredicate := predicates.LabelsMatching(map[string]string{
meta.CreatedByCapsuleLabel: controllerManager,
})
@@ -186,7 +190,16 @@ func (r *Manager) EnsureClusterRoleBindingsProvisioner(ctx context.Context) erro
listStarted := time.Now()
if err := r.Client.List(ctx, saList, client.MatchingLabels{
reader := r.reader
if reader == nil {
reader = r.Client
}
// ServiceAccount events are sourced from the metadata-only cache.
// Reading full objects from the regular cache here can observe the
// previous label value and permanently lose a promotion/demotion
// reconcile. The API reader provides an authoritative snapshot.
if err := reader.List(ctx, saList, client.MatchingLabels{
meta.OwnerPromotionLabel: meta.ValueTrue,
}); err != nil {
logOperationDuration(log, "list promoted ServiceAccounts", listStarted)
+136
View File
@@ -0,0 +1,136 @@
// Copyright 2020-2026 Project Capsule Authors
// SPDX-License-Identifier: Apache-2.0
package rbac
import (
"context"
"testing"
"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
"github.com/projectcapsule/capsule/pkg/api/meta"
"github.com/projectcapsule/capsule/pkg/runtime/configuration"
)
func TestEnsureClusterRoleBindingsProvisionerUsesAuthoritativePromotionState(t *testing.T) {
t.Parallel()
const (
configurationName = "capsule"
provisionerRole = "capsule-namespace-provisioner"
namespace = "tenant-a"
serviceAccount = "builder"
)
ctx := context.Background()
scheme := runtime.NewScheme()
for _, addToScheme := range []func(*runtime.Scheme) error{
corev1.AddToScheme,
rbacv1.AddToScheme,
capsulev1beta2.AddToScheme,
} {
if err := addToScheme(scheme); err != nil {
t.Fatal(err)
}
}
configurationObject := &capsulev1beta2.CapsuleConfiguration{
ObjectMeta: metav1.ObjectMeta{Name: configurationName},
Spec: capsulev1beta2.CapsuleConfigurationSpec{
AllowServiceAccountPromotion: true,
RBAC: &capsulev1beta2.RBACConfiguration{
ProvisionerClusterRole: provisionerRole,
},
},
}
promoted := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: serviceAccount,
Namespace: namespace,
Labels: map[string]string{
meta.OwnerPromotionLabel: meta.ValueTrue,
},
},
}
// The cached client deliberately remains stale after demotion, reproducing
// the ordering between the metadata-only watch and full-object cache.
cachedClient := fake.NewClientBuilder().
WithScheme(scheme).
WithObjects(configurationObject, promoted.DeepCopy()).
Build()
authoritativeClient := fake.NewClientBuilder().
WithScheme(scheme).
WithObjects(promoted.DeepCopy()).
Build()
cfg := configuration.NewCapsuleConfiguration(
ctx,
cachedClient,
cachedClient,
nil,
configurationName,
)
manager := &Manager{
Log: logr.Discard(),
Client: cachedClient,
Configuration: cfg,
reader: authoritativeClient,
}
if err := manager.EnsureClusterRoleBindingsProvisioner(ctx); err != nil {
t.Fatalf("initial provisioner binding reconciliation: %v", err)
}
assertServiceAccountSubject(t, cachedClient, provisionerRole, namespace, serviceAccount, true)
latest := &corev1.ServiceAccount{}
if err := authoritativeClient.Get(ctx, client.ObjectKeyFromObject(promoted), latest); err != nil {
t.Fatalf("get authoritative ServiceAccount: %v", err)
}
latest.Labels[meta.OwnerPromotionLabel] = "false"
if err := authoritativeClient.Update(ctx, latest); err != nil {
t.Fatalf("demote authoritative ServiceAccount: %v", err)
}
if err := manager.EnsureClusterRoleBindingsProvisioner(ctx); err != nil {
t.Fatalf("demotion provisioner binding reconciliation: %v", err)
}
assertServiceAccountSubject(t, cachedClient, provisionerRole, namespace, serviceAccount, false)
}
func assertServiceAccountSubject(
t *testing.T,
kubeClient client.Client,
bindingName string,
namespace string,
serviceAccount string,
want bool,
) {
t.Helper()
binding := &rbacv1.ClusterRoleBinding{}
if err := kubeClient.Get(context.Background(), client.ObjectKey{Name: bindingName}, binding); err != nil {
t.Fatalf("get ClusterRoleBinding: %v", err)
}
found := false
for _, subject := range binding.Subjects {
if subject.Kind == rbacv1.ServiceAccountKind &&
subject.Namespace == namespace &&
subject.Name == serviceAccount {
found = true
break
}
}
if found != want {
t.Fatalf("ServiceAccount subject presence = %t, want %t; subjects: %#v", found, want, binding.Subjects)
}
}
@@ -96,7 +96,13 @@ func (r *Manager) pruneGlobalResourceQuotas(
meta.NewManagedByCapsuleLabel: meta.ValueController,
meta.NewTenantLabel: tnt.Name,
})
if err := r.List(ctx, list, &client.ListOptions{LabelSelector: selector}); err != nil {
reader := client.Reader(r.Client)
if r.reader != nil {
reader = r.reader
}
if err := reader.List(ctx, list, &client.ListOptions{LabelSelector: selector}); err != nil {
return err
}
@@ -117,3 +123,13 @@ func (r *Manager) pruneGlobalResourceQuotas(
return nil
}
func hasRuleGlobalResourceQuotas(tnt *capsulev1beta2.Tenant) bool {
for _, rule := range tnt.Spec.Rules {
if rule != nil && rule.NamespaceRuleBodyNamespace != nil && len(rule.Quota) > 0 {
return true
}
}
return false
}
@@ -59,6 +59,16 @@ func TestSyncGlobalResourceQuotasGeneratesAndPrunesRuleQuotas(t *testing.T) {
if generated.Labels[meta.RuleQuotaLabel] != "shared-compute" {
t.Fatalf("rule quota label = %q, want shared-compute", generated.Labels[meta.RuleQuotaLabel])
}
if generated.Labels[meta.NewManagedByCapsuleLabel] != meta.ValueController {
t.Fatalf(
"managed-by label = %q, want %q",
generated.Labels[meta.NewManagedByCapsuleLabel],
meta.ValueController,
)
}
if !metav1.IsControlledBy(generated, tnt) {
t.Fatalf("generated GlobalResourceQuota is not controlled by Tenant %q", tnt.Name)
}
selector := generated.Spec.NamespaceSelectors[0].LabelSelector
if selector.MatchLabels[meta.TenantLabel] != tnt.Name || selector.MatchLabels["tier"] != "paid" {
t.Fatalf("generated selector = %#v", selector)
+9
View File
@@ -367,6 +367,15 @@ func (r *Manager) reconcile(ctx context.Context, log logr.Logger, instance *caps
errs = append(errs, fmt.Errorf("namespace(s) had reconciliation errors: %w", err))
}
// The managed-resource webhook intentionally denies deletion by the
// garbage collector. Remove rule-generated cluster-scoped children as
// the Capsule controller before releasing the Tenant finalizer.
if err = r.pruneGlobalResourceQuotas(ctx, instance, map[string]struct{}{}); err != nil {
errs = append(errs, fmt.Errorf("cannot delete rule global resource quotas: %w", err))
return errors.Join(errs...)
}
if err = r.ensureMetadata(ctx, instance); err != nil {
errs = append(errs, fmt.Errorf("cannot ensure metadata: %w", err))
}
+5 -1
View File
@@ -22,7 +22,11 @@ func (r *Manager) ensureMetadata(ctx context.Context, tnt *capsulev1beta2.Tenant
tnt.Labels[meta.TenantNameLabel] = tnt.Name
}
if len(tnt.Status.Spaces) == 0 {
// Rule-generated GlobalResourceQuotas are cluster scoped and protected from
// garbage-collector deletion by the managed-resource webhook. Keep the
// Tenant around until its controller can explicitly remove those children.
keepForRuleQuotas := tnt.DeletionTimestamp == nil && hasRuleGlobalResourceQuotas(tnt)
if len(tnt.Status.Spaces) == 0 && !keepForRuleQuotas {
controllerutil.RemoveFinalizer(tnt, meta.ControllerFinalizer)
} else {
controllerutil.AddFinalizer(tnt, meta.ControllerFinalizer)
@@ -0,0 +1,86 @@
// Copyright 2020-2026 Project Capsule Authors
// SPDX-License-Identifier: Apache-2.0
package tenant
import (
"context"
"testing"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
"github.com/projectcapsule/capsule/pkg/api/meta"
"github.com/projectcapsule/capsule/pkg/api/rules"
)
func TestEnsureMetadataKeepsFinalizerForRuleGlobalResourceQuotas(t *testing.T) {
t.Parallel()
tnt := &capsulev1beta2.Tenant{
ObjectMeta: metav1.ObjectMeta{Name: "tenant-a"},
Spec: capsulev1beta2.TenantSpec{Rules: []*rules.NamespaceRuleBodyTenant{{
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Quota: []rules.ResourceQuotaRule{{Name: "compute"}},
},
}}},
}
manager := &Manager{}
if err := manager.ensureMetadata(context.Background(), tnt); err != nil {
t.Fatalf("ensureMetadata() error = %v", err)
}
if !controllerutil.ContainsFinalizer(tnt, meta.ControllerFinalizer) {
t.Fatal("Tenant with rule-generated GlobalResourceQuota is missing the controller finalizer")
}
now := metav1.Now()
tnt.DeletionTimestamp = &now
if err := manager.ensureMetadata(context.Background(), tnt); err != nil {
t.Fatalf("ensureMetadata() while deleting error = %v", err)
}
if controllerutil.ContainsFinalizer(tnt, meta.ControllerFinalizer) {
t.Fatal("Tenant controller finalizer was retained after managed child cleanup")
}
}
func TestHasRuleGlobalResourceQuotas(t *testing.T) {
t.Parallel()
tests := []struct {
name string
rules []*rules.NamespaceRuleBodyTenant
want bool
}{
{name: "no rules"},
{name: "nil rule", rules: []*rules.NamespaceRuleBodyTenant{nil}},
{name: "rule without namespace body", rules: []*rules.NamespaceRuleBodyTenant{{}}},
{
name: "rule without quota",
rules: []*rules.NamespaceRuleBodyTenant{{
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{},
}},
},
{
name: "rule with quota",
rules: []*rules.NamespaceRuleBodyTenant{{
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
Quota: []rules.ResourceQuotaRule{{Name: "compute"}},
},
}},
want: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
tnt := &capsulev1beta2.Tenant{Spec: capsulev1beta2.TenantSpec{Rules: test.rules}}
if got := hasRuleGlobalResourceQuotas(tnt); got != test.want {
t.Fatalf("hasRuleGlobalResourceQuotas() = %t, want %t", got, test.want)
}
})
}
}
+228 -96
View File
@@ -165,51 +165,24 @@ func (r *Reconciler) ReconcileCertificates(
"ipAddresses", cert.IPsToStrings(sans.IPAddrs),
)
ca, caBundle, rotateServingCert, err := r.ensureCertificateMaterial(log, certSecret, sans)
if err != nil {
if err := r.reconcileTLSSecret(ctx, log, certSecret, sans); err != nil {
return err
}
log.V(4).Info(
"certificate requires rotation",
"rotation", rotateServingCert,
)
if rotateServingCert {
if ca == nil {
return fmt.Errorf("cannot rotate serving certificate without CA private key")
}
crt, key, err := ca.GenerateCertificate(cert.NewCertOpts(
time.Now().Add(certificateValidity),
sans,
))
if err != nil {
log.Error(err, "cannot generate serving TLS certificate")
return err
}
certSecret.Data[corev1.TLSCertKey] = crt.Bytes()
certSecret.Data[corev1.TLSPrivateKeyKey] = key.Bytes()
if err := r.validateSecretCertificate(certSecret, sans); err != nil {
return err
}
if err := r.upsertTLSSecret(ctx, certSecret); err != nil {
return err
}
}
caBundle = certSecret.Data[corev1.ServiceAccountRootCAKey]
caBundle := certSecret.Data[corev1.ServiceAccountRootCAKey]
if len(caBundle) == 0 {
return fmt.Errorf("missing %q field in %q secret", corev1.ServiceAccountRootCAKey, r.Configuration.TLSSecretName())
}
log.V(5).Info("Patching caBundle in managed CRD conversions")
log.V(5).Info("Patching caBundle in admission webhooks and managed CRD conversions")
patchGroup, groupCtx := errgroup.WithContext(ctx)
patchGroup.Go(func() error {
return r.patchMutatingWebhookConfigurationCABundle(groupCtx, caBundle)
})
patchGroup.Go(func() error {
return r.patchValidatingWebhookConfigurationCABundle(groupCtx, caBundle)
})
for key, managed := range r.conversionManagedCRDs() {
patchGroup.Go(func() error {
@@ -224,15 +197,108 @@ func (r *Reconciler) ReconcileCertificates(
return patchGroup.Wait()
}
// reconcileTLSSecret calculates certificate material from the latest persisted
// Secret inside an optimistic-concurrency retry. This is deliberately not based
// on the object supplied by the caller: every controller replica performs the
// startup reconciliation before leader election, so that object may already be
// stale by the time certificate generation finishes.
func (r *Reconciler) reconcileTLSSecret(
ctx context.Context,
log logr.Logger,
certSecret *corev1.Secret,
sans cert.CertificateSANs,
) error {
key := client.ObjectKeyFromObject(certSecret)
if key.Name == "" {
key.Name = r.Configuration.TLSSecretName()
}
if key.Namespace == "" {
key.Namespace = r.Namespace
}
err := retry.OnError(retry.DefaultBackoff, func(err error) bool {
return apierrors.IsAlreadyExists(err) || apierrors.IsConflict(err)
}, func() error {
// Use the TLS type when Capsule creates the Secret itself. If the Secret
// already exists, CreateOrUpdate loads its persisted type into desired;
// preserve it because Secret type is immutable. In particular, the Helm
// chart pre-creates an empty Opaque Secret so the controller Pod can mount
// it before this reconciliation supplies the certificate data.
desired := &corev1.Secret{
ObjectMeta: *certSecret.ObjectMeta.DeepCopy(),
Type: corev1.SecretTypeTLS,
}
desired.Name = key.Name
desired.Namespace = key.Namespace
_, err := controllerutil.CreateOrUpdate(ctx, r.Client, desired, func() error {
if desired.Labels == nil {
desired.Labels = map[string]string{}
}
if desired.Annotations == nil {
desired.Annotations = map[string]string{}
}
ca, _, rotateServingCert, err := r.ensureCertificateMaterial(log, desired, sans)
if err != nil {
return err
}
log.V(4).Info(
"certificate requires rotation",
"rotation", rotateServingCert,
)
if rotateServingCert {
if ca == nil {
return fmt.Errorf("cannot rotate serving certificate without CA private key")
}
crt, key, err := ca.GenerateCertificate(cert.NewCertOpts(
time.Now().Add(certificateValidity),
sans,
))
if err != nil {
return fmt.Errorf("generate serving TLS certificate: %w", err)
}
desired.Data[corev1.TLSCertKey] = crt.Bytes()
desired.Data[corev1.TLSPrivateKeyKey] = key.Bytes()
}
return r.validateSecretCertificate(desired, sans)
})
if err != nil {
return err
}
certSecret.ObjectMeta = desired.ObjectMeta
certSecret.Type = desired.Type
certSecret.Immutable = desired.Immutable
certSecret.Data = copySecretData(desired.Data)
return nil
})
if err != nil {
r.Log.Error(err, "cannot reconcile Capsule TLS Secret", "secret", key.String())
}
return err
}
// ensureCertificateMaterial ensures that the Secret contains a stable CA
// certificate/key pair and decides whether the serving certificate must be
// regenerated.
//
// Important behavior:
// - Missing Secret or missing ca.key creates a new CA.
// - Only a new, empty Secret bootstraps a CA.
// - Existing valid CA is reused.
// - Serving certificate renewal never rotates the CA.
// - Legacy Secrets without ca.key rotate once into the stable format.
// - Established or externally managed CA material is never replaced
// automatically, because doing so would immediately invalidate every
// published caBundle.
func (r *Reconciler) ensureCertificateMaterial(
log logr.Logger,
certSecret *corev1.Secret,
@@ -262,53 +328,51 @@ func (r *Reconciler) ensureCertificateMaterial(
case hasCABundle && hasCAKey:
loadedCA, err := cert.NewCertificateAuthorityFromBytes(caBundle, caKey)
if err != nil {
log.V(3).Info(
"Existing CA material is invalid, generating new CA",
"error", err.Error(),
return nil, nil, false, fmt.Errorf(
"TLS Secret %s contains invalid CA certificate/key material; refusing automatic CA replacement: %w",
client.ObjectKeyFromObject(certSecret).String(),
err,
)
generatedCA, generatedCABundle, generatedCAKey, err := generateCertificateAuthorityMaterial()
if err != nil {
return nil, nil, false, err
}
ca = generatedCA
caBundle = generatedCABundle
certSecret.Data[corev1.ServiceAccountRootCAKey] = generatedCABundle
certSecret.Data["ca.key"] = generatedCAKey
rotateServingCert = true
} else {
ca = loadedCA
}
ca = loadedCA
case hasCABundle && !hasCAKey:
// A CA bundle without its private key is typical of an externally
// managed Secret. Once the Capsule TLS controller is enabled, it must
// take ownership of the complete certificate lifecycle; otherwise a
// SAN change or certificate renewal cannot be recovered.
// This is an externally managed or legacy Secret. It is safe to keep
// serving while its certificate remains valid, but Capsule cannot renew
// it without the CA key. Replacing that CA in-place would make the API
// server distrust one or more running webhook replicas during rollout.
if err := r.validateSecretCertificate(certSecret, sans); err != nil {
return nil, nil, false, fmt.Errorf(
"TLS Secret %s has no CA private key and its serving certificate needs renewal; refusing automatic CA replacement: %w",
client.ObjectKeyFromObject(certSecret).String(),
err,
)
}
log.V(3).Info(
"TLS Secret contains CA bundle but no CA private key, rotating into controller-managed TLS material",
"Keeping externally managed TLS material without a CA private key",
"secret", client.ObjectKeyFromObject(certSecret).String(),
)
generatedCA, generatedCABundle, generatedCAKey, err := generateCertificateAuthorityMaterial()
if err != nil {
return nil, nil, false, err
}
return nil, caBundle, false, nil
ca = generatedCA
caBundle = generatedCABundle
certSecret.Data[corev1.ServiceAccountRootCAKey] = generatedCABundle
certSecret.Data["ca.key"] = generatedCAKey
rotateServingCert = true
case !hasCABundle && hasCAKey:
return nil, nil, false, fmt.Errorf(
"TLS Secret %s contains a CA private key but no CA certificate; refusing automatic CA replacement",
client.ObjectKeyFromObject(certSecret).String(),
)
default:
if len(certSecret.Data[corev1.TLSCertKey]) > 0 || len(certSecret.Data[corev1.TLSPrivateKeyKey]) > 0 {
return nil, nil, false, fmt.Errorf(
"TLS Secret %s contains serving certificate material but no CA certificate; refusing automatic CA replacement",
client.ObjectKeyFromObject(certSecret).String(),
)
}
log.V(10).Info(
"TLS Secret is missing CA material, generating new CA",
"TLS Secret is empty, generating initial CA",
"secret", client.ObjectKeyFromObject(certSecret).String(),
)
@@ -411,35 +475,103 @@ func generateCertificateAuthorityMaterial() (*cert.CapsuleCA, []byte, []byte, er
return ca, caCrt.Bytes(), caKey.Bytes(), nil
}
func (r *Reconciler) upsertTLSSecret(ctx context.Context, certSecret *corev1.Secret) error {
desired := &corev1.Secret{
ObjectMeta: certSecret.ObjectMeta,
Type: corev1.SecretTypeTLS,
}
func (r *Reconciler) patchMutatingWebhookConfigurationCABundle(ctx context.Context, caBundle []byte) error {
return r.patchAdmissionConfigurationCABundle(
ctx,
r.Configuration.MutatingWebhookConfigurationName(),
caBundle,
func() client.Object { return &admissionregistrationv1.MutatingWebhookConfiguration{} },
updateMutatingWebhookCABundles,
)
}
_, err := controllerutil.CreateOrUpdate(ctx, r.Client, desired, func() error {
if desired.Labels == nil {
desired.Labels = map[string]string{}
}
if desired.Annotations == nil {
desired.Annotations = map[string]string{}
}
desired.Data = copySecretData(certSecret.Data)
func (r *Reconciler) patchValidatingWebhookConfigurationCABundle(ctx context.Context, caBundle []byte) error {
return r.patchAdmissionConfigurationCABundle(
ctx,
r.Configuration.ValidatingWebhookConfigurationName(),
caBundle,
func() client.Object { return &admissionregistrationv1.ValidatingWebhookConfiguration{} },
updateValidatingWebhookCABundles,
)
}
func (r *Reconciler) patchAdmissionConfigurationCABundle(
ctx context.Context,
name string,
caBundle []byte,
newConfiguration func() client.Object,
updateCABundles func(client.Object, []byte) (bool, error),
) error {
if name == "" {
return nil
})
if err != nil {
r.Log.Error(err, "cannot update Capsule TLS Secret")
return err
}
certSecret.ObjectMeta = desired.ObjectMeta
certSecret.Data = copySecretData(desired.Data)
return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
configuration := newConfiguration()
if err := r.Get(ctx, types.NamespacedName{Name: name}, configuration); err != nil {
if apierrors.IsNotFound(err) {
return nil
}
return nil
return err
}
before, ok := configuration.DeepCopyObject().(client.Object)
if !ok {
return fmt.Errorf("admission configuration %q cannot be deep-copied as a client object", name)
}
changed, err := updateCABundles(configuration, caBundle)
if err != nil {
return err
}
if !changed {
return nil
}
return r.Patch(ctx, configuration, client.MergeFrom(before))
})
}
func updateMutatingWebhookCABundles(object client.Object, caBundle []byte) (bool, error) {
configuration, ok := object.(*admissionregistrationv1.MutatingWebhookConfiguration)
if !ok {
return false, fmt.Errorf("expected MutatingWebhookConfiguration, got %T", object)
}
changed := false
for index := range configuration.Webhooks {
if bytes.Equal(configuration.Webhooks[index].ClientConfig.CABundle, caBundle) {
continue
}
configuration.Webhooks[index].ClientConfig.CABundle = append([]byte(nil), caBundle...)
changed = true
}
return changed, nil
}
func updateValidatingWebhookCABundles(object client.Object, caBundle []byte) (bool, error) {
configuration, ok := object.(*admissionregistrationv1.ValidatingWebhookConfiguration)
if !ok {
return false, fmt.Errorf("expected ValidatingWebhookConfiguration, got %T", object)
}
changed := false
for index := range configuration.Webhooks {
if bytes.Equal(configuration.Webhooks[index].ClientConfig.CABundle, caBundle) {
continue
}
configuration.Webhooks[index].ClientConfig.CABundle = append([]byte(nil), caBundle...)
changed = true
}
return changed, nil
}
func (r *Reconciler) validateSecretCertificate(
+255 -78
View File
@@ -6,10 +6,12 @@ package tls
import (
"bytes"
"context"
"strings"
"testing"
"time"
"github.com/go-logr/logr"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
corev1 "k8s.io/api/core/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -18,123 +20,298 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client/fake"
capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
apiMeta "github.com/projectcapsule/capsule/pkg/api/meta"
runtimeadmission "github.com/projectcapsule/capsule/pkg/runtime/admission"
"github.com/projectcapsule/capsule/pkg/runtime/cert"
"github.com/projectcapsule/capsule/pkg/runtime/configuration"
)
func TestReconcileCertificatesMigratesExternalTLSSecret(t *testing.T) {
const (
testNamespace = "capsule-system"
testSecretName = "capsule-tls"
testServiceName = "capsule-webhook-service"
testMutatingConfiguration = "capsule-mutating-webhook-configuration"
testValidatingConfiguration = "capsule-validating-webhook-configuration"
)
func TestReconcileCertificatesPreservesValidExternalTLSSecret(t *testing.T) {
t.Parallel()
const (
namespace = "capsule-system"
secretName = "capsule-tls"
serviceName = "capsule-webhook-service"
)
externalCABundle, externalCertificate, externalKey := generateTestTLSMaterial(t, testWebhookSANs())
secret := testTLSSecret(externalCABundle, externalCertificate, externalKey)
reconciler, kubeClient := newTestTLSReconciler(t, secret)
if err := reconciler.ReconcileCertificates(context.Background(), logr.Discard(), secret.DeepCopy()); err != nil {
t.Fatalf("ReconcileCertificates() error = %v", err)
}
updated := getTestTLSSecret(t, kubeClient)
if len(updated.Data["ca.key"]) != 0 {
t.Fatal("external TLS Secret unexpectedly gained a CA private key")
}
assertTLSDataEqual(t, updated, externalCABundle, externalCertificate, externalKey)
}
func TestReconcileCertificatesRejectsUnsafeExternalCARotation(t *testing.T) {
t.Parallel()
ctx := context.Background()
externalCABundle, externalCertificate, externalKey := generateTestTLSMaterial(t, cert.CertificateSANs{
DNSNames: []string{serviceName + "." + namespace + ".svc"},
DNSNames: []string{testServiceName + "." + testNamespace + ".svc"},
})
secret := testTLSSecret(externalCABundle, externalCertificate, externalKey)
reconciler, kubeClient := newTestTLSReconciler(t, secret)
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: namespace,
},
Type: corev1.SecretTypeTLS,
Data: map[string][]byte{
corev1.ServiceAccountRootCAKey: externalCABundle,
corev1.TLSCertKey: externalCertificate,
corev1.TLSPrivateKeyKey: externalKey,
err := reconciler.ReconcileCertificates(context.Background(), logr.Discard(), secret.DeepCopy())
if err == nil {
t.Fatal("ReconcileCertificates() unexpectedly replaced externally managed CA material")
}
if !strings.Contains(err.Error(), "refusing automatic CA replacement") {
t.Fatalf("ReconcileCertificates() error = %v, want safe CA replacement refusal", err)
}
updated := getTestTLSSecret(t, kubeClient)
if len(updated.Data["ca.key"]) != 0 {
t.Fatal("rejected external TLS Secret unexpectedly gained a CA private key")
}
assertTLSDataEqual(t, updated, externalCABundle, externalCertificate, externalKey)
}
func TestReconcileCertificatesConcurrentReplicasAdoptPersistedWinner(t *testing.T) {
t.Parallel()
secret := testTLSSecret(nil, nil, nil)
reconciler, kubeClient := newTestTLSReconciler(t, secret)
firstReplica := secret.DeepCopy()
secondReplica := secret.DeepCopy()
start := make(chan struct{})
errors := make(chan error, 2)
for _, replica := range []*corev1.Secret{firstReplica, secondReplica} {
go func(replica *corev1.Secret) {
<-start
errors <- reconciler.ReconcileCertificates(context.Background(), logr.Discard(), replica)
}(replica)
}
close(start)
for range 2 {
if err := <-errors; err != nil {
t.Fatalf("concurrent ReconcileCertificates() error = %v", err)
}
}
finalPersisted := getTestTLSSecret(t, kubeClient)
if !bytes.Equal(
firstReplica.Data[corev1.ServiceAccountRootCAKey],
finalPersisted.Data[corev1.ServiceAccountRootCAKey],
) {
t.Fatal("first replica did not adopt the persisted CA")
}
if !bytes.Equal(
secondReplica.Data[corev1.ServiceAccountRootCAKey],
finalPersisted.Data[corev1.ServiceAccountRootCAKey],
) {
t.Fatal("second replica did not adopt the persisted CA")
}
if !bytes.Equal(firstReplica.Data[corev1.TLSCertKey], finalPersisted.Data[corev1.TLSCertKey]) ||
!bytes.Equal(secondReplica.Data[corev1.TLSCertKey], finalPersisted.Data[corev1.TLSCertKey]) {
t.Fatal("concurrent replicas did not adopt the persisted serving certificate")
}
}
func TestReconcileCertificatesPopulatesChartCreatedOpaqueSecret(t *testing.T) {
t.Parallel()
// The Helm chart must create this empty Secret before the controller Pod can
// mount it. Kubernetes does not allow changing a Secret's type afterwards,
// so reconciliation must preserve Opaque while adding valid TLS material.
secret := testTLSSecret(nil, nil, nil)
secret.Type = corev1.SecretTypeOpaque
reconciler, kubeClient := newTestTLSReconciler(t, secret)
if err := reconciler.ReconcileCertificates(context.Background(), logr.Discard(), secret.DeepCopy()); err != nil {
t.Fatalf("ReconcileCertificates() error = %v", err)
}
updated := getTestTLSSecret(t, kubeClient)
if updated.Type != corev1.SecretTypeOpaque {
t.Fatalf("reconciled Secret type = %q, want %q", updated.Type, corev1.SecretTypeOpaque)
}
if len(updated.Data[corev1.ServiceAccountRootCAKey]) == 0 ||
len(updated.Data[corev1.TLSCertKey]) == 0 ||
len(updated.Data[corev1.TLSPrivateKeyKey]) == 0 {
t.Fatal("reconciled Secret is missing generated TLS material")
}
}
func TestReconcileCertificatesPatchesEveryAdmissionCABundle(t *testing.T) {
t.Parallel()
oldCA, _, _ := generateTestTLSMaterial(t, testWebhookSANs())
secret := testTLSSecret(nil, nil, nil)
mutating := &admissionregistrationv1.MutatingWebhookConfiguration{
ObjectMeta: metav1.ObjectMeta{Name: testMutatingConfiguration},
Webhooks: []admissionregistrationv1.MutatingWebhook{
{Name: "first.mutating.projectcapsule.dev", ClientConfig: admissionregistrationv1.WebhookClientConfig{CABundle: oldCA}},
{Name: "second.mutating.projectcapsule.dev"},
},
}
validating := &admissionregistrationv1.ValidatingWebhookConfiguration{
ObjectMeta: metav1.ObjectMeta{Name: testValidatingConfiguration},
Webhooks: []admissionregistrationv1.ValidatingWebhook{
{Name: "first.validating.projectcapsule.dev", ClientConfig: admissionregistrationv1.WebhookClientConfig{CABundle: oldCA}},
{Name: "second.validating.projectcapsule.dev"},
},
}
reconciler, kubeClient := newTestTLSReconciler(t, secret, mutating, validating)
if err := reconciler.ReconcileCertificates(context.Background(), logr.Discard(), secret.DeepCopy()); err != nil {
t.Fatalf("ReconcileCertificates() error = %v", err)
}
wantCA := getTestTLSSecret(t, kubeClient).Data[corev1.ServiceAccountRootCAKey]
updatedMutating := &admissionregistrationv1.MutatingWebhookConfiguration{}
if err := kubeClient.Get(
context.Background(),
client.ObjectKey{Name: testMutatingConfiguration},
updatedMutating,
); err != nil {
t.Fatalf("get mutating webhook configuration: %v", err)
}
for index := range updatedMutating.Webhooks {
if !bytes.Equal(updatedMutating.Webhooks[index].ClientConfig.CABundle, wantCA) {
t.Fatalf("mutating webhook %q has stale caBundle", updatedMutating.Webhooks[index].Name)
}
}
updatedValidating := &admissionregistrationv1.ValidatingWebhookConfiguration{}
if err := kubeClient.Get(
context.Background(),
client.ObjectKey{Name: testValidatingConfiguration},
updatedValidating,
); err != nil {
t.Fatalf("get validating webhook configuration: %v", err)
}
for index := range updatedValidating.Webhooks {
if !bytes.Equal(updatedValidating.Webhooks[index].ClientConfig.CABundle, wantCA) {
t.Fatalf("validating webhook %q has stale caBundle", updatedValidating.Webhooks[index].Name)
}
}
}
func newTestTLSReconciler(t *testing.T, objects ...client.Object) (*Reconciler, client.Client) {
t.Helper()
configurationObject := &capsulev1beta2.CapsuleConfiguration{
ObjectMeta: metav1.ObjectMeta{Name: "capsule"},
Spec: capsulev1beta2.CapsuleConfigurationSpec{
EnableTLSReconciler: true,
CapsuleResources: capsulev1beta2.CapsuleResources{
TLSSecretName: secretName,
TLSSecretName: testSecretName,
},
Admission: capsulev1beta2.DynamicAdmission{
ServiceName: serviceName,
Mutating: &capsulev1beta2.DynamicMutatingAdmissionConfig{},
Validating: &capsulev1beta2.DynamicValidatingAdmissionConfig{},
ServiceName: testServiceName,
Mutating: &capsulev1beta2.DynamicMutatingAdmissionConfig{
DynamicAdmissionConfig: runtimeadmission.DynamicAdmissionConfig{
Name: apiMeta.RFC1123Name(testMutatingConfiguration),
},
},
Validating: &capsulev1beta2.DynamicValidatingAdmissionConfig{
DynamicAdmissionConfig: runtimeadmission.DynamicAdmissionConfig{
Name: apiMeta.RFC1123Name(testValidatingConfiguration),
},
},
},
},
}
scheme := runtime.NewScheme()
if err := corev1.AddToScheme(scheme); err != nil {
t.Fatal(err)
for _, addToScheme := range []func(*runtime.Scheme) error{
corev1.AddToScheme,
admissionregistrationv1.AddToScheme,
apiextensionsv1.AddToScheme,
capsulev1beta2.AddToScheme,
} {
if err := addToScheme(scheme); err != nil {
t.Fatal(err)
}
}
if err := apiextensionsv1.AddToScheme(scheme); err != nil {
t.Fatal(err)
objects = append(objects, configurationObject)
kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build()
cfg := configuration.NewCapsuleConfiguration(
context.Background(),
kubeClient,
kubeClient,
nil,
configurationObject.Name,
)
return &Reconciler{
Client: kubeClient,
Log: logr.Discard(),
Namespace: testNamespace,
Configuration: cfg,
}, kubeClient
}
func testTLSSecret(caBundle, certificate, key []byte) *corev1.Secret {
data := map[string][]byte{}
if len(caBundle) > 0 {
data[corev1.ServiceAccountRootCAKey] = caBundle
}
if len(certificate) > 0 {
data[corev1.TLSCertKey] = certificate
}
if len(key) > 0 {
data[corev1.TLSPrivateKeyKey] = key
}
if err := capsulev1beta2.AddToScheme(scheme); err != nil {
t.Fatal(err)
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: testSecretName, Namespace: testNamespace},
Type: corev1.SecretTypeTLS,
Data: data,
}
}
kubeClient := fake.NewClientBuilder().
WithScheme(scheme).
WithObjects(secret, configurationObject).
Build()
func getTestTLSSecret(t *testing.T, kubeClient client.Client) *corev1.Secret {
t.Helper()
reconciler := &Reconciler{
Client: kubeClient,
Log: logr.Discard(),
Namespace: namespace,
Configuration: configuration.NewCapsuleConfiguration(
ctx,
kubeClient,
kubeClient,
nil,
configurationObject.Name,
),
}
desiredSANs, err := reconciler.desiredWebhookSANs(ctx)
if err != nil {
t.Fatalf("desiredWebhookSANs() error = %v", err)
}
if err := reconciler.validateSecretCertificate(secret, desiredSANs); err == nil {
t.Fatal("external serving certificate unexpectedly satisfies all desired SANs")
}
if err := reconciler.ReconcileCertificates(ctx, logr.Discard(), secret.DeepCopy()); err != nil {
t.Fatalf("ReconcileCertificates() error = %v", err)
}
updated := &corev1.Secret{}
if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(secret), updated); err != nil {
secret := &corev1.Secret{}
if err := kubeClient.Get(
context.Background(),
client.ObjectKey{Namespace: testNamespace, Name: testSecretName},
secret,
); err != nil {
t.Fatalf("get reconciled TLS Secret: %v", err)
}
if len(updated.Data["ca.key"]) == 0 {
t.Fatal("reconciled TLS Secret does not contain ca.key")
}
return secret
}
if bytes.Equal(updated.Data[corev1.ServiceAccountRootCAKey], externalCABundle) {
t.Fatal("reconciled TLS Secret retained the external CA bundle")
}
func assertTLSDataEqual(t *testing.T, secret *corev1.Secret, caBundle, certificate, key []byte) {
t.Helper()
if bytes.Equal(updated.Data[corev1.TLSCertKey], externalCertificate) {
t.Fatal("reconciled TLS Secret retained the external serving certificate")
if !bytes.Equal(secret.Data[corev1.ServiceAccountRootCAKey], caBundle) {
t.Fatal("TLS Secret CA bundle changed")
}
if !bytes.Equal(secret.Data[corev1.TLSCertKey], certificate) {
t.Fatal("TLS Secret serving certificate changed")
}
if !bytes.Equal(secret.Data[corev1.TLSPrivateKeyKey], key) {
t.Fatal("TLS Secret serving private key changed")
}
}
if _, err := cert.NewCertificateAuthorityFromBytes(
updated.Data[corev1.ServiceAccountRootCAKey],
updated.Data["ca.key"],
); err != nil {
t.Fatalf("reconciled CA certificate/key pair is invalid: %v", err)
}
if err := reconciler.validateSecretCertificate(updated, desiredSANs); err != nil {
t.Fatalf("reconciled serving certificate is invalid: %v", err)
}
func testWebhookSANs() cert.CertificateSANs {
return cert.CertificateSANs{DNSNames: []string{
testServiceName,
testServiceName + "." + testNamespace,
testServiceName + "." + testNamespace + ".svc",
testServiceName + "." + testNamespace + ".svc.cluster.local",
}}
}
func generateTestTLSMaterial(
+229
View File
@@ -0,0 +1,229 @@
// Copyright 2020-2026 Project Capsule Authors
// SPDX-License-Identifier: Apache-2.0
package tls
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"fmt"
"net/http"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"github.com/projectcapsule/capsule/pkg/runtime/configuration"
)
type certificateGetter func(*tls.ClientHelloInfo) (*tls.Certificate, error)
// WebhookCertificateReadinessCheck prevents a webhook Pod from becoming ready
// while the certificate it is actually serving is not trusted by an admission
// configuration managed by Capsule. This closes the window between Secret
// publication, certwatcher reload, and caBundle reconciliation.
func WebhookCertificateReadinessCheck(
reader client.Reader,
cfg configuration.Configuration,
namespace string,
getCertificate certificateGetter,
) healthz.Checker {
return func(request *http.Request) error {
servingCertificate, err := getCertificate(nil)
if err != nil {
return fmt.Errorf("load webhook serving certificate: %w", err)
}
if servingCertificate == nil || len(servingCertificate.Certificate) == 0 {
return fmt.Errorf("webhook serving certificate is empty")
}
leaf, err := x509.ParseCertificate(servingCertificate.Certificate[0])
if err != nil {
return fmt.Errorf("parse webhook serving certificate: %w", err)
}
secret := &corev1.Secret{}
secretKey := types.NamespacedName{Namespace: namespace, Name: cfg.TLSSecretName()}
if err := reader.Get(request.Context(), secretKey, secret); err != nil {
return fmt.Errorf("get webhook TLS Secret %s: %w", secretKey.String(), err)
}
persistedLeaf, err := certificateFromPEM(secret.Data[corev1.TLSCertKey])
if err != nil {
return fmt.Errorf("parse serving certificate in TLS Secret %s: %w", secretKey.String(), err)
}
if !bytes.Equal(leaf.Raw, persistedLeaf.Raw) {
return fmt.Errorf("loaded webhook serving certificate is stale relative to TLS Secret %s", secretKey.String())
}
admission := cfg.Admission()
trusts := make([]admissionWebhookTrust, 0)
if admission.Mutating != nil && len(admission.Mutating.Webhooks) > 0 {
loaded, err := loadAdmissionWebhookTrust(
request.Context(),
reader,
"mutating",
string(admission.Mutating.Name),
func() client.Object { return &admissionregistrationv1.MutatingWebhookConfiguration{} },
mutatingWebhookTrust,
)
if err != nil {
return err
}
trusts = append(trusts, loaded...)
}
if admission.Validating != nil && len(admission.Validating.Webhooks) > 0 {
loaded, err := loadAdmissionWebhookTrust(
request.Context(),
reader,
"validating",
string(admission.Validating.Name),
func() client.Object { return &admissionregistrationv1.ValidatingWebhookConfiguration{} },
validatingWebhookTrust,
)
if err != nil {
return err
}
trusts = append(trusts, loaded...)
}
for _, trust := range trusts {
if err := verifyCertificateAgainstCABundle(leaf, trust.caBundle); err != nil {
return fmt.Errorf(
"%s webhook configuration %q webhook %q does not trust the serving certificate: %w",
trust.kind,
trust.configurationName,
trust.webhookName,
err,
)
}
}
return nil
}
}
func certificateFromPEM(certificatePEM []byte) (*x509.Certificate, error) {
block, _ := pem.Decode(certificatePEM)
if block == nil {
return nil, fmt.Errorf("invalid certificate PEM")
}
return x509.ParseCertificate(block.Bytes)
}
type admissionWebhookTrust struct {
kind string
configurationName string
webhookName string
caBundle []byte
}
func loadAdmissionWebhookTrust(
ctx context.Context,
reader client.Reader,
kind string,
name string,
newConfiguration func() client.Object,
extractTrust func(client.Object, string, string) ([]admissionWebhookTrust, error),
) ([]admissionWebhookTrust, error) {
if name == "" {
return nil, fmt.Errorf("%s webhook configuration name is empty", kind)
}
webhookConfiguration := newConfiguration()
if err := reader.Get(ctx, types.NamespacedName{Name: name}, webhookConfiguration); err != nil {
return nil, fmt.Errorf("get %s webhook configuration %q: %w", kind, name, err)
}
trusts, err := extractTrust(webhookConfiguration, kind, name)
if err != nil {
return nil, err
}
if len(trusts) == 0 {
return nil, fmt.Errorf("%s webhook configuration %q contains no webhooks", kind, name)
}
return trusts, nil
}
func mutatingWebhookTrust(
object client.Object,
kind string,
configurationName string,
) ([]admissionWebhookTrust, error) {
configuration, ok := object.(*admissionregistrationv1.MutatingWebhookConfiguration)
if !ok {
return nil, fmt.Errorf("expected MutatingWebhookConfiguration, got %T", object)
}
trusts := make([]admissionWebhookTrust, 0, len(configuration.Webhooks))
for index := range configuration.Webhooks {
trusts = append(trusts, admissionWebhookTrust{
kind: kind,
configurationName: configurationName,
webhookName: configuration.Webhooks[index].Name,
caBundle: configuration.Webhooks[index].ClientConfig.CABundle,
})
}
return trusts, nil
}
func validatingWebhookTrust(
object client.Object,
kind string,
configurationName string,
) ([]admissionWebhookTrust, error) {
configuration, ok := object.(*admissionregistrationv1.ValidatingWebhookConfiguration)
if !ok {
return nil, fmt.Errorf("expected ValidatingWebhookConfiguration, got %T", object)
}
trusts := make([]admissionWebhookTrust, 0, len(configuration.Webhooks))
for index := range configuration.Webhooks {
trusts = append(trusts, admissionWebhookTrust{
kind: kind,
configurationName: configurationName,
webhookName: configuration.Webhooks[index].Name,
caBundle: configuration.Webhooks[index].ClientConfig.CABundle,
})
}
return trusts, nil
}
func verifyCertificateAgainstCABundle(leaf *x509.Certificate, caBundle []byte) error {
if len(caBundle) == 0 {
return fmt.Errorf("caBundle is empty")
}
roots := x509.NewCertPool()
if !roots.AppendCertsFromPEM(caBundle) {
return fmt.Errorf("caBundle is not valid PEM certificate data")
}
if _, err := leaf.Verify(x509.VerifyOptions{
Roots: roots,
KeyUsages: []x509.ExtKeyUsage{
x509.ExtKeyUsageServerAuth,
},
}); err != nil {
return err
}
return nil
}
+170
View File
@@ -0,0 +1,170 @@
// Copyright 2020-2026 Project Capsule Authors
// SPDX-License-Identifier: Apache-2.0
package tls
import (
"context"
cryptotls "crypto/tls"
"net/http/httptest"
"strings"
"testing"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
runtimeadmission "github.com/projectcapsule/capsule/pkg/runtime/admission"
)
func TestWebhookCertificateReadinessCheckAcceptsPublishedCA(t *testing.T) {
t.Parallel()
caBundle, certificatePEM, keyPEM := generateTestTLSMaterial(t, testWebhookSANs())
validating := testValidatingWebhookConfiguration(caBundle)
reconciler, kubeClient := newTestTLSReconciler(
t,
testTLSSecret(caBundle, certificatePEM, keyPEM),
validating,
)
enableTestValidatingWebhook(t, kubeClient)
servingCertificate := parseTestServingCertificate(t, certificatePEM, keyPEM)
check := WebhookCertificateReadinessCheck(
kubeClient,
reconciler.Configuration,
testNamespace,
func(*cryptotls.ClientHelloInfo) (*cryptotls.Certificate, error) {
return &servingCertificate, nil
},
)
if err := check(httptest.NewRequest("GET", "/readyz", nil)); err != nil {
t.Fatalf("readiness check rejected matching serving certificate and caBundle: %v", err)
}
}
func TestWebhookCertificateReadinessCheckRejectsMismatchedPublishedCA(t *testing.T) {
t.Parallel()
caBundle, certificatePEM, keyPEM := generateTestTLSMaterial(t, testWebhookSANs())
wrongCABundle, _, _ := generateTestTLSMaterial(t, testWebhookSANs())
validating := testValidatingWebhookConfiguration(wrongCABundle)
reconciler, kubeClient := newTestTLSReconciler(
t,
testTLSSecret(caBundle, certificatePEM, keyPEM),
validating,
)
enableTestValidatingWebhook(t, kubeClient)
servingCertificate := parseTestServingCertificate(t, certificatePEM, keyPEM)
check := WebhookCertificateReadinessCheck(
kubeClient,
reconciler.Configuration,
testNamespace,
func(*cryptotls.ClientHelloInfo) (*cryptotls.Certificate, error) {
return &servingCertificate, nil
},
)
err := check(httptest.NewRequest("GET", "/readyz", nil))
if err == nil {
t.Fatal("readiness check accepted a caBundle that does not trust the serving certificate")
}
if !strings.Contains(err.Error(), "does not trust the serving certificate") {
t.Fatalf("readiness error = %v, want certificate trust failure", err)
}
}
func TestWebhookCertificateReadinessCheckRejectsStaleLoadedCertificate(t *testing.T) {
t.Parallel()
caBundle, persistedCertificate, persistedKey := generateTestTLSMaterial(t, testWebhookSANs())
_, staleCertificate, staleKey := generateTestTLSMaterial(t, testWebhookSANs())
reconciler, kubeClient := newTestTLSReconciler(
t,
testTLSSecret(caBundle, persistedCertificate, persistedKey),
)
loadedCertificate := parseTestServingCertificate(t, staleCertificate, staleKey)
check := WebhookCertificateReadinessCheck(
kubeClient,
reconciler.Configuration,
testNamespace,
func(*cryptotls.ClientHelloInfo) (*cryptotls.Certificate, error) {
return &loadedCertificate, nil
},
)
err := check(httptest.NewRequest("GET", "/readyz", nil))
if err == nil {
t.Fatal("readiness check accepted a stale certificate loaded by certwatcher")
}
if !strings.Contains(err.Error(), "is stale relative to TLS Secret") {
t.Fatalf("readiness error = %v, want stale loaded certificate failure", err)
}
}
func TestWebhookCertificateReadinessCheckRequiresManagedConfiguration(t *testing.T) {
t.Parallel()
caBundle, certificatePEM, keyPEM := generateTestTLSMaterial(t, testWebhookSANs())
reconciler, kubeClient := newTestTLSReconciler(t, testTLSSecret(caBundle, certificatePEM, keyPEM))
enableTestValidatingWebhook(t, kubeClient)
servingCertificate := parseTestServingCertificate(t, certificatePEM, keyPEM)
check := WebhookCertificateReadinessCheck(
kubeClient,
reconciler.Configuration,
testNamespace,
func(*cryptotls.ClientHelloInfo) (*cryptotls.Certificate, error) {
return &servingCertificate, nil
},
)
err := check(httptest.NewRequest("GET", "/readyz", nil))
if err == nil {
t.Fatal("readiness check accepted a missing managed webhook configuration")
}
if !strings.Contains(err.Error(), "get validating webhook configuration") {
t.Fatalf("readiness error = %v, want missing configuration failure", err)
}
}
func testValidatingWebhookConfiguration(caBundle []byte) *admissionregistrationv1.ValidatingWebhookConfiguration {
return &admissionregistrationv1.ValidatingWebhookConfiguration{
ObjectMeta: metav1.ObjectMeta{Name: testValidatingConfiguration},
Webhooks: []admissionregistrationv1.ValidatingWebhook{{
Name: "owners.validating.projectcapsule.dev",
ClientConfig: admissionregistrationv1.WebhookClientConfig{
CABundle: caBundle,
},
}},
}
}
func enableTestValidatingWebhook(t *testing.T, kubeClient client.Client) {
t.Helper()
configurationObject := &capsulev1beta2.CapsuleConfiguration{}
if err := kubeClient.Get(
context.Background(),
client.ObjectKey{Name: "capsule"},
configurationObject,
); err != nil {
t.Fatalf("get CapsuleConfiguration: %v", err)
}
configurationObject.Spec.Admission.Validating.Webhooks = []*runtimeadmission.ValidatingWebhook{{}}
if err := kubeClient.Update(context.Background(), configurationObject); err != nil {
t.Fatalf("update CapsuleConfiguration: %v", err)
}
}
func parseTestServingCertificate(t *testing.T, certificatePEM, keyPEM []byte) cryptotls.Certificate {
t.Helper()
servingCertificate, err := cryptotls.X509KeyPair(certificatePEM, keyPEM)
if err != nil {
t.Fatalf("parse test serving certificate: %v", err)
}
return servingCertificate
}