mirror of
https://github.com/projectcapsule/capsule.git
synced 2026-08-25 16:07:24 +00:00
feat: requests and limit policies (#2095)
* chore: save progress Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> * feat: requests and limit policies Signed-off-by: Oliver Baehler <oliver@sudo-i.net> --------- Signed-off-by: Oliver Baehler <oliver@sudo-i.net>
This commit is contained in:
@@ -42,10 +42,14 @@ type Manager struct {
|
||||
Log logr.Logger
|
||||
Client client.Client
|
||||
Configuration configuration.Configuration
|
||||
|
||||
reader client.Reader
|
||||
}
|
||||
|
||||
//nolint:revive
|
||||
func (r *Manager) SetupWithManager(ctx context.Context, mgr ctrl.Manager, ctrlConfig utils.ControllerOptions) (err error) {
|
||||
r.reader = mgr.GetAPIReader()
|
||||
|
||||
namesPredicate := predicates.LabelsMatching(map[string]string{
|
||||
meta.CreatedByCapsuleLabel: controllerManager,
|
||||
})
|
||||
@@ -186,7 +190,16 @@ func (r *Manager) EnsureClusterRoleBindingsProvisioner(ctx context.Context) erro
|
||||
|
||||
listStarted := time.Now()
|
||||
|
||||
if err := r.Client.List(ctx, saList, client.MatchingLabels{
|
||||
reader := r.reader
|
||||
if reader == nil {
|
||||
reader = r.Client
|
||||
}
|
||||
|
||||
// ServiceAccount events are sourced from the metadata-only cache.
|
||||
// Reading full objects from the regular cache here can observe the
|
||||
// previous label value and permanently lose a promotion/demotion
|
||||
// reconcile. The API reader provides an authoritative snapshot.
|
||||
if err := reader.List(ctx, saList, client.MatchingLabels{
|
||||
meta.OwnerPromotionLabel: meta.ValueTrue,
|
||||
}); err != nil {
|
||||
logOperationDuration(log, "list promoted ServiceAccounts", listStarted)
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
// Copyright 2020-2026 Project Capsule Authors
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package rbac
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
rbacv1 "k8s.io/api/rbac/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
|
||||
capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
|
||||
"github.com/projectcapsule/capsule/pkg/api/meta"
|
||||
"github.com/projectcapsule/capsule/pkg/runtime/configuration"
|
||||
)
|
||||
|
||||
func TestEnsureClusterRoleBindingsProvisionerUsesAuthoritativePromotionState(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
configurationName = "capsule"
|
||||
provisionerRole = "capsule-namespace-provisioner"
|
||||
namespace = "tenant-a"
|
||||
serviceAccount = "builder"
|
||||
)
|
||||
|
||||
ctx := context.Background()
|
||||
scheme := runtime.NewScheme()
|
||||
for _, addToScheme := range []func(*runtime.Scheme) error{
|
||||
corev1.AddToScheme,
|
||||
rbacv1.AddToScheme,
|
||||
capsulev1beta2.AddToScheme,
|
||||
} {
|
||||
if err := addToScheme(scheme); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
configurationObject := &capsulev1beta2.CapsuleConfiguration{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: configurationName},
|
||||
Spec: capsulev1beta2.CapsuleConfigurationSpec{
|
||||
AllowServiceAccountPromotion: true,
|
||||
RBAC: &capsulev1beta2.RBACConfiguration{
|
||||
ProvisionerClusterRole: provisionerRole,
|
||||
},
|
||||
},
|
||||
}
|
||||
promoted := &corev1.ServiceAccount{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: serviceAccount,
|
||||
Namespace: namespace,
|
||||
Labels: map[string]string{
|
||||
meta.OwnerPromotionLabel: meta.ValueTrue,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// The cached client deliberately remains stale after demotion, reproducing
|
||||
// the ordering between the metadata-only watch and full-object cache.
|
||||
cachedClient := fake.NewClientBuilder().
|
||||
WithScheme(scheme).
|
||||
WithObjects(configurationObject, promoted.DeepCopy()).
|
||||
Build()
|
||||
authoritativeClient := fake.NewClientBuilder().
|
||||
WithScheme(scheme).
|
||||
WithObjects(promoted.DeepCopy()).
|
||||
Build()
|
||||
cfg := configuration.NewCapsuleConfiguration(
|
||||
ctx,
|
||||
cachedClient,
|
||||
cachedClient,
|
||||
nil,
|
||||
configurationName,
|
||||
)
|
||||
manager := &Manager{
|
||||
Log: logr.Discard(),
|
||||
Client: cachedClient,
|
||||
Configuration: cfg,
|
||||
reader: authoritativeClient,
|
||||
}
|
||||
|
||||
if err := manager.EnsureClusterRoleBindingsProvisioner(ctx); err != nil {
|
||||
t.Fatalf("initial provisioner binding reconciliation: %v", err)
|
||||
}
|
||||
assertServiceAccountSubject(t, cachedClient, provisionerRole, namespace, serviceAccount, true)
|
||||
|
||||
latest := &corev1.ServiceAccount{}
|
||||
if err := authoritativeClient.Get(ctx, client.ObjectKeyFromObject(promoted), latest); err != nil {
|
||||
t.Fatalf("get authoritative ServiceAccount: %v", err)
|
||||
}
|
||||
latest.Labels[meta.OwnerPromotionLabel] = "false"
|
||||
if err := authoritativeClient.Update(ctx, latest); err != nil {
|
||||
t.Fatalf("demote authoritative ServiceAccount: %v", err)
|
||||
}
|
||||
|
||||
if err := manager.EnsureClusterRoleBindingsProvisioner(ctx); err != nil {
|
||||
t.Fatalf("demotion provisioner binding reconciliation: %v", err)
|
||||
}
|
||||
assertServiceAccountSubject(t, cachedClient, provisionerRole, namespace, serviceAccount, false)
|
||||
}
|
||||
|
||||
func assertServiceAccountSubject(
|
||||
t *testing.T,
|
||||
kubeClient client.Client,
|
||||
bindingName string,
|
||||
namespace string,
|
||||
serviceAccount string,
|
||||
want bool,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
binding := &rbacv1.ClusterRoleBinding{}
|
||||
if err := kubeClient.Get(context.Background(), client.ObjectKey{Name: bindingName}, binding); err != nil {
|
||||
t.Fatalf("get ClusterRoleBinding: %v", err)
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, subject := range binding.Subjects {
|
||||
if subject.Kind == rbacv1.ServiceAccountKind &&
|
||||
subject.Namespace == namespace &&
|
||||
subject.Name == serviceAccount {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if found != want {
|
||||
t.Fatalf("ServiceAccount subject presence = %t, want %t; subjects: %#v", found, want, binding.Subjects)
|
||||
}
|
||||
}
|
||||
@@ -96,7 +96,13 @@ func (r *Manager) pruneGlobalResourceQuotas(
|
||||
meta.NewManagedByCapsuleLabel: meta.ValueController,
|
||||
meta.NewTenantLabel: tnt.Name,
|
||||
})
|
||||
if err := r.List(ctx, list, &client.ListOptions{LabelSelector: selector}); err != nil {
|
||||
|
||||
reader := client.Reader(r.Client)
|
||||
if r.reader != nil {
|
||||
reader = r.reader
|
||||
}
|
||||
|
||||
if err := reader.List(ctx, list, &client.ListOptions{LabelSelector: selector}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -117,3 +123,13 @@ func (r *Manager) pruneGlobalResourceQuotas(
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func hasRuleGlobalResourceQuotas(tnt *capsulev1beta2.Tenant) bool {
|
||||
for _, rule := range tnt.Spec.Rules {
|
||||
if rule != nil && rule.NamespaceRuleBodyNamespace != nil && len(rule.Quota) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -59,6 +59,16 @@ func TestSyncGlobalResourceQuotasGeneratesAndPrunesRuleQuotas(t *testing.T) {
|
||||
if generated.Labels[meta.RuleQuotaLabel] != "shared-compute" {
|
||||
t.Fatalf("rule quota label = %q, want shared-compute", generated.Labels[meta.RuleQuotaLabel])
|
||||
}
|
||||
if generated.Labels[meta.NewManagedByCapsuleLabel] != meta.ValueController {
|
||||
t.Fatalf(
|
||||
"managed-by label = %q, want %q",
|
||||
generated.Labels[meta.NewManagedByCapsuleLabel],
|
||||
meta.ValueController,
|
||||
)
|
||||
}
|
||||
if !metav1.IsControlledBy(generated, tnt) {
|
||||
t.Fatalf("generated GlobalResourceQuota is not controlled by Tenant %q", tnt.Name)
|
||||
}
|
||||
selector := generated.Spec.NamespaceSelectors[0].LabelSelector
|
||||
if selector.MatchLabels[meta.TenantLabel] != tnt.Name || selector.MatchLabels["tier"] != "paid" {
|
||||
t.Fatalf("generated selector = %#v", selector)
|
||||
|
||||
@@ -367,6 +367,15 @@ func (r *Manager) reconcile(ctx context.Context, log logr.Logger, instance *caps
|
||||
errs = append(errs, fmt.Errorf("namespace(s) had reconciliation errors: %w", err))
|
||||
}
|
||||
|
||||
// The managed-resource webhook intentionally denies deletion by the
|
||||
// garbage collector. Remove rule-generated cluster-scoped children as
|
||||
// the Capsule controller before releasing the Tenant finalizer.
|
||||
if err = r.pruneGlobalResourceQuotas(ctx, instance, map[string]struct{}{}); err != nil {
|
||||
errs = append(errs, fmt.Errorf("cannot delete rule global resource quotas: %w", err))
|
||||
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
if err = r.ensureMetadata(ctx, instance); err != nil {
|
||||
errs = append(errs, fmt.Errorf("cannot ensure metadata: %w", err))
|
||||
}
|
||||
|
||||
@@ -22,7 +22,11 @@ func (r *Manager) ensureMetadata(ctx context.Context, tnt *capsulev1beta2.Tenant
|
||||
tnt.Labels[meta.TenantNameLabel] = tnt.Name
|
||||
}
|
||||
|
||||
if len(tnt.Status.Spaces) == 0 {
|
||||
// Rule-generated GlobalResourceQuotas are cluster scoped and protected from
|
||||
// garbage-collector deletion by the managed-resource webhook. Keep the
|
||||
// Tenant around until its controller can explicitly remove those children.
|
||||
keepForRuleQuotas := tnt.DeletionTimestamp == nil && hasRuleGlobalResourceQuotas(tnt)
|
||||
if len(tnt.Status.Spaces) == 0 && !keepForRuleQuotas {
|
||||
controllerutil.RemoveFinalizer(tnt, meta.ControllerFinalizer)
|
||||
} else {
|
||||
controllerutil.AddFinalizer(tnt, meta.ControllerFinalizer)
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
// Copyright 2020-2026 Project Capsule Authors
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package tenant
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
|
||||
capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
|
||||
"github.com/projectcapsule/capsule/pkg/api/meta"
|
||||
"github.com/projectcapsule/capsule/pkg/api/rules"
|
||||
)
|
||||
|
||||
func TestEnsureMetadataKeepsFinalizerForRuleGlobalResourceQuotas(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tnt := &capsulev1beta2.Tenant{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "tenant-a"},
|
||||
Spec: capsulev1beta2.TenantSpec{Rules: []*rules.NamespaceRuleBodyTenant{{
|
||||
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
|
||||
Quota: []rules.ResourceQuotaRule{{Name: "compute"}},
|
||||
},
|
||||
}}},
|
||||
}
|
||||
manager := &Manager{}
|
||||
|
||||
if err := manager.ensureMetadata(context.Background(), tnt); err != nil {
|
||||
t.Fatalf("ensureMetadata() error = %v", err)
|
||||
}
|
||||
if !controllerutil.ContainsFinalizer(tnt, meta.ControllerFinalizer) {
|
||||
t.Fatal("Tenant with rule-generated GlobalResourceQuota is missing the controller finalizer")
|
||||
}
|
||||
|
||||
now := metav1.Now()
|
||||
tnt.DeletionTimestamp = &now
|
||||
if err := manager.ensureMetadata(context.Background(), tnt); err != nil {
|
||||
t.Fatalf("ensureMetadata() while deleting error = %v", err)
|
||||
}
|
||||
if controllerutil.ContainsFinalizer(tnt, meta.ControllerFinalizer) {
|
||||
t.Fatal("Tenant controller finalizer was retained after managed child cleanup")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasRuleGlobalResourceQuotas(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
rules []*rules.NamespaceRuleBodyTenant
|
||||
want bool
|
||||
}{
|
||||
{name: "no rules"},
|
||||
{name: "nil rule", rules: []*rules.NamespaceRuleBodyTenant{nil}},
|
||||
{name: "rule without namespace body", rules: []*rules.NamespaceRuleBodyTenant{{}}},
|
||||
{
|
||||
name: "rule without quota",
|
||||
rules: []*rules.NamespaceRuleBodyTenant{{
|
||||
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{},
|
||||
}},
|
||||
},
|
||||
{
|
||||
name: "rule with quota",
|
||||
rules: []*rules.NamespaceRuleBodyTenant{{
|
||||
NamespaceRuleBodyNamespace: &rules.NamespaceRuleBodyNamespace{
|
||||
Quota: []rules.ResourceQuotaRule{{Name: "compute"}},
|
||||
},
|
||||
}},
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tnt := &capsulev1beta2.Tenant{Spec: capsulev1beta2.TenantSpec{Rules: test.rules}}
|
||||
if got := hasRuleGlobalResourceQuotas(tnt); got != test.want {
|
||||
t.Fatalf("hasRuleGlobalResourceQuotas() = %t, want %t", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -165,51 +165,24 @@ func (r *Reconciler) ReconcileCertificates(
|
||||
"ipAddresses", cert.IPsToStrings(sans.IPAddrs),
|
||||
)
|
||||
|
||||
ca, caBundle, rotateServingCert, err := r.ensureCertificateMaterial(log, certSecret, sans)
|
||||
if err != nil {
|
||||
if err := r.reconcileTLSSecret(ctx, log, certSecret, sans); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.V(4).Info(
|
||||
"certificate requires rotation",
|
||||
"rotation", rotateServingCert,
|
||||
)
|
||||
|
||||
if rotateServingCert {
|
||||
if ca == nil {
|
||||
return fmt.Errorf("cannot rotate serving certificate without CA private key")
|
||||
}
|
||||
|
||||
crt, key, err := ca.GenerateCertificate(cert.NewCertOpts(
|
||||
time.Now().Add(certificateValidity),
|
||||
sans,
|
||||
))
|
||||
if err != nil {
|
||||
log.Error(err, "cannot generate serving TLS certificate")
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
certSecret.Data[corev1.TLSCertKey] = crt.Bytes()
|
||||
certSecret.Data[corev1.TLSPrivateKeyKey] = key.Bytes()
|
||||
|
||||
if err := r.validateSecretCertificate(certSecret, sans); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := r.upsertTLSSecret(ctx, certSecret); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
caBundle = certSecret.Data[corev1.ServiceAccountRootCAKey]
|
||||
caBundle := certSecret.Data[corev1.ServiceAccountRootCAKey]
|
||||
if len(caBundle) == 0 {
|
||||
return fmt.Errorf("missing %q field in %q secret", corev1.ServiceAccountRootCAKey, r.Configuration.TLSSecretName())
|
||||
}
|
||||
|
||||
log.V(5).Info("Patching caBundle in managed CRD conversions")
|
||||
log.V(5).Info("Patching caBundle in admission webhooks and managed CRD conversions")
|
||||
|
||||
patchGroup, groupCtx := errgroup.WithContext(ctx)
|
||||
patchGroup.Go(func() error {
|
||||
return r.patchMutatingWebhookConfigurationCABundle(groupCtx, caBundle)
|
||||
})
|
||||
patchGroup.Go(func() error {
|
||||
return r.patchValidatingWebhookConfigurationCABundle(groupCtx, caBundle)
|
||||
})
|
||||
|
||||
for key, managed := range r.conversionManagedCRDs() {
|
||||
patchGroup.Go(func() error {
|
||||
@@ -224,15 +197,108 @@ func (r *Reconciler) ReconcileCertificates(
|
||||
return patchGroup.Wait()
|
||||
}
|
||||
|
||||
// reconcileTLSSecret calculates certificate material from the latest persisted
|
||||
// Secret inside an optimistic-concurrency retry. This is deliberately not based
|
||||
// on the object supplied by the caller: every controller replica performs the
|
||||
// startup reconciliation before leader election, so that object may already be
|
||||
// stale by the time certificate generation finishes.
|
||||
func (r *Reconciler) reconcileTLSSecret(
|
||||
ctx context.Context,
|
||||
log logr.Logger,
|
||||
certSecret *corev1.Secret,
|
||||
sans cert.CertificateSANs,
|
||||
) error {
|
||||
key := client.ObjectKeyFromObject(certSecret)
|
||||
if key.Name == "" {
|
||||
key.Name = r.Configuration.TLSSecretName()
|
||||
}
|
||||
|
||||
if key.Namespace == "" {
|
||||
key.Namespace = r.Namespace
|
||||
}
|
||||
|
||||
err := retry.OnError(retry.DefaultBackoff, func(err error) bool {
|
||||
return apierrors.IsAlreadyExists(err) || apierrors.IsConflict(err)
|
||||
}, func() error {
|
||||
// Use the TLS type when Capsule creates the Secret itself. If the Secret
|
||||
// already exists, CreateOrUpdate loads its persisted type into desired;
|
||||
// preserve it because Secret type is immutable. In particular, the Helm
|
||||
// chart pre-creates an empty Opaque Secret so the controller Pod can mount
|
||||
// it before this reconciliation supplies the certificate data.
|
||||
desired := &corev1.Secret{
|
||||
ObjectMeta: *certSecret.ObjectMeta.DeepCopy(),
|
||||
Type: corev1.SecretTypeTLS,
|
||||
}
|
||||
desired.Name = key.Name
|
||||
desired.Namespace = key.Namespace
|
||||
|
||||
_, err := controllerutil.CreateOrUpdate(ctx, r.Client, desired, func() error {
|
||||
if desired.Labels == nil {
|
||||
desired.Labels = map[string]string{}
|
||||
}
|
||||
|
||||
if desired.Annotations == nil {
|
||||
desired.Annotations = map[string]string{}
|
||||
}
|
||||
|
||||
ca, _, rotateServingCert, err := r.ensureCertificateMaterial(log, desired, sans)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.V(4).Info(
|
||||
"certificate requires rotation",
|
||||
"rotation", rotateServingCert,
|
||||
)
|
||||
|
||||
if rotateServingCert {
|
||||
if ca == nil {
|
||||
return fmt.Errorf("cannot rotate serving certificate without CA private key")
|
||||
}
|
||||
|
||||
crt, key, err := ca.GenerateCertificate(cert.NewCertOpts(
|
||||
time.Now().Add(certificateValidity),
|
||||
sans,
|
||||
))
|
||||
if err != nil {
|
||||
return fmt.Errorf("generate serving TLS certificate: %w", err)
|
||||
}
|
||||
|
||||
desired.Data[corev1.TLSCertKey] = crt.Bytes()
|
||||
desired.Data[corev1.TLSPrivateKeyKey] = key.Bytes()
|
||||
}
|
||||
|
||||
return r.validateSecretCertificate(desired, sans)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
certSecret.ObjectMeta = desired.ObjectMeta
|
||||
certSecret.Type = desired.Type
|
||||
certSecret.Immutable = desired.Immutable
|
||||
certSecret.Data = copySecretData(desired.Data)
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
r.Log.Error(err, "cannot reconcile Capsule TLS Secret", "secret", key.String())
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// ensureCertificateMaterial ensures that the Secret contains a stable CA
|
||||
// certificate/key pair and decides whether the serving certificate must be
|
||||
// regenerated.
|
||||
//
|
||||
// Important behavior:
|
||||
// - Missing Secret or missing ca.key creates a new CA.
|
||||
// - Only a new, empty Secret bootstraps a CA.
|
||||
// - Existing valid CA is reused.
|
||||
// - Serving certificate renewal never rotates the CA.
|
||||
// - Legacy Secrets without ca.key rotate once into the stable format.
|
||||
// - Established or externally managed CA material is never replaced
|
||||
// automatically, because doing so would immediately invalidate every
|
||||
// published caBundle.
|
||||
func (r *Reconciler) ensureCertificateMaterial(
|
||||
log logr.Logger,
|
||||
certSecret *corev1.Secret,
|
||||
@@ -262,53 +328,51 @@ func (r *Reconciler) ensureCertificateMaterial(
|
||||
case hasCABundle && hasCAKey:
|
||||
loadedCA, err := cert.NewCertificateAuthorityFromBytes(caBundle, caKey)
|
||||
if err != nil {
|
||||
log.V(3).Info(
|
||||
"Existing CA material is invalid, generating new CA",
|
||||
"error", err.Error(),
|
||||
return nil, nil, false, fmt.Errorf(
|
||||
"TLS Secret %s contains invalid CA certificate/key material; refusing automatic CA replacement: %w",
|
||||
client.ObjectKeyFromObject(certSecret).String(),
|
||||
err,
|
||||
)
|
||||
|
||||
generatedCA, generatedCABundle, generatedCAKey, err := generateCertificateAuthorityMaterial()
|
||||
if err != nil {
|
||||
return nil, nil, false, err
|
||||
}
|
||||
|
||||
ca = generatedCA
|
||||
caBundle = generatedCABundle
|
||||
|
||||
certSecret.Data[corev1.ServiceAccountRootCAKey] = generatedCABundle
|
||||
certSecret.Data["ca.key"] = generatedCAKey
|
||||
|
||||
rotateServingCert = true
|
||||
} else {
|
||||
ca = loadedCA
|
||||
}
|
||||
|
||||
ca = loadedCA
|
||||
|
||||
case hasCABundle && !hasCAKey:
|
||||
// A CA bundle without its private key is typical of an externally
|
||||
// managed Secret. Once the Capsule TLS controller is enabled, it must
|
||||
// take ownership of the complete certificate lifecycle; otherwise a
|
||||
// SAN change or certificate renewal cannot be recovered.
|
||||
// This is an externally managed or legacy Secret. It is safe to keep
|
||||
// serving while its certificate remains valid, but Capsule cannot renew
|
||||
// it without the CA key. Replacing that CA in-place would make the API
|
||||
// server distrust one or more running webhook replicas during rollout.
|
||||
if err := r.validateSecretCertificate(certSecret, sans); err != nil {
|
||||
return nil, nil, false, fmt.Errorf(
|
||||
"TLS Secret %s has no CA private key and its serving certificate needs renewal; refusing automatic CA replacement: %w",
|
||||
client.ObjectKeyFromObject(certSecret).String(),
|
||||
err,
|
||||
)
|
||||
}
|
||||
|
||||
log.V(3).Info(
|
||||
"TLS Secret contains CA bundle but no CA private key, rotating into controller-managed TLS material",
|
||||
"Keeping externally managed TLS material without a CA private key",
|
||||
"secret", client.ObjectKeyFromObject(certSecret).String(),
|
||||
)
|
||||
|
||||
generatedCA, generatedCABundle, generatedCAKey, err := generateCertificateAuthorityMaterial()
|
||||
if err != nil {
|
||||
return nil, nil, false, err
|
||||
}
|
||||
return nil, caBundle, false, nil
|
||||
|
||||
ca = generatedCA
|
||||
caBundle = generatedCABundle
|
||||
|
||||
certSecret.Data[corev1.ServiceAccountRootCAKey] = generatedCABundle
|
||||
certSecret.Data["ca.key"] = generatedCAKey
|
||||
|
||||
rotateServingCert = true
|
||||
case !hasCABundle && hasCAKey:
|
||||
return nil, nil, false, fmt.Errorf(
|
||||
"TLS Secret %s contains a CA private key but no CA certificate; refusing automatic CA replacement",
|
||||
client.ObjectKeyFromObject(certSecret).String(),
|
||||
)
|
||||
|
||||
default:
|
||||
if len(certSecret.Data[corev1.TLSCertKey]) > 0 || len(certSecret.Data[corev1.TLSPrivateKeyKey]) > 0 {
|
||||
return nil, nil, false, fmt.Errorf(
|
||||
"TLS Secret %s contains serving certificate material but no CA certificate; refusing automatic CA replacement",
|
||||
client.ObjectKeyFromObject(certSecret).String(),
|
||||
)
|
||||
}
|
||||
|
||||
log.V(10).Info(
|
||||
"TLS Secret is missing CA material, generating new CA",
|
||||
"TLS Secret is empty, generating initial CA",
|
||||
"secret", client.ObjectKeyFromObject(certSecret).String(),
|
||||
)
|
||||
|
||||
@@ -411,35 +475,103 @@ func generateCertificateAuthorityMaterial() (*cert.CapsuleCA, []byte, []byte, er
|
||||
return ca, caCrt.Bytes(), caKey.Bytes(), nil
|
||||
}
|
||||
|
||||
func (r *Reconciler) upsertTLSSecret(ctx context.Context, certSecret *corev1.Secret) error {
|
||||
desired := &corev1.Secret{
|
||||
ObjectMeta: certSecret.ObjectMeta,
|
||||
Type: corev1.SecretTypeTLS,
|
||||
}
|
||||
func (r *Reconciler) patchMutatingWebhookConfigurationCABundle(ctx context.Context, caBundle []byte) error {
|
||||
return r.patchAdmissionConfigurationCABundle(
|
||||
ctx,
|
||||
r.Configuration.MutatingWebhookConfigurationName(),
|
||||
caBundle,
|
||||
func() client.Object { return &admissionregistrationv1.MutatingWebhookConfiguration{} },
|
||||
updateMutatingWebhookCABundles,
|
||||
)
|
||||
}
|
||||
|
||||
_, err := controllerutil.CreateOrUpdate(ctx, r.Client, desired, func() error {
|
||||
if desired.Labels == nil {
|
||||
desired.Labels = map[string]string{}
|
||||
}
|
||||
|
||||
if desired.Annotations == nil {
|
||||
desired.Annotations = map[string]string{}
|
||||
}
|
||||
|
||||
desired.Data = copySecretData(certSecret.Data)
|
||||
func (r *Reconciler) patchValidatingWebhookConfigurationCABundle(ctx context.Context, caBundle []byte) error {
|
||||
return r.patchAdmissionConfigurationCABundle(
|
||||
ctx,
|
||||
r.Configuration.ValidatingWebhookConfigurationName(),
|
||||
caBundle,
|
||||
func() client.Object { return &admissionregistrationv1.ValidatingWebhookConfiguration{} },
|
||||
updateValidatingWebhookCABundles,
|
||||
)
|
||||
}
|
||||
|
||||
func (r *Reconciler) patchAdmissionConfigurationCABundle(
|
||||
ctx context.Context,
|
||||
name string,
|
||||
caBundle []byte,
|
||||
newConfiguration func() client.Object,
|
||||
updateCABundles func(client.Object, []byte) (bool, error),
|
||||
) error {
|
||||
if name == "" {
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
r.Log.Error(err, "cannot update Capsule TLS Secret")
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
certSecret.ObjectMeta = desired.ObjectMeta
|
||||
certSecret.Data = copySecretData(desired.Data)
|
||||
return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
|
||||
configuration := newConfiguration()
|
||||
if err := r.Get(ctx, types.NamespacedName{Name: name}, configuration); err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
|
||||
before, ok := configuration.DeepCopyObject().(client.Object)
|
||||
if !ok {
|
||||
return fmt.Errorf("admission configuration %q cannot be deep-copied as a client object", name)
|
||||
}
|
||||
|
||||
changed, err := updateCABundles(configuration, caBundle)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !changed {
|
||||
return nil
|
||||
}
|
||||
|
||||
return r.Patch(ctx, configuration, client.MergeFrom(before))
|
||||
})
|
||||
}
|
||||
|
||||
func updateMutatingWebhookCABundles(object client.Object, caBundle []byte) (bool, error) {
|
||||
configuration, ok := object.(*admissionregistrationv1.MutatingWebhookConfiguration)
|
||||
if !ok {
|
||||
return false, fmt.Errorf("expected MutatingWebhookConfiguration, got %T", object)
|
||||
}
|
||||
|
||||
changed := false
|
||||
|
||||
for index := range configuration.Webhooks {
|
||||
if bytes.Equal(configuration.Webhooks[index].ClientConfig.CABundle, caBundle) {
|
||||
continue
|
||||
}
|
||||
|
||||
configuration.Webhooks[index].ClientConfig.CABundle = append([]byte(nil), caBundle...)
|
||||
changed = true
|
||||
}
|
||||
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
func updateValidatingWebhookCABundles(object client.Object, caBundle []byte) (bool, error) {
|
||||
configuration, ok := object.(*admissionregistrationv1.ValidatingWebhookConfiguration)
|
||||
if !ok {
|
||||
return false, fmt.Errorf("expected ValidatingWebhookConfiguration, got %T", object)
|
||||
}
|
||||
|
||||
changed := false
|
||||
|
||||
for index := range configuration.Webhooks {
|
||||
if bytes.Equal(configuration.Webhooks[index].ClientConfig.CABundle, caBundle) {
|
||||
continue
|
||||
}
|
||||
|
||||
configuration.Webhooks[index].ClientConfig.CABundle = append([]byte(nil), caBundle...)
|
||||
changed = true
|
||||
}
|
||||
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
func (r *Reconciler) validateSecretCertificate(
|
||||
|
||||
@@ -6,10 +6,12 @@ package tls
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
@@ -18,123 +20,298 @@ import (
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
||||
|
||||
capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
|
||||
apiMeta "github.com/projectcapsule/capsule/pkg/api/meta"
|
||||
runtimeadmission "github.com/projectcapsule/capsule/pkg/runtime/admission"
|
||||
"github.com/projectcapsule/capsule/pkg/runtime/cert"
|
||||
"github.com/projectcapsule/capsule/pkg/runtime/configuration"
|
||||
)
|
||||
|
||||
func TestReconcileCertificatesMigratesExternalTLSSecret(t *testing.T) {
|
||||
const (
|
||||
testNamespace = "capsule-system"
|
||||
testSecretName = "capsule-tls"
|
||||
testServiceName = "capsule-webhook-service"
|
||||
testMutatingConfiguration = "capsule-mutating-webhook-configuration"
|
||||
testValidatingConfiguration = "capsule-validating-webhook-configuration"
|
||||
)
|
||||
|
||||
func TestReconcileCertificatesPreservesValidExternalTLSSecret(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
namespace = "capsule-system"
|
||||
secretName = "capsule-tls"
|
||||
serviceName = "capsule-webhook-service"
|
||||
)
|
||||
externalCABundle, externalCertificate, externalKey := generateTestTLSMaterial(t, testWebhookSANs())
|
||||
secret := testTLSSecret(externalCABundle, externalCertificate, externalKey)
|
||||
reconciler, kubeClient := newTestTLSReconciler(t, secret)
|
||||
|
||||
if err := reconciler.ReconcileCertificates(context.Background(), logr.Discard(), secret.DeepCopy()); err != nil {
|
||||
t.Fatalf("ReconcileCertificates() error = %v", err)
|
||||
}
|
||||
|
||||
updated := getTestTLSSecret(t, kubeClient)
|
||||
if len(updated.Data["ca.key"]) != 0 {
|
||||
t.Fatal("external TLS Secret unexpectedly gained a CA private key")
|
||||
}
|
||||
|
||||
assertTLSDataEqual(t, updated, externalCABundle, externalCertificate, externalKey)
|
||||
}
|
||||
|
||||
func TestReconcileCertificatesRejectsUnsafeExternalCARotation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
externalCABundle, externalCertificate, externalKey := generateTestTLSMaterial(t, cert.CertificateSANs{
|
||||
DNSNames: []string{serviceName + "." + namespace + ".svc"},
|
||||
DNSNames: []string{testServiceName + "." + testNamespace + ".svc"},
|
||||
})
|
||||
secret := testTLSSecret(externalCABundle, externalCertificate, externalKey)
|
||||
reconciler, kubeClient := newTestTLSReconciler(t, secret)
|
||||
|
||||
secret := &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: secretName,
|
||||
Namespace: namespace,
|
||||
},
|
||||
Type: corev1.SecretTypeTLS,
|
||||
Data: map[string][]byte{
|
||||
corev1.ServiceAccountRootCAKey: externalCABundle,
|
||||
corev1.TLSCertKey: externalCertificate,
|
||||
corev1.TLSPrivateKeyKey: externalKey,
|
||||
err := reconciler.ReconcileCertificates(context.Background(), logr.Discard(), secret.DeepCopy())
|
||||
if err == nil {
|
||||
t.Fatal("ReconcileCertificates() unexpectedly replaced externally managed CA material")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "refusing automatic CA replacement") {
|
||||
t.Fatalf("ReconcileCertificates() error = %v, want safe CA replacement refusal", err)
|
||||
}
|
||||
|
||||
updated := getTestTLSSecret(t, kubeClient)
|
||||
if len(updated.Data["ca.key"]) != 0 {
|
||||
t.Fatal("rejected external TLS Secret unexpectedly gained a CA private key")
|
||||
}
|
||||
|
||||
assertTLSDataEqual(t, updated, externalCABundle, externalCertificate, externalKey)
|
||||
}
|
||||
|
||||
func TestReconcileCertificatesConcurrentReplicasAdoptPersistedWinner(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
secret := testTLSSecret(nil, nil, nil)
|
||||
reconciler, kubeClient := newTestTLSReconciler(t, secret)
|
||||
firstReplica := secret.DeepCopy()
|
||||
secondReplica := secret.DeepCopy()
|
||||
start := make(chan struct{})
|
||||
errors := make(chan error, 2)
|
||||
|
||||
for _, replica := range []*corev1.Secret{firstReplica, secondReplica} {
|
||||
go func(replica *corev1.Secret) {
|
||||
<-start
|
||||
errors <- reconciler.ReconcileCertificates(context.Background(), logr.Discard(), replica)
|
||||
}(replica)
|
||||
}
|
||||
|
||||
close(start)
|
||||
for range 2 {
|
||||
if err := <-errors; err != nil {
|
||||
t.Fatalf("concurrent ReconcileCertificates() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
finalPersisted := getTestTLSSecret(t, kubeClient)
|
||||
if !bytes.Equal(
|
||||
firstReplica.Data[corev1.ServiceAccountRootCAKey],
|
||||
finalPersisted.Data[corev1.ServiceAccountRootCAKey],
|
||||
) {
|
||||
t.Fatal("first replica did not adopt the persisted CA")
|
||||
}
|
||||
if !bytes.Equal(
|
||||
secondReplica.Data[corev1.ServiceAccountRootCAKey],
|
||||
finalPersisted.Data[corev1.ServiceAccountRootCAKey],
|
||||
) {
|
||||
t.Fatal("second replica did not adopt the persisted CA")
|
||||
}
|
||||
if !bytes.Equal(firstReplica.Data[corev1.TLSCertKey], finalPersisted.Data[corev1.TLSCertKey]) ||
|
||||
!bytes.Equal(secondReplica.Data[corev1.TLSCertKey], finalPersisted.Data[corev1.TLSCertKey]) {
|
||||
t.Fatal("concurrent replicas did not adopt the persisted serving certificate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileCertificatesPopulatesChartCreatedOpaqueSecret(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// The Helm chart must create this empty Secret before the controller Pod can
|
||||
// mount it. Kubernetes does not allow changing a Secret's type afterwards,
|
||||
// so reconciliation must preserve Opaque while adding valid TLS material.
|
||||
secret := testTLSSecret(nil, nil, nil)
|
||||
secret.Type = corev1.SecretTypeOpaque
|
||||
reconciler, kubeClient := newTestTLSReconciler(t, secret)
|
||||
|
||||
if err := reconciler.ReconcileCertificates(context.Background(), logr.Discard(), secret.DeepCopy()); err != nil {
|
||||
t.Fatalf("ReconcileCertificates() error = %v", err)
|
||||
}
|
||||
|
||||
updated := getTestTLSSecret(t, kubeClient)
|
||||
if updated.Type != corev1.SecretTypeOpaque {
|
||||
t.Fatalf("reconciled Secret type = %q, want %q", updated.Type, corev1.SecretTypeOpaque)
|
||||
}
|
||||
if len(updated.Data[corev1.ServiceAccountRootCAKey]) == 0 ||
|
||||
len(updated.Data[corev1.TLSCertKey]) == 0 ||
|
||||
len(updated.Data[corev1.TLSPrivateKeyKey]) == 0 {
|
||||
t.Fatal("reconciled Secret is missing generated TLS material")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileCertificatesPatchesEveryAdmissionCABundle(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
oldCA, _, _ := generateTestTLSMaterial(t, testWebhookSANs())
|
||||
secret := testTLSSecret(nil, nil, nil)
|
||||
mutating := &admissionregistrationv1.MutatingWebhookConfiguration{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: testMutatingConfiguration},
|
||||
Webhooks: []admissionregistrationv1.MutatingWebhook{
|
||||
{Name: "first.mutating.projectcapsule.dev", ClientConfig: admissionregistrationv1.WebhookClientConfig{CABundle: oldCA}},
|
||||
{Name: "second.mutating.projectcapsule.dev"},
|
||||
},
|
||||
}
|
||||
validating := &admissionregistrationv1.ValidatingWebhookConfiguration{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: testValidatingConfiguration},
|
||||
Webhooks: []admissionregistrationv1.ValidatingWebhook{
|
||||
{Name: "first.validating.projectcapsule.dev", ClientConfig: admissionregistrationv1.WebhookClientConfig{CABundle: oldCA}},
|
||||
{Name: "second.validating.projectcapsule.dev"},
|
||||
},
|
||||
}
|
||||
reconciler, kubeClient := newTestTLSReconciler(t, secret, mutating, validating)
|
||||
|
||||
if err := reconciler.ReconcileCertificates(context.Background(), logr.Discard(), secret.DeepCopy()); err != nil {
|
||||
t.Fatalf("ReconcileCertificates() error = %v", err)
|
||||
}
|
||||
|
||||
wantCA := getTestTLSSecret(t, kubeClient).Data[corev1.ServiceAccountRootCAKey]
|
||||
updatedMutating := &admissionregistrationv1.MutatingWebhookConfiguration{}
|
||||
if err := kubeClient.Get(
|
||||
context.Background(),
|
||||
client.ObjectKey{Name: testMutatingConfiguration},
|
||||
updatedMutating,
|
||||
); err != nil {
|
||||
t.Fatalf("get mutating webhook configuration: %v", err)
|
||||
}
|
||||
for index := range updatedMutating.Webhooks {
|
||||
if !bytes.Equal(updatedMutating.Webhooks[index].ClientConfig.CABundle, wantCA) {
|
||||
t.Fatalf("mutating webhook %q has stale caBundle", updatedMutating.Webhooks[index].Name)
|
||||
}
|
||||
}
|
||||
|
||||
updatedValidating := &admissionregistrationv1.ValidatingWebhookConfiguration{}
|
||||
if err := kubeClient.Get(
|
||||
context.Background(),
|
||||
client.ObjectKey{Name: testValidatingConfiguration},
|
||||
updatedValidating,
|
||||
); err != nil {
|
||||
t.Fatalf("get validating webhook configuration: %v", err)
|
||||
}
|
||||
for index := range updatedValidating.Webhooks {
|
||||
if !bytes.Equal(updatedValidating.Webhooks[index].ClientConfig.CABundle, wantCA) {
|
||||
t.Fatalf("validating webhook %q has stale caBundle", updatedValidating.Webhooks[index].Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newTestTLSReconciler(t *testing.T, objects ...client.Object) (*Reconciler, client.Client) {
|
||||
t.Helper()
|
||||
|
||||
configurationObject := &capsulev1beta2.CapsuleConfiguration{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "capsule"},
|
||||
Spec: capsulev1beta2.CapsuleConfigurationSpec{
|
||||
EnableTLSReconciler: true,
|
||||
CapsuleResources: capsulev1beta2.CapsuleResources{
|
||||
TLSSecretName: secretName,
|
||||
TLSSecretName: testSecretName,
|
||||
},
|
||||
Admission: capsulev1beta2.DynamicAdmission{
|
||||
ServiceName: serviceName,
|
||||
Mutating: &capsulev1beta2.DynamicMutatingAdmissionConfig{},
|
||||
Validating: &capsulev1beta2.DynamicValidatingAdmissionConfig{},
|
||||
ServiceName: testServiceName,
|
||||
Mutating: &capsulev1beta2.DynamicMutatingAdmissionConfig{
|
||||
DynamicAdmissionConfig: runtimeadmission.DynamicAdmissionConfig{
|
||||
Name: apiMeta.RFC1123Name(testMutatingConfiguration),
|
||||
},
|
||||
},
|
||||
Validating: &capsulev1beta2.DynamicValidatingAdmissionConfig{
|
||||
DynamicAdmissionConfig: runtimeadmission.DynamicAdmissionConfig{
|
||||
Name: apiMeta.RFC1123Name(testValidatingConfiguration),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
scheme := runtime.NewScheme()
|
||||
if err := corev1.AddToScheme(scheme); err != nil {
|
||||
t.Fatal(err)
|
||||
for _, addToScheme := range []func(*runtime.Scheme) error{
|
||||
corev1.AddToScheme,
|
||||
admissionregistrationv1.AddToScheme,
|
||||
apiextensionsv1.AddToScheme,
|
||||
capsulev1beta2.AddToScheme,
|
||||
} {
|
||||
if err := addToScheme(scheme); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := apiextensionsv1.AddToScheme(scheme); err != nil {
|
||||
t.Fatal(err)
|
||||
objects = append(objects, configurationObject)
|
||||
kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build()
|
||||
cfg := configuration.NewCapsuleConfiguration(
|
||||
context.Background(),
|
||||
kubeClient,
|
||||
kubeClient,
|
||||
nil,
|
||||
configurationObject.Name,
|
||||
)
|
||||
|
||||
return &Reconciler{
|
||||
Client: kubeClient,
|
||||
Log: logr.Discard(),
|
||||
Namespace: testNamespace,
|
||||
Configuration: cfg,
|
||||
}, kubeClient
|
||||
}
|
||||
|
||||
func testTLSSecret(caBundle, certificate, key []byte) *corev1.Secret {
|
||||
data := map[string][]byte{}
|
||||
if len(caBundle) > 0 {
|
||||
data[corev1.ServiceAccountRootCAKey] = caBundle
|
||||
}
|
||||
if len(certificate) > 0 {
|
||||
data[corev1.TLSCertKey] = certificate
|
||||
}
|
||||
if len(key) > 0 {
|
||||
data[corev1.TLSPrivateKeyKey] = key
|
||||
}
|
||||
|
||||
if err := capsulev1beta2.AddToScheme(scheme); err != nil {
|
||||
t.Fatal(err)
|
||||
return &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: testSecretName, Namespace: testNamespace},
|
||||
Type: corev1.SecretTypeTLS,
|
||||
Data: data,
|
||||
}
|
||||
}
|
||||
|
||||
kubeClient := fake.NewClientBuilder().
|
||||
WithScheme(scheme).
|
||||
WithObjects(secret, configurationObject).
|
||||
Build()
|
||||
func getTestTLSSecret(t *testing.T, kubeClient client.Client) *corev1.Secret {
|
||||
t.Helper()
|
||||
|
||||
reconciler := &Reconciler{
|
||||
Client: kubeClient,
|
||||
Log: logr.Discard(),
|
||||
Namespace: namespace,
|
||||
Configuration: configuration.NewCapsuleConfiguration(
|
||||
ctx,
|
||||
kubeClient,
|
||||
kubeClient,
|
||||
nil,
|
||||
configurationObject.Name,
|
||||
),
|
||||
}
|
||||
|
||||
desiredSANs, err := reconciler.desiredWebhookSANs(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("desiredWebhookSANs() error = %v", err)
|
||||
}
|
||||
|
||||
if err := reconciler.validateSecretCertificate(secret, desiredSANs); err == nil {
|
||||
t.Fatal("external serving certificate unexpectedly satisfies all desired SANs")
|
||||
}
|
||||
|
||||
if err := reconciler.ReconcileCertificates(ctx, logr.Discard(), secret.DeepCopy()); err != nil {
|
||||
t.Fatalf("ReconcileCertificates() error = %v", err)
|
||||
}
|
||||
|
||||
updated := &corev1.Secret{}
|
||||
if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(secret), updated); err != nil {
|
||||
secret := &corev1.Secret{}
|
||||
if err := kubeClient.Get(
|
||||
context.Background(),
|
||||
client.ObjectKey{Namespace: testNamespace, Name: testSecretName},
|
||||
secret,
|
||||
); err != nil {
|
||||
t.Fatalf("get reconciled TLS Secret: %v", err)
|
||||
}
|
||||
|
||||
if len(updated.Data["ca.key"]) == 0 {
|
||||
t.Fatal("reconciled TLS Secret does not contain ca.key")
|
||||
}
|
||||
return secret
|
||||
}
|
||||
|
||||
if bytes.Equal(updated.Data[corev1.ServiceAccountRootCAKey], externalCABundle) {
|
||||
t.Fatal("reconciled TLS Secret retained the external CA bundle")
|
||||
}
|
||||
func assertTLSDataEqual(t *testing.T, secret *corev1.Secret, caBundle, certificate, key []byte) {
|
||||
t.Helper()
|
||||
|
||||
if bytes.Equal(updated.Data[corev1.TLSCertKey], externalCertificate) {
|
||||
t.Fatal("reconciled TLS Secret retained the external serving certificate")
|
||||
if !bytes.Equal(secret.Data[corev1.ServiceAccountRootCAKey], caBundle) {
|
||||
t.Fatal("TLS Secret CA bundle changed")
|
||||
}
|
||||
if !bytes.Equal(secret.Data[corev1.TLSCertKey], certificate) {
|
||||
t.Fatal("TLS Secret serving certificate changed")
|
||||
}
|
||||
if !bytes.Equal(secret.Data[corev1.TLSPrivateKeyKey], key) {
|
||||
t.Fatal("TLS Secret serving private key changed")
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := cert.NewCertificateAuthorityFromBytes(
|
||||
updated.Data[corev1.ServiceAccountRootCAKey],
|
||||
updated.Data["ca.key"],
|
||||
); err != nil {
|
||||
t.Fatalf("reconciled CA certificate/key pair is invalid: %v", err)
|
||||
}
|
||||
|
||||
if err := reconciler.validateSecretCertificate(updated, desiredSANs); err != nil {
|
||||
t.Fatalf("reconciled serving certificate is invalid: %v", err)
|
||||
}
|
||||
func testWebhookSANs() cert.CertificateSANs {
|
||||
return cert.CertificateSANs{DNSNames: []string{
|
||||
testServiceName,
|
||||
testServiceName + "." + testNamespace,
|
||||
testServiceName + "." + testNamespace + ".svc",
|
||||
testServiceName + "." + testNamespace + ".svc.cluster.local",
|
||||
}}
|
||||
}
|
||||
|
||||
func generateTestTLSMaterial(
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
// Copyright 2020-2026 Project Capsule Authors
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package tls
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/healthz"
|
||||
|
||||
"github.com/projectcapsule/capsule/pkg/runtime/configuration"
|
||||
)
|
||||
|
||||
type certificateGetter func(*tls.ClientHelloInfo) (*tls.Certificate, error)
|
||||
|
||||
// WebhookCertificateReadinessCheck prevents a webhook Pod from becoming ready
|
||||
// while the certificate it is actually serving is not trusted by an admission
|
||||
// configuration managed by Capsule. This closes the window between Secret
|
||||
// publication, certwatcher reload, and caBundle reconciliation.
|
||||
func WebhookCertificateReadinessCheck(
|
||||
reader client.Reader,
|
||||
cfg configuration.Configuration,
|
||||
namespace string,
|
||||
getCertificate certificateGetter,
|
||||
) healthz.Checker {
|
||||
return func(request *http.Request) error {
|
||||
servingCertificate, err := getCertificate(nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load webhook serving certificate: %w", err)
|
||||
}
|
||||
|
||||
if servingCertificate == nil || len(servingCertificate.Certificate) == 0 {
|
||||
return fmt.Errorf("webhook serving certificate is empty")
|
||||
}
|
||||
|
||||
leaf, err := x509.ParseCertificate(servingCertificate.Certificate[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse webhook serving certificate: %w", err)
|
||||
}
|
||||
|
||||
secret := &corev1.Secret{}
|
||||
|
||||
secretKey := types.NamespacedName{Namespace: namespace, Name: cfg.TLSSecretName()}
|
||||
if err := reader.Get(request.Context(), secretKey, secret); err != nil {
|
||||
return fmt.Errorf("get webhook TLS Secret %s: %w", secretKey.String(), err)
|
||||
}
|
||||
|
||||
persistedLeaf, err := certificateFromPEM(secret.Data[corev1.TLSCertKey])
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse serving certificate in TLS Secret %s: %w", secretKey.String(), err)
|
||||
}
|
||||
|
||||
if !bytes.Equal(leaf.Raw, persistedLeaf.Raw) {
|
||||
return fmt.Errorf("loaded webhook serving certificate is stale relative to TLS Secret %s", secretKey.String())
|
||||
}
|
||||
|
||||
admission := cfg.Admission()
|
||||
trusts := make([]admissionWebhookTrust, 0)
|
||||
|
||||
if admission.Mutating != nil && len(admission.Mutating.Webhooks) > 0 {
|
||||
loaded, err := loadAdmissionWebhookTrust(
|
||||
request.Context(),
|
||||
reader,
|
||||
"mutating",
|
||||
string(admission.Mutating.Name),
|
||||
func() client.Object { return &admissionregistrationv1.MutatingWebhookConfiguration{} },
|
||||
mutatingWebhookTrust,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
trusts = append(trusts, loaded...)
|
||||
}
|
||||
|
||||
if admission.Validating != nil && len(admission.Validating.Webhooks) > 0 {
|
||||
loaded, err := loadAdmissionWebhookTrust(
|
||||
request.Context(),
|
||||
reader,
|
||||
"validating",
|
||||
string(admission.Validating.Name),
|
||||
func() client.Object { return &admissionregistrationv1.ValidatingWebhookConfiguration{} },
|
||||
validatingWebhookTrust,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
trusts = append(trusts, loaded...)
|
||||
}
|
||||
|
||||
for _, trust := range trusts {
|
||||
if err := verifyCertificateAgainstCABundle(leaf, trust.caBundle); err != nil {
|
||||
return fmt.Errorf(
|
||||
"%s webhook configuration %q webhook %q does not trust the serving certificate: %w",
|
||||
trust.kind,
|
||||
trust.configurationName,
|
||||
trust.webhookName,
|
||||
err,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func certificateFromPEM(certificatePEM []byte) (*x509.Certificate, error) {
|
||||
block, _ := pem.Decode(certificatePEM)
|
||||
if block == nil {
|
||||
return nil, fmt.Errorf("invalid certificate PEM")
|
||||
}
|
||||
|
||||
return x509.ParseCertificate(block.Bytes)
|
||||
}
|
||||
|
||||
type admissionWebhookTrust struct {
|
||||
kind string
|
||||
configurationName string
|
||||
webhookName string
|
||||
caBundle []byte
|
||||
}
|
||||
|
||||
func loadAdmissionWebhookTrust(
|
||||
ctx context.Context,
|
||||
reader client.Reader,
|
||||
kind string,
|
||||
name string,
|
||||
newConfiguration func() client.Object,
|
||||
extractTrust func(client.Object, string, string) ([]admissionWebhookTrust, error),
|
||||
) ([]admissionWebhookTrust, error) {
|
||||
if name == "" {
|
||||
return nil, fmt.Errorf("%s webhook configuration name is empty", kind)
|
||||
}
|
||||
|
||||
webhookConfiguration := newConfiguration()
|
||||
if err := reader.Get(ctx, types.NamespacedName{Name: name}, webhookConfiguration); err != nil {
|
||||
return nil, fmt.Errorf("get %s webhook configuration %q: %w", kind, name, err)
|
||||
}
|
||||
|
||||
trusts, err := extractTrust(webhookConfiguration, kind, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(trusts) == 0 {
|
||||
return nil, fmt.Errorf("%s webhook configuration %q contains no webhooks", kind, name)
|
||||
}
|
||||
|
||||
return trusts, nil
|
||||
}
|
||||
|
||||
func mutatingWebhookTrust(
|
||||
object client.Object,
|
||||
kind string,
|
||||
configurationName string,
|
||||
) ([]admissionWebhookTrust, error) {
|
||||
configuration, ok := object.(*admissionregistrationv1.MutatingWebhookConfiguration)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("expected MutatingWebhookConfiguration, got %T", object)
|
||||
}
|
||||
|
||||
trusts := make([]admissionWebhookTrust, 0, len(configuration.Webhooks))
|
||||
for index := range configuration.Webhooks {
|
||||
trusts = append(trusts, admissionWebhookTrust{
|
||||
kind: kind,
|
||||
configurationName: configurationName,
|
||||
webhookName: configuration.Webhooks[index].Name,
|
||||
caBundle: configuration.Webhooks[index].ClientConfig.CABundle,
|
||||
})
|
||||
}
|
||||
|
||||
return trusts, nil
|
||||
}
|
||||
|
||||
func validatingWebhookTrust(
|
||||
object client.Object,
|
||||
kind string,
|
||||
configurationName string,
|
||||
) ([]admissionWebhookTrust, error) {
|
||||
configuration, ok := object.(*admissionregistrationv1.ValidatingWebhookConfiguration)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("expected ValidatingWebhookConfiguration, got %T", object)
|
||||
}
|
||||
|
||||
trusts := make([]admissionWebhookTrust, 0, len(configuration.Webhooks))
|
||||
for index := range configuration.Webhooks {
|
||||
trusts = append(trusts, admissionWebhookTrust{
|
||||
kind: kind,
|
||||
configurationName: configurationName,
|
||||
webhookName: configuration.Webhooks[index].Name,
|
||||
caBundle: configuration.Webhooks[index].ClientConfig.CABundle,
|
||||
})
|
||||
}
|
||||
|
||||
return trusts, nil
|
||||
}
|
||||
|
||||
func verifyCertificateAgainstCABundle(leaf *x509.Certificate, caBundle []byte) error {
|
||||
if len(caBundle) == 0 {
|
||||
return fmt.Errorf("caBundle is empty")
|
||||
}
|
||||
|
||||
roots := x509.NewCertPool()
|
||||
if !roots.AppendCertsFromPEM(caBundle) {
|
||||
return fmt.Errorf("caBundle is not valid PEM certificate data")
|
||||
}
|
||||
|
||||
if _, err := leaf.Verify(x509.VerifyOptions{
|
||||
Roots: roots,
|
||||
KeyUsages: []x509.ExtKeyUsage{
|
||||
x509.ExtKeyUsageServerAuth,
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
// Copyright 2020-2026 Project Capsule Authors
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package tls
|
||||
|
||||
import (
|
||||
"context"
|
||||
cryptotls "crypto/tls"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
|
||||
runtimeadmission "github.com/projectcapsule/capsule/pkg/runtime/admission"
|
||||
)
|
||||
|
||||
func TestWebhookCertificateReadinessCheckAcceptsPublishedCA(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
caBundle, certificatePEM, keyPEM := generateTestTLSMaterial(t, testWebhookSANs())
|
||||
validating := testValidatingWebhookConfiguration(caBundle)
|
||||
reconciler, kubeClient := newTestTLSReconciler(
|
||||
t,
|
||||
testTLSSecret(caBundle, certificatePEM, keyPEM),
|
||||
validating,
|
||||
)
|
||||
enableTestValidatingWebhook(t, kubeClient)
|
||||
servingCertificate := parseTestServingCertificate(t, certificatePEM, keyPEM)
|
||||
check := WebhookCertificateReadinessCheck(
|
||||
kubeClient,
|
||||
reconciler.Configuration,
|
||||
testNamespace,
|
||||
func(*cryptotls.ClientHelloInfo) (*cryptotls.Certificate, error) {
|
||||
return &servingCertificate, nil
|
||||
},
|
||||
)
|
||||
|
||||
if err := check(httptest.NewRequest("GET", "/readyz", nil)); err != nil {
|
||||
t.Fatalf("readiness check rejected matching serving certificate and caBundle: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebhookCertificateReadinessCheckRejectsMismatchedPublishedCA(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
caBundle, certificatePEM, keyPEM := generateTestTLSMaterial(t, testWebhookSANs())
|
||||
wrongCABundle, _, _ := generateTestTLSMaterial(t, testWebhookSANs())
|
||||
validating := testValidatingWebhookConfiguration(wrongCABundle)
|
||||
reconciler, kubeClient := newTestTLSReconciler(
|
||||
t,
|
||||
testTLSSecret(caBundle, certificatePEM, keyPEM),
|
||||
validating,
|
||||
)
|
||||
enableTestValidatingWebhook(t, kubeClient)
|
||||
servingCertificate := parseTestServingCertificate(t, certificatePEM, keyPEM)
|
||||
check := WebhookCertificateReadinessCheck(
|
||||
kubeClient,
|
||||
reconciler.Configuration,
|
||||
testNamespace,
|
||||
func(*cryptotls.ClientHelloInfo) (*cryptotls.Certificate, error) {
|
||||
return &servingCertificate, nil
|
||||
},
|
||||
)
|
||||
|
||||
err := check(httptest.NewRequest("GET", "/readyz", nil))
|
||||
if err == nil {
|
||||
t.Fatal("readiness check accepted a caBundle that does not trust the serving certificate")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "does not trust the serving certificate") {
|
||||
t.Fatalf("readiness error = %v, want certificate trust failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebhookCertificateReadinessCheckRejectsStaleLoadedCertificate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
caBundle, persistedCertificate, persistedKey := generateTestTLSMaterial(t, testWebhookSANs())
|
||||
_, staleCertificate, staleKey := generateTestTLSMaterial(t, testWebhookSANs())
|
||||
reconciler, kubeClient := newTestTLSReconciler(
|
||||
t,
|
||||
testTLSSecret(caBundle, persistedCertificate, persistedKey),
|
||||
)
|
||||
loadedCertificate := parseTestServingCertificate(t, staleCertificate, staleKey)
|
||||
check := WebhookCertificateReadinessCheck(
|
||||
kubeClient,
|
||||
reconciler.Configuration,
|
||||
testNamespace,
|
||||
func(*cryptotls.ClientHelloInfo) (*cryptotls.Certificate, error) {
|
||||
return &loadedCertificate, nil
|
||||
},
|
||||
)
|
||||
|
||||
err := check(httptest.NewRequest("GET", "/readyz", nil))
|
||||
if err == nil {
|
||||
t.Fatal("readiness check accepted a stale certificate loaded by certwatcher")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "is stale relative to TLS Secret") {
|
||||
t.Fatalf("readiness error = %v, want stale loaded certificate failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWebhookCertificateReadinessCheckRequiresManagedConfiguration(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
caBundle, certificatePEM, keyPEM := generateTestTLSMaterial(t, testWebhookSANs())
|
||||
reconciler, kubeClient := newTestTLSReconciler(t, testTLSSecret(caBundle, certificatePEM, keyPEM))
|
||||
enableTestValidatingWebhook(t, kubeClient)
|
||||
servingCertificate := parseTestServingCertificate(t, certificatePEM, keyPEM)
|
||||
check := WebhookCertificateReadinessCheck(
|
||||
kubeClient,
|
||||
reconciler.Configuration,
|
||||
testNamespace,
|
||||
func(*cryptotls.ClientHelloInfo) (*cryptotls.Certificate, error) {
|
||||
return &servingCertificate, nil
|
||||
},
|
||||
)
|
||||
|
||||
err := check(httptest.NewRequest("GET", "/readyz", nil))
|
||||
if err == nil {
|
||||
t.Fatal("readiness check accepted a missing managed webhook configuration")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "get validating webhook configuration") {
|
||||
t.Fatalf("readiness error = %v, want missing configuration failure", err)
|
||||
}
|
||||
}
|
||||
|
||||
func testValidatingWebhookConfiguration(caBundle []byte) *admissionregistrationv1.ValidatingWebhookConfiguration {
|
||||
return &admissionregistrationv1.ValidatingWebhookConfiguration{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: testValidatingConfiguration},
|
||||
Webhooks: []admissionregistrationv1.ValidatingWebhook{{
|
||||
Name: "owners.validating.projectcapsule.dev",
|
||||
ClientConfig: admissionregistrationv1.WebhookClientConfig{
|
||||
CABundle: caBundle,
|
||||
},
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func enableTestValidatingWebhook(t *testing.T, kubeClient client.Client) {
|
||||
t.Helper()
|
||||
|
||||
configurationObject := &capsulev1beta2.CapsuleConfiguration{}
|
||||
if err := kubeClient.Get(
|
||||
context.Background(),
|
||||
client.ObjectKey{Name: "capsule"},
|
||||
configurationObject,
|
||||
); err != nil {
|
||||
t.Fatalf("get CapsuleConfiguration: %v", err)
|
||||
}
|
||||
|
||||
configurationObject.Spec.Admission.Validating.Webhooks = []*runtimeadmission.ValidatingWebhook{{}}
|
||||
if err := kubeClient.Update(context.Background(), configurationObject); err != nil {
|
||||
t.Fatalf("update CapsuleConfiguration: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func parseTestServingCertificate(t *testing.T, certificatePEM, keyPEM []byte) cryptotls.Certificate {
|
||||
t.Helper()
|
||||
|
||||
servingCertificate, err := cryptotls.X509KeyPair(certificatePEM, keyPEM)
|
||||
if err != nil {
|
||||
t.Fatalf("parse test serving certificate: %v", err)
|
||||
}
|
||||
|
||||
return servingCertificate
|
||||
}
|
||||
Reference in New Issue
Block a user