fix: allow replications to own replications without blocking controller updates (#2107)

* chore

* perfromance improvements

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* perfromance improvements

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat(performance): removed duplicate client calls from all admission paths

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat(performance): removed duplicate client calls from all admission paths

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat(performance): removed duplicate client calls from all admission paths

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat(performance): removed duplicate client calls from all admission paths

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat(performance): removed duplicate client calls from all admission paths

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat(performance): removed duplicate client calls from all admission paths

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat(performance): removed duplicate client calls from all admission paths

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: add globalresourcequota api

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: add globalresourcequota api

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: add globalresourcequota api

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: add globalresourcequota api

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: add globalresourcequota api

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* feat: add globalresourcequota api

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* fix: remove resource rejections checks

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* fix: allow replications to own replications without blocking controller updates

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* fix: allow replications to own replications without blocking controller updates

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

* fix: allow replications to own replications without blocking controller updates

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>

---------

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>
This commit is contained in:
Oliver Bähler
2026-08-26 21:59:57 +02:00
committed by GitHub
parent 7ee293a5fd
commit da74cfdab7
25 changed files with 948 additions and 107 deletions
+12
View File
@@ -205,6 +205,12 @@ dev-setup: dev-setup-flux-handoff
--set 'certManager.generateCertificates=false' \
--set 'tls.enableController=false' \
--set 'tls.create=false' \
--set rbac.resources.create=true \
--set-string 'rbac.resources.labels.rbac\.authorization\.k8s\.io/aggregate-to-admin=true' \
--set rbac.resourcepoolclaims.create=true \
--set-string 'rbac.resourcepoolclaims.labels.rbac\.authorization\.k8s\.io/aggregate-to-admin=true' \
--set rbac.customquotas.create=true \
--set-string 'rbac.customquotas.labels.rbac\.authorization\.k8s\.io/aggregate-to-admin=true' \
--set "webhooks.exclusive=true"\
--set "webhooks.service.url=$${WEBHOOK_URL}" \
--set "webhooks.service.caBundle=$${CA_BUNDLE}" \
@@ -480,6 +486,12 @@ e2e-install: helm-controller-version ko-build-all dev-install-gw-api-crds
--set 'manager.options.leaderElection.leaseDuration=60s' \
--set 'manager.options.leaderElection.renewDeadline=40s' \
--set 'manager.rbac.minimal=true' \
--set rbac.resources.create=true \
--set-string 'rbac.resources.labels.rbac\.authorization\.k8s\.io/aggregate-to-admin=true' \
--set rbac.resourcepoolclaims.create=true \
--set-string 'rbac.resourcepoolclaims.labels.rbac\.authorization\.k8s\.io/aggregate-to-admin=true' \
--set rbac.customquotas.create=true \
--set-string 'rbac.customquotas.labels.rbac\.authorization\.k8s\.io/aggregate-to-admin=true' \
--set 'webhooks.hooks.nodes.enabled=true' \
--set "webhooks.exclusive=true"\
--set 'webhooks.hooks.calculations.enabled=true' \
+16 -12
View File
@@ -512,7 +512,7 @@ var _ = Describe("GlobalResourceQuota", Ordered, Label("globalresourcequota", "r
))
})
It("rejects direct hard-limit reductions and removals below allocated usage", func() {
It("rejects direct hard-limit reductions below allocated usage and allows removals", func() {
quotaKey := client.ObjectKey{Name: ephemeralQuotaName}
Eventually(func(g Gomega) {
current := &capsulev1beta2.GlobalResourceQuota{}
@@ -532,17 +532,6 @@ var _ = Describe("GlobalResourceQuota", Ordered, Label("globalresourcequota", "r
)))
})
By("rejecting removal of a resource with usage", func() {
current := &capsulev1beta2.GlobalResourceQuota{}
Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed())
delete(current.Spec.Quota.Hard, corev1.ResourceRequestsEphemeralStorage)
err := k8sClient.Update(ctx, current)
Expect(err).To(MatchError(ContainSubstring(
`spec.quota.hard["requests.ephemeral-storage"] cannot be removed while 600Mi is allocated`,
)))
})
By("allowing a decrease exactly to allocated usage", func() {
current := &capsulev1beta2.GlobalResourceQuota{}
Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed())
@@ -557,6 +546,21 @@ var _ = Describe("GlobalResourceQuota", Ordered, Label("globalresourcequota", "r
g.Expect(hard.Cmp(resource.MustParse("600Mi"))).To(Equal(0))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
By("allowing removal of a resource with usage", func() {
current := &capsulev1beta2.GlobalResourceQuota{}
Expect(k8sClient.Get(ctx, quotaKey, current)).To(Succeed())
delete(current.Spec.Quota.Hard, corev1.ResourceRequestsEphemeralStorage)
Expect(k8sClient.Update(ctx, current)).To(Succeed())
Eventually(func(g Gomega) {
reconciled := &capsulev1beta2.GlobalResourceQuota{}
g.Expect(k8sClient.Get(ctx, quotaKey, reconciled)).To(Succeed())
g.Expect(reconciled.Status.ObservedGeneration).To(Equal(reconciled.Generation))
g.Expect(reconciled.Spec.Quota.Hard).NotTo(HaveKey(corev1.ResourceRequestsEphemeralStorage))
g.Expect(reconciled.Status.Total.Hard).NotTo(HaveKey(corev1.ResourceRequestsEphemeralStorage))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
})
})
+77 -82
View File
@@ -160,94 +160,87 @@ var _ = Describe("rule-generated GlobalResourceQuota admission", Ordered,
EventuallyDeletion(tamperRole)
})
It("rejects a Tenant rule update that decreases or removes a hard limit while changing scope", func() {
for _, test := range []struct {
name string
mutate func(corev1.ResourceList)
message string
}{
{
name: "decrease",
mutate: func(hard corev1.ResourceList) {
hard[corev1.ResourceLimitsCPU] = resource.MustParse("0")
},
message: `rules[0].quota[0].hard["limits.cpu"] cannot be reduced from 8 to 0 while namespace selectors are changing`,
},
{
name: "removal",
mutate: func(hard corev1.ResourceList) {
delete(hard, corev1.ResourceLimitsCPU)
},
message: `rules[0].quota[0].hard["limits.cpu"] cannot be removed while namespace selectors are changing`,
},
} {
By(test.name, func() {
Eventually(func() error {
current := &capsulev1beta2.Tenant{}
if err := k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, current); err != nil {
return err
}
It("rejects a Tenant rule decrease while changing scope and allows removal", func() {
By("rejecting the explicit decrease", func() {
Eventually(func() error {
current := &capsulev1beta2.Tenant{}
if err := k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, current); err != nil {
return err
}
updated := current.DeepCopy()
updated.Spec.Rules[0].NamespaceSelector = nil
test.mutate(updated.Spec.Rules[0].Quota[0].Hard)
updated := current.DeepCopy()
updated.Spec.Rules[0].NamespaceSelector = nil
updated.Spec.Rules[0].Quota[0].Hard[corev1.ResourceLimitsCPU] = resource.MustParse("0")
return k8sClient.Update(ctx, updated)
}, defaultTimeoutInterval, defaultPollInterval).Should(MatchError(ContainSubstring(test.message)))
})
}
return k8sClient.Update(ctx, updated)
}, defaultTimeoutInterval, defaultPollInterval).Should(MatchError(ContainSubstring(
`rules[0].quota[0].hard["limits.cpu"] cannot be reduced from 8 to 0 while namespace selectors are changing`,
)))
})
persisted := &capsulev1beta2.Tenant{}
Expect(k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, persisted)).To(Succeed())
Expect(persisted.Spec.Rules[0].NamespaceSelector).NotTo(BeNil())
Expect(persisted.Spec.Rules[0].NamespaceSelector.MatchLabels).To(HaveKeyWithValue(selectorKey, "application"))
persistedLimit := persisted.Spec.Rules[0].Quota[0].Hard[corev1.ResourceLimitsCPU]
Expect(persistedLimit.Cmp(resource.MustParse("8"))).To(Equal(0))
By("allowing the resource limit to be removed", func() {
Eventually(func() error {
current := &capsulev1beta2.Tenant{}
if err := k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, current); err != nil {
return err
}
updated := current.DeepCopy()
updated.Spec.Rules[0].NamespaceSelector = nil
delete(updated.Spec.Rules[0].Quota[0].Hard, corev1.ResourceLimitsCPU)
return k8sClient.Update(ctx, updated)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
persisted := &capsulev1beta2.Tenant{}
Expect(k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, persisted)).To(Succeed())
Expect(persisted.Spec.Rules[0].NamespaceSelector).To(BeNil())
Expect(persisted.Spec.Rules[0].Quota[0].Hard).NotTo(HaveKey(corev1.ResourceLimitsCPU))
})
applicationSelector := &metav1.LabelSelector{MatchLabels: map[string]string{selectorKey: "application"}}
Eventually(func() error {
return setTenantQuota(applicationSelector, "8")
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
expectGeneratedQuota("application", "8", "0")
})
It("rejects the same unsafe scope and hard-limit changes on the generated quota", func() {
It("rejects a generated quota decrease while changing scope and allows removal", func() {
controllerClient := impersonationClient(ControllerServiceAccountFull, nil)
for _, test := range []struct {
name string
mutate func(corev1.ResourceList)
message string
}{
{
name: "decrease",
mutate: func(hard corev1.ResourceList) {
hard[corev1.ResourceLimitsCPU] = resource.MustParse("0")
},
message: `spec.quota.hard["limits.cpu"] cannot be reduced from 8 to 0 while namespace selectors are changing`,
},
{
name: "removal",
mutate: func(hard corev1.ResourceList) {
delete(hard, corev1.ResourceLimitsCPU)
},
message: `spec.quota.hard["limits.cpu"] cannot be removed while namespace selectors are changing`,
},
} {
By(test.name, func() {
Eventually(func() error {
current := &capsulev1beta2.GlobalResourceQuota{}
if err := controllerClient.Get(ctx, quotaKey, current); err != nil {
return err
}
updated := current.DeepCopy()
delete(updated.Spec.NamespaceSelectors[0].LabelSelector.MatchLabels, selectorKey)
test.mutate(updated.Spec.Quota.Hard)
By("rejecting the explicit decrease", func() {
Eventually(func() error {
current := &capsulev1beta2.GlobalResourceQuota{}
if err := controllerClient.Get(ctx, quotaKey, current); err != nil {
return err
}
return controllerClient.Update(ctx, updated)
}, defaultTimeoutInterval, defaultPollInterval).Should(MatchError(ContainSubstring(test.message)))
})
}
updated := current.DeepCopy()
delete(updated.Spec.NamespaceSelectors[0].LabelSelector.MatchLabels, selectorKey)
updated.Spec.Quota.Hard[corev1.ResourceLimitsCPU] = resource.MustParse("0")
persisted := &capsulev1beta2.GlobalResourceQuota{}
Expect(k8sClient.Get(ctx, quotaKey, persisted)).To(Succeed())
Expect(persisted.Spec.NamespaceSelectors[0].LabelSelector.MatchLabels).To(HaveKeyWithValue(selectorKey, "application"))
persistedLimit := persisted.Spec.Quota.Hard[corev1.ResourceLimitsCPU]
Expect(persistedLimit.Cmp(resource.MustParse("8"))).To(Equal(0))
return controllerClient.Update(ctx, updated)
}, defaultTimeoutInterval, defaultPollInterval).Should(MatchError(ContainSubstring(
`spec.quota.hard["limits.cpu"] cannot be reduced from 8 to 0 while namespace selectors are changing`,
)))
})
By("allowing the resource limit to be removed", func() {
Eventually(func() error {
current := &capsulev1beta2.GlobalResourceQuota{}
if err := controllerClient.Get(ctx, quotaKey, current); err != nil {
return err
}
updated := current.DeepCopy()
delete(updated.Spec.NamespaceSelectors[0].LabelSelector.MatchLabels, selectorKey)
delete(updated.Spec.Quota.Hard, corev1.ResourceLimitsCPU)
return controllerClient.Update(ctx, updated)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
expectGeneratedQuota("application", "8", "0")
})
It("allows equal or increased limits with a scope change and a later same-scope decrease", func() {
@@ -304,9 +297,11 @@ var _ = Describe("rule-generated GlobalResourceQuota admission", Ordered,
delete(current.Spec.Rules[0].Quota[0].Hard, corev1.ResourceLimitsCPU)
return k8sClient.Update(ctx, current)
}, defaultTimeoutInterval, defaultPollInterval).Should(MatchError(ContainSubstring(
`rules[0].quota[0].hard["limits.cpu"] cannot be removed while 300m is allocated`,
)))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
persisted := &capsulev1beta2.Tenant{}
Expect(k8sClient.Get(ctx, client.ObjectKey{Name: tenantName}, persisted)).To(Succeed())
Expect(persisted.Spec.Rules[0].Quota[0].Hard).NotTo(HaveKey(corev1.ResourceLimitsCPU))
applicationSelector := &metav1.LabelSelector{MatchLabels: map[string]string{selectorKey: "application"}}
Eventually(func() error {
+17 -3
View File
@@ -45,7 +45,9 @@ type CollectorOptions struct {
AllowCrossNamespaceSelection bool
Accumulator processor.Accumulator
Iterator CollectorIteratorOptions
ReplicationContext map[string]any
ValidatorNamespaces tpl.NamespaceValidator
preserveOwnerReferences bool
}
type CollectorIteratorOptions struct {
@@ -130,6 +132,10 @@ func (co *Collector) Collect(
}
}
if opts.ReplicationContext != nil {
tplContext[replicationContextKey] = opts.ReplicationContext
}
if tnt != nil {
tCtx, err := tenant.NewTenantContext(tnt, c.Scheme(), co.contextSanitizeOptions)
if err != nil {
@@ -155,6 +161,9 @@ func (co *Collector) Collect(
log.V(7).Info("available context", "context", tplContext)
authoredOpts := opts
authoredOpts.preserveOwnerReferences = true
// Run Raw Items
for rawIndex, item := range spec.RawItems {
log.V(5).Info("processing raw item", "index", rawIndex)
@@ -168,7 +177,7 @@ func (co *Collector) Collect(
log.V(7).Info("evaluated raw item", "object", p)
rawError = co.AddToAccumulation(tnt, ns, opts, spec, p, resourceIndex+"/raw-"+strconv.Itoa(rawIndex), true)
rawError = co.AddToAccumulation(tnt, ns, authoredOpts, spec, p, resourceIndex+"/raw-"+strconv.Itoa(rawIndex), true)
if rawError != nil {
syncErr = errors.Join(syncErr, rawError)
@@ -190,7 +199,7 @@ func (co *Collector) Collect(
log.V(5).Info("loaded resources", "amount", len(p))
for i, o := range p {
genError = co.AddToAccumulation(tnt, ns, opts, spec, o, resourceIndex+"/generator-"+strconv.Itoa(generatorIndex)+"-"+strconv.Itoa(i), true)
genError = co.AddToAccumulation(tnt, ns, authoredOpts, spec, o, resourceIndex+"/generator-"+strconv.Itoa(generatorIndex)+"-"+strconv.Itoa(i), true)
if genError != nil {
syncErr = errors.Join(syncErr, genError)
@@ -261,7 +270,12 @@ func (co *Collector) AddToAccumulation(
obj.SetAnnotations(dst)
}
sanitize.SanitizeUnstructured(obj, co.objectSanitizeOptions)
sanitizeOptions := co.objectSanitizeOptions
if opts.preserveOwnerReferences {
sanitizeOptions.StripOwnerreferences = false
}
sanitize.SanitizeUnstructured(obj, sanitizeOptions)
processor.AccumulatorAdd(opts.Accumulator, resource, processor.AccumulatorObject{
Object: obj,
@@ -4,6 +4,7 @@
package resources
import (
"context"
"strings"
"testing"
@@ -11,6 +12,7 @@ import (
k8smeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
@@ -104,6 +106,188 @@ func TestCollectorAddToAccumulationClusterScopedObjects(t *testing.T) {
})
}
func TestCollectorAddsReplicationMetadataToGeneratorContext(t *testing.T) {
t.Parallel()
mapper := k8smeta.NewDefaultRESTMapper([]schema.GroupVersion{{Version: "v1"}})
mapper.Add(schema.GroupVersionKind{Version: "v1", Kind: "ConfigMap"}, k8smeta.RESTScopeNamespace)
replicationContext, err := newReplicationContext(&capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{
Name: "tenant-distribution",
Namespace: "solar-system",
},
})
if err != nil {
t.Fatalf("newReplicationContext() error = %v", err)
}
acc := processor.Accumulator{}
collector := NewCollector(nil, mapper)
spec := capsulev1beta2.ResourceSpec{
Generators: []capsulev1beta2.TemplateItemSpec{{
MissingKey: "error",
Template: `apiVersion: v1
kind: ConfigMap
metadata:
name: {{ $.replications.metadata.name }}
annotations:
replication-namespace: {{ $.replications.metadata.namespace }}
`,
}},
}
err = collector.Collect(
context.Background(),
nil,
CollectorOptions{
Accumulator: acc,
ReplicationContext: replicationContext,
},
nil,
"0",
spec,
nil,
)
if err != nil {
t.Fatalf("Collect() error = %v", err)
}
if len(acc) != 1 {
t.Fatalf("Collect() accumulated %d objects, want 1", len(acc))
}
for _, item := range acc {
if item == nil || item.Objects == nil || len(*item.Objects) != 1 {
t.Fatalf("accumulated item = %#v", item)
}
object := (*item.Objects)[0].Object
if object.GetName() != "tenant-distribution" {
t.Fatalf("generated name = %q", object.GetName())
}
if object.GetAnnotations()["replication-namespace"] != "solar-system" {
t.Fatalf("generated annotations = %#v", object.GetAnnotations())
}
}
}
func TestCollectorPreservesOwnerReferencesInAuthoredResources(t *testing.T) {
t.Parallel()
mapper := k8smeta.NewDefaultRESTMapper([]schema.GroupVersion{{Version: "v1"}})
mapper.Add(schema.GroupVersionKind{Version: "v1", Kind: "ConfigMap"}, k8smeta.RESTScopeNamespace)
const objectTemplate = `apiVersion: v1
kind: ConfigMap
metadata:
name: %s
ownerReferences:
- apiVersion: capsule.clastix.io/v1beta2
kind: TenantResource
name: tenant-distribution
uid: replication-uid
controller: true
blockOwnerDeletion: true
`
spec := capsulev1beta2.ResourceSpec{
RawItems: []capsulev1beta2.RawExtension{{
RawExtension: runtime.RawExtension{Raw: []byte(`{
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": {
"name": "raw-item",
"ownerReferences": [{
"apiVersion": "capsule.clastix.io/v1beta2",
"kind": "TenantResource",
"name": "tenant-distribution",
"uid": "replication-uid",
"controller": true,
"blockOwnerDeletion": true
}]
}
}`)},
}},
Generators: []capsulev1beta2.TemplateItemSpec{{
MissingKey: "error",
Template: strings.Replace(objectTemplate, "%s", "generated-item", 1),
}},
}
acc := processor.Accumulator{}
collector := NewCollector(nil, mapper)
if err := collector.Collect(
context.Background(),
nil,
CollectorOptions{Accumulator: acc},
nil,
"0",
spec,
nil,
); err != nil {
t.Fatalf("Collect() error = %v", err)
}
if len(acc) != 2 {
t.Fatalf("Collect() accumulated %d objects, want 2", len(acc))
}
for _, item := range acc {
if item == nil || item.Objects == nil || len(*item.Objects) != 1 {
t.Fatalf("accumulated item = %#v", item)
}
object := (*item.Objects)[0].Object
ownerReferences := object.GetOwnerReferences()
if len(ownerReferences) != 1 {
t.Fatalf("%s ownerReferences = %#v, want one", object.GetName(), ownerReferences)
}
owner := ownerReferences[0]
if owner.APIVersion != "capsule.clastix.io/v1beta2" ||
owner.Kind != "TenantResource" ||
owner.Name != "tenant-distribution" ||
owner.UID != "replication-uid" ||
owner.Controller == nil || !*owner.Controller ||
owner.BlockOwnerDeletion == nil || !*owner.BlockOwnerDeletion {
t.Fatalf("%s ownerReference = %#v", object.GetName(), owner)
}
}
}
func TestCollectorStripsOwnerReferencesFromReplicatedResources(t *testing.T) {
t.Parallel()
mapper := k8smeta.NewDefaultRESTMapper([]schema.GroupVersion{{Version: "v1"}})
mapper.Add(schema.GroupVersionKind{Version: "v1", Kind: "ConfigMap"}, k8smeta.RESTScopeNamespace)
obj := newUnstructured("v1", "ConfigMap", "source", "replicated-item")
obj.SetOwnerReferences([]metav1.OwnerReference{{
APIVersion: "apps/v1",
Kind: "Deployment",
Name: "source-owner",
UID: "source-owner-uid",
}})
acc := processor.Accumulator{}
collector := NewCollector(nil, mapper)
if err := collector.AddToAccumulation(
nil,
nil,
CollectorOptions{Accumulator: acc},
capsuleResourceSpec(),
obj,
"replica",
false,
); err != nil {
t.Fatalf("AddToAccumulation() error = %v", err)
}
if ownerReferences := obj.GetOwnerReferences(); len(ownerReferences) != 0 {
t.Fatalf("ownerReferences = %#v, want none", ownerReferences)
}
}
func newUnstructured(apiVersion, kind, namespace, name string) *unstructured.Unstructured {
obj := &unstructured.Unstructured{}
obj.SetAPIVersion(apiVersion)
+6
View File
@@ -406,10 +406,16 @@ func (r *globalResourceController) gatherResources(
tnts capsulev1beta2.TenantList,
acc processor.Accumulator,
) error {
replicationContext, err := newReplicationContext(tntResource)
if err != nil {
return err
}
opts := CollectorOptions{
Accumulator: acc,
AllowCrossNamespaceSelection: true,
AllowClusterScopedObjects: true,
ReplicationContext: replicationContext,
}
// Collect Available Generated Items
@@ -349,9 +349,15 @@ func (r *NamespaceTrigger) gatherGlobalResources(
namespace *corev1.Namespace,
acc processor.Accumulator,
) error {
replicationContext, err := newReplicationContext(tntResource)
if err != nil {
return err
}
opts := CollectorOptions{
Accumulator: acc,
AllowClusterScopedObjects: true,
ReplicationContext: replicationContext,
}
for resourceIndex, resource := range tntResource.Spec.Resources {
@@ -423,6 +429,11 @@ func (r *NamespaceTrigger) gatherNamespacedResources(
namespace *corev1.Namespace,
acc processor.Accumulator,
) error {
replicationContext, err := newReplicationContext(tntResource)
if err != nil {
return err
}
// The Namespace has just been created, thus it may not have landed on the Tenant status
// yet: it is a legit replication target nonetheless, and the validator must know about it
// to not reject the items referring to it.
@@ -435,6 +446,7 @@ func (r *NamespaceTrigger) gatherNamespacedResources(
Accumulator: acc,
AllowCrossNamespaceSelection: false,
AllowClusterScopedObjects: false,
ReplicationContext: replicationContext,
ValidatorNamespaces: tpl.NewNamespaceValidator(false, allowed),
}
@@ -471,10 +471,16 @@ func (r *namespacedResourceController) gatherResources(
tnt capsulev1beta2.Tenant,
acc processor.Accumulator,
) (err error) {
replicationContext, err := newReplicationContext(tntResource)
if err != nil {
return err
}
opts := CollectorOptions{
Accumulator: acc,
AllowCrossNamespaceSelection: false,
AllowClusterScopedObjects: false,
ReplicationContext: replicationContext,
ValidatorNamespaces: tpl.NewNamespaceValidator(false, sets.New[string](tnt.Status.Namespaces...)),
}
@@ -0,0 +1,42 @@
// Copyright 2020-2026 Project Capsule Authors
// SPDX-License-Identifier: Apache-2.0
package resources
import (
"maps"
"slices"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/projectcapsule/capsule/pkg/utils"
)
const replicationContextKey = "replications"
func newReplicationContext(object metav1.Object) (map[string]any, error) {
annotations := maps.Clone(object.GetAnnotations())
delete(annotations, "kubectl.kubernetes.io/last-applied-configuration")
metadata, err := utils.ToUnstructuredMap(&metav1.ObjectMeta{
Name: object.GetName(),
GenerateName: object.GetGenerateName(),
Namespace: object.GetNamespace(),
SelfLink: object.GetSelfLink(),
UID: object.GetUID(),
ResourceVersion: object.GetResourceVersion(),
Generation: object.GetGeneration(),
CreationTimestamp: object.GetCreationTimestamp(),
DeletionTimestamp: object.GetDeletionTimestamp(),
DeletionGracePeriodSeconds: object.GetDeletionGracePeriodSeconds(),
Labels: maps.Clone(object.GetLabels()),
Annotations: annotations,
OwnerReferences: slices.Clone(object.GetOwnerReferences()),
Finalizers: slices.Clone(object.GetFinalizers()),
})
if err != nil {
return nil, err
}
return map[string]any{"metadata": metadata}, nil
}
@@ -0,0 +1,105 @@
// Copyright 2020-2026 Project Capsule Authors
// SPDX-License-Identifier: Apache-2.0
package resources
import (
"testing"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
)
func TestNewReplicationContext(t *testing.T) {
t.Parallel()
tests := []struct {
name string
object metav1.Object
wantName string
wantNamespace string
}{
{
name: "TenantResource",
object: &capsulev1beta2.TenantResource{ObjectMeta: replicationObjectMeta(
"tenant-distribution",
"solar-system",
)},
wantName: "tenant-distribution",
wantNamespace: "solar-system",
},
{
name: "GlobalTenantResource",
object: &capsulev1beta2.GlobalTenantResource{ObjectMeta: replicationObjectMeta(
"global-distribution",
"",
)},
wantName: "global-distribution",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
context, err := newReplicationContext(test.object)
if err != nil {
t.Fatalf("newReplicationContext() error = %v", err)
}
metadata, ok := context["metadata"].(map[string]any)
if !ok {
t.Fatalf("metadata = %#v", context["metadata"])
}
if metadata["name"] != test.wantName {
t.Fatalf("metadata.name = %#v, want %q", metadata["name"], test.wantName)
}
if namespace, _ := metadata["namespace"].(string); namespace != test.wantNamespace {
t.Fatalf("metadata.namespace = %q, want %q", namespace, test.wantNamespace)
}
if metadata["uid"] != "replication-uid" {
t.Fatalf("metadata.uid = %#v", metadata["uid"])
}
if metadata["generation"] != int64(3) {
t.Fatalf("metadata.generation = %#v", metadata["generation"])
}
if metadata["labels"].(map[string]any)["company.example/team"] != "platform" {
t.Fatalf("metadata.labels = %#v", metadata["labels"])
}
annotations := metadata["annotations"].(map[string]any)
if annotations["company.example/source"] != "git" {
t.Fatalf("metadata.annotations = %#v", annotations)
}
if _, exists := annotations["kubectl.kubernetes.io/last-applied-configuration"]; exists {
t.Fatalf("last-applied annotation was exposed: %#v", annotations)
}
if _, exists := metadata["managedFields"]; exists {
t.Fatalf("managedFields were exposed: %#v", metadata["managedFields"])
}
})
}
}
func replicationObjectMeta(name, namespace string) metav1.ObjectMeta {
return metav1.ObjectMeta{
Name: name,
Namespace: namespace,
UID: types.UID("replication-uid"),
ResourceVersion: "7",
Generation: 3,
Labels: map[string]string{
"company.example/team": "platform",
},
Annotations: map[string]string{
"company.example/source": "git",
"kubectl.kubernetes.io/last-applied-configuration": "large-payload",
},
Finalizers: []string{"capsule.clastix.io/replication"},
ManagedFields: []metav1.ManagedFieldsEntry{{
Manager: "capsule",
}},
}
}
+9
View File
@@ -18,6 +18,7 @@ import (
"github.com/projectcapsule/capsule/pkg/runtime/gvk"
"github.com/projectcapsule/capsule/pkg/runtime/handlers"
"github.com/projectcapsule/capsule/pkg/runtime/indexers/tenantresource"
"github.com/projectcapsule/capsule/pkg/users"
)
type replicaHandler struct{}
@@ -65,6 +66,14 @@ func (h *replicaHandler) handler(
req admission.Request,
recorder events.EventRecorder,
) *admission.Response {
// Replicated objects are applied with the replication's impersonated client,
// but controllers reconcile their own metadata, finalizers, and status with
// the Capsule manager client. This is especially relevant when the replicated
// object is itself a TenantResource or GlobalTenantResource.
if users.IsControllerServiceAccount(req.UserInfo.Username) {
return nil
}
// Checking if the object is managed by a TenantResource, local or global
ref := gvk.ResourceID{
Group: req.Kind.Group,
+10 -7
View File
@@ -26,12 +26,15 @@ func ExtraFuncMap() template.FuncMap {
// CustomFuncMap return our custom templates.
func CustomFuncMap() template.FuncMap {
return template.FuncMap{
"toToml": toTOML,
"fromToml": fromTOML,
"fromYamlArray": fromYAMLArray,
"fromJsonArray": fromJSONArray,
"deterministicUUID": deterministicUUID,
"generateAgeKey": generateAgeKey,
"generateAgePQKey": generateAgePQKey,
"toToml": toTOML,
"fromToml": fromTOML,
"fromYamlArray": fromYAMLArray,
"fromJsonArray": fromJSONArray,
"deterministicUUID": deterministicUUID,
"generateAgeKey": generateAgeKey,
"generateAgePQKey": generateAgePQKey,
"getResourceByName": getResourceByName,
"mustGetResourceByName": mustGetResourceByName,
"getResourceByNamespacedName": getResourceByNamespacedName,
}
}
@@ -17,6 +17,9 @@ func TestFuncMaps(t *testing.T) {
"deterministicUUID",
"generateAgeKey",
"generateAgePQKey",
"getResourceByName",
"mustGetResourceByName",
"getResourceByNamespacedName",
} {
if custom[name] == nil {
t.Fatalf("CustomFuncMap()[%q] is nil", name)
+101
View File
@@ -0,0 +1,101 @@
// Copyright 2020-2026 Project Capsule Authors
// SPDX-License-Identifier: Apache-2.0
package functions
import "fmt"
func getResourceByName(name string, resources any) (map[string]any, error) {
return findResource(
resources,
fmt.Sprintf("metadata.name %q", name),
func(metadata map[string]any) bool {
return metadata["name"] == name
},
)
}
func mustGetResourceByName(name string, resources any) (map[string]any, error) {
resource, err := getResourceByName(name, resources)
if err != nil {
return nil, err
}
if len(resource) == 0 {
return nil, fmt.Errorf("resource with metadata.name %q was not found", name)
}
return resource, nil
}
func getResourceByNamespacedName(namespace, name string, resources any) (map[string]any, error) {
return findResource(
resources,
fmt.Sprintf("metadata.namespace %q and metadata.name %q", namespace, name),
func(metadata map[string]any) bool {
resourceNamespace, _ := metadata["namespace"].(string)
return resourceNamespace == namespace && metadata["name"] == name
},
)
}
func findResource(
resources any,
description string,
matches func(map[string]any) bool,
) (map[string]any, error) {
items, err := resourceItems(resources)
if err != nil {
return nil, err
}
var match map[string]any
for index, item := range items {
metadata, ok := item["metadata"].(map[string]any)
if !ok {
return nil, fmt.Errorf("resource at index %d has invalid or missing metadata", index)
}
if !matches(metadata) {
continue
}
if match != nil {
return nil, fmt.Errorf("multiple resources match %s", description)
}
match = item
}
if match == nil {
return map[string]any{}, nil
}
return match, nil
}
func resourceItems(resources any) ([]map[string]any, error) {
switch value := resources.(type) {
case nil:
return nil, nil
case []map[string]any:
return value, nil
case []any:
items := make([]map[string]any, 0, len(value))
for index, item := range value {
resource, ok := item.(map[string]any)
if !ok {
return nil, fmt.Errorf("resource at index %d must be an object, got %T", index, item)
}
items = append(items, resource)
}
return items, nil
default:
return nil, fmt.Errorf("resources must be a list of objects, got %T", resources)
}
}
+154
View File
@@ -0,0 +1,154 @@
// Copyright 2020-2026 Project Capsule Authors
// SPDX-License-Identifier: Apache-2.0
package functions
import (
"strings"
"testing"
)
func TestGetResourceByName(t *testing.T) {
t.Parallel()
resources := []map[string]any{
templateResource("team-a", "tenant-management"),
templateResource("team-a", "tenant-settings"),
}
resource, err := getResourceByName("tenant-management", resources)
if err != nil {
t.Fatalf("getResourceByName() error = %v", err)
}
if resource["data"].(map[string]any)["team"] != "team-a" {
t.Fatalf("getResourceByName() = %#v", resource)
}
missing, err := getResourceByName("missing", resources)
if err != nil {
t.Fatalf("getResourceByName(missing) error = %v", err)
}
if len(missing) != 0 {
t.Fatalf("getResourceByName(missing) = %#v, want empty map", missing)
}
}
func TestMustGetResourceByName(t *testing.T) {
t.Parallel()
_, err := mustGetResourceByName("missing", []map[string]any{
templateResource("team-a", "tenant-management"),
})
if err == nil || !strings.Contains(err.Error(), `metadata.name "missing" was not found`) {
t.Fatalf("mustGetResourceByName() error = %v", err)
}
}
func TestGetResourceByNameRejectsAmbiguousMatch(t *testing.T) {
t.Parallel()
resources := []map[string]any{
templateResource("team-a", "tenant-management"),
templateResource("team-b", "tenant-management"),
}
_, err := getResourceByName("tenant-management", resources)
if err == nil || !strings.Contains(err.Error(), "multiple resources match") {
t.Fatalf("getResourceByName() error = %v", err)
}
}
func TestGetResourceByNamespacedName(t *testing.T) {
t.Parallel()
resources := []map[string]any{
templateResource("team-a", "tenant-management"),
templateResource("team-b", "tenant-management"),
}
resource, err := getResourceByNamespacedName("team-b", "tenant-management", resources)
if err != nil {
t.Fatalf("getResourceByNamespacedName() error = %v", err)
}
if resource["data"].(map[string]any)["team"] != "team-b" {
t.Fatalf("getResourceByNamespacedName() = %#v", resource)
}
clusterScoped := templateResource("", "shared")
delete(clusterScoped["metadata"].(map[string]any), "namespace")
resource, err = getResourceByNamespacedName("", "shared", []map[string]any{clusterScoped})
if err != nil {
t.Fatalf("getResourceByNamespacedName(cluster-scoped) error = %v", err)
}
if resource["metadata"].(map[string]any)["name"] != "shared" {
t.Fatalf("getResourceByNamespacedName(cluster-scoped) = %#v", resource)
}
}
func TestGetResourceSupportsJSONRoundTrippedContext(t *testing.T) {
t.Parallel()
resources := []any{
templateResource("team-a", "tenant-management"),
}
resource, err := getResourceByName("tenant-management", resources)
if err != nil {
t.Fatalf("getResourceByName() error = %v", err)
}
if resource["data"].(map[string]any)["team"] != "team-a" {
t.Fatalf("getResourceByName() = %#v", resource)
}
}
func TestGetResourceRejectsInvalidContext(t *testing.T) {
t.Parallel()
tests := []struct {
name string
resources any
wantErr string
}{
{
name: "not a list",
resources: map[string]any{},
wantErr: "resources must be a list of objects",
},
{
name: "list item is not an object",
resources: []any{"invalid"},
wantErr: "resource at index 0 must be an object",
},
{
name: "missing metadata",
resources: []map[string]any{{}},
wantErr: "resource at index 0 has invalid or missing metadata",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
_, err := getResourceByName("tenant-management", test.resources)
if err == nil || !strings.Contains(err.Error(), test.wantErr) {
t.Fatalf("getResourceByName() error = %v, want containing %q", err, test.wantErr)
}
})
}
}
func templateResource(namespace, name string) map[string]any {
return map[string]any{
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": map[string]any{
"namespace": namespace,
"name": name,
},
"data": map[string]any{
"team": namespace,
},
}
}
+18
View File
@@ -63,6 +63,24 @@ func TestRenderTemplateBytes(t *testing.T) {
tpl: `{{ .registry | upper }}/app:1`,
want: "HARBOR/app:1",
},
{
name: "finds a context resource by name",
context: map[string]any{
"mgmt": []map[string]any{
{
"metadata": map[string]any{"name": "tenant-management"},
"data": map[string]any{
"team-a": "subjects:\n - name: alice\n - name: bob\n",
},
},
},
},
key: MissingKeyOption("error"),
tpl: `{{- $resource := .mgmt | mustGetResourceByName "tenant-management" -}}
{{- $team := $resource.data | get "team-a" | fromYAML -}}
{{- range $team.subjects }}{{ .name }} {{ end -}}`,
want: "alice bob ",
},
{
name: "missing key returns execute error when missingkey error is enabled",
context: map[string]any{
+6 -1
View File
@@ -196,11 +196,16 @@ apply-user:
@$(KUBECTL) label ns green-prod env=prod --overwrite
@$(KUBECTL) get ns solar-uat >/dev/null 2>&1 || $(KUBECTL) create ns solar-uat --as alice --as-group projectcapsule.dev
@$(KUBECTL) label ns solar-uat env=test --overwrite
@$(KUBECTL) label ns solar-uat team=team-a --overwrite
@$(KUBECTL) get ns solar-test >/dev/null 2>&1 || $(KUBECTL) create ns solar-test --as alice --as-group projectcapsule.dev
@$(KUBECTL) label ns solar-test env=test --overwrite
@$(KUBECTL) label ns solar-test team=team-a --overwrite
@$(KUBECTL) get ns solar-prod >/dev/null 2>&1 || $(KUBECTL) create ns solar-prod --as alice --as-group projectcapsule.dev
@$(KUBECTL) label ns solar-prod env=prod --overwrite
@kubectl kustomize user | envsubst $(SUBSTITUTION_VARIABLES) | $(KUBECTL) apply -f -
@$(KUBECTL) label ns solar-prod team=team-b --overwrite
@$(KUBECTL) kustomize user/solar | envsubst $(SUBSTITUTION_VARIABLES) | $(KUBECTL) apply --as alice --as-group projectcapsule.dev -f -
wait-oidc: certificates ## Wait for ingress and Dex before enabling API-server OIDC.
@@ -52,6 +52,19 @@ spec:
enableController: false
create: false
name: capsule-tls
rbac:
resources:
create: true
labels:
rbac.authorization.k8s.io/aggregate-to-admin: "true"
resourcepoolclaims:
create: true
labels:
rbac.authorization.k8s.io/aggregate-to-admin: "true"
customquotas:
create: true
labels:
rbac.authorization.k8s.io/aggregate-to-admin: "true"
webhooks:
service:
caBundle: "${PLAYGROUND_CA_BUNDLE}"
+44
View File
@@ -1,5 +1,49 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: gateway-api-role-distributor
rules:
- apiGroups:
- rbac.authorization.k8s.io
resources:
- rolebindings
verbs:
- get
- list
- watch
- create
- update
- patch
- delete
- apiGroups:
- rbac.authorization.k8s.io
resources:
- clusterroles
resourceNames:
- gateway-api-role-distributor
verbs:
- get
- bind
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: capsule:controller:gateway-api-role-distributor
labels:
projectcapsule.dev/aggregate-to-controller: "true"
rules:
- apiGroups:
- rbac.authorization.k8s.io
resources:
- clusterroles
resourceNames:
- gateway-api-role-distributor
verbs:
- get
- bind
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: oidc:admin
+9
View File
@@ -17,6 +17,15 @@ spec:
- matchLabels:
customer: renewable
rules:
# Promote only the dedicated TenantResource ServiceAccount. Capsule binds
# this distributor role in every solar namespace so the TenantResource can
# create the temporary user-facing RoleBindings there.
- permissions:
promotions:
- clusterRoles:
- gateway-api-role-distributor
- admin
- namespaceSelector:
matchExpressions:
- key: env
+1 -2
View File
@@ -1,4 +1,3 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- claims/claims.yaml
resources: []
+5
View File
@@ -0,0 +1,5 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- claims.yaml
- tenantresources/
@@ -0,0 +1,94 @@
# Bootstrap resources for the temporary Gateway API access example.
#
# The ServiceAccount is promoted by the matching rule on the solar Tenant. The
# promotion creates RoleBindings for gateway-api-role-distributor in every
# solar namespace. This gives the TenantResource enough authority to distribute
# only temporary-gateway-api-httproute-editor; it does not grant the
# ServiceAccount any HTTPRoute permissions of its own.
---
apiVersion: v1
kind: ConfigMap
metadata:
name: tenant-management
namespace: solar-system
data:
team-a: |
owners:
- kind: User
name: alice
- kind: User
name: bob
team-b: |
owners:
- kind: User
name: alice
- kind: User
name: bob
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: gateway-api-role-distributor
namespace: solar-system
labels:
projectcapsule.dev/promote: "true"
---
apiVersion: capsule.clastix.io/v1beta2
kind: TenantResource
metadata:
name: tenant-distribution
namespace: solar-system
spec:
resyncPeriod: 60s
serviceAccount:
name: gateway-api-role-distributor
resources:
- context:
resources:
- index: mgmt
apiVersion: v1
kind: ConfigMap
name: tenant-management
namespace: solar-system
generators:
- template: |
{{- range $team, $config := (index $.mgmt 0 "data") }}
{{- $team_config := fromYAML $config }}
---
apiVersion: capsule.clastix.io/v1beta2
kind: TenantResource
metadata:
name: "{{ $team }}-tenant-distribution"
namespace: solar-system
ownerReferences:
- apiVersion: capsule.clastix.io/v1beta2
kind: TenantResource
name: {{ $.replications.metadata.name | quote }}
uid: {{ $.replications.metadata.uid | quote }}
controller: true
blockOwnerDeletion: true
spec:
resyncPeriod: 60s
serviceAccount:
name: gateway-api-role-distributor
resources:
- namespaceSelector:
matchLabels:
team: {{ $team }}
rawItems:
- apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: "tenant:{{`{{namespace}}`}}:cr:gateway-api"
namespace: "{{`{{namespace}}`}}"
subjects:
{{- range $team_config.owners }}
- kind: {{ .kind }}
name: {{ .name }}
apiGroup: rbac.authorization.k8s.io
{{- end }}
roleRef:
kind: ClusterRole
name: gateway-api-role-distributor
apiGroup: rbac.authorization.k8s.io
{{- end }}
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- gateway-api-role-distributor.yaml