Files
capsule/e2e/replications_tenantresource_test.go
T
Oliver BählerandGitHub bdcdcefe63 feat: add globalresourcequota api (#2068)
* feat: add globalresourcequota api

Signed-off-by: Oliver Baehler <oliver@sudo-i.net>
2026-08-10 21:25:10 +02:00

2355 lines
82 KiB
Go

package e2e
import (
"context"
"fmt"
"strings"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
"github.com/projectcapsule/capsule/pkg/api"
apimeta "github.com/projectcapsule/capsule/pkg/api/meta"
"github.com/projectcapsule/capsule/pkg/api/rbac"
capruntime "github.com/projectcapsule/capsule/pkg/api/runtime"
"github.com/projectcapsule/capsule/pkg/runtime/gvk"
"github.com/projectcapsule/capsule/pkg/template"
)
var (
resyncPeriod = metav1.Duration{Duration: 10 * time.Second}
)
const tenantResourceTargetLabel = "e2e.projectcapsule.dev/tenantresource-target"
var _ = Describe("TenantResource SSA", Ordered, Label("replications", "namespace", "tenantresource"), Ordered, func() {
var (
ctx context.Context
tnt *capsulev1beta2.Tenant
baseNamespace string
targetNamespaces []string
tenantOwner rbac.UserSpec
additionalBindingUser rbac.UserSpec
sharedSourceSecret *corev1.Secret
contextSecretOne *corev1.Secret
contextSecretTwo *corev1.Secret
)
originalConfig := &capsulev1beta2.CapsuleConfiguration{}
BeforeEach(func() {
ctx = context.Background()
baseNamespace = "e2e-tenantresource-ssa-system"
targetNamespaces = []string{"e2e-tenantresource-ssa-one", "e2e-tenantresource-ssa-two", "e2e-tenantresource-ssa-three"}
tenantOwner = rbac.UserSpec{Name: "e2e-tr-owner", Kind: rbac.OwnerKind("User")}
additionalBindingUser = rbac.UserSpec{Name: "e2e-tr-additional", Kind: rbac.OwnerKind("User")}
Expect(k8sClient.Get(context.Background(), client.ObjectKey{Name: defaultConfigurationName}, originalConfig)).To(Succeed())
tnt = &capsulev1beta2.Tenant{
ObjectMeta: metav1.ObjectMeta{
Name: "e2e-tenantresource-ssa",
Labels: map[string]string{
"env": "e2e",
},
},
Spec: capsulev1beta2.TenantSpec{
Owners: rbac.OwnerListSpec{{
CoreOwnerSpec: rbac.CoreOwnerSpec{UserSpec: tenantOwner},
}},
AdditionalRoleBindings: []rbac.AdditionalRoleBindingsSpec{{
ClusterRoleName: "admin",
Subjects: []rbacv1.Subject{{Kind: "User", Name: additionalBindingUser.Name}},
}},
},
}
sharedSourceSecret = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "seed-secret",
Namespace: baseNamespace,
Labels: map[string]string{
"replicate": "true",
"source": "static",
},
},
Type: corev1.SecretTypeOpaque,
StringData: map[string]string{"seed": "base"},
}
contextSecretOne = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "pull-secret-one",
Namespace: "e2e-tenantresource-ssa-one",
Labels: map[string]string{"pullsecret.company.com": "true"},
},
Type: corev1.SecretTypeOpaque,
StringData: map[string]string{".dockerconfigjson": "e30="},
}
contextSecretTwo = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "pull-secret-two",
Namespace: "e2e-tenantresource-ssa-one",
Labels: map[string]string{"pullsecret.company.com": "true"},
},
Type: corev1.SecretTypeOpaque,
StringData: map[string]string{".dockerconfigjson": "e30="},
}
Expect(k8sClient.Get(ctx, client.ObjectKey{Name: defaultConfigurationName}, originalConfig)).To(Succeed())
EventuallyCreation(func() error {
tnt.ResourceVersion = ""
return k8sClient.Create(ctx, tnt)
}).Should(Succeed())
TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval)
for _, ns := range append(append([]string{}, targetNamespaces...), baseNamespace) {
labels := map[string]string{apimeta.TenantLabel: tnt.GetName()}
if ns != baseNamespace {
labels[tenantResourceTargetLabel] = "true"
}
namespace := NewNamespace(ns, labels)
NamespaceCreation(namespace, tenantOwner, defaultTimeoutInterval).Should(Succeed())
NamespaceIsPartOfTenant(tnt, namespace).Should(Succeed())
}
ensureServiceAccount(baseNamespace, "default")
for _, ns := range append(append([]string{}, targetNamespaces...), baseNamespace) {
bindServiceAccountToTenantResourceManager(baseNamespace, "default", ns)
}
})
AfterEach(func() {
ModifyCapsuleConfigurationOpts(func(configuration *capsulev1beta2.CapsuleConfiguration) {
configuration.Spec = originalConfig.Spec
})
cleanupTenantResourcesWithDefaultServiceAccount(
ctx,
append([]string{baseNamespace}, targetNamespaces...)...,
)
ignoreNotFound(k8sClient.Delete(ctx, sharedSourceSecret))
ignoreNotFound(k8sClient.Delete(ctx, contextSecretOne))
ignoreNotFound(k8sClient.Delete(ctx, contextSecretTwo))
for _, ns := range append([]string{baseNamespace}, targetNamespaces...) {
ForceDeleteNamespace(ctx, ns)
}
EventuallyDeletion(tnt)
})
Context("cluster-scoped object protection", func() {
It("rejects cluster-scoped rawItems", func() {
clusterRoleName := "tr-raw-cluster-scoped"
defer func() {
ignoreNotFound(k8sClient.Delete(ctx, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: clusterRoleName}}))
}()
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{
Name: "rawitems-cluster-scoped",
Namespace: baseNamespace,
},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: resyncPeriod,
Resources: []capsulev1beta2.ResourceSpec{{
RawItems: []capsulev1beta2.RawExtension{{
RawExtension: runtime.RawExtension{
Object: &rbacv1.ClusterRole{
TypeMeta: metav1.TypeMeta{
APIVersion: rbacv1.SchemeGroupVersion.String(),
Kind: "ClusterRole",
},
ObjectMeta: metav1.ObjectMeta{
Name: clusterRoleName,
},
Rules: []rbacv1.PolicyRule{{
APIGroups: []string{""},
Resources: []string{"configmaps"},
Verbs: []string{"get"},
}},
},
},
}},
}},
},
},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceFailed(baseNamespace, tr.Name, "cluster-scoped kind rbac.authorization.k8s.io/v1/ClusterRole is not allowed")
expectClusterRoleAbsent(clusterRoleName)
})
It("rejects cluster-scoped generator output", func() {
clusterRoleName := "tr-generator-cluster-scoped"
defer func() {
ignoreNotFound(k8sClient.Delete(ctx, &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: clusterRoleName}}))
}()
tr := newGeneratorConfigMapTenantResource(baseNamespace, "generator-cluster-scoped", fmt.Sprintf(`---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: %s
rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get"]
`, clusterRoleName))
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceFailed(baseNamespace, tr.Name, "cluster-scoped kind rbac.authorization.k8s.io/v1/ClusterRole is not allowed")
expectClusterRoleAbsent(clusterRoleName)
})
It("rejects cluster-scoped namespacedItems", func() {
clusterRoleName := "tr-source-cluster-scoped"
source := &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: clusterRoleName,
Labels: map[string]string{
"replicate": "cluster-scoped",
},
},
Rules: []rbacv1.PolicyRule{{
APIGroups: []string{""},
Resources: []string{"configmaps"},
Verbs: []string{"get"},
}},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, source) }).Should(Succeed())
defer func() {
ignoreNotFound(k8sClient.Delete(ctx, source))
}()
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{
Name: "namespaceditems-cluster-scoped",
Namespace: baseNamespace,
},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: resyncPeriod,
Resources: []capsulev1beta2.ResourceSpec{{
NamespacedItems: []template.ResourceReference{{
VersionKind: capruntime.VersionKind{
APIVersion: rbacv1.SchemeGroupVersion.String(),
Kind: "ClusterRole",
},
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"replicate": "cluster-scoped",
},
},
}},
}},
},
},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceFailed(baseNamespace, tr.Name, "cluster-scoped kind rbac.authorization.k8s.io/v1/ClusterRole is not allowed")
})
})
It("skips applying resources to terminating namespaces and removes them from processedItems", func() {
terminatingNamespace := targetNamespaces[2]
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{
Name: "skip-terminating-namespace",
Namespace: baseNamespace,
},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: metav1.Duration{Duration: 5 * time.Second},
Resources: []capsulev1beta2.ResourceSpec{{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{tenantResourceTargetLabel: "true"},
},
RawItems: []capsulev1beta2.RawExtension{{
RawExtension: runtime.RawExtension{
Object: &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: "tr-skip-terminating",
},
Data: map[string]string{
"mode": "active",
},
},
},
}},
}},
},
},
}
EventuallyCreation(func() error {
return k8sClient.Create(ctx, tr)
}).Should(Succeed())
By("waiting for the initial replication to complete")
expectTenantResourceProcessedNamespaces(
baseNamespace,
tr.Name,
"tr-skip-terminating",
targetNamespaces,
)
By("establishing the resource in every active namespace")
for _, ns := range targetNamespaces {
expectConfigMapData(ns, "tr-skip-terminating", map[string]string{
"mode": "active",
})
}
releaseNamespace := holdNamespaceTerminating(ctx, terminatingNamespace)
defer releaseNamespace()
By("updating the resource after one target namespace starts terminating")
Eventually(func() error {
current := &capsulev1beta2.TenantResource{}
if err := k8sClient.Get(ctx, types.NamespacedName{
Name: tr.Name,
Namespace: tr.Namespace,
}, current); err != nil {
return err
}
current.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{
RawExtension: runtime.RawExtension{
Object: &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{Name: "tr-skip-terminating"},
Data: map[string]string{"mode": "updated"},
},
},
}
return k8sClient.Update(ctx, current)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
By("verifying non-terminating namespaces still receive updates")
for _, ns := range targetNamespaces[:2] {
expectConfigMapData(ns, "tr-skip-terminating", map[string]string{
"mode": "updated",
})
}
By("verifying the terminating namespace is skipped")
Eventually(func() error {
return k8sClient.Get(ctx, types.NamespacedName{
Name: "tr-skip-terminating",
Namespace: terminatingNamespace,
}, &corev1.ConfigMap{})
}, defaultTimeoutInterval, defaultPollInterval).Should(HaveOccurred())
Consistently(func() error {
return k8sClient.Get(ctx, types.NamespacedName{
Name: "tr-skip-terminating",
Namespace: terminatingNamespace,
}, &corev1.ConfigMap{})
}, 2*resyncPeriod.Duration, defaultPollInterval).Should(HaveOccurred())
By("verifying the terminating namespace item is not kept in processedItems")
expectTenantResourceProcessedNamespaces(
baseNamespace,
tr.Name,
"tr-skip-terminating",
targetNamespaces[:2],
)
})
Context("generators and template context", func() {
It("fails when a templated namespace resolves to a forbidden namespace", func() {
foreignSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "templated-foreign-secret",
Namespace: "kube-system",
Labels: map[string]string{
"pullsecret.company.com": "true",
},
},
Type: corev1.SecretTypeOpaque,
StringData: map[string]string{"token": "forbidden"},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, foreignSecret) }).Should(Succeed())
defer ignoreNotFound(k8sClient.Delete(ctx, foreignSecret))
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{
Name: "templated-forbidden-namespace",
Namespace: baseNamespace,
},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: resyncPeriod,
Resources: []capsulev1beta2.ResourceSpec{{
Context: &template.TemplateContext{
Resources: []*template.TemplateResourceReference{{
Index: "secrets",
ResourceReference: template.ResourceReference{
VersionKind: capruntime.VersionKind{
APIVersion: "v1",
Kind: "Secret",
},
Namespace: "{{ forbiddenNamespace }}",
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"pullsecret.company.com": "true",
},
},
},
}},
},
Generators: []capsulev1beta2.TemplateItemSpec{{
MissingKey: "zero",
Template: `---
apiVersion: v1
kind: ConfigMap
metadata:
name: templated-forbidden-context
data:
count: '{{ if $.secrets }}{{ len $.secrets }}{{ else }}0{{ end }}'
`,
}},
}},
},
},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
Eventually(func(g Gomega) {
current := &capsulev1beta2.TenantResource{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: tr.Namespace}, current)).To(Succeed())
rdy := current.Status.Conditions.GetConditionByType(apimeta.ReadyCondition)
g.Expect(rdy).ToNot(BeNil())
g.Expect(rdy.Status).To(Equal(metav1.ConditionTrue))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
It("renders generator templates with tenant and namespace data", func() {
tr := newGeneratorConfigMapTenantResource(baseNamespace, "generator-template", `---
apiVersion: v1
kind: ConfigMap
metadata:
name: generated-{{ $.namespace.metadata.name }}
data:
tenant: "{{ $.tenant.metadata.name }}"
namespace: "{{ $.namespace.metadata.name }}"
`)
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, tr.Name)
for _, ns := range targetNamespaces {
expectConfigMapData(ns, fmt.Sprintf("generated-%s", ns), map[string]string{
"tenant": tnt.Name,
"namespace": ns,
})
expectProcessedItemStatus(baseNamespace, tr.Name, configMapRID(tnt.Name, ns, fmt.Sprintf("generated-%s", ns), "0/generator-0-0"), metav1.ConditionTrue, true, "")
}
})
It("places generated objects into the current tenant namespace even when the template sets metadata.namespace to a foreign namespace", func() {
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{
Name: "generator-enforce-target-namespace",
Namespace: baseNamespace,
},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: resyncPeriod,
Resources: []capsulev1beta2.ResourceSpec{{
Generators: []capsulev1beta2.TemplateItemSpec{{
MissingKey: "error",
Template: `---
apiVersion: v1
kind: ConfigMap
metadata:
name: generated-namespace-locked
namespace: kube-system
data:
source: generator
renderedFor: "{{ $.namespace.metadata.name }}"
`,
}},
}},
},
},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, tr.Name)
By("verifying the generated object is created in each tenant namespace, not in the foreign namespace")
for _, ns := range targetNamespaces {
Eventually(func(g Gomega) {
cm := &corev1.ConfigMap{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{
Name: "generated-namespace-locked",
Namespace: ns,
}, cm)).To(Succeed())
g.Expect(cm.Namespace).To(Equal(ns))
g.Expect(cm.Data).To(HaveKeyWithValue("source", "generator"))
g.Expect(cm.Data).To(HaveKeyWithValue("renderedFor", ns))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
Consistently(func() error {
return k8sClient.Get(ctx, types.NamespacedName{
Name: "generated-namespace-locked",
Namespace: "kube-system",
}, &corev1.ConfigMap{})
}, 5*time.Second, defaultPollInterval).Should(HaveOccurred())
})
It("loads context from the explicitly referenced namespace for every rendered namespace", func() {
EventuallyCreation(func() error { return k8sClient.Create(ctx, contextSecretOne) }).Should(Succeed())
EventuallyCreation(func() error { return k8sClient.Create(ctx, contextSecretTwo) }).Should(Succeed())
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{Name: "context-loading-fixed-namespace", Namespace: baseNamespace},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: resyncPeriod,
Resources: []capsulev1beta2.ResourceSpec{{
Context: &template.TemplateContext{
Resources: []*template.TemplateResourceReference{{
Index: "secrets",
ResourceReference: template.ResourceReference{
VersionKind: capruntime.VersionKind{
APIVersion: "v1",
Kind: "Secret",
},
Namespace: "e2e-tenantresource-ssa-one",
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"pullsecret.company.com": "true",
},
},
},
}},
},
Generators: []capsulev1beta2.TemplateItemSpec{{
MissingKey: "zero",
Template: `---
apiVersion: v1
kind: ConfigMap
metadata:
name: show-context
data:
count: '{{ if $.secrets }}{{ len $.secrets }}{{ else }}0{{ end }}'
`,
}},
}},
},
},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, tr.Name)
expectConfigMapData("e2e-tenantresource-ssa-one", "show-context", map[string]string{"count": "2"})
expectConfigMapData("e2e-tenantresource-ssa-two", "show-context", map[string]string{"count": "2"})
expectConfigMapData("e2e-tenantresource-ssa-three", "show-context", map[string]string{"count": "2"})
})
It("loads context from the current iterating namespace", func() {
EventuallyCreation(func() error { return k8sClient.Create(ctx, contextSecretOne) }).Should(Succeed())
EventuallyCreation(func() error { return k8sClient.Create(ctx, contextSecretTwo) }).Should(Succeed())
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{Name: "context-loading-variable-namespace", Namespace: baseNamespace},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: resyncPeriod,
Resources: []capsulev1beta2.ResourceSpec{{
Context: &template.TemplateContext{
Resources: []*template.TemplateResourceReference{{
Index: "secrets",
ResourceReference: template.ResourceReference{
VersionKind: capruntime.VersionKind{
APIVersion: "v1",
Kind: "Secret",
},
Namespace: "{{ namespace }}",
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"pullsecret.company.com": "true",
},
},
},
}},
},
Generators: []capsulev1beta2.TemplateItemSpec{{
MissingKey: "zero",
Template: `---
apiVersion: v1
kind: ConfigMap
metadata:
name: show-context
data:
count: '{{ if $.secrets }}{{ len $.secrets }}{{ else }}0{{ end }}'
`,
}},
}},
},
},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, tr.Name)
expectConfigMapData("e2e-tenantresource-ssa-one", "show-context", map[string]string{"count": "2"})
expectConfigMapData("e2e-tenantresource-ssa-two", "show-context", map[string]string{"count": "0"})
expectConfigMapData("e2e-tenantresource-ssa-three", "show-context", map[string]string{"count": "0"})
})
It("loads context per rendered namespace", func() {
EventuallyCreation(func() error { return k8sClient.Create(ctx, contextSecretOne) }).Should(Succeed())
EventuallyCreation(func() error { return k8sClient.Create(ctx, contextSecretTwo) }).Should(Succeed())
solarTwoSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "pull-secret-e2e-tenantresource-ssa-two",
Namespace: "e2e-tenantresource-ssa-two",
Labels: map[string]string{
"pullsecret.company.com": "true",
},
},
Type: corev1.SecretTypeOpaque,
StringData: map[string]string{".dockerconfigjson": "e30="},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, solarTwoSecret) }).Should(Succeed())
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{Name: "context-variable-namespace", Namespace: baseNamespace},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: resyncPeriod,
Resources: []capsulev1beta2.ResourceSpec{{
Context: &template.TemplateContext{
Resources: []*template.TemplateResourceReference{{
Index: "secrets",
ResourceReference: template.ResourceReference{
VersionKind: capruntime.VersionKind{
APIVersion: "v1",
Kind: "Secret",
},
Namespace: "{{.namespace}}",
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"pullsecret.company.com": "true",
},
},
},
}},
},
Generators: []capsulev1beta2.TemplateItemSpec{{
MissingKey: "zero",
Template: `---
apiVersion: v1
kind: ConfigMap
metadata:
name: show-context
data:
count: '{{ if $.secrets }}{{ len $.secrets }}{{ else }}0{{ end }}'
`,
}},
}},
},
},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, tr.Name)
expectConfigMapData("e2e-tenantresource-ssa-one", "show-context", map[string]string{"count": "2"})
expectConfigMapData("e2e-tenantresource-ssa-two", "show-context", map[string]string{"count": "1"})
expectConfigMapData("e2e-tenantresource-ssa-three", "show-context", map[string]string{"count": "0"})
})
It("fails when context tries to load from a namespace outside the tenant", func() {
foreignSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "foreign-pull-secret",
Namespace: "kube-system",
Labels: map[string]string{
"pullsecret.company.com": "true",
},
},
Type: corev1.SecretTypeOpaque,
StringData: map[string]string{".dockerconfigjson": "e30="},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, foreignSecret) }).Should(Succeed())
defer ignoreNotFound(k8sClient.Delete(ctx, foreignSecret))
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{
Name: "context-forbidden-namespace",
Namespace: baseNamespace,
},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: resyncPeriod,
Resources: []capsulev1beta2.ResourceSpec{{
Context: &template.TemplateContext{
Resources: []*template.TemplateResourceReference{{
Index: "secrets",
ResourceReference: template.ResourceReference{
VersionKind: capruntime.VersionKind{
APIVersion: "v1",
Kind: "Secret",
},
Namespace: "kube-system",
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"pullsecret.company.com": "true",
},
},
},
}},
},
Generators: []capsulev1beta2.TemplateItemSpec{{
MissingKey: "zero",
Template: `---
apiVersion: v1
kind: ConfigMap
metadata:
name: forbidden-context
data:
count: '{{ if $.secrets }}{{ len $.secrets }}{{ else }}0{{ end }}'
`,
}},
}},
},
},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
Eventually(func(g Gomega) {
current := &capsulev1beta2.TenantResource{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: tr.Namespace}, current)).To(Succeed())
rdy := current.Status.Conditions.GetConditionByType(apimeta.ReadyCondition)
g.Expect(rdy).ToNot(BeNil())
g.Expect(rdy.Status).To(Equal(metav1.ConditionFalse))
g.Expect(rdy.Message).To(ContainSubstring("cross-namespace selection is not allowed"))
g.Expect(rdy.Message).To(ContainSubstring("kube-system"))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
for _, ns := range targetNamespaces {
Consistently(func() error {
return k8sClient.Get(ctx, types.NamespacedName{Name: "forbidden-context", Namespace: ns}, &corev1.ConfigMap{})
}, 5*time.Second, defaultPollInterval).Should(HaveOccurred())
}
})
It("fails when namespacedItems tries to load from a namespace outside the tenant", func() {
foreignSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "foreign-source-secret",
Namespace: "kube-system",
Labels: map[string]string{
"pullsecret.company.com": "true",
},
},
Type: corev1.SecretTypeOpaque,
StringData: map[string]string{"token": "forbidden"},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, foreignSecret) }).Should(Succeed())
defer ignoreNotFound(k8sClient.Delete(ctx, foreignSecret))
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{
Name: "namespaceditems-forbidden-namespace",
Namespace: baseNamespace,
},
Spec: capsulev1beta2.TenantResourceSpec{
ServiceAccount: &apimeta.LocalRFC1123ObjectReference{Name: apimeta.RFC1123Name("replicator")},
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: resyncPeriod,
Resources: []capsulev1beta2.ResourceSpec{{
NamespacedItems: []template.ResourceReference{{
VersionKind: capruntime.VersionKind{
APIVersion: "v1",
Kind: "Secret",
},
Namespace: "kube-system",
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"pullsecret.company.com": "true",
},
},
}},
}},
},
},
}
EnsureServiceAccount(ctx, k8sClient, tr.Spec.ServiceAccount.Name.String(), baseNamespace)
EnsureRoleAndBindingForNamespaces(ctx, k8sClient, tr.Spec.ServiceAccount.Name.String(), baseNamespace, append(targetNamespaces, baseNamespace))
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
Eventually(func(g Gomega) {
current := &capsulev1beta2.TenantResource{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: tr.Namespace}, current)).To(Succeed())
rdy := current.Status.Conditions.GetConditionByType(apimeta.ReadyCondition)
g.Expect(rdy).ToNot(BeNil())
g.Expect(rdy.Status).To(Equal(metav1.ConditionFalse))
g.Expect(rdy.Message).To(ContainSubstring("cross-namespace selection is not allowed"))
g.Expect(rdy.Message).To(ContainSubstring("kube-system"))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
for _, ns := range targetNamespaces {
Consistently(func() error {
return k8sClient.Get(ctx, types.NamespacedName{Name: "foreign-source-secret", Namespace: ns}, &corev1.Secret{})
}, 5*time.Second, defaultPollInterval).Should(HaveOccurred())
}
})
It("places rawItems into the current tenant namespace even when metadata.namespace is set to a foreign namespace", func() {
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{
Name: "rawitems-enforce-target-namespace",
Namespace: baseNamespace,
},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: resyncPeriod,
Resources: []capsulev1beta2.ResourceSpec{{
RawItems: []capsulev1beta2.RawExtension{{
RawExtension: runtime.RawExtension{
Object: &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: "raw-namespace-locked",
Namespace: "kube-system",
},
Data: map[string]string{
"source": "raw",
},
},
},
}},
}},
},
},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, tr.Name)
By("verifying the object is created in each tenant namespace, not in the foreign namespace")
for _, ns := range targetNamespaces {
Eventually(func(g Gomega) {
cm := &corev1.ConfigMap{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{
Name: "raw-namespace-locked",
Namespace: ns,
}, cm)).To(Succeed())
g.Expect(cm.Namespace).To(Equal(ns))
g.Expect(cm.Data).To(HaveKeyWithValue("source", "raw"))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
Consistently(func() error {
return k8sClient.Get(ctx, types.NamespacedName{
Name: "raw-namespace-locked",
Namespace: "kube-system",
}, &corev1.ConfigMap{})
}, 5*time.Second, defaultPollInterval).Should(HaveOccurred())
})
})
Context("multiple TenantResources targeting the same object", func() {
It("allows non-conflicting ownership without force", func() {
first := newGeneratorConfigMapTenantResource(baseNamespace, "same-object-no-force-a", `---
apiVersion: v1
kind: ConfigMap
metadata:
name: owned-together
data:
from-a: one
`)
second := newRawConfigMapTenantResource(baseNamespace, "same-object-no-force-b", map[string]string{"from-b": "two"})
second.Spec.Resources[0].RawItems[0].RawExtension.Object.(*corev1.ConfigMap).Name = "owned-together"
EventuallyCreation(func() error { return k8sClient.Create(ctx, first) }).Should(Succeed())
EventuallyCreation(func() error { return k8sClient.Create(ctx, second) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, first.Name)
expectTenantResourceReady(baseNamespace, second.Name)
for _, ns := range targetNamespaces {
expectConfigMapData(ns, "owned-together", map[string]string{"from-a": "one", "from-b": "two"})
}
})
It("fails on conflicting ownership without force", func() {
first := newRawConfigMapTenantResource(baseNamespace, "same-object-conflict-a", map[string]string{"shared": "one"})
first.Spec.Resources[0].RawItems[0].RawExtension.Object.(*corev1.ConfigMap).Name = "force-target"
second := newRawConfigMapTenantResource(baseNamespace, "same-object-conflict-b", map[string]string{"shared": "two"})
second.Spec.Resources[0].RawItems[0].RawExtension.Object.(*corev1.ConfigMap).Name = "force-target"
second.Spec.Settings.Force = ptr.To(false)
EventuallyCreation(func() error { return k8sClient.Create(ctx, first) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, first.Name)
EventuallyCreation(func() error { return k8sClient.Create(ctx, second) }).Should(Succeed())
expectTenantResourceFailed(baseNamespace, second.Name, "applying of")
for _, ns := range targetNamespaces {
expectConfigMapData(ns, "force-target", map[string]string{"shared": "one"})
}
})
It("wins conflicting ownership with force", func() {
first := newRawConfigMapTenantResource(baseNamespace, "same-object-force-a", map[string]string{"shared": "one"})
first.Spec.Resources[0].RawItems[0].RawExtension.Object.(*corev1.ConfigMap).Name = "forced-target"
second := newRawConfigMapTenantResource(baseNamespace, "same-object-force-b", map[string]string{"shared": "two"})
second.Spec.Resources[0].RawItems[0].RawExtension.Object.(*corev1.ConfigMap).Name = "forced-target"
second.Spec.Settings.Force = ptr.To(true)
EventuallyCreation(func() error { return k8sClient.Create(ctx, first) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, first.Name)
EventuallyCreation(func() error { return k8sClient.Create(ctx, second) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, second.Name)
for _, ns := range targetNamespaces {
expectConfigMapData(ns, "forced-target", map[string]string{"shared": "two"})
}
})
})
Context("namespaced item replication", func() {
It("replicates source objects and strips selector labels to avoid loops", func() {
EventuallyCreation(func() error { return k8sClient.Create(ctx, sharedSourceSecret) }).Should(Succeed())
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{Name: "selector-replication", Namespace: baseNamespace},
Spec: capsulev1beta2.TenantResourceSpec{
ServiceAccount: &apimeta.LocalRFC1123ObjectReference{Name: apimeta.RFC1123Name("replicator")},
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: resyncPeriod,
Resources: []capsulev1beta2.ResourceSpec{{
NamespacedItems: []template.ResourceReference{{
VersionKind: capruntime.VersionKind{
APIVersion: "v1",
Kind: "Secret",
},
Namespace: baseNamespace,
Selector: &metav1.LabelSelector{MatchLabels: map[string]string{
"replicate": "true",
}},
}},
}},
},
},
}
EnsureServiceAccount(ctx, k8sClient, tr.Spec.ServiceAccount.Name.String(), baseNamespace)
EnsureRoleAndBindingForNamespaces(ctx, k8sClient, tr.Spec.ServiceAccount.Name.String(), baseNamespace, append(targetNamespaces, baseNamespace))
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, tr.Name)
for _, ns := range targetNamespaces {
Eventually(func(g Gomega) {
sec := &corev1.Secret{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: sharedSourceSecret.Name, Namespace: ns}, sec)).To(Succeed())
g.Expect(sec.Labels).ToNot(HaveKey("replicate"))
g.Expect(sec.Labels).To(HaveKeyWithValue("source", "static"))
g.Expect(sec.Labels).To(HaveKeyWithValue(apimeta.CreatedByCapsuleLabel, apimeta.ValueControllerReplications))
g.Expect(sec.Labels).To(HaveKeyWithValue(apimeta.NewManagedByCapsuleLabel, apimeta.ValueControllerReplications))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
})
})
Context("apply lifecycle with prune enabled", func() {
It("applies, updates and prunes raw items", Label("skip-on-openshift"), func() {
tr := newRawConfigMapTenantResource(baseNamespace, "raw-prune-enabled", map[string]string{
"mode": "before",
"foo": "one",
})
tr.Spec.PruningOnDelete = ptr.To(true)
By("creating the TenantResource")
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, tr.Name)
By("verifying the created ConfigMaps and status entries")
for _, ns := range targetNamespaces {
expectConfigMapData(ns, "shared-config", map[string]string{"mode": "before", "foo": "one"})
expectManagedLabelsOnConfigMap(ns, "shared-config", true)
expectProcessedItemStatus(baseNamespace, tr.Name, configMapRID(tnt.Name, ns, "shared-config", "0/raw-0"), metav1.ConditionTrue, true, "")
}
By("updating the applied data")
Eventually(func() error {
current := &capsulev1beta2.TenantResource{}
if err := k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: baseNamespace}, current); err != nil {
return err
}
current.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{
RawExtension: runtime.RawExtension{
Object: &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: "shared-config",
},
Data: map[string]string{
"mode": "after",
"foo": "two",
"bar": "three",
},
},
},
}
return k8sClient.Update(ctx, current)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
for _, ns := range targetNamespaces {
expectConfigMapData(ns, "shared-config", map[string]string{"mode": "after", "foo": "two", "bar": "three"})
expectProcessedItemApplied(baseNamespace, tr.Name, configMapRID(tnt.Name, ns, "shared-config", "0/raw-0"))
}
By("deleting the TenantResource and pruning created objects")
Expect(k8sClient.Delete(ctx, tr)).To(Succeed())
for _, ns := range targetNamespaces {
expectConfigMapDeleted(ns, "shared-config")
}
})
})
It("places generated objects into the current tenant namespace even when the template sets metadata.namespace to a foreign namespace", func() {
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{
Name: "generator-enforce-target-namespace",
Namespace: baseNamespace,
},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: resyncPeriod,
Resources: []capsulev1beta2.ResourceSpec{{
Generators: []capsulev1beta2.TemplateItemSpec{{
MissingKey: "error",
Template: `---
apiVersion: v1
kind: ConfigMap
metadata:
name: generated-namespace-locked
namespace: kube-system
data:
source: generator
renderedFor: "{{ $.namespace.metadata.name }}"
`,
}},
}},
},
},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, tr.Name)
By("verifying the generated object is created in each tenant namespace, not in the foreign namespace")
for _, ns := range targetNamespaces {
Eventually(func(g Gomega) {
cm := &corev1.ConfigMap{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{
Name: "generated-namespace-locked",
Namespace: ns,
}, cm)).To(Succeed())
g.Expect(cm.Namespace).To(Equal(ns))
g.Expect(cm.Data).To(HaveKeyWithValue("source", "generator"))
g.Expect(cm.Data).To(HaveKeyWithValue("renderedFor", ns))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
Consistently(func() error {
return k8sClient.Get(ctx, types.NamespacedName{
Name: "generated-namespace-locked",
Namespace: "kube-system",
}, &corev1.ConfigMap{})
}, 5*time.Second, defaultPollInterval).Should(HaveOccurred())
})
It("merge rawItems and generators when they target the same object", func() {
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{
Name: "raw-and-generator-same-object",
Namespace: baseNamespace,
},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: resyncPeriod,
Resources: []capsulev1beta2.ResourceSpec{{
RawItems: []capsulev1beta2.RawExtension{{
RawExtension: runtime.RawExtension{
Object: &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: "raw-generated-shared",
},
Data: map[string]string{
"static": "raw",
},
},
},
}},
Generators: []capsulev1beta2.TemplateItemSpec{{
MissingKey: "zero",
Template: `---
apiVersion: v1
kind: ConfigMap
metadata:
name: raw-generated-shared
data:
generated-{{ $.namespace.metadata.name }}: "true"
`,
}},
}},
},
},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, tr.Name)
By("verifying the object contains both raw and generated data in every target namespace")
for _, ns := range targetNamespaces {
expectConfigMapData(ns, "raw-generated-shared", map[string]string{
"static": "raw",
"generated-" + ns: "true",
})
}
})
It("allows multiple TenantResources to adopt and co-manage the same preexisting object with non-conflicting fields", func() {
By("creating the preexisting object in all target namespaces")
for _, ns := range targetNamespaces {
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "shared-adopted-config",
Namespace: ns,
},
Data: map[string]string{
"existing": "true",
},
}
EventuallyCreation(func() error {
cm.ResourceVersion = ""
return k8sClient.Create(ctx, cm)
}).Should(Succeed())
}
trA := newRawConfigMapTenantResource(baseNamespace, "adopt-shared-a", map[string]string{
"foo": "one",
})
trA.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{
RawExtension: runtime.RawExtension{
Object: &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: "shared-adopted-config",
},
Data: map[string]string{
"foo": "one",
},
},
},
}
trA.Spec.Settings.Adopt = ptr.To(true)
trB := newRawConfigMapTenantResource(baseNamespace, "adopt-shared-b", map[string]string{
"bar": "two",
})
trB.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{
RawExtension: runtime.RawExtension{
Object: &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: "shared-adopted-config",
},
Data: map[string]string{
"bar": "two",
},
},
},
}
trB.Spec.Settings.Adopt = ptr.To(true)
By("creating both TenantResources")
EventuallyCreation(func() error { return k8sClient.Create(ctx, trA) }).Should(Succeed())
EventuallyCreation(func() error { return k8sClient.Create(ctx, trB) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, trA.Name)
expectTenantResourceReady(baseNamespace, trB.Name)
By("verifying the final object contains the merged fields and is adopted, not created")
for _, ns := range targetNamespaces {
Eventually(func(g Gomega) {
cm := &corev1.ConfigMap{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "shared-adopted-config", Namespace: ns}, cm)).To(Succeed())
g.Expect(cm.Data).To(HaveKeyWithValue("existing", "true"))
g.Expect(cm.Data).To(HaveKeyWithValue("foo", "one"))
g.Expect(cm.Data).To(HaveKeyWithValue("bar", "two"))
g.Expect(cm.Labels).To(HaveKeyWithValue(apimeta.NewManagedByCapsuleLabel, apimeta.ValueControllerReplications))
g.Expect(cm.Labels).ToNot(HaveKey(apimeta.CreatedByCapsuleLabel))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
})
It("aligns objects created by the legacy resource label implementation", func() {
By("creating legacy-labelled objects in all target namespaces")
for _, ns := range targetNamespaces {
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "legacy-aligned-config",
Namespace: ns,
Labels: map[string]string{
"capsule.clastix.io/resources": "0",
apimeta.TenantLabel: tnt.GetName(),
},
},
Data: map[string]string{
"legacy": "true",
},
}
EventuallyCreation(func() error {
cm.ResourceVersion = ""
return k8sClient.Create(ctx, cm)
}).Should(Succeed())
}
tr := newRawConfigMapTenantResource(baseNamespace, "legacy-alignment", map[string]string{
"mode": "new-controller",
"foo": "bar",
})
tr.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{
RawExtension: runtime.RawExtension{
Object: &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: "legacy-aligned-config",
},
Data: map[string]string{
"mode": "new-controller",
"foo": "bar",
},
},
},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, tr.Name)
By("verifying the object was aligned to the new implementation")
for _, ns := range targetNamespaces {
Eventually(func(g Gomega) {
cm := &corev1.ConfigMap{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{
Name: "legacy-aligned-config",
Namespace: ns,
}, cm)).To(Succeed())
g.Expect(cm.Data).To(HaveKeyWithValue("mode", "new-controller"))
g.Expect(cm.Data).To(HaveKeyWithValue("foo", "bar"))
// legacy marker still present from the old object
g.Expect(cm.Labels).To(HaveKeyWithValue(apimeta.TenantLabel, tnt.GetName()))
// new implementation metadata
g.Expect(cm.Labels).To(HaveKeyWithValue(apimeta.CreatedByCapsuleLabel, apimeta.ValueControllerReplications))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
})
Context("impersonation", func() {
It("reflects the resolved service account in status", func() {
tr := newRawConfigMapTenantResource(baseNamespace, "sa-resolution", map[string]string{"mode": "default-service-account"})
tr.Spec.ServiceAccount = nil
By("creating the TenantResource without an explicit ServiceAccount")
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
By("defaulting to the configured tenant service account")
expectResolvedServiceAccount(baseNamespace, tr.Name, "default", baseNamespace)
By("overriding with an explicit service account on the TenantResource")
Eventually(func() error {
current := &capsulev1beta2.TenantResource{}
if err := k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: baseNamespace}, current); err != nil {
return err
}
current.Spec.ServiceAccount = &apimeta.LocalRFC1123ObjectReference{Name: apimeta.RFC1123Name("custom-account")}
return k8sClient.Update(ctx, current)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
expectResolvedServiceAccount(baseNamespace, tr.Name, "custom-account", baseNamespace)
By("removing the explicit override again")
Eventually(func() error {
current := &capsulev1beta2.TenantResource{}
if err := k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: baseNamespace}, current); err != nil {
return err
}
current.Spec.ServiceAccount = nil
return k8sClient.Update(ctx, current)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
expectResolvedServiceAccount(baseNamespace, tr.Name, "default", baseNamespace)
})
It("fails to apply raw items when the impersonated service account cannot create target resources", func() {
saName := "restricted-creator"
ensureServiceAccount(baseNamespace, saName)
// Intentionally do not grant create/update/patch on configmaps in tenant namespaces.
tr := newRawConfigMapTenantResource(baseNamespace, "sa-no-create", map[string]string{
"mode": "blocked",
})
tr.Spec.ServiceAccount = &apimeta.LocalRFC1123ObjectReference{
Name: apimeta.RFC1123Name(saName),
}
renameFirstTenantResourceRawConfigMap(tr, "blocked-create-config")
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectResolvedServiceAccount(baseNamespace, tr.Name, saName, baseNamespace)
expectTenantResourceFailed(baseNamespace, tr.Name, "applying of")
for _, ns := range targetNamespaces {
expectConfigMapAbsent(ns, "blocked-create-config")
}
})
It("fails to render generators when the impersonated service account cannot read context resources", func() {
saName := "restricted-context-reader"
ensureServiceAccount(baseNamespace, saName)
// Create context source secret.
sec := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "ctx-secret",
Namespace: "e2e-tenantresource-ssa-one",
Labels: map[string]string{
"pullsecret.company.com": "true",
},
},
Type: corev1.SecretTypeOpaque,
StringData: map[string]string{".dockerconfigjson": "e30="},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, sec) }).Should(Succeed())
// Grant write on ConfigMaps in target namespaces if you want to isolate the failure to context loading.
for _, ns := range targetNamespaces {
bindServiceAccountToConfigMapWriter(baseNamespace, saName, ns)
}
// But do NOT grant get/list on secrets in e2e-tenantresource-ssa-one.
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{Name: "sa-no-context-read", Namespace: baseNamespace},
Spec: capsulev1beta2.TenantResourceSpec{
ServiceAccount: &apimeta.LocalRFC1123ObjectReference{
Name: apimeta.RFC1123Name(saName),
},
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
ResyncPeriod: resyncPeriod,
PruningOnDelete: ptr.To(true),
Resources: []capsulev1beta2.ResourceSpec{{
Context: &template.TemplateContext{
Resources: []*template.TemplateResourceReference{{
Index: "secrets",
ResourceReference: template.ResourceReference{
VersionKind: capruntime.VersionKind{
APIVersion: "v1",
Kind: "Secret",
},
Namespace: "e2e-tenantresource-ssa-one",
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"pullsecret.company.com": "true",
},
},
},
}},
},
Generators: []capsulev1beta2.TemplateItemSpec{{
MissingKey: "error",
Template: `---
apiVersion: v1
kind: ConfigMap
metadata:
name: blocked-context
data:
count: '{{ if $.secrets }}{{ len $.secrets }}{{ else }}0{{ end }}'
`,
}},
}},
},
},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectResolvedServiceAccount(baseNamespace, tr.Name, saName, baseNamespace)
expectTenantResourceFailed(baseNamespace, tr.Name, "forbidden")
for _, ns := range targetNamespaces {
expectConfigMapAbsent(ns, "blocked-context")
}
})
It("only allows the managing TenantResource service account to update created objects", func() {
saName := "tr-update-guard"
targetNamespace := targetNamespaces[0]
configMapName := "tr-admission-protected"
ensureServiceAccount(baseNamespace, saName)
for _, ns := range append(targetNamespaces, baseNamespace) {
bindServiceAccountToConfigMapWriter(baseNamespace, saName, ns)
}
tr := newRawConfigMapTenantResource(baseNamespace, "sa-update-guard", map[string]string{
"mode": "managed",
})
tr.Spec.ServiceAccount = &apimeta.LocalRFC1123ObjectReference{
Name: apimeta.RFC1123Name(saName),
}
renameFirstTenantResourceRawConfigMap(tr, configMapName)
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, tr.Name)
expectConfigMapData(targetNamespace, configMapName, map[string]string{"mode": "managed"})
expectManagedLabelsOnConfigMap(targetNamespace, configMapName, true)
expectProcessedItemStatus(
baseNamespace,
tr.Name,
configMapRID(tnt.Name, targetNamespace, configMapName, "0/raw-0"),
metav1.ConditionTrue,
true,
"",
)
tenantOwnerClient := impersonationClient(tenantOwner.Name, withDefaultGroups([]string{tenantOwner.Name}))
Eventually(func() error {
cm := &corev1.ConfigMap{}
if err := tenantOwnerClient.Get(ctx, types.NamespacedName{Name: configMapName, Namespace: targetNamespace}, cm); err != nil {
return err
}
cm.Data["mode"] = "tenant-owner"
err := tenantOwnerClient.Update(ctx, cm)
if err == nil {
return fmt.Errorf("expected tenant owner update to be denied")
}
if apierrors.IsConflict(err) {
return err
}
if !apierrors.IsForbidden(err) {
return fmt.Errorf("expected forbidden error, got: %w", err)
}
if !strings.Contains(err.Error(), "managed by a tenant capsule replication") {
return fmt.Errorf("expected tenant replication admission denial, got: %w", err)
}
return nil
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
replicationClient := impersonationClient(
serviceAccountUsername(baseNamespace, saName),
serviceAccountGroups(baseNamespace),
)
Eventually(func() error {
cm := &corev1.ConfigMap{}
if err := replicationClient.Get(ctx, types.NamespacedName{Name: configMapName, Namespace: targetNamespace}, cm); err != nil {
return err
}
cm.Data["mode"] = "service-account"
return replicationClient.Update(ctx, cm)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
It("fails to prune replicated resources when the impersonated service account cannot delete them", func() {
saCreate := "creator-ok"
saNoDelete := "creator-no-delete"
ensureServiceAccount(baseNamespace, saCreate)
ensureServiceAccount(baseNamespace, saNoDelete)
// Phase 1: creator can fully reconcile objects in every targeted namespace.
for _, ns := range append(targetNamespaces, baseNamespace) {
bindServiceAccountToConfigMapWriter(baseNamespace, saCreate, ns)
}
// Phase 2 SA can still read/apply, but has no delete verb.
for _, ns := range append(targetNamespaces, baseNamespace) {
bindServiceAccountToConfigMapWriter(baseNamespace, saNoDelete, ns)
}
tr := newRawConfigMapTenantResource(baseNamespace, "sa-no-prune", map[string]string{
"mode": "created",
})
tr.Spec.ServiceAccount = &apimeta.LocalRFC1123ObjectReference{
Name: apimeta.RFC1123Name(saCreate),
}
tr.Spec.PruningOnDelete = ptr.To(true)
renameFirstTenantResourceRawConfigMap(tr, "prune-protected")
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, tr.Name)
for _, ns := range append(targetNamespaces, baseNamespace) {
expectConfigMapData(ns, "prune-protected", map[string]string{"mode": "created"})
}
// Switch to the SA that cannot delete.
Eventually(func() error {
current := &capsulev1beta2.TenantResource{}
if err := k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: baseNamespace}, current); err != nil {
return err
}
current.Spec.ServiceAccount = &apimeta.LocalRFC1123ObjectReference{
Name: apimeta.RFC1123Name(saNoDelete),
}
return k8sClient.Update(ctx, current)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
Eventually(func(g Gomega) {
current := &capsulev1beta2.TenantResource{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: baseNamespace}, current)).To(Succeed())
g.Expect(current.Status.ServiceAccount).ToNot(BeNil())
g.Expect(current.Status.ServiceAccount.Name).To(Equal(apimeta.RFC1123Name(saNoDelete)))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
Expect(k8sClient.Delete(ctx, tr)).To(Succeed())
// Objects should remain because prune cannot delete them.
for _, ns := range append(targetNamespaces, baseNamespace) {
Eventually(func(g Gomega) {
cm := &corev1.ConfigMap{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "prune-protected", Namespace: ns}, cm)).To(Succeed())
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
// Switch to the SA that can for cleanup delete.
Eventually(func() error {
current := &capsulev1beta2.TenantResource{}
if err := k8sClient.Get(ctx, types.NamespacedName{Name: tr.Name, Namespace: baseNamespace}, current); err != nil {
return err
}
current.Spec.ServiceAccount = &apimeta.LocalRFC1123ObjectReference{
Name: apimeta.RFC1123Name(saNoDelete),
}
return k8sClient.Update(ctx, current)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
It("fails to replicate namespacedItems when the impersonated service account cannot read source resources", func() {
saName := "restricted-source-reader"
ensureServiceAccount(baseNamespace, saName)
source := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "source-secret",
Namespace: baseNamespace,
Labels: map[string]string{
"replicate": "true",
},
},
Type: corev1.SecretTypeOpaque,
StringData: map[string]string{"token": "abc"},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, source) }).Should(Succeed())
// Allow write into targets only, but do not grant read on source namespace secrets.
for _, ns := range targetNamespaces {
bindServiceAccountToSecretWriter(baseNamespace, saName, ns)
}
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{Name: "sa-no-namespaceditem-read", Namespace: baseNamespace},
Spec: capsulev1beta2.TenantResourceSpec{
ServiceAccount: &apimeta.LocalRFC1123ObjectReference{
Name: apimeta.RFC1123Name(saName),
},
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
ResyncPeriod: resyncPeriod,
PruningOnDelete: ptr.To(true),
Resources: []capsulev1beta2.ResourceSpec{{
NamespacedItems: []template.ResourceReference{{
VersionKind: capruntime.VersionKind{
APIVersion: "v1",
Kind: "Secret",
},
Namespace: baseNamespace,
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"replicate": "true",
},
},
}},
}},
},
},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectResolvedServiceAccount(baseNamespace, tr.Name, saName, baseNamespace)
expectTenantResourceFailed(baseNamespace, tr.Name, "forbidden")
for _, ns := range targetNamespaces {
expectSecretAbsent(ns, "source-secret")
}
})
})
Context("advanced TenantResource ownership and namespace behavior", func() {
It("fails when multiple TenantResources target the same preexisting object without adoption", func() {
By("creating the preexisting object in all target namespaces")
for _, ns := range targetNamespaces {
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "shared-no-adopt-config",
Namespace: ns,
},
Data: map[string]string{
"existing": "true",
},
}
EventuallyCreation(func() error {
cm.ResourceVersion = ""
return k8sClient.Create(ctx, cm)
}).Should(Succeed())
}
trA := newRawConfigMapTenantResource(baseNamespace, "no-adopt-shared-a", map[string]string{
"foo": "one",
})
trA.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{
RawExtension: runtime.RawExtension{
Object: &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: "shared-no-adopt-config",
},
Data: map[string]string{
"foo": "one",
},
},
},
}
trA.Spec.Settings.Adopt = ptr.To(false)
trB := newRawConfigMapTenantResource(baseNamespace, "no-adopt-shared-b", map[string]string{
"bar": "two",
})
trB.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{
RawExtension: runtime.RawExtension{
Object: &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: "shared-no-adopt-config",
},
Data: map[string]string{
"bar": "two",
},
},
},
}
trB.Spec.Settings.Adopt = ptr.To(false)
By("creating both TenantResources")
EventuallyCreation(func() error { return k8sClient.Create(ctx, trA) }).Should(Succeed())
EventuallyCreation(func() error { return k8sClient.Create(ctx, trB) }).Should(Succeed())
expectTenantResourceFailed(baseNamespace, trA.Name, "applying of 3 resources failed")
expectTenantResourceFailed(baseNamespace, trB.Name, "applying of 3 resources failed")
By("verifying the preexisting object remains unchanged")
for _, ns := range targetNamespaces {
Eventually(func(g Gomega) {
cm := &corev1.ConfigMap{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "shared-no-adopt-config", Namespace: ns}, cm)).To(Succeed())
g.Expect(cm.Data).To(Equal(map[string]string{
"existing": "true",
}))
g.Expect(cm.Labels).ToNot(HaveKey(apimeta.NewManagedByCapsuleLabel))
g.Expect(cm.Labels).ToNot(HaveKey(apimeta.CreatedByCapsuleLabel))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
})
It("forces rawItems into the current iterating tenant namespace regardless of metadata.namespace", func() {
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{
Name: "raw-target-namespace",
Namespace: baseNamespace,
},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: resyncPeriod,
Resources: []capsulev1beta2.ResourceSpec{{
RawItems: []capsulev1beta2.RawExtension{{
RawExtension: runtime.RawExtension{
Object: &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: "raw-namespace-enforced",
Namespace: "kube-system",
},
Data: map[string]string{
"source": "raw",
},
},
},
}},
}},
},
},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, tr.Name)
By("verifying the ConfigMap exists in each tenant namespace")
for _, ns := range targetNamespaces {
Eventually(func(g Gomega) {
cm := &corev1.ConfigMap{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "raw-namespace-enforced", Namespace: ns}, cm)).To(Succeed())
g.Expect(cm.Namespace).To(Equal(ns))
g.Expect(cm.Data).To(HaveKeyWithValue("source", "raw"))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
By("verifying the ConfigMap was not created in the foreign namespace")
expectConfigMapAbsent("kube-system", "raw-namespace-enforced")
})
})
Context("apply lifecycle with prune disabled", func() {
It("applies, updates and keeps objects while removing managed ownership", func() {
tr := newRawConfigMapTenantResource(baseNamespace, "raw-prune-disabled", map[string]string{"mode": "keep"})
tr.Spec.PruningOnDelete = ptr.To(false)
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, tr.Name)
for _, ns := range targetNamespaces {
expectConfigMapData(ns, "shared-config", map[string]string{"mode": "keep"})
expectManagedLabelsOnConfigMap(ns, "shared-config", true)
}
Expect(k8sClient.Delete(ctx, tr)).To(Succeed())
By("verifying the ConfigMaps remain but are no longer managed")
for _, ns := range targetNamespaces {
Eventually(func(g Gomega) {
cm := &corev1.ConfigMap{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "shared-config", Namespace: ns}, cm)).To(Succeed())
g.Expect(cm.Labels).To(HaveKeyWithValue(apimeta.CreatedByCapsuleLabel, apimeta.ValueControllerReplications))
g.Expect(cm.Labels).ToNot(HaveKey(apimeta.NewManagedByCapsuleLabel))
g.Expect(cm.Data).To(HaveKeyWithValue("mode", "keep"))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
})
})
Context("adoption", func() {
It("fails without adopt and succeeds with adopt", func() {
preexisting := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: "adopt-me", Namespace: "e2e-tenantresource-ssa-one"},
Data: map[string]string{"existing": "true"},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, preexisting) }).Should(Succeed())
withoutAdopt := newGeneratorConfigMapTenantResource(baseNamespace, "adopt-disabled", `---
apiVersion: v1
kind: ConfigMap
metadata:
name: adopt-me
data:
source: generator
namespace: "{{ $.namespace.metadata.name }}"
`)
withoutAdopt.Spec.PruningOnDelete = ptr.To(true)
withoutAdopt.Spec.Settings.Adopt = ptr.To(false)
EventuallyCreation(func() error { return k8sClient.Create(ctx, withoutAdopt) }).Should(Succeed())
expectTenantResourceFailed(baseNamespace, withoutAdopt.Name, "applying of")
expectProcessedItemStatus(baseNamespace, withoutAdopt.Name, configMapRID(tnt.Name, "e2e-tenantresource-ssa-one", "adopt-me", "0/generator-0-0"), metav1.ConditionFalse, false, "cannot be adopted")
By("recreating a second resource with adoption enabled")
withAdopt := newGeneratorConfigMapTenantResource(baseNamespace, "adopt-enabled", `---
apiVersion: v1
kind: ConfigMap
metadata:
name: adopt-me
data:
source: generator
namespace: "{{ $.namespace.metadata.name }}"
`)
withAdopt.Spec.PruningOnDelete = ptr.To(true)
withAdopt.Spec.Settings.Adopt = ptr.To(true)
EventuallyCreation(func() error { return k8sClient.Create(ctx, withAdopt) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, withAdopt.Name)
expectConfigMapData("e2e-tenantresource-ssa-one", "adopt-me", map[string]string{"source": "generator", "namespace": "e2e-tenantresource-ssa-one"})
Eventually(func(g Gomega) {
cm := &corev1.ConfigMap{}
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "adopt-me", Namespace: "e2e-tenantresource-ssa-one"}, cm)).To(Succeed())
g.Expect(cm.Labels).To(HaveKeyWithValue(apimeta.NewManagedByCapsuleLabel, apimeta.ValueControllerReplications))
g.Expect(cm.Labels).ToNot(HaveKey(apimeta.CreatedByCapsuleLabel))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
expectProcessedItemStatus(baseNamespace, withAdopt.Name, configMapRID(tnt.Name, "e2e-tenantresource-ssa-one", "adopt-me", "0/generator-0-0"), metav1.ConditionTrue, false, "")
})
})
Context("same object within one TenantResource", func() {
It("merges non-conflicting fields from generator and raw item", func() {
tr := &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{Name: "same-object-merge", Namespace: baseNamespace},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
PruningOnDelete: ptr.To(true),
ResyncPeriod: resyncPeriod,
Resources: []capsulev1beta2.ResourceSpec{{
Generators: []capsulev1beta2.TemplateItemSpec{{
MissingKey: "error",
Template: `---
apiVersion: v1
kind: ConfigMap
metadata:
name: common-config
data:
generated-{{ $.namespace.metadata.name }}: from-generator
`,
}},
RawItems: []capsulev1beta2.RawExtension{{RawExtension: runtime.RawExtension{Object: &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"},
ObjectMeta: metav1.ObjectMeta{Name: "common-config"},
Data: map[string]string{"additional-data": "raw"},
}}}},
}},
},
},
}
EventuallyCreation(func() error { return k8sClient.Create(ctx, tr) }).Should(Succeed())
expectTenantResourceReady(baseNamespace, tr.Name)
for _, ns := range targetNamespaces {
expectConfigMapData(ns, "common-config", map[string]string{
fmt.Sprintf("generated-%s", ns): "from-generator",
"additional-data": "raw",
})
}
})
})
})
func newRawConfigMapTenantResource(namespace, name string, data map[string]string) *capsulev1beta2.TenantResource {
return &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
ResyncPeriod: resyncPeriod,
PruningOnDelete: ptr.To(true),
Resources: []capsulev1beta2.ResourceSpec{{
RawItems: []capsulev1beta2.RawExtension{{RawExtension: runtime.RawExtension{Object: &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "ConfigMap"},
ObjectMeta: metav1.ObjectMeta{Name: "shared-config"},
Data: data,
}}}},
AdditionalMetadata: &api.AdditionalMetadataSpec{Labels: map[string]string{"extra-label": "set-by-tr"}},
}},
},
},
}
}
func newGeneratorConfigMapTenantResource(namespace, name, tpl string) *capsulev1beta2.TenantResource {
return &capsulev1beta2.TenantResource{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace},
Spec: capsulev1beta2.TenantResourceSpec{
TenantResourceCommonSpec: capsulev1beta2.TenantResourceCommonSpec{
ResyncPeriod: resyncPeriod,
PruningOnDelete: ptr.To(true),
Resources: []capsulev1beta2.ResourceSpec{{
Generators: []capsulev1beta2.TemplateItemSpec{{MissingKey: "zero", Template: tpl}},
}},
},
},
}
}
func getTenantResource(namespace, name string) *capsulev1beta2.TenantResource {
tr := &capsulev1beta2.TenantResource{}
Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, tr)).To(Succeed())
return tr
}
func expectTenantResourceReady(namespace, name string) {
Eventually(func(g Gomega) {
tr := &capsulev1beta2.TenantResource{}
g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, tr)).To(Succeed())
rdy := tr.Status.Conditions.GetConditionByType(apimeta.ReadyCondition)
g.Expect(rdy).ToNot(BeNil())
g.Expect(rdy.Status).To(Equal(metav1.ConditionTrue))
g.Expect(tr.Status.Size).To(BeNumerically(">", 0))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
func expectTenantResourceFailed(namespace, name, contains string) {
Eventually(func(g Gomega) {
tr := &capsulev1beta2.TenantResource{}
g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, tr)).To(Succeed())
rdy := tr.Status.Conditions.GetConditionByType(apimeta.ReadyCondition)
g.Expect(rdy).ToNot(BeNil())
g.Expect(rdy.Status).To(Equal(metav1.ConditionFalse))
g.Expect(strings.ToLower(rdy.Message)).To(ContainSubstring(strings.ToLower(contains)))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
func expectTenantResourceProcessedNamespaces(namespace, name, itemName string, expected []string) {
Eventually(func(g Gomega) {
tr := &capsulev1beta2.TenantResource{}
g.Expect(k8sClient.Get(
context.Background(),
types.NamespacedName{Name: name, Namespace: namespace},
tr,
)).To(Succeed())
ready := tr.Status.Conditions.GetConditionByType(apimeta.ReadyCondition)
g.Expect(ready).NotTo(BeNil(), "TenantResource %s/%s has no Ready condition", namespace, name)
if ready == nil {
return
}
g.Expect(ready.Status).To(
Equal(metav1.ConditionTrue),
"TenantResource %s/%s reconciliation failed: %s",
namespace,
name,
ready.Message,
)
g.Expect(tr.Status.ObservedGeneration).To(Equal(tr.Generation))
processedNamespaces := make([]string, 0, len(tr.Status.ProcessedItems))
for _, item := range tr.Status.ProcessedItems {
g.Expect(item.Name).To(Equal(itemName))
g.Expect(item.Status).To(Equal(metav1.ConditionTrue), item.Message)
processedNamespaces = append(processedNamespaces, item.Namespace)
}
g.Expect(processedNamespaces).To(ConsistOf(expected))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
func expectResolvedServiceAccount(namespace, name, saName, saNamespace string) {
Eventually(func(g Gomega) {
tr := &capsulev1beta2.TenantResource{}
g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, tr)).To(Succeed())
g.Expect(tr.Status.ServiceAccount).ToNot(BeNil())
g.Expect(tr.Status.ServiceAccount.Name).To(Equal(apimeta.RFC1123Name(saName)))
g.Expect(tr.Status.ServiceAccount.Namespace).To(Equal(apimeta.RFC1123SubdomainName(saNamespace)))
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
func cleanupTenantResourcesWithDefaultServiceAccount(ctx context.Context, namespaces ...string) {
for _, namespace := range namespaces {
Eventually(func() error {
resources := &capsulev1beta2.TenantResourceList{}
if err := k8sClient.List(ctx, resources, client.InNamespace(namespace)); err != nil {
return err
}
if len(resources.Items) == 0 {
return nil
}
for i := range resources.Items {
resource := &resources.Items[i]
if resource.Spec.ServiceAccount != nil {
resource.Spec.ServiceAccount = nil
if err := k8sClient.Update(ctx, resource); err != nil && !apierrors.IsNotFound(err) {
return err
}
continue
}
if !resource.DeletionTimestamp.IsZero() {
continue
}
if err := k8sClient.Delete(ctx, resource); err != nil && !apierrors.IsNotFound(err) {
return err
}
}
return fmt.Errorf(
"namespace %q still has %d TenantResource(s)",
namespace,
len(resources.Items),
)
}, defaultTerminationTimeoutInterval, defaultPollInterval).Should(Succeed())
}
}
func expectConfigMapData(namespace, name string, expected map[string]string) {
Eventually(func(g Gomega) {
cm := &corev1.ConfigMap{}
g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, cm)).To(
Succeed(),
"expected ConfigMap %s/%s",
namespace,
name,
)
for k, v := range expected {
g.Expect(cm.Data).To(HaveKeyWithValue(k, v))
}
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
func expectConfigMapDeleted(namespace, name string) {
Eventually(func() error {
cm := &corev1.ConfigMap{}
err := k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, cm)
return client.IgnoreNotFound(err)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
Consistently(func() bool {
cm := &corev1.ConfigMap{}
err := k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, cm)
return client.IgnoreNotFound(err) == nil
}, 3*time.Second, defaultPollInterval).Should(BeTrue())
}
func expectManagedLabelsOnConfigMap(namespace, name string, created bool) {
Eventually(func(g Gomega) {
cm := &corev1.ConfigMap{}
g.Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, cm)).To(Succeed())
g.Expect(cm.Labels).To(HaveKeyWithValue(apimeta.NewManagedByCapsuleLabel, apimeta.ValueControllerReplications))
if created {
g.Expect(cm.Labels).To(HaveKeyWithValue(apimeta.CreatedByCapsuleLabel, apimeta.ValueControllerReplications))
}
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
func expectProcessedItemApplied(namespace, trName string, rid gvk.ResourceID) {
Eventually(func(g Gomega) {
tr := getTenantResource(namespace, trName)
item := tr.Status.ProcessedItems.GetItem(rid)
g.Expect(item).ToNot(BeNil())
g.Expect(item.LastApply).ToNot(BeNil())
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
func expectProcessedItemStatus(namespace, trName string, rid gvk.ResourceID, cond metav1.ConditionStatus, created bool, msgContains string) {
Eventually(func(g Gomega) {
tr := getTenantResource(namespace, trName)
item := tr.Status.ProcessedItems.GetItem(rid)
g.Expect(item).ToNot(BeNil(), "processed item %+v not found", rid)
g.Expect(item.ObjectReferenceStatusCondition.Status).To(Equal(cond))
g.Expect(item.ObjectReferenceStatusCondition.Type).To(Equal(apimeta.ReadyCondition))
g.Expect(item.ObjectReferenceStatusCondition.Created).To(Equal(created))
if msgContains != "" {
g.Expect(strings.ToLower(item.ObjectReferenceStatusCondition.Message)).To(ContainSubstring(strings.ToLower(msgContains)))
}
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
func configMapRID(tenant, namespace, name, origin string) gvk.ResourceID {
return gvk.ResourceID{
Version: "v1",
Kind: "ConfigMap",
Name: name,
Namespace: namespace,
TenantResourceIDWithOrigin: gvk.TenantResourceIDWithOrigin{
Origin: origin,
TenantResourceID: gvk.TenantResourceID{Tenant: tenant},
},
}
}
func expectConfigMapAbsent(namespace, name string) {
Consistently(func() error {
return k8sClient.Get(context.Background(), types.NamespacedName{
Name: name,
Namespace: namespace,
}, &corev1.ConfigMap{})
}, 5*time.Second, defaultPollInterval).Should(HaveOccurred())
}
func expectSecretAbsent(namespace, name string) {
Consistently(func() error {
return k8sClient.Get(context.Background(), types.NamespacedName{
Name: name,
Namespace: namespace,
}, &corev1.Secret{})
}, 5*time.Second, defaultPollInterval).Should(HaveOccurred())
}
func expectClusterRoleAbsent(name string) {
Consistently(func() error {
return k8sClient.Get(context.Background(), types.NamespacedName{Name: name}, &rbacv1.ClusterRole{})
}, 5*time.Second, defaultPollInterval).Should(HaveOccurred())
}
func renameFirstTenantResourceRawConfigMap(tr *capsulev1beta2.TenantResource, name string) {
tr.Spec.Resources[0].RawItems[0] = capsulev1beta2.RawExtension{
RawExtension: runtime.RawExtension{
Object: &corev1.ConfigMap{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "ConfigMap",
},
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Data: tr.Spec.Resources[0].RawItems[0].RawExtension.Object.(*corev1.ConfigMap).Data,
},
},
}
}
func bindServiceAccountToNamespacedResource(
saNamespace, saName, targetNamespace string,
resources, verbs []string,
) {
ctx := context.Background()
resourceKey := strings.Join(resources, "-")
roleName := fmt.Sprintf("sa-%s-%s-%s", saName, resourceKey, targetNamespace)
roleBindingName := fmt.Sprintf("sa-%s-%s-%s-binding", saName, resourceKey, targetNamespace)
role := &rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: roleName,
Namespace: targetNamespace,
},
Rules: []rbacv1.PolicyRule{{
APIGroups: []string{""},
Resources: resources,
Verbs: verbs,
}},
}
roleBinding := &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: roleBindingName,
Namespace: targetNamespace,
},
Subjects: []rbacv1.Subject{{
Kind: "ServiceAccount",
Name: saName,
Namespace: saNamespace,
}},
RoleRef: rbacv1.RoleRef{
APIGroup: rbacv1.GroupName,
Kind: "Role",
Name: roleName,
},
}
Eventually(func() error {
current := &rbacv1.Role{}
err := k8sClient.Get(ctx, types.NamespacedName{Name: roleName, Namespace: targetNamespace}, current)
if apierrors.IsNotFound(err) {
return k8sClient.Create(ctx, role)
}
if err != nil {
return err
}
current.Rules = role.Rules
return k8sClient.Update(ctx, current)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
Eventually(func() error {
current := &rbacv1.RoleBinding{}
err := k8sClient.Get(ctx, types.NamespacedName{Name: roleBindingName, Namespace: targetNamespace}, current)
if apierrors.IsNotFound(err) {
return k8sClient.Create(ctx, roleBinding)
}
if err != nil {
return err
}
current.Subjects = roleBinding.Subjects
current.RoleRef = roleBinding.RoleRef
return k8sClient.Update(ctx, current)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
func bindServiceAccountToSecretWriter(saNamespace, saName, targetNamespace string) {
bindServiceAccountToNamespacedResource(
saNamespace,
saName,
targetNamespace,
[]string{"secrets"},
[]string{"get", "list", "watch", "create", "update", "patch"},
)
}
func bindServiceAccountToSecretReader(saNamespace, saName, targetNamespace string) {
bindServiceAccountToNamespacedResource(
saNamespace,
saName,
targetNamespace,
[]string{"secrets"},
[]string{"get", "list", "watch"},
)
}
func bindServiceAccountToConfigMapWriter(saNamespace, saName, targetNamespace string) {
bindServiceAccountToNamespacedResource(
saNamespace,
saName,
targetNamespace,
[]string{"configmaps"},
[]string{"get", "list", "watch", "create", "update", "patch"},
)
}
func bindServiceAccountToConfigMapDeleter(saNamespace, saName, targetNamespace string) {
bindServiceAccountToNamespacedResource(
saNamespace,
saName,
targetNamespace,
[]string{"configmaps"},
[]string{"get", "list", "watch", "delete"},
)
}
func bindServiceAccountToTenantResourceManager(saNamespace, saName, targetNamespace string) {
bindServiceAccountToNamespacedResource(
saNamespace,
saName,
targetNamespace,
[]string{"configmaps", "secrets"},
[]string{"get", "list", "watch", "create", "update", "patch", "delete"},
)
}
func ensureServiceAccount(namespace, name string) {
ctx := context.Background()
sa := &corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
}
Eventually(func() error {
current := &corev1.ServiceAccount{}
err := k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, current)
if apierrors.IsNotFound(err) {
return k8sClient.Create(ctx, sa)
}
return err
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
func serviceAccountUsername(namespace, name string) string {
return fmt.Sprintf("system:serviceaccount:%s:%s", namespace, name)
}
func serviceAccountGroups(namespace string) []string {
return []string{
"system:authenticated",
"system:serviceaccounts",
fmt.Sprintf("system:serviceaccounts:%s", namespace),
}
}