mirror of
https://github.com/projectcapsule/capsule.git
synced 2026-02-14 18:09:58 +00:00
Scaffolding e2e testing (#56)
* Implementing generic e2e features * Adding changes upon e2e benchmarking
This commit is contained in:
committed by
GitHub
parent
3f5e23bf00
commit
9969864141
@@ -16,6 +16,25 @@ limitations under the License.
|
||||
|
||||
package v1alpha1
|
||||
|
||||
func (t Tenant) IsFull() bool {
|
||||
import (
|
||||
"sort"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
func (t *Tenant) IsFull() bool {
|
||||
return t.Status.Namespaces.Len() >= int(t.Spec.NamespaceQuota)
|
||||
}
|
||||
|
||||
func (t *Tenant) AssignNamespaces(namespaces []corev1.Namespace) {
|
||||
var l []string
|
||||
for _, ns := range namespaces {
|
||||
if ns.Status.Phase == corev1.NamespaceActive {
|
||||
l = append(l, ns.GetName())
|
||||
}
|
||||
}
|
||||
sort.Strings(l)
|
||||
|
||||
t.Status.Namespaces = l
|
||||
t.Status.Size = uint(len(l))
|
||||
}
|
||||
|
||||
@@ -1,182 +0,0 @@
|
||||
/*
|
||||
Copyright 2020 Clastix Labs.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/util/retry"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/builder"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/event"
|
||||
"sigs.k8s.io/controller-runtime/pkg/predicate"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
"github.com/clastix/capsule/api/v1alpha1"
|
||||
)
|
||||
|
||||
type NamespaceReconciler struct {
|
||||
client.Client
|
||||
Log logr.Logger
|
||||
Scheme *runtime.Scheme
|
||||
}
|
||||
|
||||
func (r *NamespaceReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
For(&corev1.Namespace{}, builder.WithPredicates(predicate.Funcs{
|
||||
CreateFunc: func(event event.CreateEvent) (ok bool) {
|
||||
ok, _ = getCapsuleReference(event.Meta.GetOwnerReferences())
|
||||
return
|
||||
},
|
||||
DeleteFunc: func(deleteEvent event.DeleteEvent) (ok bool) {
|
||||
ok, _ = getCapsuleReference(deleteEvent.Meta.GetOwnerReferences())
|
||||
return
|
||||
},
|
||||
UpdateFunc: func(updateEvent event.UpdateEvent) (ok bool) {
|
||||
ok, _ = getCapsuleReference(updateEvent.MetaNew.GetOwnerReferences())
|
||||
return
|
||||
},
|
||||
GenericFunc: func(genericEvent event.GenericEvent) (ok bool) {
|
||||
ok, _ = getCapsuleReference(genericEvent.Meta.GetOwnerReferences())
|
||||
return
|
||||
},
|
||||
})).
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
func getCapsuleReference(refs []v1.OwnerReference) (ok bool, reference *v1.OwnerReference) {
|
||||
for _, r := range refs {
|
||||
if r.APIVersion == v1alpha1.GroupVersion.String() {
|
||||
return true, r.DeepCopy()
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (r *NamespaceReconciler) removeNamespace(name string, tenant *v1alpha1.Tenant) {
|
||||
c := tenant.Status.Namespaces.DeepCopy()
|
||||
sort.Sort(c)
|
||||
i := sort.SearchStrings(c, name)
|
||||
// namespace already removed, do nothing
|
||||
if i > c.Len() || i == c.Len() {
|
||||
r.Log.Info("Namespace has been already removed")
|
||||
return
|
||||
}
|
||||
// namespace is there, removing it
|
||||
r.Log.Info("Removing Namespace from Tenant status")
|
||||
tenant.Status.Namespaces = []string{}
|
||||
tenant.Status.Namespaces = append(tenant.Status.Namespaces, c[:i]...)
|
||||
tenant.Status.Namespaces = append(tenant.Status.Namespaces, c[i+1:]...)
|
||||
}
|
||||
|
||||
func (r *NamespaceReconciler) addNamespace(name string, tenant *v1alpha1.Tenant) {
|
||||
c := tenant.Status.Namespaces.DeepCopy()
|
||||
sort.Sort(c)
|
||||
i := sort.SearchStrings(c, name)
|
||||
// namespace already there, nothing to do
|
||||
if i < c.Len() && c[i] == name {
|
||||
r.Log.Info("Namespace has been already added")
|
||||
return
|
||||
}
|
||||
// missing namespace, let's append it
|
||||
r.Log.Info("Adding Namespace to Tenant status")
|
||||
if i == 0 {
|
||||
tenant.Status.Namespaces = []string{name}
|
||||
} else {
|
||||
tenant.Status.Namespaces = v1alpha1.NamespaceList{}
|
||||
tenant.Status.Namespaces = append(tenant.Status.Namespaces, c[:i]...)
|
||||
tenant.Status.Namespaces = append(tenant.Status.Namespaces, name)
|
||||
}
|
||||
tenant.Status.Namespaces = append(tenant.Status.Namespaces, c[i:]...)
|
||||
}
|
||||
|
||||
func (r NamespaceReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
|
||||
r.Log = r.Log.WithValues("Request.Name", request.Name)
|
||||
r.Log.Info("Reconciling Namespace")
|
||||
|
||||
// Fetch the Namespace instance
|
||||
ns := &corev1.Namespace{}
|
||||
if err := r.Get(context.TODO(), request.NamespacedName, ns); err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
// Request object not found, could have been deleted after reconcile request.
|
||||
// Owned objects are automatically garbage collected. For additional cleanup logic use finalizers.
|
||||
// Return and don't requeue
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
// Error reading the object - requeue the request.
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
_, or := getCapsuleReference(ns.OwnerReferences)
|
||||
t := &v1alpha1.Tenant{}
|
||||
if err := r.Client.Get(context.TODO(), types.NamespacedName{Name: or.Name}, t); err != nil {
|
||||
// Error reading the object - requeue the request.
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
if err := r.ensureLabel(ns, t.Name); err != nil {
|
||||
r.Log.Error(err, "cannot update Namespace label")
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
if err := r.updateTenantStatus(ns, t); err != nil {
|
||||
r.Log.Error(err, "cannot update Tenant status")
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
r.Log.Info("Namespace reconciliation processed")
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
func (r *NamespaceReconciler) ensureLabel(ns *corev1.Namespace, tenantName string) error {
|
||||
capsuleLabel, err := v1alpha1.GetTypeLabel(&v1alpha1.Tenant{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ns.Labels == nil {
|
||||
ns.Labels = make(map[string]string)
|
||||
}
|
||||
tl, ok := ns.Labels[capsuleLabel]
|
||||
if !ok || tl != tenantName {
|
||||
return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
|
||||
ns.Labels[capsuleLabel] = tenantName
|
||||
return r.Client.Update(context.TODO(), ns, &client.UpdateOptions{})
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *NamespaceReconciler) updateTenantStatus(ns *corev1.Namespace, tenant *v1alpha1.Tenant) error {
|
||||
return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
|
||||
switch ns.Status.Phase {
|
||||
case corev1.NamespaceTerminating:
|
||||
r.removeNamespace(ns.Name, tenant)
|
||||
case corev1.NamespaceActive:
|
||||
r.addNamespace(ns.Name, tenant)
|
||||
}
|
||||
|
||||
return r.Client.Status().Update(context.TODO(), tenant, &client.UpdateOptions{})
|
||||
})
|
||||
}
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/selection"
|
||||
@@ -62,12 +63,12 @@ func (r *TenantReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
Complete(r)
|
||||
}
|
||||
|
||||
func (r TenantReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
|
||||
func (r TenantReconciler) Reconcile(request ctrl.Request) (result ctrl.Result, err error) {
|
||||
r.Log = r.Log.WithValues("Request.Name", request.Name)
|
||||
|
||||
// Fetch the Tenant instance
|
||||
instance := &capsulev1alpha1.Tenant{}
|
||||
err := r.Get(context.TODO(), request.NamespacedName, instance)
|
||||
err = r.Get(context.TODO(), request.NamespacedName, instance)
|
||||
if err != nil {
|
||||
if errors.IsNotFound(err) {
|
||||
r.Log.Info("Request object not found, could have been deleted after reconcile request")
|
||||
@@ -77,6 +78,13 @@ func (r TenantReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
// Ensuring all namespaces are collected
|
||||
r.Log.Info("Ensuring all Namespaces are collected")
|
||||
if err := r.collectNamespaces(instance); err != nil {
|
||||
r.Log.Error(err, "Cannot collect Namespace resources")
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
|
||||
r.Log.Info("Starting processing of Namespaces", "items", instance.Status.Namespaces.Len())
|
||||
if err := r.syncNamespaces(instance); err != nil {
|
||||
r.Log.Error(err, "Cannot sync Namespace items")
|
||||
@@ -120,7 +128,7 @@ func (r TenantReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
|
||||
}
|
||||
|
||||
r.Log.Info("Tenant reconciling completed")
|
||||
return ctrl.Result{}, nil
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
|
||||
// pruningResources is taking care of removing the no more requested sub-resources as LimitRange, ResourceQuota or
|
||||
@@ -202,8 +210,8 @@ func (r *TenantReconciler) resourceQuotasUpdate(resourceName corev1.ResourceName
|
||||
if e != nil {
|
||||
// We had an error and we mark the whole transaction as failed
|
||||
// to process it another time acording to the Tenant controller back-off factor.
|
||||
r.Log.Error(e, "Cannot update outer ResourceQuotas", "resourceName", resourceName.String())
|
||||
err = fmt.Errorf("update of outer ResourceQuota items has failed")
|
||||
r.Log.Error(err, "Cannot update outer ResourceQuotas", "resourceName", resourceName.String())
|
||||
}
|
||||
}
|
||||
return
|
||||
@@ -313,6 +321,9 @@ func (r *TenantReconciler) syncResourceQuotas(tenant *capsulev1alpha1.Tenant) er
|
||||
// restoring the default one for all the elements,
|
||||
// also for the reconciliated one.
|
||||
for i := range rql.Items {
|
||||
if rql.Items[i].Spec.Hard == nil {
|
||||
rql.Items[i].Spec.Hard = map[corev1.ResourceName]resource.Quantity{}
|
||||
}
|
||||
rql.Items[i].Spec.Hard[rn] = q.Hard[rn]
|
||||
}
|
||||
target.Spec = q
|
||||
@@ -381,7 +392,7 @@ func (r *TenantReconciler) syncLimitRanges(tenant *capsulev1alpha1.Tenant) error
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *TenantReconciler) syncNamespace(namespace string, ingressClasses []string, storageClasses []string, wg *sync.WaitGroup, channel chan error) {
|
||||
func (r *TenantReconciler) syncNamespace(namespace string, ingressClasses []string, storageClasses []string, tenantLabel string, wg *sync.WaitGroup, channel chan error) {
|
||||
defer wg.Done()
|
||||
|
||||
t := &corev1.Namespace{}
|
||||
@@ -395,6 +406,14 @@ func (r *TenantReconciler) syncNamespace(namespace string, ingressClasses []stri
|
||||
}
|
||||
t.Annotations[capsulev1alpha1.AvailableIngressClassesAnnotation] = strings.Join(ingressClasses, ",")
|
||||
t.Annotations[capsulev1alpha1.AvailableStorageClassesAnnotation] = strings.Join(storageClasses, ",")
|
||||
if t.Labels == nil {
|
||||
t.Labels = make(map[string]string)
|
||||
}
|
||||
capsuleLabel, err := capsulev1alpha1.GetTypeLabel(&capsulev1alpha1.Tenant{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
t.Labels[capsuleLabel] = tenantLabel
|
||||
return r.Client.Update(context.TODO(), t, &client.UpdateOptions{})
|
||||
})
|
||||
}
|
||||
@@ -407,7 +426,7 @@ func (r *TenantReconciler) syncNamespaces(tenant *capsulev1alpha1.Tenant) (err e
|
||||
wg.Add(tenant.Status.Namespaces.Len())
|
||||
|
||||
for _, ns := range tenant.Status.Namespaces {
|
||||
go r.syncNamespace(ns, tenant.Spec.IngressClasses, tenant.Spec.StorageClasses, wg, ch)
|
||||
go r.syncNamespace(ns, tenant.Spec.IngressClasses, tenant.Spec.StorageClasses, tenant.GetName(), wg, ch)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
@@ -565,6 +584,26 @@ func (r *TenantReconciler) ensureNodeSelector(tenant *capsulev1alpha1.Tenant) (e
|
||||
func (r *TenantReconciler) ensureNamespaceCount(tenant *capsulev1alpha1.Tenant) error {
|
||||
return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
|
||||
tenant.Status.Size = uint(tenant.Status.Namespaces.Len())
|
||||
return r.Client.Status().Update(context.TODO(), tenant, &client.UpdateOptions{})
|
||||
found := &capsulev1alpha1.Tenant{}
|
||||
if err := r.Client.Get(context.TODO(), types.NamespacedName{Name: tenant.GetName()}, found); err != nil {
|
||||
return err
|
||||
}
|
||||
found.Status.Size = tenant.Status.Size
|
||||
return r.Client.Status().Update(context.TODO(), found, &client.UpdateOptions{})
|
||||
})
|
||||
}
|
||||
|
||||
func (r *TenantReconciler) collectNamespaces(tenant *capsulev1alpha1.Tenant) (err error) {
|
||||
nl := &corev1.NamespaceList{}
|
||||
err = r.Client.List(context.TODO(), nl, client.MatchingFieldsSelector{
|
||||
Selector: fields.OneTermEqualSelector(".metadata.ownerReferences[*].capsule", tenant.GetName()),
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
tenant.AssignNamespaces(nl.Items)
|
||||
_, err = controllerutil.CreateOrUpdate(context.TODO(), r.Client, tenant.DeepCopy(), func() error {
|
||||
return r.Client.Status().Update(context.TODO(), tenant, &client.UpdateOptions{})
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
134
e2e/ingress_class_test.go
Normal file
134
e2e/ingress_class_test.go
Normal file
@@ -0,0 +1,134 @@
|
||||
//+build e2e
|
||||
|
||||
/*
|
||||
Copyright 2020 Clastix Labs.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
v1beta12 "k8s.io/api/extensions/v1beta1"
|
||||
networkingv1 "k8s.io/api/networking/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/utils/pointer"
|
||||
|
||||
"github.com/clastix/capsule/api/v1alpha1"
|
||||
)
|
||||
|
||||
var _ = Describe("when Tenant handles Ingress classes", func() {
|
||||
tnt := &v1alpha1.Tenant{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "ingress-class",
|
||||
},
|
||||
Spec: v1alpha1.TenantSpec{
|
||||
Owner: "ingress",
|
||||
StorageClasses: []string{},
|
||||
IngressClasses: []string{
|
||||
"nginx",
|
||||
"haproxy",
|
||||
},
|
||||
LimitRanges: []corev1.LimitRangeSpec{},
|
||||
NamespaceQuota: 3,
|
||||
NodeSelector: map[string]string{},
|
||||
NetworkPolicies: []networkingv1.NetworkPolicySpec{},
|
||||
ResourceQuota: []corev1.ResourceQuotaSpec{},
|
||||
},
|
||||
}
|
||||
JustBeforeEach(func() {
|
||||
tnt.ResourceVersion = ""
|
||||
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
|
||||
})
|
||||
JustAfterEach(func() {
|
||||
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
|
||||
})
|
||||
It("should block non allowed Ingress class", func() {
|
||||
ns := NewNamespace("ingress-class-disallowed")
|
||||
cs := ownerClient(tnt)
|
||||
|
||||
NamespaceCreationShouldSucceed(ns, tnt)
|
||||
NamespaceShouldBeManagedByTenant(ns, tnt)
|
||||
|
||||
By("non-specifying the class", func() {
|
||||
Eventually(func() (err error) {
|
||||
i := &v1beta12.Ingress{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "denied-ingress",
|
||||
},
|
||||
Spec: v1beta12.IngressSpec{
|
||||
Backend: &v1beta12.IngressBackend{
|
||||
ServiceName: "foo",
|
||||
ServicePort: intstr.FromInt(8080),
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err = cs.ExtensionsV1beta1().Ingresses(ns.GetName()).Create(context.TODO(), i, metav1.CreateOptions{})
|
||||
return
|
||||
}, 30*time.Second, time.Second).ShouldNot(Succeed())
|
||||
})
|
||||
By("specifying a forbidden class", func() {
|
||||
Eventually(func() (err error) {
|
||||
i := &v1beta12.Ingress{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "denied-ingress",
|
||||
},
|
||||
Spec: v1beta12.IngressSpec{
|
||||
IngressClassName: pointer.StringPtr("the-worst-ingress-available"),
|
||||
Backend: &v1beta12.IngressBackend{
|
||||
ServiceName: "foo",
|
||||
ServicePort: intstr.FromInt(8080),
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err = cs.ExtensionsV1beta1().Ingresses(ns.GetName()).Create(context.TODO(), i, metav1.CreateOptions{})
|
||||
return
|
||||
}, 30*time.Second, time.Second).ShouldNot(Succeed())
|
||||
})
|
||||
})
|
||||
It("should allow enabled Ingress class", func() {
|
||||
ns := NewNamespace("ingress-class-allowed")
|
||||
cs := ownerClient(tnt)
|
||||
|
||||
NamespaceCreationShouldSucceed(ns, tnt)
|
||||
NamespaceShouldBeManagedByTenant(ns, tnt)
|
||||
|
||||
By("using an available class", func() {
|
||||
for _, c := range tnt.Spec.IngressClasses {
|
||||
Eventually(func() (err error) {
|
||||
i := &v1beta12.Ingress{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: c,
|
||||
},
|
||||
Spec: v1beta12.IngressSpec{
|
||||
IngressClassName: pointer.StringPtr(c),
|
||||
Backend: &v1beta12.IngressBackend{
|
||||
ServiceName: "foo",
|
||||
ServicePort: intstr.FromInt(8080),
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err = cs.ExtensionsV1beta1().Ingresses(ns.GetName()).Create(context.TODO(), i, metav1.CreateOptions{})
|
||||
return
|
||||
}, 30*time.Second, time.Second).Should(Succeed())
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
43
e2e/missing_tenant_test.go
Normal file
43
e2e/missing_tenant_test.go
Normal file
@@ -0,0 +1,43 @@
|
||||
//+build e2e
|
||||
|
||||
/*
|
||||
Copyright 2020 Clastix Labs.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/clastix/capsule/api/v1alpha1"
|
||||
)
|
||||
|
||||
var _ = Describe("Namespace creation with no Tenant assigned", func() {
|
||||
It("should fail", func() {
|
||||
tnt := &v1alpha1.Tenant{
|
||||
Spec: v1alpha1.TenantSpec{
|
||||
Owner: "missing",
|
||||
},
|
||||
}
|
||||
ns := NewNamespace("no-namespace")
|
||||
cs := ownerClient(tnt)
|
||||
_, err := cs.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
|
||||
Expect(err).ShouldNot(Succeed())
|
||||
})
|
||||
})
|
||||
58
e2e/new_namespace_test.go
Normal file
58
e2e/new_namespace_test.go
Normal file
@@ -0,0 +1,58 @@
|
||||
//+build e2e
|
||||
|
||||
/*
|
||||
Copyright 2020 Clastix Labs.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/clastix/capsule/api/v1alpha1"
|
||||
)
|
||||
|
||||
var _ = Describe("creating a Namespace as Tenant owner", func() {
|
||||
tnt := &v1alpha1.Tenant{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "tenant-assigned",
|
||||
},
|
||||
Spec: v1alpha1.TenantSpec{
|
||||
Owner: "alice",
|
||||
StorageClasses: []string{},
|
||||
IngressClasses: []string{},
|
||||
LimitRanges: []corev1.LimitRangeSpec{},
|
||||
NamespaceQuota: 10,
|
||||
NodeSelector: map[string]string{},
|
||||
ResourceQuota: []corev1.ResourceQuotaSpec{},
|
||||
},
|
||||
}
|
||||
JustBeforeEach(func() {
|
||||
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
|
||||
})
|
||||
JustAfterEach(func() {
|
||||
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
|
||||
})
|
||||
It("should be available in Tenant namespaces list", func() {
|
||||
ns := NewNamespace("new-namespace")
|
||||
NamespaceCreationShouldSucceed(ns, tnt)
|
||||
NamespaceShouldBeManagedByTenant(ns, tnt)
|
||||
})
|
||||
})
|
||||
68
e2e/overquota_namespace_test.go
Normal file
68
e2e/overquota_namespace_test.go
Normal file
@@ -0,0 +1,68 @@
|
||||
//+build e2e
|
||||
|
||||
/*
|
||||
Copyright 2020 Clastix Labs.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/clastix/capsule/api/v1alpha1"
|
||||
)
|
||||
|
||||
var _ = Describe("creating a Namespace over-quota", func() {
|
||||
tnt := &v1alpha1.Tenant{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "overquota-tenant",
|
||||
},
|
||||
Spec: v1alpha1.TenantSpec{
|
||||
Owner: "bob",
|
||||
StorageClasses: []string{},
|
||||
IngressClasses: []string{},
|
||||
LimitRanges: []corev1.LimitRangeSpec{},
|
||||
NamespaceQuota: 3,
|
||||
NodeSelector: map[string]string{},
|
||||
ResourceQuota: []corev1.ResourceQuotaSpec{},
|
||||
},
|
||||
}
|
||||
JustBeforeEach(func() {
|
||||
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
|
||||
})
|
||||
JustAfterEach(func() {
|
||||
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
|
||||
})
|
||||
It("should fail", func() {
|
||||
By("creating three Namespaces", func() {
|
||||
for _, name := range []string{"bob-dev", "bob-staging", "bob-production"} {
|
||||
ns := NewNamespace(name)
|
||||
NamespaceCreationShouldSucceed(ns, tnt)
|
||||
NamespaceShouldBeManagedByTenant(ns, tnt)
|
||||
}
|
||||
})
|
||||
|
||||
ns := NewNamespace("bob-fail")
|
||||
cs := ownerClient(tnt)
|
||||
_, err := cs.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
|
||||
Expect(err).ShouldNot(Succeed())
|
||||
|
||||
})
|
||||
})
|
||||
213
e2e/owner_webhooks_test.go
Normal file
213
e2e/owner_webhooks_test.go
Normal file
@@ -0,0 +1,213 @@
|
||||
//+build e2e
|
||||
|
||||
/*
|
||||
Copyright 2020 Clastix Labs.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
networkingv1 "k8s.io/api/networking/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
|
||||
"github.com/clastix/capsule/api/v1alpha1"
|
||||
)
|
||||
|
||||
var _ = Describe("when Tenant owner interacts with the webhooks", func() {
|
||||
tnt := &v1alpha1.Tenant{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "tenant-owner",
|
||||
},
|
||||
Spec: v1alpha1.TenantSpec{
|
||||
Owner: "ruby",
|
||||
StorageClasses: []string{
|
||||
"cephfs",
|
||||
"glusterfs",
|
||||
},
|
||||
IngressClasses: []string{},
|
||||
LimitRanges: []corev1.LimitRangeSpec{
|
||||
{
|
||||
Limits: []corev1.LimitRangeItem{
|
||||
{
|
||||
Type: corev1.LimitTypePod,
|
||||
Min: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("50m"),
|
||||
corev1.ResourceMemory: resource.MustParse("5Mi"),
|
||||
},
|
||||
Max: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
NamespaceQuota: 3,
|
||||
NodeSelector: map[string]string{},
|
||||
NetworkPolicies: []networkingv1.NetworkPolicySpec{
|
||||
{
|
||||
Egress: []networkingv1.NetworkPolicyEgressRule{
|
||||
{
|
||||
To: []networkingv1.NetworkPolicyPeer{
|
||||
{
|
||||
IPBlock: &networkingv1.IPBlock{
|
||||
CIDR: "0.0.0.0/0",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
PodSelector: metav1.LabelSelector{},
|
||||
PolicyTypes: []networkingv1.PolicyType{
|
||||
networkingv1.PolicyTypeIngress,
|
||||
networkingv1.PolicyTypeEgress,
|
||||
},
|
||||
},
|
||||
},
|
||||
ResourceQuota: []corev1.ResourceQuotaSpec{
|
||||
{
|
||||
Hard: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourcePods: resource.MustParse("10"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
JustBeforeEach(func() {
|
||||
tnt.ResourceVersion = ""
|
||||
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
|
||||
})
|
||||
JustAfterEach(func() {
|
||||
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
|
||||
})
|
||||
It("should disallow deletions", func() {
|
||||
By("blocking Capsule Limit ranges", func() {
|
||||
ns := NewNamespace("limit-range-disallow")
|
||||
NamespaceCreationShouldSucceed(ns, tnt)
|
||||
NamespaceShouldBeManagedByTenant(ns, tnt)
|
||||
|
||||
lr := &corev1.LimitRange{}
|
||||
Eventually(func() error {
|
||||
n := fmt.Sprintf("capsule-%s-0", tnt.GetName())
|
||||
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns.GetName()}, lr)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
|
||||
cs := ownerClient(tnt)
|
||||
Expect(cs.CoreV1().LimitRanges(ns.GetName()).Delete(context.TODO(), lr.Name, metav1.DeleteOptions{})).ShouldNot(Succeed())
|
||||
})
|
||||
By("blocking Capsule Network Policy", func() {
|
||||
ns := NewNamespace("network-policy-disallow")
|
||||
NamespaceCreationShouldSucceed(ns, tnt)
|
||||
NamespaceShouldBeManagedByTenant(ns, tnt)
|
||||
|
||||
np := &networkingv1.NetworkPolicy{}
|
||||
Eventually(func() error {
|
||||
n := fmt.Sprintf("capsule-%s-0", tnt.GetName())
|
||||
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns.GetName()}, np)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
|
||||
cs := ownerClient(tnt)
|
||||
Expect(cs.NetworkingV1().NetworkPolicies(ns.GetName()).Delete(context.TODO(), np.Name, metav1.DeleteOptions{})).ShouldNot(Succeed())
|
||||
})
|
||||
By("blocking blocking Capsule Resource Quota", func() {
|
||||
ns := NewNamespace("resource-quota-disallow")
|
||||
NamespaceCreationShouldSucceed(ns, tnt)
|
||||
NamespaceShouldBeManagedByTenant(ns, tnt)
|
||||
|
||||
rq := &corev1.ResourceQuota{}
|
||||
Eventually(func() error {
|
||||
n := fmt.Sprintf("capsule-%s-0", tnt.GetName())
|
||||
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns.GetName()}, rq)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
|
||||
cs := ownerClient(tnt)
|
||||
Expect(cs.NetworkingV1().NetworkPolicies(ns.GetName()).Delete(context.TODO(), rq.Name, metav1.DeleteOptions{})).ShouldNot(Succeed())
|
||||
})
|
||||
})
|
||||
It("should allow listing", func() {
|
||||
By("Limit Range resources", func() {
|
||||
ns := NewNamespace("limit-range-list")
|
||||
NamespaceCreationShouldSucceed(ns, tnt)
|
||||
NamespaceShouldBeManagedByTenant(ns, tnt)
|
||||
|
||||
Eventually(func() (err error) {
|
||||
cs := ownerClient(tnt)
|
||||
_, err = cs.CoreV1().LimitRanges(ns.GetName()).List(context.TODO(), metav1.ListOptions{})
|
||||
return
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
})
|
||||
By("Network Policy resources", func() {
|
||||
ns := NewNamespace("network-policy-list")
|
||||
NamespaceCreationShouldSucceed(ns, tnt)
|
||||
NamespaceShouldBeManagedByTenant(ns, tnt)
|
||||
|
||||
Eventually(func() (err error) {
|
||||
cs := ownerClient(tnt)
|
||||
_, err = cs.NetworkingV1().NetworkPolicies(ns.GetName()).List(context.TODO(), metav1.ListOptions{})
|
||||
return
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
})
|
||||
By("Resource Quota resources", func() {
|
||||
ns := NewNamespace("resource-quota-list")
|
||||
NamespaceCreationShouldSucceed(ns, tnt)
|
||||
NamespaceShouldBeManagedByTenant(ns, tnt)
|
||||
|
||||
Eventually(func() (err error) {
|
||||
cs := ownerClient(tnt)
|
||||
_, err = cs.NetworkingV1().NetworkPolicies(ns.GetName()).List(context.TODO(), metav1.ListOptions{})
|
||||
return
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
})
|
||||
})
|
||||
It("should allow all actions to Tenant owner Network Policy resources", func() {
|
||||
ns := NewNamespace("network-policy-allow")
|
||||
NamespaceCreationShouldSucceed(ns, tnt)
|
||||
NamespaceShouldBeManagedByTenant(ns, tnt)
|
||||
|
||||
cs := ownerClient(tnt)
|
||||
np := &networkingv1.NetworkPolicy{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "custom-network-policy",
|
||||
},
|
||||
Spec: tnt.Spec.NetworkPolicies[0],
|
||||
}
|
||||
By("creating", func() {
|
||||
Eventually(func() (err error) {
|
||||
_, err = cs.NetworkingV1().NetworkPolicies(ns.GetName()).Create(context.TODO(), np, metav1.CreateOptions{})
|
||||
return
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
})
|
||||
By("updating", func() {
|
||||
Eventually(func() (err error) {
|
||||
np.Spec.Egress = []networkingv1.NetworkPolicyEgressRule{}
|
||||
_, err = cs.NetworkingV1().NetworkPolicies(ns.GetName()).Update(context.TODO(), np, metav1.UpdateOptions{})
|
||||
return
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
})
|
||||
By("deleting", func() {
|
||||
Eventually(func() (err error) {
|
||||
return cs.NetworkingV1().NetworkPolicies(ns.GetName()).Delete(context.TODO(), np.Name, metav1.DeleteOptions{})
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
})
|
||||
})
|
||||
})
|
||||
230
e2e/resource_quota_exceeded_test.go
Normal file
230
e2e/resource_quota_exceeded_test.go
Normal file
@@ -0,0 +1,230 @@
|
||||
//+build e2e
|
||||
|
||||
/*
|
||||
Copyright 2020 Clastix Labs.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
v1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
networkingv1 "k8s.io/api/networking/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/utils/pointer"
|
||||
|
||||
"github.com/clastix/capsule/api/v1alpha1"
|
||||
)
|
||||
|
||||
var _ = Describe("exceeding Tenant resource quota", func() {
|
||||
tnt := &v1alpha1.Tenant{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "tenant-resources-changes",
|
||||
},
|
||||
Spec: v1alpha1.TenantSpec{
|
||||
Owner: "bobby",
|
||||
StorageClasses: []string{},
|
||||
IngressClasses: []string{},
|
||||
LimitRanges: []corev1.LimitRangeSpec{
|
||||
{
|
||||
Limits: []corev1.LimitRangeItem{
|
||||
{
|
||||
Type: corev1.LimitTypePod,
|
||||
Min: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("50m"),
|
||||
corev1.ResourceMemory: resource.MustParse("5Mi"),
|
||||
},
|
||||
Max: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: corev1.LimitTypeContainer,
|
||||
Default: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("200m"),
|
||||
corev1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
},
|
||||
DefaultRequest: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("100m"),
|
||||
corev1.ResourceMemory: resource.MustParse("10Mi"),
|
||||
},
|
||||
Min: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("50m"),
|
||||
corev1.ResourceMemory: resource.MustParse("5Mi"),
|
||||
},
|
||||
Max: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: corev1.LimitTypePersistentVolumeClaim,
|
||||
Min: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceStorage: resource.MustParse("1Gi"),
|
||||
},
|
||||
Max: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceStorage: resource.MustParse("10Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
NetworkPolicies: []networkingv1.NetworkPolicySpec{},
|
||||
NamespaceQuota: 2,
|
||||
NodeSelector: map[string]string{},
|
||||
ResourceQuota: []corev1.ResourceQuotaSpec{
|
||||
{
|
||||
Hard: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceLimitsCPU: resource.MustParse("8"),
|
||||
corev1.ResourceLimitsMemory: resource.MustParse("16Gi"),
|
||||
corev1.ResourceRequestsCPU: resource.MustParse("8"),
|
||||
corev1.ResourceRequestsMemory: resource.MustParse("16Gi"),
|
||||
},
|
||||
Scopes: []corev1.ResourceQuotaScope{
|
||||
corev1.ResourceQuotaScopeNotTerminating,
|
||||
},
|
||||
},
|
||||
{
|
||||
Hard: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourcePods: resource.MustParse("10"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Hard: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceRequestsStorage: resource.MustParse("100Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
nsl := []string{"easy", "peasy"}
|
||||
JustBeforeEach(func() {
|
||||
tnt.ResourceVersion = ""
|
||||
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
|
||||
By("creating the Namespaces", func() {
|
||||
for _, i := range nsl {
|
||||
ns := NewNamespace(i)
|
||||
NamespaceCreationShouldSucceed(ns, tnt)
|
||||
NamespaceShouldBeManagedByTenant(ns, tnt)
|
||||
}
|
||||
})
|
||||
})
|
||||
JustAfterEach(func() {
|
||||
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
|
||||
})
|
||||
It("should block new Pods if limit is reached", func() {
|
||||
cs := ownerClient(tnt)
|
||||
for _, namespace := range nsl {
|
||||
Eventually(func() (err error) {
|
||||
d := &v1.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-pause",
|
||||
},
|
||||
Spec: v1.DeploymentSpec{
|
||||
Replicas: pointer.Int32Ptr(5),
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"app": "pause",
|
||||
},
|
||||
},
|
||||
Template: corev1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"app": "pause",
|
||||
},
|
||||
},
|
||||
Spec: corev1.PodSpec{
|
||||
Containers: []corev1.Container{
|
||||
{
|
||||
Name: "my-pause",
|
||||
Image: "gcr.io/google_containers/pause-amd64:3.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err = cs.AppsV1().Deployments(namespace).Create(context.TODO(), d, metav1.CreateOptions{})
|
||||
return
|
||||
}, 15*time.Second, time.Second).Should(Succeed())
|
||||
}
|
||||
for _, ns := range nsl {
|
||||
n := fmt.Sprintf("capsule-%s-1", tnt.GetName())
|
||||
rq := &corev1.ResourceQuota{}
|
||||
By("retrieving the Resource Quota", func() {
|
||||
Eventually(func() error {
|
||||
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, rq)
|
||||
}, 15*time.Second, time.Second).Should(Succeed())
|
||||
})
|
||||
By("ensuring the status has been blocked with actual usage", func() {
|
||||
Eventually(func() corev1.ResourceList {
|
||||
_ = k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, rq)
|
||||
return rq.Status.Hard
|
||||
}, 15*time.Second, time.Second).Should(Equal(rq.Status.Used))
|
||||
})
|
||||
By("creating an exceeded Pod", func() {
|
||||
d := &v1.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "my-exceeded",
|
||||
},
|
||||
Spec: v1.DeploymentSpec{
|
||||
Replicas: pointer.Int32Ptr(5),
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"app": "exceeded",
|
||||
},
|
||||
},
|
||||
Template: corev1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"app": "exceeded",
|
||||
},
|
||||
},
|
||||
Spec: corev1.PodSpec{
|
||||
Containers: []corev1.Container{
|
||||
{
|
||||
Name: "my-exceeded",
|
||||
Image: "gcr.io/google_containers/pause-amd64:3.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err := cs.AppsV1().Deployments(ns).Create(context.TODO(), d, metav1.CreateOptions{})
|
||||
Expect(err).Should(Succeed())
|
||||
Eventually(func() (condition *v1.DeploymentCondition) {
|
||||
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: d.GetName(), Namespace: ns}, d)).Should(Succeed())
|
||||
for _, i := range d.Status.Conditions {
|
||||
if i.Type == v1.DeploymentReplicaFailure {
|
||||
condition = &i
|
||||
break
|
||||
}
|
||||
}
|
||||
return
|
||||
}, 30*time.Second, time.Second).ShouldNot(BeNil())
|
||||
})
|
||||
}
|
||||
})
|
||||
})
|
||||
70
e2e/selecting_non_owned_tenant_test.go
Normal file
70
e2e/selecting_non_owned_tenant_test.go
Normal file
@@ -0,0 +1,70 @@
|
||||
//+build e2e
|
||||
|
||||
/*
|
||||
Copyright 2020 Clastix Labs.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/clastix/capsule/api/v1alpha1"
|
||||
)
|
||||
|
||||
var _ = Describe("creating a Namespace trying to select a third Tenant", func() {
|
||||
tnt := &v1alpha1.Tenant{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "tenant-non-owned",
|
||||
},
|
||||
Spec: v1alpha1.TenantSpec{
|
||||
Owner: "undefined",
|
||||
StorageClasses: []string{},
|
||||
IngressClasses: []string{},
|
||||
LimitRanges: []corev1.LimitRangeSpec{},
|
||||
NamespaceQuota: 10,
|
||||
NodeSelector: map[string]string{},
|
||||
ResourceQuota: []corev1.ResourceQuotaSpec{},
|
||||
},
|
||||
}
|
||||
JustBeforeEach(func() {
|
||||
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
|
||||
})
|
||||
JustAfterEach(func() {
|
||||
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
|
||||
})
|
||||
It("should fail", func() {
|
||||
var ns *corev1.Namespace
|
||||
|
||||
By("assigning to the Namespace the Capsule Tenant label", func() {
|
||||
l, err := v1alpha1.GetTypeLabel(&v1alpha1.Tenant{})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
ns := NewNamespace("tenant-non-owned-ns")
|
||||
ns.SetLabels(map[string]string{
|
||||
l: tnt.Name,
|
||||
})
|
||||
})
|
||||
|
||||
cs := ownerClient(&v1alpha1.Tenant{Spec: v1alpha1.TenantSpec{Owner: "dale"}})
|
||||
_, err := cs.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
|
||||
Expect(err).ShouldNot(Succeed())
|
||||
})
|
||||
})
|
||||
81
e2e/selecting_tenant_test.go
Normal file
81
e2e/selecting_tenant_test.go
Normal file
@@ -0,0 +1,81 @@
|
||||
//+build e2e
|
||||
|
||||
/*
|
||||
Copyright 2020 Clastix Labs.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/clastix/capsule/api/v1alpha1"
|
||||
)
|
||||
|
||||
var _ = Describe("creating a Namespace with Tenant selector", func() {
|
||||
t1 := &v1alpha1.Tenant{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "tenant-one",
|
||||
},
|
||||
Spec: v1alpha1.TenantSpec{
|
||||
Owner: "john",
|
||||
StorageClasses: []string{},
|
||||
IngressClasses: []string{},
|
||||
LimitRanges: []corev1.LimitRangeSpec{},
|
||||
NamespaceQuota: 10,
|
||||
NodeSelector: map[string]string{},
|
||||
ResourceQuota: []corev1.ResourceQuotaSpec{},
|
||||
},
|
||||
}
|
||||
t2 := &v1alpha1.Tenant{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "tenant-two",
|
||||
},
|
||||
Spec: v1alpha1.TenantSpec{
|
||||
Owner: "john",
|
||||
StorageClasses: []string{},
|
||||
IngressClasses: []string{},
|
||||
LimitRanges: []corev1.LimitRangeSpec{},
|
||||
NamespaceQuota: 10,
|
||||
NodeSelector: map[string]string{},
|
||||
ResourceQuota: []corev1.ResourceQuotaSpec{},
|
||||
},
|
||||
}
|
||||
JustBeforeEach(func() {
|
||||
Expect(k8sClient.Create(context.TODO(), t1)).Should(Succeed())
|
||||
Expect(k8sClient.Create(context.TODO(), t2)).Should(Succeed())
|
||||
})
|
||||
JustAfterEach(func() {
|
||||
Expect(k8sClient.Delete(context.TODO(), t1)).Should(Succeed())
|
||||
Expect(k8sClient.Delete(context.TODO(), t2)).Should(Succeed())
|
||||
})
|
||||
It("should be assigned to the selected Tenant", func() {
|
||||
ns := NewNamespace("tenant-2-ns")
|
||||
By("assigning to the Namespace the Capsule Tenant label", func() {
|
||||
l, err := v1alpha1.GetTypeLabel(&v1alpha1.Tenant{})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
ns.Labels = map[string]string{
|
||||
l: t2.Name,
|
||||
}
|
||||
})
|
||||
NamespaceCreationShouldSucceed(ns, t2)
|
||||
NamespaceShouldBeManagedByTenant(ns, t2)
|
||||
})
|
||||
})
|
||||
135
e2e/storage_class_test.go
Normal file
135
e2e/storage_class_test.go
Normal file
@@ -0,0 +1,135 @@
|
||||
//+build e2e
|
||||
|
||||
/*
|
||||
Copyright 2020 Clastix Labs.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
networkingv1 "k8s.io/api/networking/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/utils/pointer"
|
||||
|
||||
"github.com/clastix/capsule/api/v1alpha1"
|
||||
)
|
||||
|
||||
var _ = Describe("when Tenant handles Storage classes", func() {
|
||||
tnt := &v1alpha1.Tenant{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "storage-class",
|
||||
},
|
||||
Spec: v1alpha1.TenantSpec{
|
||||
Owner: "storage",
|
||||
StorageClasses: []string{
|
||||
"cephfs",
|
||||
"glusterfs",
|
||||
},
|
||||
IngressClasses: []string{},
|
||||
LimitRanges: []corev1.LimitRangeSpec{},
|
||||
NamespaceQuota: 3,
|
||||
NodeSelector: map[string]string{},
|
||||
NetworkPolicies: []networkingv1.NetworkPolicySpec{},
|
||||
ResourceQuota: []corev1.ResourceQuotaSpec{},
|
||||
},
|
||||
}
|
||||
JustBeforeEach(func() {
|
||||
tnt.ResourceVersion = ""
|
||||
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
|
||||
})
|
||||
JustAfterEach(func() {
|
||||
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
|
||||
})
|
||||
It("should block non allowed Storage Class", func() {
|
||||
ns := NewNamespace("storage-class-disallowed")
|
||||
NamespaceCreationShouldSucceed(ns, tnt)
|
||||
NamespaceShouldBeManagedByTenant(ns, tnt)
|
||||
|
||||
By("non-specifying the class", func() {
|
||||
Eventually(func() (err error) {
|
||||
cs := ownerClient(tnt)
|
||||
p := &corev1.PersistentVolumeClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "denied-pvc",
|
||||
},
|
||||
Spec: corev1.PersistentVolumeClaimSpec{
|
||||
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceStorage: resource.MustParse("3Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err = cs.CoreV1().PersistentVolumeClaims(ns.GetName()).Create(context.TODO(), p, metav1.CreateOptions{})
|
||||
return
|
||||
}, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed())
|
||||
})
|
||||
By("specifying a forbidden class", func() {
|
||||
Eventually(func() (err error) {
|
||||
cs := ownerClient(tnt)
|
||||
p := &corev1.PersistentVolumeClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "mighty-storage",
|
||||
},
|
||||
Spec: corev1.PersistentVolumeClaimSpec{
|
||||
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceStorage: resource.MustParse("3Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err = cs.CoreV1().PersistentVolumeClaims(ns.GetName()).Create(context.TODO(), p, metav1.CreateOptions{})
|
||||
return
|
||||
}, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed())
|
||||
})
|
||||
})
|
||||
It("should allow enabled Storage Class", func() {
|
||||
ns := NewNamespace("storage-class-allowed")
|
||||
cs := ownerClient(tnt)
|
||||
|
||||
NamespaceCreationShouldSucceed(ns, tnt)
|
||||
NamespaceShouldBeManagedByTenant(ns, tnt)
|
||||
|
||||
for _, c := range tnt.Spec.StorageClasses {
|
||||
Eventually(func() (err error) {
|
||||
p := &corev1.PersistentVolumeClaim{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: c,
|
||||
},
|
||||
Spec: corev1.PersistentVolumeClaimSpec{
|
||||
StorageClassName: pointer.StringPtr(c),
|
||||
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceStorage: resource.MustParse("3Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err = cs.CoreV1().PersistentVolumeClaims(ns.GetName()).Create(context.TODO(), p, metav1.CreateOptions{})
|
||||
return
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -1,3 +1,5 @@
|
||||
//+build e2e
|
||||
|
||||
/*
|
||||
Copyright 2020 Clastix Labs.
|
||||
|
||||
@@ -14,7 +16,7 @@ See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package controllers
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
@@ -22,16 +24,17 @@ import (
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/kubernetes/scheme"
|
||||
"k8s.io/client-go/rest"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client/config"
|
||||
"sigs.k8s.io/controller-runtime/pkg/envtest"
|
||||
"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
|
||||
logf "sigs.k8s.io/controller-runtime/pkg/log"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log/zap"
|
||||
|
||||
capsulev1alpha "github.com/clastix/capsule/api/v1alpha1"
|
||||
// +kubebuilder:scaffold:imports
|
||||
)
|
||||
|
||||
// These tests use Ginkgo (BDD-style Go testing framework). Refer to
|
||||
@@ -55,6 +58,9 @@ var _ = BeforeSuite(func(done Done) {
|
||||
By("bootstrapping test environment")
|
||||
testEnv = &envtest.Environment{
|
||||
CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")},
|
||||
UseExistingCluster: func(v bool) *bool {
|
||||
return &v
|
||||
}(true),
|
||||
}
|
||||
|
||||
var err error
|
||||
@@ -65,8 +71,6 @@ var _ = BeforeSuite(func(done Done) {
|
||||
err = capsulev1alpha.AddToScheme(scheme.Scheme)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
// +kubebuilder:scaffold:scheme
|
||||
|
||||
k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(k8sClient).ToNot(BeNil())
|
||||
@@ -76,6 +80,15 @@ var _ = BeforeSuite(func(done Done) {
|
||||
|
||||
var _ = AfterSuite(func() {
|
||||
By("tearing down the test environment")
|
||||
err := testEnv.Stop()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(testEnv.Stop()).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
func ownerClient(tenant *capsulev1alpha.Tenant) (cs kubernetes.Interface) {
|
||||
c, err := config.GetConfig()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
c.Impersonate.Groups = []string{capsulev1alpha.GroupVersion.Group}
|
||||
c.Impersonate.UserName = tenant.Spec.Owner
|
||||
cs, err = kubernetes.NewForConfig(c)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
return
|
||||
}
|
||||
239
e2e/tenant_resources_changes_test.go
Normal file
239
e2e/tenant_resources_changes_test.go
Normal file
@@ -0,0 +1,239 @@
|
||||
//+build e2e
|
||||
|
||||
/*
|
||||
Copyright 2020 Clastix Labs.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
networkingv1 "k8s.io/api/networking/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
"github.com/clastix/capsule/api/v1alpha1"
|
||||
)
|
||||
|
||||
var _ = Describe("changing Tenant managed Kubernetes resources", func() {
|
||||
tnt := &v1alpha1.Tenant{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "tenant-resources-changes",
|
||||
},
|
||||
Spec: v1alpha1.TenantSpec{
|
||||
Owner: "laura",
|
||||
StorageClasses: []string{},
|
||||
IngressClasses: []string{},
|
||||
LimitRanges: []corev1.LimitRangeSpec{
|
||||
{
|
||||
Limits: []corev1.LimitRangeItem{
|
||||
{
|
||||
Type: corev1.LimitTypePod,
|
||||
Min: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("50m"),
|
||||
corev1.ResourceMemory: resource.MustParse("5Mi"),
|
||||
},
|
||||
Max: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: corev1.LimitTypeContainer,
|
||||
Default: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("200m"),
|
||||
corev1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
},
|
||||
DefaultRequest: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("100m"),
|
||||
corev1.ResourceMemory: resource.MustParse("10Mi"),
|
||||
},
|
||||
Min: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("50m"),
|
||||
corev1.ResourceMemory: resource.MustParse("5Mi"),
|
||||
},
|
||||
Max: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: corev1.LimitTypePersistentVolumeClaim,
|
||||
Min: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceStorage: resource.MustParse("1Gi"),
|
||||
},
|
||||
Max: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceStorage: resource.MustParse("10Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
NetworkPolicies: []networkingv1.NetworkPolicySpec{
|
||||
{
|
||||
Ingress: []networkingv1.NetworkPolicyIngressRule{
|
||||
{
|
||||
From: []networkingv1.NetworkPolicyPeer{
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"capsule.clastix.io/tenant": "tenant-resources",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
PodSelector: &metav1.LabelSelector{},
|
||||
},
|
||||
{
|
||||
IPBlock: &networkingv1.IPBlock{
|
||||
CIDR: "192.168.0.0/12",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Egress: []networkingv1.NetworkPolicyEgressRule{
|
||||
{
|
||||
To: []networkingv1.NetworkPolicyPeer{
|
||||
{
|
||||
IPBlock: &networkingv1.IPBlock{
|
||||
CIDR: "0.0.0.0/0",
|
||||
Except: []string{
|
||||
"192.168.0.0/12",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
PodSelector: metav1.LabelSelector{},
|
||||
PolicyTypes: []networkingv1.PolicyType{
|
||||
networkingv1.PolicyTypeIngress,
|
||||
networkingv1.PolicyTypeEgress,
|
||||
},
|
||||
},
|
||||
},
|
||||
NamespaceQuota: 4,
|
||||
NodeSelector: map[string]string{
|
||||
"kubernetes.io/os": "linux",
|
||||
},
|
||||
ResourceQuota: []corev1.ResourceQuotaSpec{
|
||||
{
|
||||
Hard: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceLimitsCPU: resource.MustParse("8"),
|
||||
corev1.ResourceLimitsMemory: resource.MustParse("16Gi"),
|
||||
corev1.ResourceRequestsCPU: resource.MustParse("8"),
|
||||
corev1.ResourceRequestsMemory: resource.MustParse("16Gi"),
|
||||
},
|
||||
Scopes: []corev1.ResourceQuotaScope{
|
||||
corev1.ResourceQuotaScopeNotTerminating,
|
||||
},
|
||||
},
|
||||
{
|
||||
Hard: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourcePods: resource.MustParse("10"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Hard: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceRequestsStorage: resource.MustParse("100Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
nsl := []string{"fire", "walk", "with", "me"}
|
||||
JustBeforeEach(func() {
|
||||
tnt.ResourceVersion = ""
|
||||
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
|
||||
By("creating the Namespaces", func() {
|
||||
for _, i := range nsl {
|
||||
ns := NewNamespace(i)
|
||||
NamespaceCreationShouldSucceed(ns, tnt)
|
||||
NamespaceShouldBeManagedByTenant(ns, tnt)
|
||||
}
|
||||
})
|
||||
})
|
||||
JustAfterEach(func() {
|
||||
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
|
||||
})
|
||||
It("should reapply the original resources upon third party change", func() {
|
||||
for _, ns := range nsl {
|
||||
By("changing Limit Range resources", func() {
|
||||
for i, s := range tnt.Spec.LimitRanges {
|
||||
n := fmt.Sprintf("capsule-%s-%d", tnt.GetName(), i)
|
||||
lr := &corev1.LimitRange{}
|
||||
Eventually(func() error {
|
||||
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, lr)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
|
||||
c := lr.DeepCopy()
|
||||
c.Spec.Limits = []corev1.LimitRangeItem{}
|
||||
Expect(k8sClient.Update(context.TODO(), c, &client.UpdateOptions{})).Should(Succeed())
|
||||
|
||||
Eventually(func() corev1.LimitRangeSpec {
|
||||
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, lr)).Should(Succeed())
|
||||
return lr.Spec
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Equal(s))
|
||||
}
|
||||
})
|
||||
By("changing Network Policy resources", func() {
|
||||
for i, s := range tnt.Spec.NetworkPolicies {
|
||||
n := fmt.Sprintf("capsule-%s-%d", tnt.GetName(), i)
|
||||
np := &networkingv1.NetworkPolicy{}
|
||||
Eventually(func() error {
|
||||
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, np)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
Expect(np.Spec).Should(Equal(s))
|
||||
|
||||
c := np.DeepCopy()
|
||||
c.Spec.Egress = []networkingv1.NetworkPolicyEgressRule{}
|
||||
c.Spec.Ingress = []networkingv1.NetworkPolicyIngressRule{}
|
||||
Expect(k8sClient.Update(context.TODO(), c, &client.UpdateOptions{})).Should(Succeed())
|
||||
|
||||
Eventually(func() networkingv1.NetworkPolicySpec {
|
||||
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, np)).Should(Succeed())
|
||||
return np.Spec
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Equal(s))
|
||||
}
|
||||
})
|
||||
By("changing Resource Quota resources", func() {
|
||||
for i, s := range tnt.Spec.ResourceQuota {
|
||||
n := fmt.Sprintf("capsule-%s-%d", tnt.GetName(), i)
|
||||
rq := &corev1.ResourceQuota{}
|
||||
Eventually(func() error {
|
||||
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, rq)
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
|
||||
c := rq.DeepCopy()
|
||||
c.Spec.Hard = map[corev1.ResourceName]resource.Quantity{}
|
||||
Expect(k8sClient.Update(context.TODO(), c, &client.UpdateOptions{})).Should(Succeed())
|
||||
|
||||
Eventually(func() corev1.ResourceQuotaSpec {
|
||||
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, rq)).Should(Succeed())
|
||||
return rq.Spec
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Equal(s))
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
})
|
||||
226
e2e/tenant_resources_test.go
Normal file
226
e2e/tenant_resources_test.go
Normal file
@@ -0,0 +1,226 @@
|
||||
//+build e2e
|
||||
|
||||
/*
|
||||
Copyright 2020 Clastix Labs.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/api/networking/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
|
||||
"github.com/clastix/capsule/api/v1alpha1"
|
||||
)
|
||||
|
||||
var _ = Describe("creating namespaces within a Tenant with resources", func() {
|
||||
tnt := &v1alpha1.Tenant{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "tenant-resources",
|
||||
},
|
||||
Spec: v1alpha1.TenantSpec{
|
||||
Owner: "john",
|
||||
StorageClasses: []string{},
|
||||
IngressClasses: []string{},
|
||||
LimitRanges: []corev1.LimitRangeSpec{
|
||||
{
|
||||
Limits: []corev1.LimitRangeItem{
|
||||
{
|
||||
Type: corev1.LimitTypePod,
|
||||
Min: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("50m"),
|
||||
corev1.ResourceMemory: resource.MustParse("5Mi"),
|
||||
},
|
||||
Max: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: corev1.LimitTypeContainer,
|
||||
Default: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("200m"),
|
||||
corev1.ResourceMemory: resource.MustParse("100Mi"),
|
||||
},
|
||||
DefaultRequest: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("100m"),
|
||||
corev1.ResourceMemory: resource.MustParse("10Mi"),
|
||||
},
|
||||
Min: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("50m"),
|
||||
corev1.ResourceMemory: resource.MustParse("5Mi"),
|
||||
},
|
||||
Max: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceCPU: resource.MustParse("1"),
|
||||
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: corev1.LimitTypePersistentVolumeClaim,
|
||||
Min: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceStorage: resource.MustParse("1Gi"),
|
||||
},
|
||||
Max: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceStorage: resource.MustParse("10Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
NetworkPolicies: []v1.NetworkPolicySpec{
|
||||
{
|
||||
Ingress: []v1.NetworkPolicyIngressRule{
|
||||
{
|
||||
From: []v1.NetworkPolicyPeer{
|
||||
{
|
||||
NamespaceSelector: &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"capsule.clastix.io/tenant": "tenant-resources",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
PodSelector: &metav1.LabelSelector{},
|
||||
},
|
||||
{
|
||||
IPBlock: &v1.IPBlock{
|
||||
CIDR: "192.168.0.0/12",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Egress: []v1.NetworkPolicyEgressRule{
|
||||
{
|
||||
To: []v1.NetworkPolicyPeer{
|
||||
{
|
||||
IPBlock: &v1.IPBlock{
|
||||
CIDR: "0.0.0.0/0",
|
||||
Except: []string{
|
||||
"192.168.0.0/12",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
PodSelector: metav1.LabelSelector{},
|
||||
PolicyTypes: []v1.PolicyType{
|
||||
v1.PolicyTypeIngress,
|
||||
v1.PolicyTypeEgress,
|
||||
},
|
||||
},
|
||||
},
|
||||
NamespaceQuota: 3,
|
||||
NodeSelector: map[string]string{
|
||||
"kubernetes.io/os": "linux",
|
||||
},
|
||||
ResourceQuota: []corev1.ResourceQuotaSpec{
|
||||
{
|
||||
Hard: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceLimitsCPU: resource.MustParse("8"),
|
||||
corev1.ResourceLimitsMemory: resource.MustParse("16Gi"),
|
||||
corev1.ResourceRequestsCPU: resource.MustParse("8"),
|
||||
corev1.ResourceRequestsMemory: resource.MustParse("16Gi"),
|
||||
},
|
||||
Scopes: []corev1.ResourceQuotaScope{
|
||||
corev1.ResourceQuotaScopeNotTerminating,
|
||||
},
|
||||
},
|
||||
{
|
||||
Hard: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourcePods: resource.MustParse("10"),
|
||||
},
|
||||
},
|
||||
{
|
||||
Hard: map[corev1.ResourceName]resource.Quantity{
|
||||
corev1.ResourceRequestsStorage: resource.MustParse("100Gi"),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
nsl := []string{"bim", "bum", "bam"}
|
||||
JustBeforeEach(func() {
|
||||
tnt.ResourceVersion = ""
|
||||
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
|
||||
|
||||
By("creating the Namespaces", func() {
|
||||
for _, i := range nsl {
|
||||
ns := NewNamespace(i)
|
||||
NamespaceCreationShouldSucceed(ns, tnt)
|
||||
NamespaceShouldBeManagedByTenant(ns, tnt)
|
||||
}
|
||||
})
|
||||
})
|
||||
JustAfterEach(func() {
|
||||
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
|
||||
})
|
||||
It("should contains all replicated resources", func() {
|
||||
for _, name := range nsl {
|
||||
By("checking Limit Range resources", func() {
|
||||
for i, s := range tnt.Spec.LimitRanges {
|
||||
n := fmt.Sprintf("capsule-%s-%d", tnt.GetName(), i)
|
||||
lr := &corev1.LimitRange{}
|
||||
Eventually(func() error {
|
||||
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: name}, lr)
|
||||
}, 10*time.Second, time.Second).Should(Succeed())
|
||||
Expect(lr.Spec).Should(Equal(s))
|
||||
}
|
||||
})
|
||||
By("checking Network Policy resources", func() {
|
||||
for i, s := range tnt.Spec.NetworkPolicies {
|
||||
n := fmt.Sprintf("capsule-%s-%d", tnt.GetName(), i)
|
||||
np := &v1.NetworkPolicy{}
|
||||
Eventually(func() error {
|
||||
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: name}, np)
|
||||
}, 10*time.Second, time.Second).Should(Succeed())
|
||||
Expect(np.Spec).Should(Equal(s))
|
||||
}
|
||||
})
|
||||
By("checking the Namespace scheduler annotation", func() {
|
||||
var selector []string
|
||||
for k, v := range tnt.Spec.NodeSelector {
|
||||
selector = append(selector, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
Eventually(func() string {
|
||||
ns := &corev1.Namespace{}
|
||||
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: name}, ns)).Should(Succeed())
|
||||
return ns.GetAnnotations()["scheduler.alpha.kubernetes.io/node-selector"]
|
||||
}, 10*time.Second, time.Second).Should(Equal(strings.Join(selector, ",")))
|
||||
})
|
||||
By("checking the Resource Quota resources", func() {
|
||||
for i, s := range tnt.Spec.ResourceQuota {
|
||||
n := fmt.Sprintf("capsule-%s-%d", tnt.GetName(), i)
|
||||
rq := &corev1.ResourceQuota{}
|
||||
Eventually(func() error {
|
||||
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: name}, rq)
|
||||
}, 10*time.Second, time.Second).Should(Succeed())
|
||||
Expect(rq.Spec).Should(Equal(s))
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
})
|
||||
59
e2e/utils_test.go
Normal file
59
e2e/utils_test.go
Normal file
@@ -0,0 +1,59 @@
|
||||
//+build e2e
|
||||
|
||||
/*
|
||||
Copyright 2020 Clastix Labs.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/gomega"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
|
||||
"github.com/clastix/capsule/api/v1alpha1"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultTimeoutInterval = 10 * time.Second
|
||||
defaultPollInterval = time.Second
|
||||
)
|
||||
|
||||
func NewNamespace(name string) *corev1.Namespace {
|
||||
return &corev1.Namespace{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func NamespaceCreationShouldSucceed(ns *corev1.Namespace, t *v1alpha1.Tenant) {
|
||||
cs := ownerClient(t)
|
||||
Eventually(func() (err error) {
|
||||
_, err = cs.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
|
||||
return
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
|
||||
}
|
||||
|
||||
func NamespaceShouldBeManagedByTenant(ns *corev1.Namespace, t *v1alpha1.Tenant) {
|
||||
Eventually(func() v1alpha1.NamespaceList {
|
||||
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: t.GetName()}, t)).Should(Succeed())
|
||||
return t.Status.Namespaces
|
||||
}, defaultTimeoutInterval, defaultPollInterval).Should(ContainElement(ns.GetName()))
|
||||
}
|
||||
1
go.mod
1
go.mod
@@ -11,5 +11,6 @@ require (
|
||||
k8s.io/api v0.18.2
|
||||
k8s.io/apimachinery v0.18.2
|
||||
k8s.io/client-go v0.18.2
|
||||
k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89
|
||||
sigs.k8s.io/controller-runtime v0.6.0
|
||||
)
|
||||
|
||||
9
main.go
9
main.go
@@ -125,15 +125,6 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err = (&controllers.NamespaceReconciler{
|
||||
Client: mgr.GetClient(),
|
||||
Log: ctrl.Log.WithName("controllers").WithName("Namespace"),
|
||||
Scheme: mgr.GetScheme(),
|
||||
}).SetupWithManager(mgr); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "Namespace")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err = (&secret.CaReconciler{
|
||||
Client: mgr.GetClient(),
|
||||
Log: ctrl.Log.WithName("controllers").WithName("CA"),
|
||||
|
||||
@@ -16,8 +16,10 @@ limitations under the License.
|
||||
|
||||
package indexer
|
||||
|
||||
import "github.com/clastix/capsule/pkg/indexer/tenant"
|
||||
import (
|
||||
"github.com/clastix/capsule/pkg/indexer/namespace"
|
||||
)
|
||||
|
||||
func init() {
|
||||
AddToIndexerFuncs = append(AddToIndexerFuncs, tenant.OwnerReference{})
|
||||
AddToIndexerFuncs = append(AddToIndexerFuncs, namespace.OwnerReference{})
|
||||
}
|
||||
@@ -20,4 +20,5 @@ import "github.com/clastix/capsule/pkg/indexer/tenant"
|
||||
|
||||
func init() {
|
||||
AddToIndexerFuncs = append(AddToIndexerFuncs, tenant.NamespacesReference{})
|
||||
AddToIndexerFuncs = append(AddToIndexerFuncs, tenant.OwnerReference{})
|
||||
}
|
||||
49
pkg/indexer/namespace/namespaces.go
Normal file
49
pkg/indexer/namespace/namespaces.go
Normal file
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
Copyright 2020 Clastix Labs.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package namespace
|
||||
|
||||
import (
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
"github.com/clastix/capsule/api/v1alpha1"
|
||||
)
|
||||
|
||||
type OwnerReference struct {
|
||||
}
|
||||
|
||||
func (o OwnerReference) Object() runtime.Object {
|
||||
return &v1.Namespace{}
|
||||
}
|
||||
|
||||
func (o OwnerReference) Field() string {
|
||||
return ".metadata.ownerReferences[*].capsule"
|
||||
}
|
||||
|
||||
func (o OwnerReference) Func() client.IndexerFunc {
|
||||
return func(object runtime.Object) []string {
|
||||
var res []string
|
||||
ns := object.(*v1.Namespace)
|
||||
for _, or := range ns.OwnerReferences {
|
||||
if or.APIVersion == v1alpha1.GroupVersion.String() {
|
||||
res = append(res, or.Name)
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user