Scaffolding e2e testing (#56)

* Implementing generic e2e features

* Adding changes upon e2e benchmarking
This commit is contained in:
Dario Tranchitella
2020-08-21 14:55:48 +02:00
committed by GitHub
parent 3f5e23bf00
commit 9969864141
21 changed files with 1696 additions and 207 deletions

View File

@@ -16,6 +16,25 @@ limitations under the License.
package v1alpha1
func (t Tenant) IsFull() bool {
import (
"sort"
corev1 "k8s.io/api/core/v1"
)
func (t *Tenant) IsFull() bool {
return t.Status.Namespaces.Len() >= int(t.Spec.NamespaceQuota)
}
func (t *Tenant) AssignNamespaces(namespaces []corev1.Namespace) {
var l []string
for _, ns := range namespaces {
if ns.Status.Phase == corev1.NamespaceActive {
l = append(l, ns.GetName())
}
}
sort.Strings(l)
t.Status.Namespaces = l
t.Status.Size = uint(len(l))
}

View File

@@ -1,182 +0,0 @@
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controllers
import (
"context"
"sort"
"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/retry"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"github.com/clastix/capsule/api/v1alpha1"
)
type NamespaceReconciler struct {
client.Client
Log logr.Logger
Scheme *runtime.Scheme
}
func (r *NamespaceReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&corev1.Namespace{}, builder.WithPredicates(predicate.Funcs{
CreateFunc: func(event event.CreateEvent) (ok bool) {
ok, _ = getCapsuleReference(event.Meta.GetOwnerReferences())
return
},
DeleteFunc: func(deleteEvent event.DeleteEvent) (ok bool) {
ok, _ = getCapsuleReference(deleteEvent.Meta.GetOwnerReferences())
return
},
UpdateFunc: func(updateEvent event.UpdateEvent) (ok bool) {
ok, _ = getCapsuleReference(updateEvent.MetaNew.GetOwnerReferences())
return
},
GenericFunc: func(genericEvent event.GenericEvent) (ok bool) {
ok, _ = getCapsuleReference(genericEvent.Meta.GetOwnerReferences())
return
},
})).
Complete(r)
}
func getCapsuleReference(refs []v1.OwnerReference) (ok bool, reference *v1.OwnerReference) {
for _, r := range refs {
if r.APIVersion == v1alpha1.GroupVersion.String() {
return true, r.DeepCopy()
}
}
return false, nil
}
func (r *NamespaceReconciler) removeNamespace(name string, tenant *v1alpha1.Tenant) {
c := tenant.Status.Namespaces.DeepCopy()
sort.Sort(c)
i := sort.SearchStrings(c, name)
// namespace already removed, do nothing
if i > c.Len() || i == c.Len() {
r.Log.Info("Namespace has been already removed")
return
}
// namespace is there, removing it
r.Log.Info("Removing Namespace from Tenant status")
tenant.Status.Namespaces = []string{}
tenant.Status.Namespaces = append(tenant.Status.Namespaces, c[:i]...)
tenant.Status.Namespaces = append(tenant.Status.Namespaces, c[i+1:]...)
}
func (r *NamespaceReconciler) addNamespace(name string, tenant *v1alpha1.Tenant) {
c := tenant.Status.Namespaces.DeepCopy()
sort.Sort(c)
i := sort.SearchStrings(c, name)
// namespace already there, nothing to do
if i < c.Len() && c[i] == name {
r.Log.Info("Namespace has been already added")
return
}
// missing namespace, let's append it
r.Log.Info("Adding Namespace to Tenant status")
if i == 0 {
tenant.Status.Namespaces = []string{name}
} else {
tenant.Status.Namespaces = v1alpha1.NamespaceList{}
tenant.Status.Namespaces = append(tenant.Status.Namespaces, c[:i]...)
tenant.Status.Namespaces = append(tenant.Status.Namespaces, name)
}
tenant.Status.Namespaces = append(tenant.Status.Namespaces, c[i:]...)
}
func (r NamespaceReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
r.Log = r.Log.WithValues("Request.Name", request.Name)
r.Log.Info("Reconciling Namespace")
// Fetch the Namespace instance
ns := &corev1.Namespace{}
if err := r.Get(context.TODO(), request.NamespacedName, ns); err != nil {
if errors.IsNotFound(err) {
// Request object not found, could have been deleted after reconcile request.
// Owned objects are automatically garbage collected. For additional cleanup logic use finalizers.
// Return and don't requeue
return reconcile.Result{}, nil
}
// Error reading the object - requeue the request.
return reconcile.Result{}, err
}
_, or := getCapsuleReference(ns.OwnerReferences)
t := &v1alpha1.Tenant{}
if err := r.Client.Get(context.TODO(), types.NamespacedName{Name: or.Name}, t); err != nil {
// Error reading the object - requeue the request.
return reconcile.Result{}, err
}
if err := r.ensureLabel(ns, t.Name); err != nil {
r.Log.Error(err, "cannot update Namespace label")
return reconcile.Result{}, err
}
if err := r.updateTenantStatus(ns, t); err != nil {
r.Log.Error(err, "cannot update Tenant status")
return reconcile.Result{}, err
}
r.Log.Info("Namespace reconciliation processed")
return reconcile.Result{}, nil
}
func (r *NamespaceReconciler) ensureLabel(ns *corev1.Namespace, tenantName string) error {
capsuleLabel, err := v1alpha1.GetTypeLabel(&v1alpha1.Tenant{})
if err != nil {
return err
}
if ns.Labels == nil {
ns.Labels = make(map[string]string)
}
tl, ok := ns.Labels[capsuleLabel]
if !ok || tl != tenantName {
return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
ns.Labels[capsuleLabel] = tenantName
return r.Client.Update(context.TODO(), ns, &client.UpdateOptions{})
})
}
return nil
}
func (r *NamespaceReconciler) updateTenantStatus(ns *corev1.Namespace, tenant *v1alpha1.Tenant) error {
return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
switch ns.Status.Phase {
case corev1.NamespaceTerminating:
r.removeNamespace(ns.Name, tenant)
case corev1.NamespaceActive:
r.addNamespace(ns.Name, tenant)
}
return r.Client.Status().Update(context.TODO(), tenant, &client.UpdateOptions{})
})
}

View File

@@ -31,6 +31,7 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/selection"
@@ -62,12 +63,12 @@ func (r *TenantReconciler) SetupWithManager(mgr ctrl.Manager) error {
Complete(r)
}
func (r TenantReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
func (r TenantReconciler) Reconcile(request ctrl.Request) (result ctrl.Result, err error) {
r.Log = r.Log.WithValues("Request.Name", request.Name)
// Fetch the Tenant instance
instance := &capsulev1alpha1.Tenant{}
err := r.Get(context.TODO(), request.NamespacedName, instance)
err = r.Get(context.TODO(), request.NamespacedName, instance)
if err != nil {
if errors.IsNotFound(err) {
r.Log.Info("Request object not found, could have been deleted after reconcile request")
@@ -77,6 +78,13 @@ func (r TenantReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
return reconcile.Result{}, err
}
// Ensuring all namespaces are collected
r.Log.Info("Ensuring all Namespaces are collected")
if err := r.collectNamespaces(instance); err != nil {
r.Log.Error(err, "Cannot collect Namespace resources")
return reconcile.Result{}, err
}
r.Log.Info("Starting processing of Namespaces", "items", instance.Status.Namespaces.Len())
if err := r.syncNamespaces(instance); err != nil {
r.Log.Error(err, "Cannot sync Namespace items")
@@ -120,7 +128,7 @@ func (r TenantReconciler) Reconcile(request ctrl.Request) (ctrl.Result, error) {
}
r.Log.Info("Tenant reconciling completed")
return ctrl.Result{}, nil
return ctrl.Result{}, err
}
// pruningResources is taking care of removing the no more requested sub-resources as LimitRange, ResourceQuota or
@@ -202,8 +210,8 @@ func (r *TenantReconciler) resourceQuotasUpdate(resourceName corev1.ResourceName
if e != nil {
// We had an error and we mark the whole transaction as failed
// to process it another time acording to the Tenant controller back-off factor.
r.Log.Error(e, "Cannot update outer ResourceQuotas", "resourceName", resourceName.String())
err = fmt.Errorf("update of outer ResourceQuota items has failed")
r.Log.Error(err, "Cannot update outer ResourceQuotas", "resourceName", resourceName.String())
}
}
return
@@ -313,6 +321,9 @@ func (r *TenantReconciler) syncResourceQuotas(tenant *capsulev1alpha1.Tenant) er
// restoring the default one for all the elements,
// also for the reconciliated one.
for i := range rql.Items {
if rql.Items[i].Spec.Hard == nil {
rql.Items[i].Spec.Hard = map[corev1.ResourceName]resource.Quantity{}
}
rql.Items[i].Spec.Hard[rn] = q.Hard[rn]
}
target.Spec = q
@@ -381,7 +392,7 @@ func (r *TenantReconciler) syncLimitRanges(tenant *capsulev1alpha1.Tenant) error
return nil
}
func (r *TenantReconciler) syncNamespace(namespace string, ingressClasses []string, storageClasses []string, wg *sync.WaitGroup, channel chan error) {
func (r *TenantReconciler) syncNamespace(namespace string, ingressClasses []string, storageClasses []string, tenantLabel string, wg *sync.WaitGroup, channel chan error) {
defer wg.Done()
t := &corev1.Namespace{}
@@ -395,6 +406,14 @@ func (r *TenantReconciler) syncNamespace(namespace string, ingressClasses []stri
}
t.Annotations[capsulev1alpha1.AvailableIngressClassesAnnotation] = strings.Join(ingressClasses, ",")
t.Annotations[capsulev1alpha1.AvailableStorageClassesAnnotation] = strings.Join(storageClasses, ",")
if t.Labels == nil {
t.Labels = make(map[string]string)
}
capsuleLabel, err := capsulev1alpha1.GetTypeLabel(&capsulev1alpha1.Tenant{})
if err != nil {
return err
}
t.Labels[capsuleLabel] = tenantLabel
return r.Client.Update(context.TODO(), t, &client.UpdateOptions{})
})
}
@@ -407,7 +426,7 @@ func (r *TenantReconciler) syncNamespaces(tenant *capsulev1alpha1.Tenant) (err e
wg.Add(tenant.Status.Namespaces.Len())
for _, ns := range tenant.Status.Namespaces {
go r.syncNamespace(ns, tenant.Spec.IngressClasses, tenant.Spec.StorageClasses, wg, ch)
go r.syncNamespace(ns, tenant.Spec.IngressClasses, tenant.Spec.StorageClasses, tenant.GetName(), wg, ch)
}
wg.Wait()
@@ -565,6 +584,26 @@ func (r *TenantReconciler) ensureNodeSelector(tenant *capsulev1alpha1.Tenant) (e
func (r *TenantReconciler) ensureNamespaceCount(tenant *capsulev1alpha1.Tenant) error {
return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
tenant.Status.Size = uint(tenant.Status.Namespaces.Len())
return r.Client.Status().Update(context.TODO(), tenant, &client.UpdateOptions{})
found := &capsulev1alpha1.Tenant{}
if err := r.Client.Get(context.TODO(), types.NamespacedName{Name: tenant.GetName()}, found); err != nil {
return err
}
found.Status.Size = tenant.Status.Size
return r.Client.Status().Update(context.TODO(), found, &client.UpdateOptions{})
})
}
func (r *TenantReconciler) collectNamespaces(tenant *capsulev1alpha1.Tenant) (err error) {
nl := &corev1.NamespaceList{}
err = r.Client.List(context.TODO(), nl, client.MatchingFieldsSelector{
Selector: fields.OneTermEqualSelector(".metadata.ownerReferences[*].capsule", tenant.GetName()),
})
if err != nil {
return
}
tenant.AssignNamespaces(nl.Items)
_, err = controllerutil.CreateOrUpdate(context.TODO(), r.Client, tenant.DeepCopy(), func() error {
return r.Client.Status().Update(context.TODO(), tenant, &client.UpdateOptions{})
})
return
}

134
e2e/ingress_class_test.go Normal file
View File

@@ -0,0 +1,134 @@
//+build e2e
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"context"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
v1beta12 "k8s.io/api/extensions/v1beta1"
networkingv1 "k8s.io/api/networking/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/utils/pointer"
"github.com/clastix/capsule/api/v1alpha1"
)
var _ = Describe("when Tenant handles Ingress classes", func() {
tnt := &v1alpha1.Tenant{
ObjectMeta: metav1.ObjectMeta{
Name: "ingress-class",
},
Spec: v1alpha1.TenantSpec{
Owner: "ingress",
StorageClasses: []string{},
IngressClasses: []string{
"nginx",
"haproxy",
},
LimitRanges: []corev1.LimitRangeSpec{},
NamespaceQuota: 3,
NodeSelector: map[string]string{},
NetworkPolicies: []networkingv1.NetworkPolicySpec{},
ResourceQuota: []corev1.ResourceQuotaSpec{},
},
}
JustBeforeEach(func() {
tnt.ResourceVersion = ""
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
})
JustAfterEach(func() {
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
})
It("should block non allowed Ingress class", func() {
ns := NewNamespace("ingress-class-disallowed")
cs := ownerClient(tnt)
NamespaceCreationShouldSucceed(ns, tnt)
NamespaceShouldBeManagedByTenant(ns, tnt)
By("non-specifying the class", func() {
Eventually(func() (err error) {
i := &v1beta12.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "denied-ingress",
},
Spec: v1beta12.IngressSpec{
Backend: &v1beta12.IngressBackend{
ServiceName: "foo",
ServicePort: intstr.FromInt(8080),
},
},
}
_, err = cs.ExtensionsV1beta1().Ingresses(ns.GetName()).Create(context.TODO(), i, metav1.CreateOptions{})
return
}, 30*time.Second, time.Second).ShouldNot(Succeed())
})
By("specifying a forbidden class", func() {
Eventually(func() (err error) {
i := &v1beta12.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: "denied-ingress",
},
Spec: v1beta12.IngressSpec{
IngressClassName: pointer.StringPtr("the-worst-ingress-available"),
Backend: &v1beta12.IngressBackend{
ServiceName: "foo",
ServicePort: intstr.FromInt(8080),
},
},
}
_, err = cs.ExtensionsV1beta1().Ingresses(ns.GetName()).Create(context.TODO(), i, metav1.CreateOptions{})
return
}, 30*time.Second, time.Second).ShouldNot(Succeed())
})
})
It("should allow enabled Ingress class", func() {
ns := NewNamespace("ingress-class-allowed")
cs := ownerClient(tnt)
NamespaceCreationShouldSucceed(ns, tnt)
NamespaceShouldBeManagedByTenant(ns, tnt)
By("using an available class", func() {
for _, c := range tnt.Spec.IngressClasses {
Eventually(func() (err error) {
i := &v1beta12.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: c,
},
Spec: v1beta12.IngressSpec{
IngressClassName: pointer.StringPtr(c),
Backend: &v1beta12.IngressBackend{
ServiceName: "foo",
ServicePort: intstr.FromInt(8080),
},
},
}
_, err = cs.ExtensionsV1beta1().Ingresses(ns.GetName()).Create(context.TODO(), i, metav1.CreateOptions{})
return
}, 30*time.Second, time.Second).Should(Succeed())
}
})
})
})

View File

@@ -0,0 +1,43 @@
//+build e2e
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"context"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/clastix/capsule/api/v1alpha1"
)
var _ = Describe("Namespace creation with no Tenant assigned", func() {
It("should fail", func() {
tnt := &v1alpha1.Tenant{
Spec: v1alpha1.TenantSpec{
Owner: "missing",
},
}
ns := NewNamespace("no-namespace")
cs := ownerClient(tnt)
_, err := cs.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
Expect(err).ShouldNot(Succeed())
})
})

58
e2e/new_namespace_test.go Normal file
View File

@@ -0,0 +1,58 @@
//+build e2e
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"context"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/clastix/capsule/api/v1alpha1"
)
var _ = Describe("creating a Namespace as Tenant owner", func() {
tnt := &v1alpha1.Tenant{
ObjectMeta: metav1.ObjectMeta{
Name: "tenant-assigned",
},
Spec: v1alpha1.TenantSpec{
Owner: "alice",
StorageClasses: []string{},
IngressClasses: []string{},
LimitRanges: []corev1.LimitRangeSpec{},
NamespaceQuota: 10,
NodeSelector: map[string]string{},
ResourceQuota: []corev1.ResourceQuotaSpec{},
},
}
JustBeforeEach(func() {
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
})
JustAfterEach(func() {
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
})
It("should be available in Tenant namespaces list", func() {
ns := NewNamespace("new-namespace")
NamespaceCreationShouldSucceed(ns, tnt)
NamespaceShouldBeManagedByTenant(ns, tnt)
})
})

View File

@@ -0,0 +1,68 @@
//+build e2e
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"context"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/clastix/capsule/api/v1alpha1"
)
var _ = Describe("creating a Namespace over-quota", func() {
tnt := &v1alpha1.Tenant{
ObjectMeta: metav1.ObjectMeta{
Name: "overquota-tenant",
},
Spec: v1alpha1.TenantSpec{
Owner: "bob",
StorageClasses: []string{},
IngressClasses: []string{},
LimitRanges: []corev1.LimitRangeSpec{},
NamespaceQuota: 3,
NodeSelector: map[string]string{},
ResourceQuota: []corev1.ResourceQuotaSpec{},
},
}
JustBeforeEach(func() {
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
})
JustAfterEach(func() {
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
})
It("should fail", func() {
By("creating three Namespaces", func() {
for _, name := range []string{"bob-dev", "bob-staging", "bob-production"} {
ns := NewNamespace(name)
NamespaceCreationShouldSucceed(ns, tnt)
NamespaceShouldBeManagedByTenant(ns, tnt)
}
})
ns := NewNamespace("bob-fail")
cs := ownerClient(tnt)
_, err := cs.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
Expect(err).ShouldNot(Succeed())
})
})

213
e2e/owner_webhooks_test.go Normal file
View File

@@ -0,0 +1,213 @@
//+build e2e
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"context"
"fmt"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"github.com/clastix/capsule/api/v1alpha1"
)
var _ = Describe("when Tenant owner interacts with the webhooks", func() {
tnt := &v1alpha1.Tenant{
ObjectMeta: metav1.ObjectMeta{
Name: "tenant-owner",
},
Spec: v1alpha1.TenantSpec{
Owner: "ruby",
StorageClasses: []string{
"cephfs",
"glusterfs",
},
IngressClasses: []string{},
LimitRanges: []corev1.LimitRangeSpec{
{
Limits: []corev1.LimitRangeItem{
{
Type: corev1.LimitTypePod,
Min: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("50m"),
corev1.ResourceMemory: resource.MustParse("5Mi"),
},
Max: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
},
},
},
},
NamespaceQuota: 3,
NodeSelector: map[string]string{},
NetworkPolicies: []networkingv1.NetworkPolicySpec{
{
Egress: []networkingv1.NetworkPolicyEgressRule{
{
To: []networkingv1.NetworkPolicyPeer{
{
IPBlock: &networkingv1.IPBlock{
CIDR: "0.0.0.0/0",
},
},
},
},
},
PodSelector: metav1.LabelSelector{},
PolicyTypes: []networkingv1.PolicyType{
networkingv1.PolicyTypeIngress,
networkingv1.PolicyTypeEgress,
},
},
},
ResourceQuota: []corev1.ResourceQuotaSpec{
{
Hard: map[corev1.ResourceName]resource.Quantity{
corev1.ResourcePods: resource.MustParse("10"),
},
},
},
},
}
JustBeforeEach(func() {
tnt.ResourceVersion = ""
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
})
JustAfterEach(func() {
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
})
It("should disallow deletions", func() {
By("blocking Capsule Limit ranges", func() {
ns := NewNamespace("limit-range-disallow")
NamespaceCreationShouldSucceed(ns, tnt)
NamespaceShouldBeManagedByTenant(ns, tnt)
lr := &corev1.LimitRange{}
Eventually(func() error {
n := fmt.Sprintf("capsule-%s-0", tnt.GetName())
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns.GetName()}, lr)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
cs := ownerClient(tnt)
Expect(cs.CoreV1().LimitRanges(ns.GetName()).Delete(context.TODO(), lr.Name, metav1.DeleteOptions{})).ShouldNot(Succeed())
})
By("blocking Capsule Network Policy", func() {
ns := NewNamespace("network-policy-disallow")
NamespaceCreationShouldSucceed(ns, tnt)
NamespaceShouldBeManagedByTenant(ns, tnt)
np := &networkingv1.NetworkPolicy{}
Eventually(func() error {
n := fmt.Sprintf("capsule-%s-0", tnt.GetName())
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns.GetName()}, np)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
cs := ownerClient(tnt)
Expect(cs.NetworkingV1().NetworkPolicies(ns.GetName()).Delete(context.TODO(), np.Name, metav1.DeleteOptions{})).ShouldNot(Succeed())
})
By("blocking blocking Capsule Resource Quota", func() {
ns := NewNamespace("resource-quota-disallow")
NamespaceCreationShouldSucceed(ns, tnt)
NamespaceShouldBeManagedByTenant(ns, tnt)
rq := &corev1.ResourceQuota{}
Eventually(func() error {
n := fmt.Sprintf("capsule-%s-0", tnt.GetName())
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns.GetName()}, rq)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
cs := ownerClient(tnt)
Expect(cs.NetworkingV1().NetworkPolicies(ns.GetName()).Delete(context.TODO(), rq.Name, metav1.DeleteOptions{})).ShouldNot(Succeed())
})
})
It("should allow listing", func() {
By("Limit Range resources", func() {
ns := NewNamespace("limit-range-list")
NamespaceCreationShouldSucceed(ns, tnt)
NamespaceShouldBeManagedByTenant(ns, tnt)
Eventually(func() (err error) {
cs := ownerClient(tnt)
_, err = cs.CoreV1().LimitRanges(ns.GetName()).List(context.TODO(), metav1.ListOptions{})
return
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
By("Network Policy resources", func() {
ns := NewNamespace("network-policy-list")
NamespaceCreationShouldSucceed(ns, tnt)
NamespaceShouldBeManagedByTenant(ns, tnt)
Eventually(func() (err error) {
cs := ownerClient(tnt)
_, err = cs.NetworkingV1().NetworkPolicies(ns.GetName()).List(context.TODO(), metav1.ListOptions{})
return
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
By("Resource Quota resources", func() {
ns := NewNamespace("resource-quota-list")
NamespaceCreationShouldSucceed(ns, tnt)
NamespaceShouldBeManagedByTenant(ns, tnt)
Eventually(func() (err error) {
cs := ownerClient(tnt)
_, err = cs.NetworkingV1().NetworkPolicies(ns.GetName()).List(context.TODO(), metav1.ListOptions{})
return
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
})
It("should allow all actions to Tenant owner Network Policy resources", func() {
ns := NewNamespace("network-policy-allow")
NamespaceCreationShouldSucceed(ns, tnt)
NamespaceShouldBeManagedByTenant(ns, tnt)
cs := ownerClient(tnt)
np := &networkingv1.NetworkPolicy{
ObjectMeta: metav1.ObjectMeta{
Name: "custom-network-policy",
},
Spec: tnt.Spec.NetworkPolicies[0],
}
By("creating", func() {
Eventually(func() (err error) {
_, err = cs.NetworkingV1().NetworkPolicies(ns.GetName()).Create(context.TODO(), np, metav1.CreateOptions{})
return
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
By("updating", func() {
Eventually(func() (err error) {
np.Spec.Egress = []networkingv1.NetworkPolicyEgressRule{}
_, err = cs.NetworkingV1().NetworkPolicies(ns.GetName()).Update(context.TODO(), np, metav1.UpdateOptions{})
return
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
By("deleting", func() {
Eventually(func() (err error) {
return cs.NetworkingV1().NetworkPolicies(ns.GetName()).Delete(context.TODO(), np.Name, metav1.DeleteOptions{})
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
})
})
})

View File

@@ -0,0 +1,230 @@
//+build e2e
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"context"
"fmt"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
v1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/utils/pointer"
"github.com/clastix/capsule/api/v1alpha1"
)
var _ = Describe("exceeding Tenant resource quota", func() {
tnt := &v1alpha1.Tenant{
ObjectMeta: metav1.ObjectMeta{
Name: "tenant-resources-changes",
},
Spec: v1alpha1.TenantSpec{
Owner: "bobby",
StorageClasses: []string{},
IngressClasses: []string{},
LimitRanges: []corev1.LimitRangeSpec{
{
Limits: []corev1.LimitRangeItem{
{
Type: corev1.LimitTypePod,
Min: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("50m"),
corev1.ResourceMemory: resource.MustParse("5Mi"),
},
Max: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
},
{
Type: corev1.LimitTypeContainer,
Default: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("200m"),
corev1.ResourceMemory: resource.MustParse("100Mi"),
},
DefaultRequest: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("100m"),
corev1.ResourceMemory: resource.MustParse("10Mi"),
},
Min: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("50m"),
corev1.ResourceMemory: resource.MustParse("5Mi"),
},
Max: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
},
{
Type: corev1.LimitTypePersistentVolumeClaim,
Min: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceStorage: resource.MustParse("1Gi"),
},
Max: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceStorage: resource.MustParse("10Gi"),
},
},
},
},
},
NetworkPolicies: []networkingv1.NetworkPolicySpec{},
NamespaceQuota: 2,
NodeSelector: map[string]string{},
ResourceQuota: []corev1.ResourceQuotaSpec{
{
Hard: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceLimitsCPU: resource.MustParse("8"),
corev1.ResourceLimitsMemory: resource.MustParse("16Gi"),
corev1.ResourceRequestsCPU: resource.MustParse("8"),
corev1.ResourceRequestsMemory: resource.MustParse("16Gi"),
},
Scopes: []corev1.ResourceQuotaScope{
corev1.ResourceQuotaScopeNotTerminating,
},
},
{
Hard: map[corev1.ResourceName]resource.Quantity{
corev1.ResourcePods: resource.MustParse("10"),
},
},
{
Hard: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceRequestsStorage: resource.MustParse("100Gi"),
},
},
},
},
}
nsl := []string{"easy", "peasy"}
JustBeforeEach(func() {
tnt.ResourceVersion = ""
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
By("creating the Namespaces", func() {
for _, i := range nsl {
ns := NewNamespace(i)
NamespaceCreationShouldSucceed(ns, tnt)
NamespaceShouldBeManagedByTenant(ns, tnt)
}
})
})
JustAfterEach(func() {
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
})
It("should block new Pods if limit is reached", func() {
cs := ownerClient(tnt)
for _, namespace := range nsl {
Eventually(func() (err error) {
d := &v1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "my-pause",
},
Spec: v1.DeploymentSpec{
Replicas: pointer.Int32Ptr(5),
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "pause",
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": "pause",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "my-pause",
Image: "gcr.io/google_containers/pause-amd64:3.0",
},
},
},
},
},
}
_, err = cs.AppsV1().Deployments(namespace).Create(context.TODO(), d, metav1.CreateOptions{})
return
}, 15*time.Second, time.Second).Should(Succeed())
}
for _, ns := range nsl {
n := fmt.Sprintf("capsule-%s-1", tnt.GetName())
rq := &corev1.ResourceQuota{}
By("retrieving the Resource Quota", func() {
Eventually(func() error {
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, rq)
}, 15*time.Second, time.Second).Should(Succeed())
})
By("ensuring the status has been blocked with actual usage", func() {
Eventually(func() corev1.ResourceList {
_ = k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, rq)
return rq.Status.Hard
}, 15*time.Second, time.Second).Should(Equal(rq.Status.Used))
})
By("creating an exceeded Pod", func() {
d := &v1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "my-exceeded",
},
Spec: v1.DeploymentSpec{
Replicas: pointer.Int32Ptr(5),
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"app": "exceeded",
},
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": "exceeded",
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "my-exceeded",
Image: "gcr.io/google_containers/pause-amd64:3.0",
},
},
},
},
},
}
_, err := cs.AppsV1().Deployments(ns).Create(context.TODO(), d, metav1.CreateOptions{})
Expect(err).Should(Succeed())
Eventually(func() (condition *v1.DeploymentCondition) {
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: d.GetName(), Namespace: ns}, d)).Should(Succeed())
for _, i := range d.Status.Conditions {
if i.Type == v1.DeploymentReplicaFailure {
condition = &i
break
}
}
return
}, 30*time.Second, time.Second).ShouldNot(BeNil())
})
}
})
})

View File

@@ -0,0 +1,70 @@
//+build e2e
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"context"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/clastix/capsule/api/v1alpha1"
)
var _ = Describe("creating a Namespace trying to select a third Tenant", func() {
tnt := &v1alpha1.Tenant{
ObjectMeta: metav1.ObjectMeta{
Name: "tenant-non-owned",
},
Spec: v1alpha1.TenantSpec{
Owner: "undefined",
StorageClasses: []string{},
IngressClasses: []string{},
LimitRanges: []corev1.LimitRangeSpec{},
NamespaceQuota: 10,
NodeSelector: map[string]string{},
ResourceQuota: []corev1.ResourceQuotaSpec{},
},
}
JustBeforeEach(func() {
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
})
JustAfterEach(func() {
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
})
It("should fail", func() {
var ns *corev1.Namespace
By("assigning to the Namespace the Capsule Tenant label", func() {
l, err := v1alpha1.GetTypeLabel(&v1alpha1.Tenant{})
Expect(err).ToNot(HaveOccurred())
ns := NewNamespace("tenant-non-owned-ns")
ns.SetLabels(map[string]string{
l: tnt.Name,
})
})
cs := ownerClient(&v1alpha1.Tenant{Spec: v1alpha1.TenantSpec{Owner: "dale"}})
_, err := cs.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
Expect(err).ShouldNot(Succeed())
})
})

View File

@@ -0,0 +1,81 @@
//+build e2e
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"context"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/clastix/capsule/api/v1alpha1"
)
var _ = Describe("creating a Namespace with Tenant selector", func() {
t1 := &v1alpha1.Tenant{
ObjectMeta: metav1.ObjectMeta{
Name: "tenant-one",
},
Spec: v1alpha1.TenantSpec{
Owner: "john",
StorageClasses: []string{},
IngressClasses: []string{},
LimitRanges: []corev1.LimitRangeSpec{},
NamespaceQuota: 10,
NodeSelector: map[string]string{},
ResourceQuota: []corev1.ResourceQuotaSpec{},
},
}
t2 := &v1alpha1.Tenant{
ObjectMeta: metav1.ObjectMeta{
Name: "tenant-two",
},
Spec: v1alpha1.TenantSpec{
Owner: "john",
StorageClasses: []string{},
IngressClasses: []string{},
LimitRanges: []corev1.LimitRangeSpec{},
NamespaceQuota: 10,
NodeSelector: map[string]string{},
ResourceQuota: []corev1.ResourceQuotaSpec{},
},
}
JustBeforeEach(func() {
Expect(k8sClient.Create(context.TODO(), t1)).Should(Succeed())
Expect(k8sClient.Create(context.TODO(), t2)).Should(Succeed())
})
JustAfterEach(func() {
Expect(k8sClient.Delete(context.TODO(), t1)).Should(Succeed())
Expect(k8sClient.Delete(context.TODO(), t2)).Should(Succeed())
})
It("should be assigned to the selected Tenant", func() {
ns := NewNamespace("tenant-2-ns")
By("assigning to the Namespace the Capsule Tenant label", func() {
l, err := v1alpha1.GetTypeLabel(&v1alpha1.Tenant{})
Expect(err).ToNot(HaveOccurred())
ns.Labels = map[string]string{
l: t2.Name,
}
})
NamespaceCreationShouldSucceed(ns, t2)
NamespaceShouldBeManagedByTenant(ns, t2)
})
})

135
e2e/storage_class_test.go Normal file
View File

@@ -0,0 +1,135 @@
//+build e2e
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"context"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/pointer"
"github.com/clastix/capsule/api/v1alpha1"
)
var _ = Describe("when Tenant handles Storage classes", func() {
tnt := &v1alpha1.Tenant{
ObjectMeta: metav1.ObjectMeta{
Name: "storage-class",
},
Spec: v1alpha1.TenantSpec{
Owner: "storage",
StorageClasses: []string{
"cephfs",
"glusterfs",
},
IngressClasses: []string{},
LimitRanges: []corev1.LimitRangeSpec{},
NamespaceQuota: 3,
NodeSelector: map[string]string{},
NetworkPolicies: []networkingv1.NetworkPolicySpec{},
ResourceQuota: []corev1.ResourceQuotaSpec{},
},
}
JustBeforeEach(func() {
tnt.ResourceVersion = ""
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
})
JustAfterEach(func() {
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
})
It("should block non allowed Storage Class", func() {
ns := NewNamespace("storage-class-disallowed")
NamespaceCreationShouldSucceed(ns, tnt)
NamespaceShouldBeManagedByTenant(ns, tnt)
By("non-specifying the class", func() {
Eventually(func() (err error) {
cs := ownerClient(tnt)
p := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "denied-pvc",
},
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
Resources: corev1.ResourceRequirements{
Requests: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceStorage: resource.MustParse("3Gi"),
},
},
},
}
_, err = cs.CoreV1().PersistentVolumeClaims(ns.GetName()).Create(context.TODO(), p, metav1.CreateOptions{})
return
}, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed())
})
By("specifying a forbidden class", func() {
Eventually(func() (err error) {
cs := ownerClient(tnt)
p := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "mighty-storage",
},
Spec: corev1.PersistentVolumeClaimSpec{
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
Resources: corev1.ResourceRequirements{
Requests: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceStorage: resource.MustParse("3Gi"),
},
},
},
}
_, err = cs.CoreV1().PersistentVolumeClaims(ns.GetName()).Create(context.TODO(), p, metav1.CreateOptions{})
return
}, defaultTimeoutInterval, defaultPollInterval).ShouldNot(Succeed())
})
})
It("should allow enabled Storage Class", func() {
ns := NewNamespace("storage-class-allowed")
cs := ownerClient(tnt)
NamespaceCreationShouldSucceed(ns, tnt)
NamespaceShouldBeManagedByTenant(ns, tnt)
for _, c := range tnt.Spec.StorageClasses {
Eventually(func() (err error) {
p := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: c,
},
Spec: corev1.PersistentVolumeClaimSpec{
StorageClassName: pointer.StringPtr(c),
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
Resources: corev1.ResourceRequirements{
Requests: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceStorage: resource.MustParse("3Gi"),
},
},
},
}
_, err = cs.CoreV1().PersistentVolumeClaims(ns.GetName()).Create(context.TODO(), p, metav1.CreateOptions{})
return
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
})
})

View File

@@ -1,3 +1,5 @@
//+build e2e
/*
Copyright 2020 Clastix Labs.
@@ -14,7 +16,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package controllers
package e2e
import (
"path/filepath"
@@ -22,16 +24,17 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
"sigs.k8s.io/controller-runtime/pkg/envtest"
"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
capsulev1alpha "github.com/clastix/capsule/api/v1alpha1"
// +kubebuilder:scaffold:imports
)
// These tests use Ginkgo (BDD-style Go testing framework). Refer to
@@ -55,6 +58,9 @@ var _ = BeforeSuite(func(done Done) {
By("bootstrapping test environment")
testEnv = &envtest.Environment{
CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")},
UseExistingCluster: func(v bool) *bool {
return &v
}(true),
}
var err error
@@ -65,8 +71,6 @@ var _ = BeforeSuite(func(done Done) {
err = capsulev1alpha.AddToScheme(scheme.Scheme)
Expect(err).NotTo(HaveOccurred())
// +kubebuilder:scaffold:scheme
k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
Expect(err).ToNot(HaveOccurred())
Expect(k8sClient).ToNot(BeNil())
@@ -76,6 +80,15 @@ var _ = BeforeSuite(func(done Done) {
var _ = AfterSuite(func() {
By("tearing down the test environment")
err := testEnv.Stop()
Expect(err).ToNot(HaveOccurred())
Expect(testEnv.Stop()).ToNot(HaveOccurred())
})
func ownerClient(tenant *capsulev1alpha.Tenant) (cs kubernetes.Interface) {
c, err := config.GetConfig()
Expect(err).ToNot(HaveOccurred())
c.Impersonate.Groups = []string{capsulev1alpha.GroupVersion.Group}
c.Impersonate.UserName = tenant.Spec.Owner
cs, err = kubernetes.NewForConfig(c)
Expect(err).ToNot(HaveOccurred())
return
}

View File

@@ -0,0 +1,239 @@
//+build e2e
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"context"
"fmt"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/clastix/capsule/api/v1alpha1"
)
var _ = Describe("changing Tenant managed Kubernetes resources", func() {
tnt := &v1alpha1.Tenant{
ObjectMeta: metav1.ObjectMeta{
Name: "tenant-resources-changes",
},
Spec: v1alpha1.TenantSpec{
Owner: "laura",
StorageClasses: []string{},
IngressClasses: []string{},
LimitRanges: []corev1.LimitRangeSpec{
{
Limits: []corev1.LimitRangeItem{
{
Type: corev1.LimitTypePod,
Min: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("50m"),
corev1.ResourceMemory: resource.MustParse("5Mi"),
},
Max: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
},
{
Type: corev1.LimitTypeContainer,
Default: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("200m"),
corev1.ResourceMemory: resource.MustParse("100Mi"),
},
DefaultRequest: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("100m"),
corev1.ResourceMemory: resource.MustParse("10Mi"),
},
Min: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("50m"),
corev1.ResourceMemory: resource.MustParse("5Mi"),
},
Max: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
},
{
Type: corev1.LimitTypePersistentVolumeClaim,
Min: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceStorage: resource.MustParse("1Gi"),
},
Max: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceStorage: resource.MustParse("10Gi"),
},
},
},
},
},
NetworkPolicies: []networkingv1.NetworkPolicySpec{
{
Ingress: []networkingv1.NetworkPolicyIngressRule{
{
From: []networkingv1.NetworkPolicyPeer{
{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"capsule.clastix.io/tenant": "tenant-resources",
},
},
},
{
PodSelector: &metav1.LabelSelector{},
},
{
IPBlock: &networkingv1.IPBlock{
CIDR: "192.168.0.0/12",
},
},
},
},
},
Egress: []networkingv1.NetworkPolicyEgressRule{
{
To: []networkingv1.NetworkPolicyPeer{
{
IPBlock: &networkingv1.IPBlock{
CIDR: "0.0.0.0/0",
Except: []string{
"192.168.0.0/12",
},
},
},
},
},
},
PodSelector: metav1.LabelSelector{},
PolicyTypes: []networkingv1.PolicyType{
networkingv1.PolicyTypeIngress,
networkingv1.PolicyTypeEgress,
},
},
},
NamespaceQuota: 4,
NodeSelector: map[string]string{
"kubernetes.io/os": "linux",
},
ResourceQuota: []corev1.ResourceQuotaSpec{
{
Hard: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceLimitsCPU: resource.MustParse("8"),
corev1.ResourceLimitsMemory: resource.MustParse("16Gi"),
corev1.ResourceRequestsCPU: resource.MustParse("8"),
corev1.ResourceRequestsMemory: resource.MustParse("16Gi"),
},
Scopes: []corev1.ResourceQuotaScope{
corev1.ResourceQuotaScopeNotTerminating,
},
},
{
Hard: map[corev1.ResourceName]resource.Quantity{
corev1.ResourcePods: resource.MustParse("10"),
},
},
{
Hard: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceRequestsStorage: resource.MustParse("100Gi"),
},
},
},
},
}
nsl := []string{"fire", "walk", "with", "me"}
JustBeforeEach(func() {
tnt.ResourceVersion = ""
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
By("creating the Namespaces", func() {
for _, i := range nsl {
ns := NewNamespace(i)
NamespaceCreationShouldSucceed(ns, tnt)
NamespaceShouldBeManagedByTenant(ns, tnt)
}
})
})
JustAfterEach(func() {
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
})
It("should reapply the original resources upon third party change", func() {
for _, ns := range nsl {
By("changing Limit Range resources", func() {
for i, s := range tnt.Spec.LimitRanges {
n := fmt.Sprintf("capsule-%s-%d", tnt.GetName(), i)
lr := &corev1.LimitRange{}
Eventually(func() error {
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, lr)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
c := lr.DeepCopy()
c.Spec.Limits = []corev1.LimitRangeItem{}
Expect(k8sClient.Update(context.TODO(), c, &client.UpdateOptions{})).Should(Succeed())
Eventually(func() corev1.LimitRangeSpec {
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, lr)).Should(Succeed())
return lr.Spec
}, defaultTimeoutInterval, defaultPollInterval).Should(Equal(s))
}
})
By("changing Network Policy resources", func() {
for i, s := range tnt.Spec.NetworkPolicies {
n := fmt.Sprintf("capsule-%s-%d", tnt.GetName(), i)
np := &networkingv1.NetworkPolicy{}
Eventually(func() error {
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, np)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
Expect(np.Spec).Should(Equal(s))
c := np.DeepCopy()
c.Spec.Egress = []networkingv1.NetworkPolicyEgressRule{}
c.Spec.Ingress = []networkingv1.NetworkPolicyIngressRule{}
Expect(k8sClient.Update(context.TODO(), c, &client.UpdateOptions{})).Should(Succeed())
Eventually(func() networkingv1.NetworkPolicySpec {
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, np)).Should(Succeed())
return np.Spec
}, defaultTimeoutInterval, defaultPollInterval).Should(Equal(s))
}
})
By("changing Resource Quota resources", func() {
for i, s := range tnt.Spec.ResourceQuota {
n := fmt.Sprintf("capsule-%s-%d", tnt.GetName(), i)
rq := &corev1.ResourceQuota{}
Eventually(func() error {
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, rq)
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
c := rq.DeepCopy()
c.Spec.Hard = map[corev1.ResourceName]resource.Quantity{}
Expect(k8sClient.Update(context.TODO(), c, &client.UpdateOptions{})).Should(Succeed())
Eventually(func() corev1.ResourceQuotaSpec {
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: ns}, rq)).Should(Succeed())
return rq.Spec
}, defaultTimeoutInterval, defaultPollInterval).Should(Equal(s))
}
})
}
})
})

View File

@@ -0,0 +1,226 @@
//+build e2e
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"context"
"fmt"
"strings"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
"k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"github.com/clastix/capsule/api/v1alpha1"
)
var _ = Describe("creating namespaces within a Tenant with resources", func() {
tnt := &v1alpha1.Tenant{
ObjectMeta: metav1.ObjectMeta{
Name: "tenant-resources",
},
Spec: v1alpha1.TenantSpec{
Owner: "john",
StorageClasses: []string{},
IngressClasses: []string{},
LimitRanges: []corev1.LimitRangeSpec{
{
Limits: []corev1.LimitRangeItem{
{
Type: corev1.LimitTypePod,
Min: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("50m"),
corev1.ResourceMemory: resource.MustParse("5Mi"),
},
Max: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
},
{
Type: corev1.LimitTypeContainer,
Default: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("200m"),
corev1.ResourceMemory: resource.MustParse("100Mi"),
},
DefaultRequest: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("100m"),
corev1.ResourceMemory: resource.MustParse("10Mi"),
},
Min: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("50m"),
corev1.ResourceMemory: resource.MustParse("5Mi"),
},
Max: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
},
{
Type: corev1.LimitTypePersistentVolumeClaim,
Min: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceStorage: resource.MustParse("1Gi"),
},
Max: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceStorage: resource.MustParse("10Gi"),
},
},
},
},
},
NetworkPolicies: []v1.NetworkPolicySpec{
{
Ingress: []v1.NetworkPolicyIngressRule{
{
From: []v1.NetworkPolicyPeer{
{
NamespaceSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"capsule.clastix.io/tenant": "tenant-resources",
},
},
},
{
PodSelector: &metav1.LabelSelector{},
},
{
IPBlock: &v1.IPBlock{
CIDR: "192.168.0.0/12",
},
},
},
},
},
Egress: []v1.NetworkPolicyEgressRule{
{
To: []v1.NetworkPolicyPeer{
{
IPBlock: &v1.IPBlock{
CIDR: "0.0.0.0/0",
Except: []string{
"192.168.0.0/12",
},
},
},
},
},
},
PodSelector: metav1.LabelSelector{},
PolicyTypes: []v1.PolicyType{
v1.PolicyTypeIngress,
v1.PolicyTypeEgress,
},
},
},
NamespaceQuota: 3,
NodeSelector: map[string]string{
"kubernetes.io/os": "linux",
},
ResourceQuota: []corev1.ResourceQuotaSpec{
{
Hard: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceLimitsCPU: resource.MustParse("8"),
corev1.ResourceLimitsMemory: resource.MustParse("16Gi"),
corev1.ResourceRequestsCPU: resource.MustParse("8"),
corev1.ResourceRequestsMemory: resource.MustParse("16Gi"),
},
Scopes: []corev1.ResourceQuotaScope{
corev1.ResourceQuotaScopeNotTerminating,
},
},
{
Hard: map[corev1.ResourceName]resource.Quantity{
corev1.ResourcePods: resource.MustParse("10"),
},
},
{
Hard: map[corev1.ResourceName]resource.Quantity{
corev1.ResourceRequestsStorage: resource.MustParse("100Gi"),
},
},
},
},
}
nsl := []string{"bim", "bum", "bam"}
JustBeforeEach(func() {
tnt.ResourceVersion = ""
Expect(k8sClient.Create(context.TODO(), tnt)).Should(Succeed())
By("creating the Namespaces", func() {
for _, i := range nsl {
ns := NewNamespace(i)
NamespaceCreationShouldSucceed(ns, tnt)
NamespaceShouldBeManagedByTenant(ns, tnt)
}
})
})
JustAfterEach(func() {
Expect(k8sClient.Delete(context.TODO(), tnt)).Should(Succeed())
})
It("should contains all replicated resources", func() {
for _, name := range nsl {
By("checking Limit Range resources", func() {
for i, s := range tnt.Spec.LimitRanges {
n := fmt.Sprintf("capsule-%s-%d", tnt.GetName(), i)
lr := &corev1.LimitRange{}
Eventually(func() error {
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: name}, lr)
}, 10*time.Second, time.Second).Should(Succeed())
Expect(lr.Spec).Should(Equal(s))
}
})
By("checking Network Policy resources", func() {
for i, s := range tnt.Spec.NetworkPolicies {
n := fmt.Sprintf("capsule-%s-%d", tnt.GetName(), i)
np := &v1.NetworkPolicy{}
Eventually(func() error {
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: name}, np)
}, 10*time.Second, time.Second).Should(Succeed())
Expect(np.Spec).Should(Equal(s))
}
})
By("checking the Namespace scheduler annotation", func() {
var selector []string
for k, v := range tnt.Spec.NodeSelector {
selector = append(selector, fmt.Sprintf("%s=%s", k, v))
}
Eventually(func() string {
ns := &corev1.Namespace{}
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: name}, ns)).Should(Succeed())
return ns.GetAnnotations()["scheduler.alpha.kubernetes.io/node-selector"]
}, 10*time.Second, time.Second).Should(Equal(strings.Join(selector, ",")))
})
By("checking the Resource Quota resources", func() {
for i, s := range tnt.Spec.ResourceQuota {
n := fmt.Sprintf("capsule-%s-%d", tnt.GetName(), i)
rq := &corev1.ResourceQuota{}
Eventually(func() error {
return k8sClient.Get(context.TODO(), types.NamespacedName{Name: n, Namespace: name}, rq)
}, 10*time.Second, time.Second).Should(Succeed())
Expect(rq.Spec).Should(Equal(s))
}
})
}
})
})

59
e2e/utils_test.go Normal file
View File

@@ -0,0 +1,59 @@
//+build e2e
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"context"
"time"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"github.com/clastix/capsule/api/v1alpha1"
)
const (
defaultTimeoutInterval = 10 * time.Second
defaultPollInterval = time.Second
)
func NewNamespace(name string) *corev1.Namespace {
return &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
}
}
func NamespaceCreationShouldSucceed(ns *corev1.Namespace, t *v1alpha1.Tenant) {
cs := ownerClient(t)
Eventually(func() (err error) {
_, err = cs.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
return
}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
}
func NamespaceShouldBeManagedByTenant(ns *corev1.Namespace, t *v1alpha1.Tenant) {
Eventually(func() v1alpha1.NamespaceList {
Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: t.GetName()}, t)).Should(Succeed())
return t.Status.Namespaces
}, defaultTimeoutInterval, defaultPollInterval).Should(ContainElement(ns.GetName()))
}

1
go.mod
View File

@@ -11,5 +11,6 @@ require (
k8s.io/api v0.18.2
k8s.io/apimachinery v0.18.2
k8s.io/client-go v0.18.2
k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89
sigs.k8s.io/controller-runtime v0.6.0
)

View File

@@ -125,15 +125,6 @@ func main() {
os.Exit(1)
}
if err = (&controllers.NamespaceReconciler{
Client: mgr.GetClient(),
Log: ctrl.Log.WithName("controllers").WithName("Namespace"),
Scheme: mgr.GetScheme(),
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "Namespace")
os.Exit(1)
}
if err = (&secret.CaReconciler{
Client: mgr.GetClient(),
Log: ctrl.Log.WithName("controllers").WithName("CA"),

View File

@@ -16,8 +16,10 @@ limitations under the License.
package indexer
import "github.com/clastix/capsule/pkg/indexer/tenant"
import (
"github.com/clastix/capsule/pkg/indexer/namespace"
)
func init() {
AddToIndexerFuncs = append(AddToIndexerFuncs, tenant.OwnerReference{})
AddToIndexerFuncs = append(AddToIndexerFuncs, namespace.OwnerReference{})
}

View File

@@ -20,4 +20,5 @@ import "github.com/clastix/capsule/pkg/indexer/tenant"
func init() {
AddToIndexerFuncs = append(AddToIndexerFuncs, tenant.NamespacesReference{})
AddToIndexerFuncs = append(AddToIndexerFuncs, tenant.OwnerReference{})
}

View File

@@ -0,0 +1,49 @@
/*
Copyright 2020 Clastix Labs.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package namespace
import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/clastix/capsule/api/v1alpha1"
)
type OwnerReference struct {
}
func (o OwnerReference) Object() runtime.Object {
return &v1.Namespace{}
}
func (o OwnerReference) Field() string {
return ".metadata.ownerReferences[*].capsule"
}
func (o OwnerReference) Func() client.IndexerFunc {
return func(object runtime.Object) []string {
var res []string
ns := object.(*v1.Namespace)
for _, or := range ns.OwnerReferences {
if or.APIVersion == v1alpha1.GroupVersion.String() {
res = append(res, or.Name)
}
}
return res
}
}