capsule/e2e/performance_scalability_test.go

// Copyright 2020-2023 Project Capsule Authors.
// SPDX-License-Identifier: Apache-2.0

package e2e

import (
	"context"
	"fmt"

	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"

	capsulev1beta2 "github.com/projectcapsule/capsule/api/v1beta2"
	"github.com/projectcapsule/capsule/pkg/api/meta"
	"github.com/projectcapsule/capsule/pkg/api/rbac"
)

var _ = Describe("verify scalability", Ordered, Label("performance", "scalability"), func() {
	tnt := &capsulev1beta2.Tenant{
		ObjectMeta: metav1.ObjectMeta{
			Name: "e2e-perf-scalability",
		},
		Spec: capsulev1beta2.TenantSpec{
			Owners: rbac.OwnerListSpec{
				{
					CoreOwnerSpec: rbac.CoreOwnerSpec{
						UserSpec: rbac.UserSpec{
							Name: "e2e-perf-scalability",
							Kind: "User",
						},
					},
				},
			},
		},
	}

	JustBeforeEach(func() {
		EventuallyCreation(func() error {
			tnt.ResourceVersion = ""
			return k8sClient.Create(context.TODO(), tnt)
		}).Should(Succeed())

		TenantReady(tnt, metav1.ConditionTrue, defaultTimeoutInterval)
	})
	JustAfterEach(func() {
		EventuallyDeletion(tnt)
	})

	It("verify lifecycle (scalability)", Label("skip-on-openshift"), func() {
		const amount = 50
		const podsPerNamespace int32 = 1

		getTenant := func() *capsulev1beta2.Tenant {
			t := &capsulev1beta2.Tenant{}
			Expect(k8sClient.Get(context.TODO(), types.NamespacedName{Name: tnt.GetName()}, t)).To(Succeed())

			return t
		}

		waitTenantNamespacesReady := func(namespaces []*corev1.Namespace) {
			Eventually(func() error {
				t := getTenant()

				if t.Status.Size != uint(len(namespaces)) {
					return fmt.Errorf("tenant size=%d, want %d", t.Status.Size, len(namespaces))
				}

				for _, ns := range namespaces {
					inst := t.Status.GetInstance(&capsulev1beta2.TenantStatusNamespaceItem{
						Name: ns.GetName(),
						UID:  ns.GetUID(),
					})
					if inst == nil {
						return fmt.Errorf("instance not found for ns=%q uid=%q", ns.GetName(), ns.GetUID())
					}

					cond := inst.Conditions.GetConditionByType(meta.ReadyCondition)
					if cond == nil {
						return fmt.Errorf("namespace %q missing %q condition", ns.GetName(), meta.ReadyCondition)
					}

					if cond.Status != metav1.ConditionTrue {
						return fmt.Errorf("namespace %q ready status=%q, want %q", ns.GetName(), cond.Status, metav1.ConditionTrue)
					}

					if cond.Reason != meta.SucceededReason {
						return fmt.Errorf("namespace %q ready reason=%q, want %q", ns.GetName(), cond.Reason, meta.SucceededReason)
					}
				}

				return nil
			}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())
		}

		waitTenantNamespacesGone := func() {
			Eventually(func() uint {
				return getTenant().Status.Size
			}, defaultTimeoutInterval, defaultPollInterval).Should(Equal(uint(0)))
		}

		namespaces := make([]*corev1.Namespace, 0, amount)

		By("creating namespaces")
		for i := 0; i < amount; i++ {
			ns := NewNamespace(fmt.Sprintf("e2e-scale-%d", i), map[string]string{
				meta.TenantLabel: tnt.GetName(),
			})

			NamespaceCreation(ns, tnt.Spec.Owners[0].UserSpec, defaultTimeoutInterval).Should(Succeed())
			namespaces = append(namespaces, ns)
		}

		By("waiting for all namespaces to be reflected in tenant status")
		waitTenantNamespacesReady(namespaces)

		By("creating low-impact deployments")
		for _, ns := range namespaces {
			dep := newTrafficDeployment(ns.GetName(), podsPerNamespace)
			EventuallyCreation(func() error {
				return k8sClient.Create(context.TODO(), dep)
			}).Should(Succeed())
		}

		By("waiting for deployments")
		for _, ns := range namespaces {
			waitDeploymentReady(context.TODO(), ns.GetName(), "traffic-pause", podsPerNamespace)
		}

		By("deleting namespaces")
		for _, ns := range namespaces {
			Expect(k8sClient.Delete(context.TODO(), ns)).To(Succeed())
		}

		By("waiting for tenant status to be empty")
		waitTenantNamespacesGone()
	})

})

func newTrafficDeployment(ns string, replicas int32) *appsv1.Deployment {
	labels := map[string]string{"app": "traffic-pause"}

	return &appsv1.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "traffic-pause",
			Namespace: ns,
		},
		Spec: appsv1.DeploymentSpec{
			Replicas: &replicas,
			Selector: &metav1.LabelSelector{MatchLabels: labels},
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{Labels: labels},
				Spec: corev1.PodSpec{
					SecurityContext: nobodyPodSecurityContext(),
					Containers: []corev1.Container{
						{
							Name:            "pause",
							Image:           "registry.k8s.io/pause:3.9",
							SecurityContext: restrictedContainerSecurityContext(),
						},
					},
				},
			},
		},
	}
}

func scaleDeployment(ctx context.Context, ns, name string, replicas int32) {
	Eventually(func() error {
		d := &appsv1.Deployment{}
		if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, d); err != nil {
			return err
		}
		*d.Spec.Replicas = replicas
		return k8sClient.Update(ctx, d)
	}, defaultTimeoutInterval, defaultPollInterval).Should(Succeed())

	waitDeploymentReady(ctx, ns, name, replicas)
}

func waitDeploymentReady(ctx context.Context, ns, name string, replicas int32) {
	Eventually(func() (int32, error) {
		d := &appsv1.Deployment{}
		err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, d)
		if err != nil {
			return 0, err
		}
		return d.Status.ReadyReplicas, nil
	}, defaultTimeoutInterval, defaultPollInterval).Should(Equal(replicas))
}