k3k/tests/e2e/tests_suite_test.go

package k3k_test

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"testing"
	"time"

	"github.com/testcontainers/testcontainers-go"
	"github.com/testcontainers/testcontainers-go/modules/k3s"
	"k8s.io/apimachinery/pkg/api/resource"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/remotecommand"
	"k8s.io/kubernetes/pkg/api/v1/pod"
	"k8s.io/utils/ptr"
	"sigs.k8s.io/controller-runtime/pkg/client"

	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	networkingv1 "k8s.io/api/networking/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	fwclient "github.com/rancher/k3k/tests/framework/client"
	fwcontainer "github.com/rancher/k3k/tests/framework/container"
	fwlog "github.com/rancher/k3k/tests/framework/log"

	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"
)

const (
	k3kNamespace = "k3k-system"

	k3sVersion    = "v1.35.2-k3s1"
	k3sOldVersion = "v1.35.0-k3s1"

	e2eTestLabel           = "e2e"
	slowTestsLabel         = "slow"
	updateTestsLabel       = "update"
	persistenceTestsLabel  = "persistence"
	networkingTestsLabel   = "networking"
	statusTestsLabel       = "status"
	certificatesTestsLabel = "certificates"
	registryTestsLabel     = "registry"

	registryImage               = "registry:2"
	registryCACertSecretName    = "private-registry-ca-cert"
	registryCertSecretName      = "private-registry-cert"
	registryConfigSecretName    = "private-registry-config"
	k3sRegistryConfigSecretName = "k3s-registry-config"
)

func TestTests(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Tests Suite")
}

var (
	k3sContainer   *k3s.K3sContainer
	hostIP         string
	restcfg        *rest.Config
	k8s            *kubernetes.Clientset
	k8sClient      client.Client
	kubeconfigPath string
)

var _ = BeforeSuite(func() {
	ctx := context.Background()

	GinkgoWriter.Println("GOCOVERDIR:", os.Getenv("GOCOVERDIR"))

	_, dockerInstallEnabled := os.LookupEnv("K3K_DOCKER_INSTALL")

	if dockerInstallEnabled {
		repo := os.Getenv("REPO")
		if repo == "" {
			repo = "rancher"
		}

		installK3SDocker(ctx, repo+"/k3k", repo+"/k3k-kubelet")
		initKubernetesClient(ctx)
		installK3kChart(repo+"/k3k", repo+"/k3k-kubelet")
	} else {
		initKubernetesClient(ctx)
	}

	patchPVC(ctx, k8s)
})

func installK3SDocker(ctx context.Context, controllerImage, kubeletImage string) {
	k3sHostVersion := os.Getenv("K3S_HOST_VERSION")
	if k3sHostVersion == "" {
		k3sHostVersion = k3sVersion
	}

	k3sContainer, kubeconfigPath = fwcontainer.SetupK3s(ctx, k3sHostVersion, controllerImage, kubeletImage)
}

func initKubernetesClient(ctx context.Context) {
	scheme := fwclient.NewScheme()
	config, err := fwclient.InitFromKubeconfig(ctx, scheme, k3sContainer)
	Expect(err).NotTo(HaveOccurred())

	hostIP = config.HostIP
	restcfg = config.RestConfig
	k8s = config.Clientset
	k8sClient = config.Client
}

func installK3kChart(controllerImage, kubeletImage string) {
	installer := fwcontainer.NewHelmInstaller(controllerImage, kubeletImage, kubeconfigPath)
	kubeconfig, err := os.ReadFile(kubeconfigPath)
	Expect(err).NotTo(HaveOccurred())

	restClientGetter, err := fwclient.NewRESTClientGetter(kubeconfig)
	Expect(err).NotTo(HaveOccurred())

	installer.InstallK3kChart(restClientGetter)
}

func patchPVC(ctx context.Context, clientset *kubernetes.Clientset) {
	deployments, err := clientset.AppsV1().Deployments(k3kNamespace).List(ctx, metav1.ListOptions{})
	Expect(err).To(Not(HaveOccurred()))
	Expect(deployments.Items).To(HaveLen(1))

	k3kDeployment := &deployments.Items[0]

	pvc := &corev1.PersistentVolumeClaim{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "coverage-data-pvc",
			Namespace: k3kNamespace,
		},
		Spec: corev1.PersistentVolumeClaimSpec{
			AccessModes: []corev1.PersistentVolumeAccessMode{
				corev1.ReadWriteOnce,
			},
			Resources: corev1.VolumeResourceRequirements{
				Requests: corev1.ResourceList{
					corev1.ResourceStorage: resource.MustParse("100M"),
				},
			},
		},
	}

	_, err = clientset.CoreV1().PersistentVolumeClaims(k3kNamespace).Create(ctx, pvc, metav1.CreateOptions{})
	Expect(client.IgnoreAlreadyExists(err)).To(Not(HaveOccurred()))

	k3kSpec := k3kDeployment.Spec.Template.Spec

	// check if the Deployment has already the volume for the coverage
	for _, volumes := range k3kSpec.Volumes {
		if volumes.Name == "tmp-covdata" {
			return
		}
	}

	k3kSpec.Volumes = append(k3kSpec.Volumes, corev1.Volume{
		Name: "tmp-covdata",
		VolumeSource: corev1.VolumeSource{
			PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
				ClaimName: "coverage-data-pvc",
			},
		},
	})

	k3kSpec.Containers[0].VolumeMounts = append(k3kSpec.Containers[0].VolumeMounts, corev1.VolumeMount{
		Name:      "tmp-covdata",
		MountPath: "/tmp/covdata",
	})

	k3kSpec.Containers[0].Env = append(k3kSpec.Containers[0].Env, corev1.EnvVar{
		Name:  "GOCOVERDIR",
		Value: "/tmp/covdata",
	})

	k3kDeployment.Spec.Template.Spec = k3kSpec

	_, err = clientset.AppsV1().Deployments(k3kNamespace).Update(ctx, k3kDeployment, metav1.UpdateOptions{})
	Expect(err).To(Not(HaveOccurred()))

	Eventually(func(g Gomega) {
		GinkgoWriter.Println("Checking K3k deployment status")

		dep, err := clientset.AppsV1().Deployments(k3kNamespace).Get(ctx, k3kDeployment.Name, metav1.GetOptions{})
		g.Expect(err).To(Not(HaveOccurred()))
		g.Expect(dep.Generation).To(Equal(dep.Status.ObservedGeneration))

		var availableCond appsv1.DeploymentCondition

		for _, cond := range dep.Status.Conditions {
			if cond.Type == appsv1.DeploymentAvailable {
				availableCond = cond
				break
			}
		}

		g.Expect(availableCond.Type).To(Equal(appsv1.DeploymentAvailable))
		g.Expect(availableCond.Status).To(Equal(corev1.ConditionTrue))
	}).
		WithPolling(time.Second).
		WithTimeout(time.Second * 30).
		Should(Succeed())
}

var _ = AfterSuite(func() {
	ctx := context.Background()

	goCoverDir := os.Getenv("GOCOVERDIR")
	if goCoverDir == "" {
		goCoverDir = path.Join(os.TempDir(), "covdata")
		Expect(os.MkdirAll(goCoverDir, 0o755)).To(Succeed())
	}

	dumpK3kCoverageData(ctx, goCoverDir)

	if k3sContainer != nil {
		// dump k3s logs
		k3sLogs, err := k3sContainer.Logs(ctx)
		Expect(err).To(Not(HaveOccurred()))
		fwlog.WriteLogs("k3s.log", k3sLogs)

		// dump k3k controller logs
		k3kLogs := fwlog.GetK3kPodLogs(ctx, k8sClient, k8s, k3kNamespace)
		fwlog.WriteLogs("k3k.log", k3kLogs)

		testcontainers.CleanupContainer(GinkgoTB(), k3sContainer)
	}
})

// dumpK3kCoverageData will kill the K3k controller container to force it to dump the coverage data.
// It will then download the files with kubectl cp into the specified folder. If the folder doesn't exists it will be created.
func dumpK3kCoverageData(ctx context.Context, folder string) {
	By("Restarting k3k controller")

	var podList corev1.PodList

	err := k8sClient.List(ctx, &podList, &client.ListOptions{Namespace: k3kNamespace})
	Expect(err).To(Not(HaveOccurred()))
	Expect(podList.Items).NotTo(BeEmpty())

	k3kPod := podList.Items[0]
	k3kContainerName := k3kPod.Spec.Containers[0].Name

	By("Restarting k3k controller " + k3kPod.Name + "/" + k3kContainerName)

	cmd := exec.Command("kubectl", "exec", "-n", k3kNamespace, k3kPod.Name, "-c", k3kContainerName, "--", "/bin/sh", "-c", "kill 1")
	output, err := cmd.CombinedOutput()
	Expect(err).NotTo(HaveOccurred(), string(output))

	By("Waiting to be ready again")

	Eventually(func(g Gomega) {
		key := types.NamespacedName{
			Namespace: k3kNamespace,
			Name:      k3kPod.Name,
		}

		var controllerPod corev1.Pod

		err = k8sClient.Get(ctx, key, &controllerPod)
		g.Expect(err).To(Not(HaveOccurred()))

		_, cond := pod.GetPodCondition(&controllerPod.Status, corev1.PodReady)
		g.Expect(cond).NotTo(BeNil())
		g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
	}).
		MustPassRepeatedly(5).
		WithPolling(time.Second * 2).
		WithTimeout(time.Minute * 2).
		Should(Succeed())

	By("Controller is ready again, dumping coverage data")

	GinkgoWriter.Printf("Downloading covdata from k3k controller %s/%s to %s\n", k3kNamespace, k3kPod.Name, folder)

	tarPod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "tar",
			Namespace: k3kNamespace,
		},
		Spec: corev1.PodSpec{
			Containers: []corev1.Container{{
				Name:    "tar",
				Image:   "busybox",
				Command: []string{"/bin/sh", "-c", "sleep 3600"},
				VolumeMounts: []corev1.VolumeMount{{
					Name:      "tmp-covdata",
					MountPath: "/tmp/covdata",
				}},
			}},
			Volumes: []corev1.Volume{{
				Name: "tmp-covdata",
				VolumeSource: corev1.VolumeSource{
					PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
						ClaimName: "coverage-data-pvc",
					},
				},
			}},
		},
	}

	_, err = k8s.CoreV1().Pods(k3kNamespace).Create(ctx, tarPod, metav1.CreateOptions{})
	Expect(client.IgnoreAlreadyExists(err)).To(Not(HaveOccurred()))

	By("Waiting for tar pod to be ready")

	Eventually(func(g Gomega) {
		err = k8sClient.Get(ctx, client.ObjectKeyFromObject(tarPod), tarPod)
		g.Expect(err).To(Not(HaveOccurred()))

		_, cond := pod.GetPodCondition(&tarPod.Status, corev1.PodReady)
		g.Expect(cond).NotTo(BeNil())
		g.Expect(cond.Status).To(BeEquivalentTo(metav1.ConditionTrue))
	}).
		WithPolling(time.Second).
		WithTimeout(time.Minute).
		Should(Succeed())

	By("Copying covdata from tar pod")

	cmd = exec.Command("kubectl", "cp", fmt.Sprintf("%s/%s:/tmp/covdata", k3kNamespace, tarPod.Name), folder)
	output, err = cmd.CombinedOutput()
	Expect(err).NotTo(HaveOccurred(), string(output))

	Expect(k8sClient.Delete(ctx, tarPod)).To(Succeed())
}

func readFileWithinPod(ctx context.Context, client *kubernetes.Clientset, config *rest.Config, name, namespace, path string) ([]byte, error) {
	command := []string{"cat", path}

	output := new(bytes.Buffer)

	stderr, err := podExec(ctx, client, config, namespace, name, command, nil, output)
	if err != nil || len(stderr) > 0 {
		return nil, fmt.Errorf("failed to read the following file %s: %v", path, err)
	}

	return output.Bytes(), nil
}

func podExec(ctx context.Context, clientset *kubernetes.Clientset, config *rest.Config, namespace, name string, command []string, stdin io.Reader, stdout io.Writer) ([]byte, error) {
	req := clientset.CoreV1().RESTClient().Post().
		Resource("pods").
		Name(name).
		Namespace(namespace).
		SubResource("exec")
	scheme := runtime.NewScheme()

	if err := corev1.AddToScheme(scheme); err != nil {
		return nil, fmt.Errorf("error adding to scheme: %v", err)
	}

	parameterCodec := runtime.NewParameterCodec(scheme)

	req.VersionedParams(&corev1.PodExecOptions{
		Command: command,
		Stdin:   stdin != nil,
		Stdout:  stdout != nil,
		Stderr:  true,
		TTY:     false,
	}, parameterCodec)

	exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL())
	if err != nil {
		return nil, fmt.Errorf("error while creating Executor: %v", err)
	}

	var stderr bytes.Buffer

	err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{
		Stdin:  stdin,
		Stdout: stdout,
		Stderr: &stderr,
		Tty:    false,
	})
	if err != nil {
		return nil, fmt.Errorf("error in Stream: %v", err)
	}

	return stderr.Bytes(), nil
}

func caCertSecret(name, namespace string, crt, key []byte) *corev1.Secret {
	return &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: namespace,
		},
		TypeMeta: metav1.TypeMeta{
			Kind:       "Secret",
			APIVersion: "v1",
		},
		Data: map[string][]byte{
			"tls.crt": crt,
			"tls.key": key,
		},
	}
}

func privateRegistry(ctx context.Context, namespace string) error {
	caCrtMap := map[string]string{
		"tls.crt": filepath.Join("testdata", "registry", "certs", "ca.crt"),
		"tls.key": filepath.Join("testdata", "registry", "certs", "ca.key"),
	}

	caSecret, err := buildRegistryConfigSecret(caCrtMap, namespace, registryCACertSecretName, true)
	if err != nil {
		return err
	}

	if err := k8sClient.Create(ctx, caSecret); err != nil {
		return err
	}

	registryCrtMap := map[string]string{
		"tls.crt": filepath.Join("testdata", "registry", "certs", "registry.crt"),
		"tls.key": filepath.Join("testdata", "registry", "certs", "registry.key"),
	}

	registrySecret, err := buildRegistryConfigSecret(registryCrtMap, namespace, registryCertSecretName, true)
	if err != nil {
		return err
	}

	if err := k8sClient.Create(ctx, registrySecret); err != nil {
		return err
	}

	configMap := map[string]string{
		"config.yml": filepath.Join("testdata", "registry", "config.yml"),
	}

	configSecret, err := buildRegistryConfigSecret(configMap, namespace, registryConfigSecretName, false)
	if err != nil {
		return err
	}

	if err := k8sClient.Create(ctx, configSecret); err != nil {
		return err
	}

	k3sRegistryConfig := map[string]string{
		"registries.yaml": filepath.Join("testdata", "registry", "registries.yaml"),
	}

	k3sRegistrySecret, err := buildRegistryConfigSecret(k3sRegistryConfig, namespace, k3sRegistryConfigSecretName, false)
	if err != nil {
		return err
	}

	if err := k8sClient.Create(ctx, k3sRegistrySecret); err != nil {
		return err
	}

	registryDeployment := &appsv1.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "private-registry",
			Namespace: namespace,
		},
		TypeMeta: metav1.TypeMeta{
			Kind:       "Deployment",
			APIVersion: "apps/v1",
		},
		Spec: appsv1.DeploymentSpec{
			Replicas: ptr.To(int32(1)),
			Selector: &metav1.LabelSelector{
				MatchLabels: map[string]string{
					"app": "private-registry",
				},
			},
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: map[string]string{
						"app": "private-registry",
					},
				},
				Spec: corev1.PodSpec{
					Containers: []corev1.Container{
						{
							Name:  "private-registry",
							Image: registryImage,
							VolumeMounts: []corev1.VolumeMount{
								{
									Name:      "config",
									MountPath: "/etc/docker/registry/",
								},
								{
									Name:      "ca-cert",
									MountPath: "/etc/docker/registry/ssl/ca",
								},
								{
									Name:      "registry-cert",
									MountPath: "/etc/docker/registry/ssl/registry",
								},
							},
						},
					},
					Volumes: []corev1.Volume{
						{
							Name: "config",
							VolumeSource: corev1.VolumeSource{
								Secret: &corev1.SecretVolumeSource{
									SecretName: "private-registry-config",
								},
							},
						},
						{
							Name: "ca-cert",
							VolumeSource: corev1.VolumeSource{
								Secret: &corev1.SecretVolumeSource{
									SecretName: "private-registry-ca-cert",
								},
							},
						},
						{
							Name: "registry-cert",
							VolumeSource: corev1.VolumeSource{
								Secret: &corev1.SecretVolumeSource{
									SecretName: "private-registry-cert",
								},
							},
						},
					},
				},
			},
		},
	}

	if err := k8sClient.Create(ctx, registryDeployment); err != nil {
		return err
	}

	registryService := &corev1.Service{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "private-registry",
			Namespace: namespace,
		},
		TypeMeta: metav1.TypeMeta{
			Kind:       "Service",
			APIVersion: "v1",
		},
		Spec: corev1.ServiceSpec{
			Selector: map[string]string{
				"app": "private-registry",
			},
			Ports: []corev1.ServicePort{
				{
					Name:       "registry-port",
					Port:       5000,
					TargetPort: intstr.FromInt(5000),
				},
			},
		},
	}

	return k8sClient.Create(ctx, registryService)
}

func buildRegistryNetPolicy(ctx context.Context, namespace string) error {
	np := networkingv1.NetworkPolicy{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "private-registry-test-netpol",
			Namespace: namespace,
		},
		Spec: networkingv1.NetworkPolicySpec{
			PodSelector: metav1.LabelSelector{
				MatchLabels: map[string]string{
					"role": "server",
				},
			},
			PolicyTypes: []networkingv1.PolicyType{
				networkingv1.PolicyTypeEgress,
			},
			Egress: []networkingv1.NetworkPolicyEgressRule{
				{
					To: []networkingv1.NetworkPolicyPeer{
						{
							IPBlock: &networkingv1.IPBlock{
								CIDR: "10.0.0.0/8",
							},
						},
					},
				},
			},
		},
	}

	return k8sClient.Create(ctx, &np)
}

func buildRegistryConfigSecret(tlsMap map[string]string, namespace, name string, tlsSecret bool) (*corev1.Secret, error) {
	secretType := corev1.SecretTypeOpaque
	if tlsSecret {
		secretType = corev1.SecretTypeTLS
	}

	data := make(map[string][]byte)

	for key, path := range tlsMap {
		b, err := os.ReadFile(path)
		if err != nil {
			return nil, err
		}

		data[key] = b
	}

	secret := &corev1.Secret{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Secret",
			APIVersion: "v1",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      name,
			Namespace: namespace,
		},
		Type: secretType,
		Data: data,
	}

	return secret, nil
}