feat: add support for multiple Datastores (#961)

* feat: add support for multiple Datastores

* docs: add guide for datastore overrides

* feat(datastore): add e2e test for dataStoreOverrides

* ci: reclaim disk space from runner to fix flaky tests

internal/builders/controlplane/deployment.go

@@ -52,9 +52,15 @@ const (
 	kineInitContainerName     = "chmod"
 )
 
+type DataStoreOverrides struct {
+	Resource  string
+	DataStore kamajiv1alpha1.DataStore
+}
+
 type Deployment struct {
 	KineContainerImage string
 	DataStore          kamajiv1alpha1.DataStore
+	DataStoreOverrides []DataStoreOverrides
 	Client             client.Client
 }
 
@@ -711,11 +717,29 @@ func (d Deployment) buildKubeAPIServerCommand(tenantControlPlane kamajiv1alpha1.
 		desiredArgs["--etcd-keyfile"] = "/etc/kubernetes/pki/etcd/server.key"
 	}
 
+	if len(d.DataStoreOverrides) != 0 {
+		desiredArgs["--etcd-servers-overrides"] = d.etcdServersOverrides()
+	}
+
 	// Order matters, here: extraArgs could try to overwrite some arguments managed by Kamaji and that would be crucial.
 	// Adding as first element of the array of maps, we're sure that these overrides will be sanitized by our configuration.
 	return utilities.MergeMaps(current, desiredArgs, extraArgs)
 }
 
+func (d Deployment) etcdServersOverrides() string {
+	dataStoreOverridesEndpoints := make([]string, 0, len(d.DataStoreOverrides))
+	for _, dso := range d.DataStoreOverrides {
+		httpsEndpoints := make([]string, 0, len(dso.DataStore.Spec.Endpoints))
+
+		for _, ep := range dso.DataStore.Spec.Endpoints {
+			httpsEndpoints = append(httpsEndpoints, fmt.Sprintf("https://%s", ep))
+		}
+		dataStoreOverridesEndpoints = append(dataStoreOverridesEndpoints, fmt.Sprintf("%s#%s", dso.Resource, strings.Join(httpsEndpoints, ";")))
+	}
+
+	return strings.Join(dataStoreOverridesEndpoints, ",")
+}
+
 func (d Deployment) secretProjection(secretName, certKeyName, keyName string) *corev1.SecretProjection {
 	return &corev1.SecretProjection{
 		LocalObjectReference: corev1.LocalObjectReference{

internal/builders/controlplane/deployment_test.go

@@ -0,0 +1,46 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controlplane
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("Controlplane Deployment", func() {
+	var d Deployment
+	BeforeEach(func() {
+		d = Deployment{
+			DataStoreOverrides: []DataStoreOverrides{{
+				Resource: "/events",
+				DataStore: kamajiv1alpha1.DataStore{
+					Spec: kamajiv1alpha1.DataStoreSpec{
+						Endpoints: kamajiv1alpha1.Endpoints{"etcd-0", "etcd-1", "etcd-2"},
+					},
+				},
+			}},
+		}
+	})
+
+	Describe("DataStoreOverrides flag generation", func() {
+		It("should generate valid --etcd-servers-overrides value", func() {
+			etcdSerVersOverrides := d.etcdServersOverrides()
+			Expect(etcdSerVersOverrides).To(Equal("/events#https://etcd-0;https://etcd-1;https://etcd-2"))
+		})
+		It("should generate valid --etcd-servers-overrides value with 2 DataStoreOverrides", func() {
+			d.DataStoreOverrides = append(d.DataStoreOverrides, DataStoreOverrides{
+				Resource: "/pods",
+				DataStore: kamajiv1alpha1.DataStore{
+					Spec: kamajiv1alpha1.DataStoreSpec{
+						Endpoints: kamajiv1alpha1.Endpoints{"etcd-3", "etcd-4", "etcd-5"},
+					},
+				},
+			})
+			etcdSerVersOverrides := d.etcdServersOverrides()
+			Expect(etcdSerVersOverrides).To(Equal("/events#https://etcd-0;https://etcd-1;https://etcd-2,/pods#https://etcd-3;https://etcd-4;https://etcd-5"))
+		})
+	})
+})

internal/resources/datastore/datastore_storage_config.go

@@ -30,6 +30,7 @@ type Config struct {
 	Client     client.Client
 	ConnString string
 	DataStore  kamajiv1alpha1.DataStore
+	IsOverride bool
 }
 
 func (r *Config) GetHistogram() prometheus.Histogram {
@@ -103,10 +104,12 @@ func (r *Config) GetName() string {
 }
 
 func (r *Config) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	tenantControlPlane.Status.Storage.Driver = string(r.DataStore.Spec.Driver)
-	tenantControlPlane.Status.Storage.DataStoreName = r.DataStore.GetName()
-	tenantControlPlane.Status.Storage.Config.SecretName = r.resource.GetName()
-	tenantControlPlane.Status.Storage.Config.Checksum = utilities.GetObjectChecksum(r.resource)
+	if !r.IsOverride {
+		tenantControlPlane.Status.Storage.Driver = string(r.DataStore.Spec.Driver)
+		tenantControlPlane.Status.Storage.DataStoreName = r.DataStore.GetName()
+		tenantControlPlane.Status.Storage.Config.SecretName = r.resource.GetName()
+		tenantControlPlane.Status.Storage.Config.Checksum = utilities.GetObjectChecksum(r.resource)
+	}
 
 	return nil
 }

internal/resources/k8s_deployment_resource.go

@@ -22,6 +22,7 @@ type KubernetesDeploymentResource struct {
 	resource           *appsv1.Deployment
 	Client             client.Client
 	DataStore          kamajiv1alpha1.DataStore
+	DataStoreOverrides []builder.DataStoreOverrides
 	KineContainerImage string
 }
 
@@ -66,6 +67,7 @@ func (r *KubernetesDeploymentResource) mutate(ctx context.Context, tenantControl
 			Client:             r.Client,
 			DataStore:          r.DataStore,
 			KineContainerImage: r.KineContainerImage,
+			DataStoreOverrides: r.DataStoreOverrides,
 		}).Build(ctx, r.resource, *tenantControlPlane)
 
 		return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())

internal/webhook/handlers/tcp_datastore.go

@@ -30,7 +30,7 @@ func (t TenantControlPlaneDataStore) OnCreate(object runtime.Object) AdmissionRe
 			return nil, t.check(ctx, tcp.Spec.DataStore)
 		}
 
-		return nil, nil
+		return nil, t.checkDataStoreOverrides(ctx, tcp)
 	}
 }
 
@@ -61,3 +61,19 @@ func (t TenantControlPlaneDataStore) check(ctx context.Context, dataStoreName st
 
 	return nil
 }
+
+func (t TenantControlPlaneDataStore) checkDataStoreOverrides(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) error {
+	overrideCheck := make(map[string]struct{}, 0)
+	for _, ds := range tcp.Spec.DataStoreOverrides {
+		if _, exists := overrideCheck[ds.Resource]; !exists {
+			overrideCheck[ds.Resource] = struct{}{}
+		} else {
+			return fmt.Errorf("duplicate resource override in Spec.DataStoreOverrides")
+		}
+		if err := t.check(ctx, ds.DataStore); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/webhook/handlers/tcp_datastore_test.go

@@ -0,0 +1,94 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package handlers
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("TCP Datastore webhook", func() {
+	var (
+		ctx context.Context
+		t   TenantControlPlaneDataStore
+		tcp *kamajiv1alpha1.TenantControlPlane
+	)
+	BeforeEach(func() {
+		scheme := runtime.NewScheme()
+		utilruntime.Must(kamajiv1alpha1.AddToScheme(scheme))
+
+		ctx = context.Background()
+		t = TenantControlPlaneDataStore{
+			Client: fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(&kamajiv1alpha1.DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "foo",
+				},
+			}, &kamajiv1alpha1.DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "bar",
+				},
+			}).Build(),
+		}
+		tcp = &kamajiv1alpha1.TenantControlPlane{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "tcp",
+				Namespace: "default",
+			},
+			Spec: kamajiv1alpha1.TenantControlPlaneSpec{},
+		}
+	})
+	Describe("validation should succeed without DataStoreOverrides", func() {
+		It("should validate TCP without DataStoreOverrides", func() {
+			err := t.checkDataStoreOverrides(ctx, tcp)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+
+	Describe("validation should fail with duplicate resources in DataStoreOverrides", func() {
+		It("should fail to validate TCP with duplicate resources in DataStoreOverrides", func() {
+			tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
+				Resource:  "/event",
+				DataStore: "foo",
+			}, {
+				Resource:  "/event",
+				DataStore: "bar",
+			}}
+			err := t.checkDataStoreOverrides(ctx, tcp)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("validation should succeed with valid DataStoreOverrides", func() {
+		It("should validate TCP with valid DataStoreOverrides", func() {
+			tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
+				Resource:  "/leases",
+				DataStore: "foo",
+			}, {
+				Resource:  "/event",
+				DataStore: "bar",
+			}}
+			err := t.checkDataStoreOverrides(ctx, tcp)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+
+	Describe("validation should fail with nonexistent DataStoreOverrides", func() {
+		It("should fail to validate TCP with nonexistent DataStoreOverrides", func() {
+			tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
+				Resource:  "/leases",
+				DataStore: "baz",
+			}}
+			err := t.checkDataStoreOverrides(ctx, tcp)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+})

internal/webhook/handlers/tcp_deployment.go

@@ -67,6 +67,20 @@ func (t TenantControlPlaneDeployment) OnUpdate(newObject runtime.Object, oldObje
 		}
 		t.DeploymentBuilder.DataStore = ds
 
+		dataStoreOverrides := make([]controlplane.DataStoreOverrides, 0, len(tcp.Spec.DataStoreOverrides))
+
+		for _, dso := range tcp.Spec.DataStoreOverrides {
+			ds := kamajiv1alpha1.DataStore{}
+			if err := t.Client.Get(ctx, types.NamespacedName{Name: dso.DataStore}, &ds); err != nil {
+				return nil, err
+			}
+			dataStoreOverrides = append(dataStoreOverrides, controlplane.DataStoreOverrides{
+				Resource:  dso.Resource,
+				DataStore: ds,
+			})
+		}
+		t.DeploymentBuilder.DataStoreOverrides = dataStoreOverrides
+
 		deployment := appsv1.Deployment{}
 		deployment.Name = tcp.Name
 		deployment.Namespace = tcp.Namespace