mirror of
https://github.com/clastix/kamaji.git
synced 2026-02-14 10:00:02 +00:00
feat: using datastore api for backing storage driver
This commit is contained in:
96
controllers/datastore_controller.go
Normal file
96
controllers/datastore_controller.go
Normal file
@@ -0,0 +1,96 @@
|
||||
// Copyright 2022 Clastix Labs
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
k8serrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
controllerruntime "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/builder"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/event"
|
||||
"sigs.k8s.io/controller-runtime/pkg/predicate"
|
||||
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
||||
|
||||
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
|
||||
)
|
||||
|
||||
type DataStore struct {
|
||||
client client.Client
|
||||
// TenantControlPlaneTrigger is the channel used to communicate across the controllers:
|
||||
// if a Data Source is updated we have to be sure that the reconciliation of the certificates content
|
||||
// for each Tenant Control Plane is put in place properly.
|
||||
TenantControlPlaneTrigger TenantControlPlaneChannel
|
||||
// ResourceName is the DataStore object that should be watched for changes.
|
||||
ResourceName string
|
||||
}
|
||||
|
||||
//+kubebuilder:rbac:groups=kamaji.clastix.io,resources=datastores,verbs=get;list;watch;create;update;patch;delete
|
||||
//+kubebuilder:rbac:groups=kamaji.clastix.io,resources=datastores/status,verbs=get;update;patch
|
||||
|
||||
func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
|
||||
ds := kamajiv1alpha1.DataStore{}
|
||||
if err := r.client.Get(ctx, request.NamespacedName, &ds); err != nil {
|
||||
if k8serrors.IsNotFound(err) {
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
// A Data Source can trigger several Tenant Control Planes and requires a minimum validation:
|
||||
// we have to ensure the data provided by the Data Source is valid and referencing an existing Secret object.
|
||||
if _, err := ds.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, r.client); err != nil {
|
||||
return reconcile.Result{}, errors.Wrap(err, "invalid Certificate Authority data")
|
||||
}
|
||||
|
||||
if _, err := ds.Spec.TLSConfig.ClientCertificate.Certificate.GetContent(ctx, r.client); err != nil {
|
||||
return reconcile.Result{}, errors.Wrap(err, "invalid Client Certificate data")
|
||||
}
|
||||
|
||||
if _, err := ds.Spec.TLSConfig.ClientCertificate.PrivateKey.GetContent(ctx, r.client); err != nil {
|
||||
return reconcile.Result{}, errors.Wrap(err, "invalid Client Certificate data")
|
||||
}
|
||||
|
||||
tcpList := kamajiv1alpha1.TenantControlPlaneList{}
|
||||
|
||||
if err := r.client.List(ctx, &tcpList); err != nil {
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
// Updating the status with the list of Tenant Control Plane using the following Data Source
|
||||
tcpSets := sets.NewString()
|
||||
for _, tcp := range tcpList.Items {
|
||||
tcpSets.Insert(getNamespacedName(tcp.GetNamespace(), tcp.GetName()).String())
|
||||
}
|
||||
|
||||
ds.Status.UsedBy = tcpSets.List()
|
||||
|
||||
if err := r.client.Status().Update(ctx, &ds); err != nil {
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
// Triggering the reconciliation of the Tenant Control Plane upon a Secret change
|
||||
for _, i := range tcpList.Items {
|
||||
tcp := i
|
||||
|
||||
r.TenantControlPlaneTrigger <- event.GenericEvent{Object: &tcp}
|
||||
}
|
||||
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
func (r *DataStore) InjectClient(client client.Client) error {
|
||||
r.client = client
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *DataStore) SetupWithManager(mgr controllerruntime.Manager) error {
|
||||
return controllerruntime.NewControllerManagedBy(mgr).
|
||||
For(&kamajiv1alpha1.DataStore{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
|
||||
return object.GetName() == r.ResourceName
|
||||
}))).
|
||||
Complete(r)
|
||||
}
|
||||
@@ -1,10 +1,10 @@
|
||||
// Copyright 2022 Clastix Labs
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/google/uuid"
|
||||
@@ -15,11 +15,6 @@ import (
|
||||
"github.com/clastix/kamaji/internal/resources"
|
||||
"github.com/clastix/kamaji/internal/resources/konnectivity"
|
||||
"github.com/clastix/kamaji/internal/sql"
|
||||
"github.com/clastix/kamaji/internal/types"
|
||||
)
|
||||
|
||||
const (
|
||||
separator = ","
|
||||
)
|
||||
|
||||
type GroupResourceBuilderConfiguration struct {
|
||||
@@ -28,6 +23,7 @@ type GroupResourceBuilderConfiguration struct {
|
||||
tcpReconcilerConfig TenantControlPlaneReconcilerConfig
|
||||
tenantControlPlane kamajiv1alpha1.TenantControlPlane
|
||||
DBConnection sql.DBConnection
|
||||
DataStore kamajiv1alpha1.DataStore
|
||||
}
|
||||
|
||||
type GroupDeleteableResourceBuilderConfiguration struct {
|
||||
@@ -41,25 +37,25 @@ type GroupDeleteableResourceBuilderConfiguration struct {
|
||||
// GetResources returns a list of resources that will be used to provide tenant control planes
|
||||
// Currently there is only a default approach
|
||||
// TODO: the idea of this function is to become a factory to return the group of resources according to the given configuration.
|
||||
func GetResources(config GroupResourceBuilderConfiguration) []resources.Resource {
|
||||
return getDefaultResources(config)
|
||||
func GetResources(config GroupResourceBuilderConfiguration, dataStore kamajiv1alpha1.DataStore) []resources.Resource {
|
||||
return getDefaultResources(config, dataStore)
|
||||
}
|
||||
|
||||
// GetDeletableResources returns a list of resources that have to be deleted when tenant control planes are deleted
|
||||
// Currently there is only a default approach
|
||||
// TODO: the idea of this function is to become a factory to return the group of deleteable resources according to the given configuration.
|
||||
func GetDeletableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource {
|
||||
return getDefaultDeleteableResources(config)
|
||||
func GetDeletableResources(config GroupDeleteableResourceBuilderConfiguration, dataStore kamajiv1alpha1.DataStore) []resources.DeleteableResource {
|
||||
return getDefaultDeleteableResources(config, dataStore)
|
||||
}
|
||||
|
||||
func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.Resource {
|
||||
func getDefaultResources(config GroupResourceBuilderConfiguration, dataStore kamajiv1alpha1.DataStore) []resources.Resource {
|
||||
resources := append(getUpgradeResources(config.client, config.tenantControlPlane), getKubernetesServiceResources(config.client, config.tenantControlPlane)...)
|
||||
resources = append(resources, getKubeadmConfigResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
|
||||
resources = append(resources, getKubeadmConfigResources(config.client, getTmpDirectory(config.tcpReconcilerConfig.TmpBaseDirectory, config.tenantControlPlane), dataStore)...)
|
||||
resources = append(resources, getKubernetesCertificatesResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
|
||||
resources = append(resources, getKubeconfigResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
|
||||
resources = append(resources, getKubernetesStorageResources(config.client, config.log, config.tcpReconcilerConfig, config.DBConnection, config.tenantControlPlane)...)
|
||||
resources = append(resources, getKubernetesStorageResources(config.client, config.log, config.tcpReconcilerConfig, config.DBConnection, config.tenantControlPlane, dataStore)...)
|
||||
resources = append(resources, getInternalKonnectivityResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...)
|
||||
resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
|
||||
resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, dataStore)...)
|
||||
resources = append(resources, getKubernetesIngressResources(config.client, config.tenantControlPlane)...)
|
||||
resources = append(resources, getKubeadmPhaseResources(config.client, config.log, config.tenantControlPlane)...)
|
||||
resources = append(resources, getKubeadmAddonResources(config.client, config.log, config.tenantControlPlane)...)
|
||||
@@ -68,20 +64,18 @@ func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.R
|
||||
return resources
|
||||
}
|
||||
|
||||
func getDefaultDeleteableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource {
|
||||
switch config.tcpReconcilerConfig.ETCDStorageType {
|
||||
case types.ETCD:
|
||||
func getDefaultDeleteableResources(config GroupDeleteableResourceBuilderConfiguration, dataStore kamajiv1alpha1.DataStore) []resources.DeleteableResource {
|
||||
switch dataStore.Spec.Driver {
|
||||
case kamajiv1alpha1.EtcdDriver:
|
||||
return []resources.DeleteableResource{
|
||||
&resources.ETCDSetupResource{
|
||||
Name: "etcd-setup",
|
||||
Client: config.client,
|
||||
Log: config.log,
|
||||
ETCDClientCertsSecret: getNamespacedName(config.tcpReconcilerConfig.ETCDClientSecretNamespace, config.tcpReconcilerConfig.ETCDClientSecretName),
|
||||
ETCDCACertsSecret: getNamespacedName(config.tcpReconcilerConfig.ETCDCASecretNamespace, config.tcpReconcilerConfig.ETCDCASecretName),
|
||||
Endpoints: getArrayFromString(config.tcpReconcilerConfig.ETCDEndpoints),
|
||||
Name: "etcd-setup",
|
||||
Client: config.client,
|
||||
Log: config.log,
|
||||
DataStore: dataStore,
|
||||
},
|
||||
}
|
||||
case types.KineMySQL, types.KinePostgreSQL:
|
||||
case kamajiv1alpha1.KineMySQLDriver, kamajiv1alpha1.KinePostgreSQLDriver:
|
||||
return []resources.DeleteableResource{
|
||||
&resources.SQLSetup{
|
||||
Client: config.client,
|
||||
@@ -111,14 +105,22 @@ func getKubernetesServiceResources(c client.Client, tenantControlPlane kamajiv1a
|
||||
}
|
||||
}
|
||||
|
||||
func getKubeadmConfigResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
|
||||
func getKubeadmConfigResources(c client.Client, tmpDirectory string, dataStore kamajiv1alpha1.DataStore) []resources.Resource {
|
||||
var endpoints []string
|
||||
|
||||
switch dataStore.Spec.Driver {
|
||||
case kamajiv1alpha1.EtcdDriver:
|
||||
endpoints = dataStore.Spec.Endpoints
|
||||
default:
|
||||
endpoints = []string{"127.0.0.1:2379"}
|
||||
}
|
||||
|
||||
return []resources.Resource{
|
||||
&resources.KubeadmConfigResource{
|
||||
Name: "kubeadmconfig",
|
||||
ETCDs: getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
|
||||
ETCDCompactionInterval: tcpReconcilerConfig.ETCDCompactionInterval,
|
||||
Client: c,
|
||||
TmpDirectory: getTmpDirectory(tcpReconcilerConfig.TmpBaseDirectory, tenantControlPlane),
|
||||
Name: "kubeadmconfig",
|
||||
ETCDs: endpoints,
|
||||
Client: c,
|
||||
TmpDirectory: tmpDirectory,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -190,16 +192,15 @@ func getKubeconfigResources(c client.Client, log logr.Logger, tcpReconcilerConfi
|
||||
}
|
||||
}
|
||||
|
||||
func getKubernetesStorageResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dbConnection sql.DBConnection, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
|
||||
switch tcpReconcilerConfig.ETCDStorageType {
|
||||
case types.ETCD:
|
||||
func getKubernetesStorageResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dbConnection sql.DBConnection, tenantControlPlane kamajiv1alpha1.TenantControlPlane, ds kamajiv1alpha1.DataStore) []resources.Resource {
|
||||
switch ds.Spec.Driver {
|
||||
case kamajiv1alpha1.EtcdDriver:
|
||||
return []resources.Resource{
|
||||
&resources.ETCDCACertificatesResource{
|
||||
Name: "etcd-ca-certificates",
|
||||
Client: c,
|
||||
Log: log,
|
||||
ETCDCASecretName: tcpReconcilerConfig.ETCDCASecretName,
|
||||
ETCDCASecretNamespace: tcpReconcilerConfig.ETCDCASecretNamespace,
|
||||
Name: "etcd-ca-certificates",
|
||||
Client: c,
|
||||
Log: log,
|
||||
DataStore: ds,
|
||||
},
|
||||
&resources.ETCDCertificatesResource{
|
||||
Name: "etcd-certificates",
|
||||
@@ -207,15 +208,13 @@ func getKubernetesStorageResources(c client.Client, log logr.Logger, tcpReconcil
|
||||
Log: log,
|
||||
},
|
||||
&resources.ETCDSetupResource{
|
||||
Name: "etcd-setup",
|
||||
Client: c,
|
||||
Log: log,
|
||||
ETCDClientCertsSecret: getNamespacedName(tcpReconcilerConfig.ETCDClientSecretNamespace, tcpReconcilerConfig.ETCDClientSecretName),
|
||||
ETCDCACertsSecret: getNamespacedName(tcpReconcilerConfig.ETCDCASecretNamespace, tcpReconcilerConfig.ETCDCASecretName),
|
||||
Endpoints: getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
|
||||
Name: "etcd-setup",
|
||||
Client: c,
|
||||
Log: log,
|
||||
DataStore: ds,
|
||||
},
|
||||
}
|
||||
case types.KineMySQL, types.KinePostgreSQL:
|
||||
case kamajiv1alpha1.KineMySQLDriver, kamajiv1alpha1.KinePostgreSQLDriver:
|
||||
return []resources.Resource{
|
||||
&resources.SQLStorageConfig{
|
||||
Client: c,
|
||||
@@ -231,11 +230,9 @@ func getKubernetesStorageResources(c client.Client, log logr.Logger, tcpReconcil
|
||||
Driver: dbConnection.Driver(),
|
||||
},
|
||||
&resources.SQLCertificate{
|
||||
Client: c,
|
||||
Name: "sql-certificate",
|
||||
StorageType: tcpReconcilerConfig.ETCDStorageType,
|
||||
SQLConfigSecretName: tcpReconcilerConfig.KineSecretName,
|
||||
SQLConfigSecretNamespace: tcpReconcilerConfig.KineSecretNamespace,
|
||||
Client: c,
|
||||
Name: "sql-certificate",
|
||||
DataStore: ds,
|
||||
},
|
||||
}
|
||||
default:
|
||||
@@ -243,14 +240,22 @@ func getKubernetesStorageResources(c client.Client, log logr.Logger, tcpReconcil
|
||||
}
|
||||
}
|
||||
|
||||
func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource {
|
||||
func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dataStore kamajiv1alpha1.DataStore) []resources.Resource {
|
||||
var endpoints []string
|
||||
|
||||
switch dataStore.Spec.Driver {
|
||||
case kamajiv1alpha1.EtcdDriver:
|
||||
endpoints = dataStore.Spec.Endpoints
|
||||
default:
|
||||
endpoints = []string{"127.0.0.1:2379"}
|
||||
}
|
||||
|
||||
return []resources.Resource{
|
||||
&resources.KubernetesDeploymentResource{
|
||||
Client: c,
|
||||
ETCDEndpoints: getArrayFromString(tcpReconcilerConfig.ETCDEndpoints),
|
||||
ETCDCompactionInterval: tcpReconcilerConfig.ETCDCompactionInterval,
|
||||
ETCDStorageType: tcpReconcilerConfig.ETCDStorageType,
|
||||
KineContainerImage: tcpReconcilerConfig.KineContainerImage,
|
||||
Client: c,
|
||||
ETCDEndpoints: endpoints,
|
||||
DataStoreDriver: dataStore.Spec.Driver,
|
||||
KineContainerImage: tcpReconcilerConfig.KineContainerImage,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -346,13 +351,6 @@ func getInternalKonnectivityResources(c client.Client, log logr.Logger, tcpRecon
|
||||
}
|
||||
}
|
||||
|
||||
func getArrayFromString(s string) []string {
|
||||
var a []string
|
||||
a = append(a, strings.Split(s, separator)...)
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
func getNamespacedName(namespace string, name string) k8stypes.NamespacedName {
|
||||
return k8stypes.NamespacedName{Namespace: namespace, Name: name}
|
||||
}
|
||||
|
||||
@@ -8,75 +8,92 @@ import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"strings"
|
||||
"net"
|
||||
"strconv"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
k8stypes "k8s.io/apimachinery/pkg/types"
|
||||
|
||||
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
|
||||
"github.com/clastix/kamaji/internal/sql"
|
||||
"github.com/clastix/kamaji/internal/types"
|
||||
)
|
||||
|
||||
func (r *TenantControlPlaneReconciler) getStorageConnection(ctx context.Context) (sql.DBConnection, error) {
|
||||
func (r *TenantControlPlaneReconciler) getStorageConnection(ctx context.Context, ds kamajiv1alpha1.DataStore) (sql.DBConnection, error) {
|
||||
var driver sql.Driver
|
||||
var dbName string
|
||||
|
||||
// TODO: https://github.com/clastix/kamaji/issues/67
|
||||
switch r.Config.ETCDStorageType {
|
||||
case types.ETCD:
|
||||
switch ds.Spec.Driver {
|
||||
case kamajiv1alpha1.EtcdDriver:
|
||||
return nil, nil
|
||||
case types.KineMySQL:
|
||||
case kamajiv1alpha1.KineMySQLDriver:
|
||||
driver = sql.MySQL
|
||||
dbName = "mysql"
|
||||
case types.KinePostgreSQL:
|
||||
case kamajiv1alpha1.KinePostgreSQLDriver:
|
||||
driver = sql.PostgreSQL
|
||||
default:
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
secret := &corev1.Secret{}
|
||||
namespacedName := k8stypes.NamespacedName{Namespace: r.Config.KineSecretNamespace, Name: r.Config.KineSecretName}
|
||||
if err := r.Client.Get(ctx, namespacedName, secret); err != nil {
|
||||
ca, err := ds.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, r.Client)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if t := "kamaji.clastix.io/kine"; string(secret.Type) != t {
|
||||
return nil, fmt.Errorf("expecting a secret of type %s", t)
|
||||
crt, err := ds.Spec.TLSConfig.ClientCertificate.Certificate.GetContent(ctx, r.Client)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
keys := []string{"ca.crt", "server.crt", "server.key", "username", "password"}
|
||||
|
||||
if secret.Data == nil {
|
||||
return nil, fmt.Errorf("the Kine secret %s/%s is missing all the required keys (%s)", secret.GetNamespace(), secret.GetName(), strings.Join(keys, ","))
|
||||
}
|
||||
|
||||
for _, key := range keys {
|
||||
if _, ok := secret.Data[key]; !ok {
|
||||
return nil, fmt.Errorf("missing required key %s for the Kine secret %s/%s", key, secret.GetNamespace(), secret.GetName())
|
||||
}
|
||||
key, err := ds.Spec.TLSConfig.ClientCertificate.PrivateKey.GetContent(ctx, r.Client)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rootCAs := x509.NewCertPool()
|
||||
if ok := rootCAs.AppendCertsFromPEM(secret.Data["ca.crt"]); !ok {
|
||||
if ok := rootCAs.AppendCertsFromPEM(ca); !ok {
|
||||
return nil, fmt.Errorf("error create root CA for the DB connector")
|
||||
}
|
||||
|
||||
certificate, err := tls.X509KeyPair(secret.Data["server.crt"], secret.Data["server.key"])
|
||||
certificate, err := tls.X509KeyPair(crt, key)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot retrieve x.509 key pair from the Kine Secret")
|
||||
}
|
||||
|
||||
var user, password string
|
||||
if auth := ds.Spec.BasicAuth; auth != nil {
|
||||
u, err := auth.Username.GetContent(ctx, r.Client)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
user = string(u)
|
||||
|
||||
p, err := auth.Password.GetContent(ctx, r.Client)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
password = string(p)
|
||||
}
|
||||
|
||||
host, stringPort, err := net.SplitHostPort(ds.Spec.Endpoints[0])
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot retrieve host-port pair from DataStore endpoints")
|
||||
}
|
||||
|
||||
port, err := strconv.Atoi(stringPort)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "cannot convert port from string for the given DataStore")
|
||||
}
|
||||
|
||||
return sql.GetDBConnection(
|
||||
sql.ConnectionConfig{
|
||||
SQLDriver: driver,
|
||||
User: string(secret.Data["username"]),
|
||||
Password: string(secret.Data["password"]),
|
||||
Host: r.Config.KineHost,
|
||||
Port: r.Config.KinePort,
|
||||
User: user,
|
||||
Password: password,
|
||||
Host: host,
|
||||
Port: port,
|
||||
DBName: dbName,
|
||||
TLSConfig: &tls.Config{
|
||||
ServerName: r.Config.KineHost,
|
||||
ServerName: host,
|
||||
RootCAs: rootCAs,
|
||||
Certificates: []tls.Certificate{certificate},
|
||||
},
|
||||
|
||||
8
controllers/tcp_channel.go
Normal file
8
controllers/tcp_channel.go
Normal file
@@ -0,0 +1,8 @@
|
||||
// Copyright 2022 Clastix Labs
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package controllers
|
||||
|
||||
import "sigs.k8s.io/controller-runtime/pkg/event"
|
||||
|
||||
type TenantControlPlaneChannel chan event.GenericEvent
|
||||
@@ -7,22 +7,25 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
networkingv1 "k8s.io/api/networking/v1"
|
||||
k8serrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
k8stypes "k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
"sigs.k8s.io/controller-runtime/pkg/event"
|
||||
"sigs.k8s.io/controller-runtime/pkg/handler"
|
||||
"sigs.k8s.io/controller-runtime/pkg/log"
|
||||
"sigs.k8s.io/controller-runtime/pkg/source"
|
||||
|
||||
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
|
||||
kamajierrors "github.com/clastix/kamaji/internal/errors"
|
||||
"github.com/clastix/kamaji/internal/resources"
|
||||
"github.com/clastix/kamaji/internal/sql"
|
||||
"github.com/clastix/kamaji/internal/types"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -32,26 +35,16 @@ const (
|
||||
// TenantControlPlaneReconciler reconciles a TenantControlPlane object.
|
||||
type TenantControlPlaneReconciler struct {
|
||||
client.Client
|
||||
Scheme *runtime.Scheme
|
||||
Config TenantControlPlaneReconcilerConfig
|
||||
Scheme *runtime.Scheme
|
||||
Config TenantControlPlaneReconcilerConfig
|
||||
TriggerChan TenantControlPlaneChannel
|
||||
}
|
||||
|
||||
// TenantControlPlaneReconcilerConfig gives the necessary configuration for TenantControlPlaneReconciler.
|
||||
type TenantControlPlaneReconcilerConfig struct {
|
||||
ETCDStorageType types.ETCDStorageType
|
||||
ETCDCASecretName string
|
||||
ETCDCASecretNamespace string
|
||||
ETCDClientSecretName string
|
||||
ETCDClientSecretNamespace string
|
||||
ETCDEndpoints string
|
||||
ETCDCompactionInterval string
|
||||
TmpBaseDirectory string
|
||||
DBConnection sql.DBConnection
|
||||
KineSecretName string
|
||||
KineSecretNamespace string
|
||||
KineHost string
|
||||
KinePort int
|
||||
KineContainerImage string
|
||||
DataStoreName string
|
||||
KineContainerImage string
|
||||
TmpBaseDirectory string
|
||||
}
|
||||
|
||||
//+kubebuilder:rbac:groups=kamaji.clastix.io,resources=tenantcontrolplanes,verbs=get;list;watch;create;update;patch;delete
|
||||
@@ -82,7 +75,12 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
|
||||
return ctrl.Result{}, nil
|
||||
}
|
||||
|
||||
dbConnection, err := r.getStorageConnection(ctx)
|
||||
ds := kamajiv1alpha1.DataStore{}
|
||||
if err = r.Client.Get(ctx, k8stypes.NamespacedName{Name: r.Config.DataStoreName}, &ds); err != nil {
|
||||
return ctrl.Result{}, errors.Wrap(err, "cannot retrieve kamajiv1alpha.DataStore object")
|
||||
}
|
||||
|
||||
dbConnection, err := r.getStorageConnection(ctx, ds)
|
||||
if err != nil {
|
||||
return ctrl.Result{}, err
|
||||
}
|
||||
@@ -104,7 +102,7 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
|
||||
tenantControlPlane: *tenantControlPlane,
|
||||
DBConnection: dbConnection,
|
||||
}
|
||||
registeredDeletableResources := GetDeletableResources(groupDeleteableResourceBuilderConfiguration)
|
||||
registeredDeletableResources := GetDeletableResources(groupDeleteableResourceBuilderConfiguration, ds)
|
||||
|
||||
for _, resource := range registeredDeletableResources {
|
||||
if err := resources.HandleDeletion(ctx, resource, tenantControlPlane); err != nil {
|
||||
@@ -134,9 +132,10 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
|
||||
log: log,
|
||||
tcpReconcilerConfig: r.Config,
|
||||
tenantControlPlane: *tenantControlPlane,
|
||||
DataStore: ds,
|
||||
DBConnection: dbConnection,
|
||||
}
|
||||
registeredResources := GetResources(groupResourceBuilderConfiguration)
|
||||
registeredResources := GetResources(groupResourceBuilderConfiguration, ds)
|
||||
|
||||
for _, resource := range registeredResources {
|
||||
result, err := resources.Handle(ctx, resource, tenantControlPlane)
|
||||
@@ -171,6 +170,14 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
|
||||
// SetupWithManager sets up the controller with the Manager.
|
||||
func (r *TenantControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error {
|
||||
return ctrl.NewControllerManagedBy(mgr).
|
||||
Watches(&source.Channel{Source: r.TriggerChan}, handler.Funcs{GenericFunc: func(genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) {
|
||||
limitingInterface.AddRateLimited(ctrl.Request{
|
||||
NamespacedName: k8stypes.NamespacedName{
|
||||
Namespace: genericEvent.Object.GetNamespace(),
|
||||
Name: genericEvent.Object.GetName(),
|
||||
},
|
||||
})
|
||||
}}).
|
||||
For(&kamajiv1alpha1.TenantControlPlane{}).
|
||||
Owns(&corev1.Secret{}).
|
||||
Owns(&corev1.ConfigMap{}).
|
||||
|
||||
@@ -18,7 +18,6 @@ import (
|
||||
"k8s.io/utils/pointer"
|
||||
|
||||
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
|
||||
"github.com/clastix/kamaji/internal/types"
|
||||
"github.com/clastix/kamaji/internal/utilities"
|
||||
)
|
||||
|
||||
@@ -51,7 +50,7 @@ type Deployment struct {
|
||||
Address string
|
||||
ETCDEndpoints []string
|
||||
ETCDCompactionInterval string
|
||||
ETCDStorageType types.ETCDStorageType
|
||||
ETCDStorageType kamajiv1alpha1.Driver
|
||||
KineContainerImage string
|
||||
}
|
||||
|
||||
@@ -117,7 +116,7 @@ func (d *Deployment) buildPKIVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1
|
||||
},
|
||||
}
|
||||
|
||||
if d.ETCDStorageType == types.ETCD {
|
||||
if d.ETCDStorageType == kamajiv1alpha1.EtcdDriver {
|
||||
sources = append(sources, corev1.VolumeProjection{
|
||||
Secret: d.secretProjection(tcp.Status.Certificates.ETCD.APIServer.SecretName, constants.APIServerEtcdClientCertName, constants.APIServerEtcdClientKeyName),
|
||||
})
|
||||
@@ -550,8 +549,7 @@ func (d *Deployment) buildKubeAPIServerCommand(tenantControlPlane *kamajiv1alpha
|
||||
"--tls-private-key-file": path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKeyName),
|
||||
}
|
||||
|
||||
if d.ETCDStorageType == types.ETCD {
|
||||
desiredArgs["--etcd-compaction-interval"] = d.ETCDCompactionInterval
|
||||
if d.ETCDStorageType == kamajiv1alpha1.EtcdDriver {
|
||||
desiredArgs["--etcd-cafile"] = path.Join(v1beta3.DefaultCertificatesDir, constants.EtcdCACertName)
|
||||
desiredArgs["--etcd-certfile"] = path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientCertName)
|
||||
desiredArgs["--etcd-keyfile"] = path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientKeyName)
|
||||
@@ -601,7 +599,7 @@ func (d *Deployment) removeKineVolumes(podSpec *corev1.PodSpec) {
|
||||
}
|
||||
|
||||
func (d *Deployment) buildKineVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
|
||||
if d.ETCDStorageType == types.ETCD {
|
||||
if d.ETCDStorageType == kamajiv1alpha1.EtcdDriver {
|
||||
d.removeKineVolumes(podSpec)
|
||||
|
||||
return
|
||||
@@ -648,7 +646,7 @@ func (d *Deployment) removeKineContainers(podSpec *corev1.PodSpec) {
|
||||
}
|
||||
|
||||
func (d *Deployment) buildKine(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
|
||||
if d.ETCDStorageType == types.ETCD {
|
||||
if d.ETCDStorageType == kamajiv1alpha1.EtcdDriver {
|
||||
d.removeKineContainers(podSpec)
|
||||
|
||||
return
|
||||
@@ -668,9 +666,9 @@ func (d *Deployment) buildKine(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.Tena
|
||||
}
|
||||
|
||||
switch d.ETCDStorageType {
|
||||
case types.KineMySQL:
|
||||
case kamajiv1alpha1.KineMySQLDriver:
|
||||
args["--endpoint"] = "mysql://$(DB_USER):$(DB_PASSWORD)@tcp($(DB_HOST):$(DB_PORT))/$(DB_SCHEMA)"
|
||||
case types.KinePostgreSQL:
|
||||
case kamajiv1alpha1.KinePostgreSQLDriver:
|
||||
args["--endpoint"] = "postgres://$(DB_USER):$(DB_PASSWORD)@$(DB_HOST):$(DB_PORT)/$(DB_SCHEMA)"
|
||||
}
|
||||
|
||||
|
||||
@@ -21,18 +21,10 @@ var (
|
||||
)
|
||||
|
||||
const (
|
||||
envPrefix = "KAMAJI"
|
||||
defaultETCDStorageType = "etcd"
|
||||
defaultETCDCASecretName = "etcd-certs"
|
||||
defaultETCDCASecretNamespace = "kamaji-system"
|
||||
defaultETCDEndpoints = "etcd-server:2379"
|
||||
defaultETCDCompactionInterval = "0"
|
||||
defaultETCDClientSecretName = "root-client-certs"
|
||||
defaultETCDClientSecretNamespace = "kamaji-system"
|
||||
defaultTmpDirectory = "/tmp/kamaji"
|
||||
defaultKineSecretName = "kine-secret"
|
||||
defaultKineSecretNamespace = "kamaji-system"
|
||||
defaultKineImage = "rancher/kine:v0.9.2-amd64"
|
||||
envPrefix = "KAMAJI"
|
||||
defaultTmpDirectory = "/tmp/kamaji"
|
||||
defaultKineImage = "rancher/kine:v0.9.2-amd64"
|
||||
defaultDataStore = "etcd"
|
||||
)
|
||||
|
||||
func InitConfig() (*viper.Viper, error) {
|
||||
@@ -44,19 +36,9 @@ func InitConfig() (*viper.Viper, error) {
|
||||
flag.String("health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
|
||||
flag.Bool("leader-elect", false, "Enable leader election for controller manager. "+
|
||||
"Enabling this will ensure there is only one active controller manager.")
|
||||
flag.String("etcd-storage-type", defaultETCDStorageType, "Type of storage for ETCD (i.e etcd, kine-mysql, kine-psql)")
|
||||
flag.String("etcd-ca-secret-name", defaultETCDCASecretName, "Name of the secret which contains CA's certificate and private key.")
|
||||
flag.String("etcd-ca-secret-namespace", defaultETCDCASecretNamespace, "Namespace of the secret which contains CA's certificate and private key.")
|
||||
flag.String("etcd-client-secret-name", defaultETCDClientSecretName, "Name of the secret which contains ETCD client certificates")
|
||||
flag.String("etcd-client-secret-namespace", defaultETCDClientSecretNamespace, "Name of the namespace where the secret which contains ETCD client certificates is")
|
||||
flag.String("etcd-endpoints", defaultETCDEndpoints, "Comma-separated list with ETCD endpoints (i.e. https://etcd-0.etcd.kamaji-system.svc.cluster.local,https://etcd-1.etcd.kamaji-system.svc.cluster.local,https://etcd-2.etcd.kamaji-system.svc.cluster.local)")
|
||||
flag.String("etcd-compaction-interval", defaultETCDCompactionInterval, "ETCD Compaction interval (i.e. \"5m0s\"). (default: \"0\" (disabled))")
|
||||
flag.String("tmp-directory", defaultTmpDirectory, "Directory which will be used to work with temporary files.")
|
||||
flag.String("kine-secret-name", defaultKineSecretName, "Name of the secret which contains the Kine configuration.")
|
||||
flag.String("kine-secret-namespace", defaultKineSecretNamespace, "Name of the namespace where the secret which contains the Kine configuration.")
|
||||
flag.String("kine-host", "", "Host where the DB used by Kine is working.")
|
||||
flag.Int("kine-port", 0, "Port where the DB used by Kine is listening to.")
|
||||
flag.String("kine-image", defaultKineImage, "Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies)")
|
||||
flag.String("datastore", defaultDataStore, "The default DataStore that should be used by Kamaji to setup the required storage")
|
||||
|
||||
// Setup zap configuration
|
||||
opts := zap.Options{
|
||||
@@ -84,45 +66,15 @@ func InitConfig() (*viper.Viper, error) {
|
||||
if err := config.BindEnv("leader-elect", fmt.Sprintf("%s_LEADER_ELECTION", envPrefix)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := config.BindEnv("etcd-storage-type", fmt.Sprintf("%s_ETCD_STORAGE_TYPE", envPrefix)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := config.BindEnv("etcd-ca-secret-name", fmt.Sprintf("%s_ETCD_CA_SECRET_NAME", envPrefix)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := config.BindEnv("etcd-ca-secret-namespace", fmt.Sprintf("%s_ETCD_CA_SECRET_NAMESPACE", envPrefix)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := config.BindEnv("etcd-client-secret-name", fmt.Sprintf("%s_ETCD_CLIENT_SECRET_NAME", envPrefix)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := config.BindEnv("etcd-client-secret-namespace", fmt.Sprintf("%s_ETCD_CLIENT_SECRET_NAMESPACE", envPrefix)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := config.BindEnv("etcd-endpoints", fmt.Sprintf("%s_ETCD_ENDPOINTS", envPrefix)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := config.BindEnv("etcd-compaction-interval", fmt.Sprintf("%s_ETCD_COMPACTION_INTERVAL", envPrefix)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := config.BindEnv("tmp-directory", fmt.Sprintf("%s_TMP_DIRECTORY", envPrefix)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := config.BindEnv("kine-secret-name", fmt.Sprintf("%s_KINE_SECRET_NAME", envPrefix)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := config.BindEnv("kine-secret-namespace", fmt.Sprintf("%s_KINE_SECRET_NAMESPACE", envPrefix)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := config.BindEnv("kine-host", fmt.Sprintf("%s_KINE_HOST", envPrefix)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := config.BindEnv("kine-port", fmt.Sprintf("%s_KINE_PORT", envPrefix)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := config.BindEnv("kine-image", fmt.Sprintf("%s_KINE_IMAGE", envPrefix)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := config.BindEnv("datastore", fmt.Sprintf("%s_DATASTORE", envPrefix)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Setup config file
|
||||
if cfgFile != "" {
|
||||
|
||||
@@ -90,7 +90,7 @@ func getKubeadmClusterConfiguration(params Parameters) kubeadmapi.ClusterConfigu
|
||||
}, params.TenantControlPlaneCertSANs...),
|
||||
ControlPlaneComponent: kubeadmapi.ControlPlaneComponent{
|
||||
ExtraArgs: map[string]string{
|
||||
"etcd-compaction-interval": params.ETCDCompactionInterval,
|
||||
"etcd-compaction-interval": "0s",
|
||||
"etcd-prefix": fmt.Sprintf("/%s", params.TenantControlPlaneName),
|
||||
},
|
||||
},
|
||||
|
||||
@@ -44,7 +44,6 @@ type Parameters struct {
|
||||
TenantControlPlaneVersion string
|
||||
TenantControlPlaneCGroupDriver string
|
||||
ETCDs []string
|
||||
ETCDCompactionInterval string
|
||||
CertificatesDir string
|
||||
KubeconfigDir string
|
||||
}
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"github.com/go-logr/logr"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
k8stypes "k8s.io/apimachinery/pkg/types"
|
||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
@@ -22,12 +21,11 @@ import (
|
||||
)
|
||||
|
||||
type ETCDCACertificatesResource struct {
|
||||
resource *corev1.Secret
|
||||
Client client.Client
|
||||
Log logr.Logger
|
||||
Name string
|
||||
ETCDCASecretName string
|
||||
ETCDCASecretNamespace string
|
||||
resource *corev1.Secret
|
||||
Client client.Client
|
||||
Log logr.Logger
|
||||
Name string
|
||||
DataStore kamajiv1alpha1.DataStore
|
||||
}
|
||||
|
||||
func (r *ETCDCACertificatesResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
|
||||
@@ -83,6 +81,10 @@ func (r *ETCDCACertificatesResource) getPrefixedName(tenantControlPlane *kamajiv
|
||||
|
||||
func (r *ETCDCACertificatesResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
|
||||
return func() error {
|
||||
if r.DataStore.Spec.TLSConfig.CertificateAuthority.PrivateKey == nil {
|
||||
return fmt.Errorf("missing private key, cannot generate certificate for the given tenant control plane")
|
||||
}
|
||||
|
||||
if etcdStatus := tenantControlPlane.Status.Certificates.ETCD; etcdStatus != nil && len(etcdStatus.CA.Checksum) > 0 && etcdStatus.CA.Checksum == r.resource.GetAnnotations()["checksum"] {
|
||||
isValid, err := etcd.IsETCDCertificateAndKeyPairValid(r.resource.Data[kubeadmconstants.CACertName], r.resource.Data[kubeadmconstants.CAKeyName])
|
||||
if err != nil {
|
||||
@@ -94,15 +96,19 @@ func (r *ETCDCACertificatesResource) mutate(ctx context.Context, tenantControlPl
|
||||
}
|
||||
}
|
||||
|
||||
etcdCASecretNamespacedName := k8stypes.NamespacedName{Namespace: r.ETCDCASecretNamespace, Name: r.ETCDCASecretName}
|
||||
etcdCASecret := &corev1.Secret{}
|
||||
if err := r.Client.Get(ctx, etcdCASecretNamespacedName, etcdCASecret); err != nil {
|
||||
ca, err := r.DataStore.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, r.Client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
key, err := r.DataStore.Spec.TLSConfig.CertificateAuthority.PrivateKey.GetContent(ctx, r.Client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r.resource.Data = map[string][]byte{
|
||||
kubeadmconstants.CACertName: etcdCASecret.Data[kubeadmconstants.CACertName],
|
||||
kubeadmconstants.CAKeyName: etcdCASecret.Data[kubeadmconstants.CAKeyName],
|
||||
kubeadmconstants.CACertName: ca,
|
||||
kubeadmconstants.CAKeyName: key,
|
||||
}
|
||||
|
||||
r.resource.SetLabels(utilities.KamajiLabels())
|
||||
|
||||
@@ -8,9 +8,6 @@ import (
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
etcdclient "go.etcd.io/etcd/client/v3"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
k8stypes "k8s.io/apimachinery/pkg/types"
|
||||
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
|
||||
@@ -18,23 +15,17 @@ import (
|
||||
"github.com/clastix/kamaji/internal/etcd"
|
||||
)
|
||||
|
||||
const (
|
||||
caKeyName = kubeadmconstants.CACertName
|
||||
)
|
||||
|
||||
type etcdSetupResource struct {
|
||||
role etcd.Role
|
||||
user etcd.User
|
||||
}
|
||||
|
||||
type ETCDSetupResource struct {
|
||||
resource *etcdSetupResource
|
||||
Client client.Client
|
||||
Log logr.Logger
|
||||
Name string
|
||||
Endpoints []string
|
||||
ETCDClientCertsSecret k8stypes.NamespacedName
|
||||
ETCDCACertsSecret k8stypes.NamespacedName
|
||||
resource *etcdSetupResource
|
||||
Client client.Client
|
||||
Log logr.Logger
|
||||
Name string
|
||||
DataStore kamajiv1alpha1.DataStore
|
||||
}
|
||||
|
||||
func (r *ETCDSetupResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
|
||||
@@ -144,21 +135,26 @@ func (r *ETCDSetupResource) reconcile(ctx context.Context) (controllerutil.Opera
|
||||
}
|
||||
|
||||
func (r *ETCDSetupResource) getETCDClient(ctx context.Context) (*etcdclient.Client, error) {
|
||||
var certsClientSecret corev1.Secret
|
||||
if err := r.Client.Get(ctx, r.ETCDClientCertsSecret, &certsClientSecret); err != nil {
|
||||
ca, err := r.DataStore.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, r.Client)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var certsCASecret corev1.Secret
|
||||
if err := r.Client.Get(ctx, r.ETCDCACertsSecret, &certsCASecret); err != nil {
|
||||
crt, err := r.DataStore.Spec.TLSConfig.ClientCertificate.Certificate.GetContent(ctx, r.Client)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
key, err := r.DataStore.Spec.TLSConfig.ClientCertificate.PrivateKey.GetContent(ctx, r.Client)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
config := etcd.Config{
|
||||
ETCDCertificate: certsClientSecret.Data[corev1.TLSCertKey],
|
||||
ETCDPrivateKey: certsClientSecret.Data[corev1.TLSPrivateKeyKey],
|
||||
ETCDCA: certsCASecret.Data[caKeyName],
|
||||
Endpoints: r.Endpoints,
|
||||
ETCDCertificate: crt,
|
||||
ETCDPrivateKey: key,
|
||||
ETCDCA: ca,
|
||||
Endpoints: r.DataStore.Spec.Endpoints,
|
||||
}
|
||||
|
||||
return etcd.NewClient(config)
|
||||
|
||||
@@ -14,18 +14,16 @@ import (
|
||||
|
||||
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
|
||||
builder "github.com/clastix/kamaji/internal/builders/controlplane"
|
||||
"github.com/clastix/kamaji/internal/types"
|
||||
"github.com/clastix/kamaji/internal/utilities"
|
||||
)
|
||||
|
||||
type KubernetesDeploymentResource struct {
|
||||
resource *appsv1.Deployment
|
||||
Client client.Client
|
||||
ETCDStorageType types.ETCDStorageType
|
||||
ETCDEndpoints []string
|
||||
ETCDCompactionInterval string
|
||||
Name string
|
||||
KineContainerImage string
|
||||
resource *appsv1.Deployment
|
||||
Client client.Client
|
||||
DataStoreDriver kamajiv1alpha1.Driver
|
||||
ETCDEndpoints []string
|
||||
Name string
|
||||
KineContainerImage string
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
|
||||
@@ -65,11 +63,10 @@ func (r *KubernetesDeploymentResource) mutate(ctx context.Context, tenantControl
|
||||
}
|
||||
|
||||
d := builder.Deployment{
|
||||
Address: address,
|
||||
ETCDEndpoints: r.ETCDEndpoints,
|
||||
ETCDCompactionInterval: r.ETCDCompactionInterval,
|
||||
ETCDStorageType: r.ETCDStorageType,
|
||||
KineContainerImage: r.KineContainerImage,
|
||||
Address: address,
|
||||
ETCDEndpoints: r.ETCDEndpoints,
|
||||
ETCDStorageType: r.DataStoreDriver,
|
||||
KineContainerImage: r.KineContainerImage,
|
||||
}
|
||||
d.SetLabels(r.resource, utilities.MergeMaps(utilities.CommonLabels(tenantControlPlane.GetName()), tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Labels))
|
||||
d.SetAnnotations(r.resource, utilities.MergeMaps(r.resource.Annotations, tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Annotations))
|
||||
@@ -135,7 +132,7 @@ func (r *KubernetesDeploymentResource) deploymentTemplateLabels(ctx context.Cont
|
||||
"component.kamaji.clastix.io/scheduler-kubeconfig": hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.KubeConfig.Scheduler.SecretName),
|
||||
}
|
||||
|
||||
if r.ETCDStorageType == types.ETCD {
|
||||
if r.DataStoreDriver == kamajiv1alpha1.EtcdDriver {
|
||||
labels["component.kamaji.clastix.io/etcd-ca-certificates"] = hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.CA.SecretName)
|
||||
labels["component.kamaji.clastix.io/etcd-certificates"] = hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName)
|
||||
}
|
||||
|
||||
@@ -33,12 +33,11 @@ const (
|
||||
)
|
||||
|
||||
type KubernetesDeploymentResource struct {
|
||||
resource *appsv1.Deployment
|
||||
Client client.Client
|
||||
ETCDStorageType types.ETCDStorageType
|
||||
ETCDEndpoints []string
|
||||
ETCDCompactionInterval string
|
||||
Name string
|
||||
resource *appsv1.Deployment
|
||||
Client client.Client
|
||||
ETCDStorageType types.ETCDStorageType
|
||||
ETCDEndpoints []string
|
||||
Name string
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
|
||||
|
||||
@@ -19,12 +19,11 @@ import (
|
||||
)
|
||||
|
||||
type KubeadmConfigResource struct {
|
||||
resource *corev1.ConfigMap
|
||||
Client client.Client
|
||||
Name string
|
||||
ETCDs []string
|
||||
ETCDCompactionInterval string
|
||||
TmpDirectory string
|
||||
resource *corev1.ConfigMap
|
||||
Client client.Client
|
||||
Name string
|
||||
ETCDs []string
|
||||
TmpDirectory string
|
||||
}
|
||||
|
||||
func (r *KubeadmConfigResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
|
||||
@@ -98,7 +97,6 @@ func (r *KubeadmConfigResource) mutate(tenantControlPlane *kamajiv1alpha1.Tenant
|
||||
TenantControlPlaneServiceCIDR: tenantControlPlane.Spec.NetworkProfile.ServiceCIDR,
|
||||
TenantControlPlaneVersion: tenantControlPlane.Spec.Kubernetes.Version,
|
||||
ETCDs: r.ETCDs,
|
||||
ETCDCompactionInterval: r.ETCDCompactionInterval,
|
||||
CertificatesDir: r.TmpDirectory,
|
||||
}
|
||||
|
||||
|
||||
@@ -5,11 +5,9 @@ package resources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
k8stypes "k8s.io/apimachinery/pkg/types"
|
||||
ctrl "sigs.k8s.io/controller-runtime"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
@@ -20,12 +18,11 @@ import (
|
||||
)
|
||||
|
||||
type SQLCertificate struct {
|
||||
resource *corev1.Secret
|
||||
Client client.Client
|
||||
Name string
|
||||
StorageType types.ETCDStorageType
|
||||
SQLConfigSecretName string
|
||||
SQLConfigSecretNamespace string
|
||||
resource *corev1.Secret
|
||||
Client client.Client
|
||||
Name string
|
||||
StorageType types.ETCDStorageType
|
||||
DataStore kamajiv1alpha1.DataStore
|
||||
}
|
||||
|
||||
func (r *SQLCertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
|
||||
@@ -82,15 +79,25 @@ func (r *SQLCertificate) UpdateTenantControlPlaneStatus(ctx context.Context, ten
|
||||
|
||||
func (r *SQLCertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
|
||||
return func() error {
|
||||
sqlConfig := &corev1.Secret{}
|
||||
namespacedName := k8stypes.NamespacedName{Namespace: r.SQLConfigSecretNamespace, Name: r.SQLConfigSecretName}
|
||||
if err := r.Client.Get(ctx, namespacedName, sqlConfig); err != nil {
|
||||
return err
|
||||
ca, err := r.DataStore.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, r.Client)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
checksum, err := r.buildSecret(ctx, *sqlConfig)
|
||||
crt, err := r.DataStore.Spec.TLSConfig.ClientCertificate.Certificate.GetContent(ctx, r.Client)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil
|
||||
}
|
||||
|
||||
key, err := r.DataStore.Spec.TLSConfig.ClientCertificate.PrivateKey.GetContent(ctx, r.Client)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
r.resource.Data = map[string][]byte{
|
||||
"ca.crt": ca,
|
||||
"server.crt": crt,
|
||||
"server.key": key,
|
||||
}
|
||||
|
||||
annotations := r.resource.GetAnnotations()
|
||||
@@ -98,7 +105,7 @@ func (r *SQLCertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv
|
||||
annotations = map[string]string{}
|
||||
}
|
||||
|
||||
annotations["checksum"] = checksum
|
||||
annotations["checksum"] = utilities.CalculateConfigMapChecksum(r.resource.StringData)
|
||||
|
||||
r.resource.SetAnnotations(annotations)
|
||||
|
||||
@@ -114,30 +121,3 @@ func (r *SQLCertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv
|
||||
return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
|
||||
}
|
||||
}
|
||||
|
||||
func (r *SQLCertificate) buildSecret(ctx context.Context, sqlConfig corev1.Secret) (checksum string, err error) {
|
||||
switch r.StorageType {
|
||||
case types.KineMySQL, types.KinePostgreSQL:
|
||||
keys := []string{"ca.crt", "server.crt", "server.key"}
|
||||
|
||||
return r.buildKineSecret(ctx, keys, sqlConfig)
|
||||
default:
|
||||
return "", fmt.Errorf("storage type %s is not implemented", r.StorageType)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *SQLCertificate) buildKineSecret(ctx context.Context, keys []string, sqlConfig corev1.Secret) (string, error) {
|
||||
checksumMap := map[string]string{}
|
||||
|
||||
for _, key := range keys {
|
||||
value, ok := sqlConfig.Data[key]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%s is not in sql config secret", key)
|
||||
}
|
||||
|
||||
r.resource.Data[key] = value
|
||||
checksumMap[key] = string(value)
|
||||
}
|
||||
|
||||
return utilities.CalculateConfigMapChecksum(checksumMap), nil
|
||||
}
|
||||
|
||||
14
kamaji.yaml
14
kamaji.yaml
@@ -1,16 +1,6 @@
|
||||
etcd-storage-type: etcd
|
||||
etcd-ca-secret-name: "etcd-certs"
|
||||
etcd-ca-secret-namespace: kamaji-system
|
||||
etcd-endpoints: https://etcd-0.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-1.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-2.etcd.kamaji-system.svc.cluster.local:2379
|
||||
etcd-client-secret-name: root-client-certs
|
||||
etcd-client-secret-namespaced: kamaji-system
|
||||
etcd-compaction: "0"
|
||||
metrics-bind-address: :8080
|
||||
health-probe-bind-address: :8081
|
||||
leader-elect: false
|
||||
tmp-directory: /tmp/kamaji
|
||||
kine-mysql-secret-name: "mysql-config"
|
||||
kine-mysql-secret-namespace: kamaji-system
|
||||
kine-mysql-host: localhost
|
||||
kine-mysql-port: 3306
|
||||
kine-image: rancher/kine:v0.9.2-amd64
|
||||
datastore: etcd
|
||||
kine-image: rancher/kine:v0.9.2-amd64
|
||||
|
||||
25
main.go
25
main.go
@@ -20,7 +20,6 @@ import (
|
||||
"github.com/clastix/kamaji/controllers"
|
||||
"github.com/clastix/kamaji/internal"
|
||||
"github.com/clastix/kamaji/internal/config"
|
||||
"github.com/clastix/kamaji/internal/types"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -60,24 +59,22 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
tcpChannel := make(controllers.TenantControlPlaneChannel)
|
||||
|
||||
if err = (&controllers.DataStore{TenantControlPlaneTrigger: tcpChannel, ResourceName: conf.GetString("datastore")}).SetupWithManager(mgr); err != nil {
|
||||
setupLog.Error(err, "unable to create controller", "controller", "DataStore")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
reconciler := &controllers.TenantControlPlaneReconciler{
|
||||
Client: mgr.GetClient(),
|
||||
Scheme: mgr.GetScheme(),
|
||||
Config: controllers.TenantControlPlaneReconcilerConfig{
|
||||
ETCDStorageType: types.ParseETCDStorageType(conf.GetString("etcd-storage-type")),
|
||||
ETCDCASecretName: conf.GetString("etcd-ca-secret-name"),
|
||||
ETCDCASecretNamespace: conf.GetString("etcd-ca-secret-namespace"),
|
||||
ETCDClientSecretName: conf.GetString("etcd-client-secret-name"),
|
||||
ETCDClientSecretNamespace: conf.GetString("etcd-client-secret-namespace"),
|
||||
ETCDEndpoints: types.ParseETCDEndpoint(conf),
|
||||
ETCDCompactionInterval: conf.GetString("etcd-compaction-interval"),
|
||||
TmpBaseDirectory: conf.GetString("tmp-directory"),
|
||||
KineSecretName: conf.GetString("kine-secret-name"),
|
||||
KineSecretNamespace: conf.GetString("kine-secret-namespace"),
|
||||
KineHost: conf.GetString("kine-host"),
|
||||
KinePort: conf.GetInt("kine-port"),
|
||||
KineContainerImage: conf.GetString("kine-image"),
|
||||
DataStoreName: conf.GetString("datastore"),
|
||||
KineContainerImage: conf.GetString("kine-image"),
|
||||
TmpBaseDirectory: conf.GetString("tmp-directory"),
|
||||
},
|
||||
TriggerChan: tcpChannel,
|
||||
}
|
||||
|
||||
if err := reconciler.SetupWithManager(mgr); err != nil {
|
||||
|
||||
Reference in New Issue
Block a user