Compare commits

...

8 Commits

Author SHA1 Message Date
Dario Tranchitella
cac1631523 feat: rotating certificates via annotation (#877)
* fix(kubeconfig): checking certificate authority data for validity

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

* feat: rotating certificates via annotation

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

* docs: rotating certificates via annotation

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

---------

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>
2025-07-21 09:23:29 +02:00
Dario Tranchitella
d1eb860918 feat!: support for konnectivity deployment mode (#875)
* feat(konnectivity): support for deployment mode

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

* feat(helm)!: support for konnectivity deployment mode

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

* chore(sample): support for konnectivity deployment mode

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

* docs: support for konnectivity deployment mode

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

---------

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>
2025-07-21 09:21:35 +02:00
dependabot[bot]
6c76bd6a97 feat(deps): bump github.com/testcontainers/testcontainers-go (#878)
Bumps [github.com/testcontainers/testcontainers-go](https://github.com/testcontainers/testcontainers-go) from 0.37.0 to 0.38.0.
- [Release notes](https://github.com/testcontainers/testcontainers-go/releases)
- [Commits](https://github.com/testcontainers/testcontainers-go/compare/v0.37.0...v0.38.0)

---
updated-dependencies:
- dependency-name: github.com/testcontainers/testcontainers-go
  dependency-version: 0.38.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-07-18 14:42:18 +02:00
dependabot[bot]
462d52332c feat(deps): bump github.com/spf13/pflag from 1.0.6 to 1.0.7 (#884)
Bumps [github.com/spf13/pflag](https://github.com/spf13/pflag) from 1.0.6 to 1.0.7.
- [Release notes](https://github.com/spf13/pflag/releases)
- [Commits](https://github.com/spf13/pflag/compare/v1.0.6...v1.0.7)

---
updated-dependencies:
- dependency-name: github.com/spf13/pflag
  dependency-version: 1.0.7
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-07-18 14:41:49 +02:00
Mario Valderrama
63a29b4b59 fix: typo in llms.txt (#879) 2025-07-16 11:36:21 +02:00
Dario Tranchitella
e366dc3959 feat: pausing reconciliation of controlled objects (#874)
* feat: pausing reconciliation of controlled objects

Objects such as TenantControlPlane and Secret can be annotated with
kamaji.clastix.io/paused to prevent controllers from processing them.

This will stop reconciling objects for debugging or other purposes.
Annotation value is irrelevant, just the key presence is evaluated.

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

* docs: pausing reconciliation of controlled objects

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

* chore(logs): typo for deleted resources

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

---------

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>
2025-07-16 10:44:48 +02:00
Dario Tranchitella
0ab8843418 feat(chore): support for customising container repository via ldflags (#873)
Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>
2025-07-14 13:38:09 +02:00
dependabot[bot]
ce5fe906aa feat(deps): bump github.com/docker/docker (#869)
Bumps [github.com/docker/docker](https://github.com/docker/docker) from 28.3.0+incompatible to 28.3.2+incompatible.
- [Release notes](https://github.com/docker/docker/releases)
- [Commits](https://github.com/docker/docker/compare/v28.3.0...v28.3.2)

---
updated-dependencies:
- dependency-name: github.com/docker/docker
  dependency-version: 28.3.2+incompatible
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2025-07-13 19:29:12 +02:00
40 changed files with 610 additions and 172 deletions

View File

@@ -0,0 +1,10 @@
// Copyright 2022 Clastix Labs
// SPDX-License-Identifier: Apache-2.0
package v1alpha1
const (
// PausedReconciliationAnnotation is an annotation that can be applied to
// Tenant Control Plane objects to prevent the controller from processing such a resource.
PausedReconciliationAnnotation = "kamaji.clastix.io/paused"
)

View File

@@ -122,6 +122,12 @@ type ExternalKubernetesObjectStatus struct {
LastUpdate metav1.Time `json:"lastUpdate,omitempty"`
}
type KonnectivityAgentStatus struct {
ExternalKubernetesObjectStatus `json:",inline"`
Mode KonnectivityAgentMode `json:"mode,omitempty"`
}
// KonnectivityStatus defines the status of Konnectivity as Addon.
type KonnectivityStatus struct {
Enabled bool `json:"enabled"`
@@ -130,7 +136,7 @@ type KonnectivityStatus struct {
Kubeconfig KubeconfigStatus `json:"kubeconfig,omitempty"`
ServiceAccount ExternalKubernetesObjectStatus `json:"sa,omitempty"`
ClusterRoleBinding ExternalKubernetesObjectStatus `json:"clusterrolebinding,omitempty"`
Agent ExternalKubernetesObjectStatus `json:"agent,omitempty"`
Agent KonnectivityAgentStatus `json:"agent,omitempty"`
Service KubernetesServiceStatus `json:"service,omitempty"`
}

View File

@@ -236,6 +236,15 @@ type KonnectivityServerSpec struct {
ExtraArgs ExtraArgs `json:"extraArgs,omitempty"`
}
type KonnectivityAgentMode string
var (
KonnectivityAgentModeDaemonSet KonnectivityAgentMode = "DaemonSet"
KonnectivityAgentModeDeployment KonnectivityAgentMode = "Deployment"
)
//+kubebuilder:validation:XValidation:rule="!(self.mode == 'DaemonSet' && has(self.replicas) && self.replicas != 0) && !(self.mode == 'Deployment' && self.replicas == 0)",message="replicas must be 0 when mode is DaemonSet, and greater than 0 when mode is Deployment"
type KonnectivityAgentSpec struct {
// AgentImage defines the container image for Konnectivity's agent.
//+kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-agent
@@ -248,13 +257,21 @@ type KonnectivityAgentSpec struct {
//+kubebuilder:default={{key: "CriticalAddonsOnly", operator: "Exists"}}
Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
ExtraArgs ExtraArgs `json:"extraArgs,omitempty"`
// Mode allows specifying the Agent deployment mode: Deployment, or DaemonSet (default).
//+kubebuilder:default="DaemonSet"
//+kubebuilder:validation:Enum=DaemonSet;Deployment
Mode KonnectivityAgentMode `json:"mode,omitempty"`
// Replicas defines the number of replicas when Mode is Deployment.
// Must be 0 if Mode is DaemonSet.
//+kubebuilder:validation:Optional
Replicas int32 `json:"replicas,omitempty"`
}
// KonnectivitySpec defines the spec for Konnectivity.
type KonnectivitySpec struct {
//+kubebuilder:default={version:"v0.28.6",image:"registry.k8s.io/kas-network-proxy/proxy-server",port:8132}
KonnectivityServerSpec KonnectivityServerSpec `json:"server,omitempty"`
//+kubebuilder:default={version:"v0.28.6",image:"registry.k8s.io/kas-network-proxy/proxy-agent"}
//+kubebuilder:default={version:"v0.28.6",image:"registry.k8s.io/kas-network-proxy/proxy-agent",mode:"DaemonSet"}
KonnectivityAgentSpec KonnectivityAgentSpec `json:"agent,omitempty"`
}

View File

@@ -808,6 +808,22 @@ func (in *KonnectivityAgentSpec) DeepCopy() *KonnectivityAgentSpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *KonnectivityAgentStatus) DeepCopyInto(out *KonnectivityAgentStatus) {
*out = *in
in.ExternalKubernetesObjectStatus.DeepCopyInto(&out.ExternalKubernetesObjectStatus)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KonnectivityAgentStatus.
func (in *KonnectivityAgentStatus) DeepCopy() *KonnectivityAgentStatus {
if in == nil {
return nil
}
out := new(KonnectivityAgentStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *KonnectivityConfigMap) DeepCopyInto(out *KonnectivityConfigMap) {
*out = *in

View File

@@ -96,6 +96,7 @@ spec:
agent:
default:
image: registry.k8s.io/kas-network-proxy/proxy-agent
mode: DaemonSet
version: v0.28.6
properties:
extraArgs:
@@ -111,6 +112,19 @@ spec:
default: registry.k8s.io/kas-network-proxy/proxy-agent
description: AgentImage defines the container image for Konnectivity's agent.
type: string
mode:
default: DaemonSet
description: 'Mode allows specifying the Agent deployment mode: Deployment, or DaemonSet (default).'
enum:
- DaemonSet
- Deployment
type: string
replicas:
description: |-
Replicas defines the number of replicas when Mode is Deployment.
Must be 0 if Mode is DaemonSet.
format: int32
type: integer
tolerations:
default:
- key: CriticalAddonsOnly
@@ -160,6 +174,9 @@ spec:
description: Version for Konnectivity agent.
type: string
type: object
x-kubernetes-validations:
- message: replicas must be 0 when mode is DaemonSet, and greater than 0 when mode is Deployment
rule: '!(self.mode == ''DaemonSet'' && has(self.replicas) && self.replicas != 0) && !(self.mode == ''Deployment'' && self.replicas == 0)'
server:
default:
image: registry.k8s.io/kas-network-proxy/proxy-server
@@ -6685,6 +6702,8 @@ spec:
description: Last time when k8s object was updated
format: date-time
type: string
mode:
type: string
name:
type: string
namespace:

View File

@@ -306,7 +306,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
cmd.Flags().StringVar(&tmpDirectory, "tmp-directory", "/tmp/kamaji", "Directory which will be used to work with temporary files.")
cmd.Flags().StringVar(&kineImage, "kine-image", "rancher/kine:v0.11.10-amd64", "Container image along with tag to use for the Kine sidecar container (used only if etcd-storage-type is set to one of kine strategies).")
cmd.Flags().StringVar(&datastore, "datastore", "", "Optional, the default DataStore that should be used by Kamaji to setup the required storage of Tenant Control Planes with undeclared DataStore.")
cmd.Flags().StringVar(&migrateJobImage, "migrate-image", fmt.Sprintf("clastix/kamaji:%s", internal.GitTag), "Specify the container image to launch when a TenantControlPlane is migrated to a new datastore.")
cmd.Flags().StringVar(&migrateJobImage, "migrate-image", fmt.Sprintf("%s/clastix/kamaji:%s", internal.ContainerRepository, internal.GitTag), "Specify the container image to launch when a TenantControlPlane is migrated to a new datastore.")
cmd.Flags().IntVar(&maxConcurrentReconciles, "max-concurrent-tcp-reconciles", 1, "Specify the number of workers for the Tenant Control Plane controller (beware of CPU consumption)")
cmd.Flags().StringVar(&managerNamespace, "pod-namespace", os.Getenv("POD_NAMESPACE"), "The Kubernetes Namespace on which the Operator is running in, required for the TenantControlPlane migration jobs.")
cmd.Flags().StringVar(&managerServiceName, "webhook-service-name", "kamaji-webhook-service", "The Kamaji webhook server Service name which is used to get validation webhooks, required for the TenantControlPlane migration jobs.")

View File

@@ -22,3 +22,5 @@ spec:
konnectivity:
server:
port: 8132
agent:
mode: DaemonSet

View File

@@ -24,6 +24,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/reconcile"
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
"github.com/clastix/kamaji/controllers/utils"
"github.com/clastix/kamaji/internal/constants"
"github.com/clastix/kamaji/internal/crypto"
"github.com/clastix/kamaji/internal/utilities"
@@ -41,19 +42,25 @@ func (s *CertificateLifecycle) Reconcile(ctx context.Context, request reconcile.
logger.Info("starting CertificateLifecycle handling")
secret := corev1.Secret{}
err := s.client.Get(ctx, request.NamespacedName, &secret)
if k8serrors.IsNotFound(err) {
logger.Info("resource have been deleted, skipping")
var secret corev1.Secret
if err := s.client.Get(ctx, request.NamespacedName, &secret); err != nil {
if k8serrors.IsNotFound(err) {
logger.Info("resource may have been deleted, skipping")
return reconcile.Result{}, nil
}
return reconcile.Result{}, nil
}
if err != nil {
logger.Error(err, "cannot retrieve the required resource")
return reconcile.Result{}, err
}
if utils.IsPaused(&secret) {
logger.Info("paused reconciliation, no further actions")
return reconcile.Result{}, nil
}
checkType, ok := secret.GetLabels()[constants.ControllerLabelResource]
if !ok {
logger.Info("missing controller label, shouldn't happen")
@@ -62,14 +69,15 @@ func (s *CertificateLifecycle) Reconcile(ctx context.Context, request reconcile.
}
var crt *x509.Certificate
var err error
switch checkType {
case "x509":
case utilities.CertificateX509Label:
crt, err = s.extractCertificateFromBareSecret(secret)
case "kubeconfig":
case utilities.CertificateKubeconfigLabel:
crt, err = s.extractCertificateFromKubeconfig(secret)
default:
err = fmt.Errorf("unsupported strategy, %s", checkType)
return reconcile.Result{}, fmt.Errorf("unsupported strategy, %q", checkType)
}
if err != nil {
@@ -144,7 +152,7 @@ func (s *CertificateLifecycle) extractCertificateFromKubeconfig(secret corev1.Se
func (s *CertificateLifecycle) SetupWithManager(mgr controllerruntime.Manager) error {
s.client = mgr.GetClient()
supportedStrategies := sets.New[string]("x509", "kubeconfig")
supportedStrategies := sets.New[string](utilities.CertificateX509Label, utilities.CertificateKubeconfigLabel)
return controllerruntime.NewControllerManagedBy(mgr).
For(&corev1.Secret{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {

View File

@@ -43,7 +43,7 @@ func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (r
var ds kamajiv1alpha1.DataStore
if err := r.Client.Get(ctx, request.NamespacedName, &ds); err != nil {
if k8serrors.IsNotFound(err) {
logger.Info("resource have been deleted, skipping")
logger.Info("resource may have been deleted, skipping")
return reconcile.Result{}, nil
}
@@ -53,6 +53,12 @@ func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (r
return reconcile.Result{}, err
}
if utils.IsPaused(&ds) {
logger.Info("paused reconciliation, no further actions")
return reconcile.Result{}, nil
}
var tcpList kamajiv1alpha1.TenantControlPlaneList
updateErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {

View File

@@ -7,6 +7,7 @@ import (
"context"
"github.com/go-logr/logr"
"github.com/pkg/errors"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
@@ -23,6 +24,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"
sooterrors "github.com/clastix/kamaji/controllers/soot/controllers/errors"
"github.com/clastix/kamaji/controllers/utils"
"github.com/clastix/kamaji/internal/kubeadm"
"github.com/clastix/kamaji/internal/resources"
@@ -39,7 +41,11 @@ type CoreDNS struct {
func (c *CoreDNS) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
tcp, err := c.GetTenantControlPlaneFunc()
if err != nil {
c.Logger.Error(err, "cannot retrieve TenantControlPlane")
if errors.Is(err, sooterrors.ErrPausedReconciliation) {
c.Logger.Info(err.Error())
return reconcile.Result{}, nil
}
return reconcile.Result{}, err
}

View File

@@ -0,0 +1,10 @@
// Copyright 2022 Clastix Labs
// SPDX-License-Identifier: Apache-2.0
package errors
import (
"github.com/pkg/errors"
)
var ErrPausedReconciliation = errors.New("paused reconciliation, no further actions")

View File

@@ -7,6 +7,7 @@ import (
"context"
"github.com/go-logr/logr"
"github.com/pkg/errors"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/rbac/v1"
@@ -25,6 +26,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/source"
"github.com/clastix/kamaji/controllers"
sooterrors "github.com/clastix/kamaji/controllers/soot/controllers/errors"
"github.com/clastix/kamaji/controllers/utils"
"github.com/clastix/kamaji/internal/resources"
"github.com/clastix/kamaji/internal/resources/konnectivity"
@@ -40,6 +42,12 @@ type KonnectivityAgent struct {
func (k *KonnectivityAgent) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
tcp, err := k.GetTenantControlPlaneFunc()
if err != nil {
if errors.Is(err, sooterrors.ErrPausedReconciliation) {
k.Logger.Info(err.Error())
return reconcile.Result{}, nil
}
k.Logger.Error(err, "cannot retrieve TenantControlPlane")
return reconcile.Result{}, err

View File

@@ -7,6 +7,7 @@ import (
"context"
"github.com/go-logr/logr"
"github.com/pkg/errors"
"k8s.io/utils/ptr"
controllerruntime "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -19,6 +20,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"
sooterrors "github.com/clastix/kamaji/controllers/soot/controllers/errors"
"github.com/clastix/kamaji/controllers/utils"
"github.com/clastix/kamaji/internal/resources"
)
@@ -34,6 +36,12 @@ type KubeadmPhase struct {
func (k *KubeadmPhase) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
tcp, err := k.GetTenantControlPlaneFunc()
if err != nil {
if errors.Is(err, sooterrors.ErrPausedReconciliation) {
k.logger.Info(err.Error())
return reconcile.Result{}, nil
}
return reconcile.Result{}, err
}

View File

@@ -7,6 +7,7 @@ import (
"context"
"github.com/go-logr/logr"
"github.com/pkg/errors"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
@@ -23,6 +24,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"
sooterrors "github.com/clastix/kamaji/controllers/soot/controllers/errors"
"github.com/clastix/kamaji/controllers/utils"
"github.com/clastix/kamaji/internal/kubeadm"
"github.com/clastix/kamaji/internal/resources"
@@ -39,6 +41,12 @@ type KubeProxy struct {
func (k *KubeProxy) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
tcp, err := k.GetTenantControlPlaneFunc()
if err != nil {
if errors.Is(err, sooterrors.ErrPausedReconciliation) {
k.Logger.Info(err.Error())
return reconcile.Result{}, nil
}
k.Logger.Error(err, "cannot retrieve TenantControlPlane")
return reconcile.Result{}, err

View File

@@ -9,8 +9,9 @@ import (
"time"
"github.com/go-logr/logr"
"github.com/pkg/errors"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
"k8s.io/apimachinery/pkg/api/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
pointer "k8s.io/utils/ptr"
controllerruntime "sigs.k8s.io/controller-runtime"
@@ -25,6 +26,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/source"
"github.com/clastix/kamaji/api/v1alpha1"
sooterrors "github.com/clastix/kamaji/controllers/soot/controllers/errors"
"github.com/clastix/kamaji/controllers/utils"
"github.com/clastix/kamaji/internal/utilities"
)
@@ -42,6 +44,12 @@ type Migrate struct {
func (m *Migrate) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
tcp, err := m.GetTenantControlPlaneFunc()
if err != nil {
if errors.Is(err, sooterrors.ErrPausedReconciliation) {
m.Logger.Info(err.Error())
return reconcile.Result{}, nil
}
return reconcile.Result{}, err
}
// Cannot detect the status of the TenantControlPlane, enqueuing back
@@ -67,7 +75,7 @@ func (m *Migrate) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile
func (m *Migrate) cleanup(ctx context.Context) error {
if err := m.Client.Delete(ctx, m.object()); err != nil {
if errors.IsNotFound(err) {
if apierrors.IsNotFound(err) {
return nil
}

View File

@@ -29,6 +29,7 @@ import (
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
"github.com/clastix/kamaji/controllers/finalizers"
"github.com/clastix/kamaji/controllers/soot/controllers"
"github.com/clastix/kamaji/controllers/soot/controllers/errors"
"github.com/clastix/kamaji/controllers/utils"
"github.com/clastix/kamaji/internal/resources"
"github.com/clastix/kamaji/internal/utilities"
@@ -69,6 +70,10 @@ func (m *Manager) retrieveTenantControlPlane(ctx context.Context, request reconc
return nil, err
}
if utils.IsPaused(tcp) {
return nil, errors.ErrPausedReconciliation
}
return tcp, nil
}
}

View File

@@ -85,7 +85,7 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
tenantControlPlane, err := r.getTenantControlPlane(ctx, req.NamespacedName)()
if k8serrors.IsNotFound(err) {
log.Info("resource have been deleted, skipping")
log.Info("resource may have been deleted, skipping")
return reconcile.Result{}, nil
}
@@ -95,6 +95,12 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
return reconcile.Result{}, err
}
if utils.IsPaused(tenantControlPlane) {
log.Info("paused reconciliation, no further actions")
return ctrl.Result{}, nil
}
releaser, err := mutex.Acquire(r.mutexSpec(tenantControlPlane))
if err != nil {
switch {

View File

@@ -0,0 +1,19 @@
// Copyright 2022 Clastix Labs
// SPDX-License-Identifier: Apache-2.0
package utils
import (
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/clastix/kamaji/api/v1alpha1"
)
func IsPaused(obj client.Object) bool {
if obj.GetAnnotations() == nil {
return false
}
_, paused := obj.GetAnnotations()[v1alpha1.PausedReconciliationAnnotation]
return paused
}

View File

@@ -1,22 +1,32 @@
# Konnectivity
In traditional Kubernetes deployments, the control plane components need to communicate directly with worker nodes for various operations like executing commands in pods, retrieving logs, or managing port forwards. However, in many real-world environments, especially those spanning multiple networks or cloud providers, direct communication isn't always possible or desirable. This is where Konnectivity comes in.
In traditional Kubernetes deployments, the control plane components need to communicate directly with worker nodes for various operations
like executing commands in pods, retrieving logs, or managing port forwards.
However, in many real-world environments, especially those spanning multiple networks or cloud providers,
direct communication isn't always possible or desirable. This is where Konnectivity comes in.
## Understanding Konnectivity in Kamaji
Kamaji integrates [Konnectivity](https://kubernetes.io/docs/concepts/architecture/control-plane-node-communication/) as a core component of its architecture. Each Tenant Control Plane pod includes a konnectivity-server running as a sidecar container, which establishes and maintains secure tunnels with agents running on the worker nodes. This design ensures reliable communication even in complex network environments.
Kamaji integrates [Konnectivity](https://kubernetes.io/docs/concepts/architecture/control-plane-node-communication/) as a core component of its architecture.
Each Tenant Control Plane pod includes a konnectivity-server running as a sidecar container,
which establishes and maintains secure tunnels with agents running on the worker nodes.
This design ensures reliable communication even in complex network environments.
The Konnectivity service consists of two main components:
1. **Konnectivity Server:**
Runs alongside the control plane components in each Tenant Control Plane pod and is exposed on port 8132. It manages connections from worker nodes and routes traffic appropriately.
Runs alongside the control plane components in each Tenant Control Plane pod and is exposed on port 8132.
It manages connections from worker nodes and routes traffic appropriately.
2. **Konnectivity Agent:**
Runs on each worker node and initiates outbound connections to its control plane's Konnectivity server. These connections are maintained to create a reliable tunnel for all control plane to worker node communication.
Runs on worker nodes as _DaemonSet_ or _Deployment_ and initiates outbound connections to its control plane's Konnectivity server.
These connections are maintained to create a reliable tunnel for all control plane to worker node communications.
## How It Works
When a worker node joins a Tenant Cluster, the Konnectivity agents automatically establish connections to their designated Konnectivity server. These connections are maintained continuously, ensuring reliable communication paths between the control plane and worker nodes.
When a worker node joins a Tenant Cluster, the Konnectivity agents automatically establish connections to their designated Konnectivity server.
These connections are maintained continuously, ensuring reliable communication paths between the control plane and worker nodes.
All traffic from the control plane to worker nodes flows through these established tunnels, enabling operations such as:
@@ -28,10 +38,51 @@ All traffic from the control plane to worker nodes flows through these establish
## Configuration and Management
Konnectivity is enabled by default in Kamaji, as it's considered a best practice for modern Kubernetes deployments. However, it can be disabled if your environment has different requirements or if you need to use alternative networking solutions.
Konnectivity is enabled by default in Kamaji, as it's considered a best practice for modern Kubernetes deployments.
However, it can be disabled if your environment has different requirements, or if you need to use alternative networking solutions.
The service is automatically configured when worker nodes join a cluster, without requiring any operational overhead. The connection details are managed as part of the standard node bootstrap process, making it transparent to cluster operators and users.
The service is automatically configured when worker nodes join a cluster, without requiring any operational overhead.
The connection details are managed as part of the standard node bootstrap process,
making it transparent to cluster operators and users.
## Agent delivery mode
You can customise the Konnectivity Agent delivery mode via the Tenant Control Plane definition
using the field `tenantcontrolplane.spec.addons.konnectivity.agent.mode`.
```yaml
apiVersion: kamaji.clastix.io/v1alpha1
kind: TenantControlPlane
metadata:
name: konnectivity-example
spec:
controlPlane:
deployment:
replicas: 2
service:
serviceType: LoadBalancer
kubernetes:
version: "v1.33.0"
networkProfile:
port: 6443
addons:
konnectivity:
server:
port: 8132
agent:
## DaemonSet, Deployment
mode: DaemonSet
## When mode is Deployment, specify the desired Agent replicas
# replicas: 2
```
Available strategies are the following:
- `DaemonSet`: runs on every node
- `Deployment`: useful to decrease the resource footprint in certain workloads cluster,
it allows customising also the amount of deployed replicas via the field
`tenantcontrolplane.spec.addons.konnectivity.agent.replicas`.
---
By integrating Konnectivity as a core feature, Kamaji ensures that your Tenant Clusters can operate reliably and securely across any network topology, making it easier to build and manage distributed Kubernetes environments at scale.
By integrating Konnectivity as a core feature, Kamaji ensures that your Tenant Clusters can operate reliably and securely across any network topology,
making it easier to build and manage distributed Kubernetes environments at scale.

View File

@@ -19,50 +19,44 @@ All the certificates are created with the `kubeadm` defaults, thus their validit
## How to rotate certificates
If you need to manually rotate one of these certificates, the required operation is the deletion for the given Secret.
All certificates can be rotated at the same time, or one by one: this is possible by annotating resources using
the well-known annotation `certs.kamaji.clastix.io/rotate`.
```
$: kubectl get secret
NAME TYPE DATA AGE
k8s-126-admin-kubeconfig Opaque 1 12m
k8s-126-api-server-certificate Opaque 2 12m
k8s-126-api-server-kubelet-client-certificate Opaque 2 3h45m
k8s-126-ca Opaque 4 3h45m
k8s-126-controller-manager-kubeconfig Opaque 1 3h45m
k8s-126-datastore-certificate Opaque 3 3h45m
k8s-126-datastore-config Opaque 4 3h45m
k8s-126-front-proxy-ca-certificate Opaque 2 3h45m
k8s-126-front-proxy-client-certificate Opaque 2 3h45m
k8s-126-konnectivity-certificate kubernetes.io/tls 2 3h45m
k8s-126-konnectivity-kubeconfig Opaque 1 3h45m
k8s-126-sa-certificate Opaque 2 3h45m
k8s-126-scheduler-kubeconfig Opaque 1 3h45m
k8s-133-admin-kubeconfig Opaque 1 12m
k8s-133-api-server-certificate Opaque 2 12m
k8s-133-api-server-kubelet-client-certificate Opaque 2 3h45m
k8s-133-ca Opaque 4 3h45m
k8s-133-controller-manager-kubeconfig Opaque 1 3h45m
k8s-133-datastore-certificate Opaque 3 3h45m
k8s-133-datastore-config Opaque 4 3h45m
k8s-133-front-proxy-ca-certificate Opaque 2 3h45m
k8s-133-front-proxy-client-certificate Opaque 2 3h45m
k8s-133-konnectivity-certificate kubernetes.io/tls 2 3h45m
k8s-133-konnectivity-kubeconfig Opaque 1 3h45m
k8s-133-sa-certificate Opaque 2 3h45m
k8s-133-scheduler-kubeconfig Opaque 1 3h45m
```
Once this operation is performed, Kamaji will be notified of the missing certificate, and it will create it back.
Once this operation is performed, Kamaji will trigger a certificate renewal,
reporting the rotation date time as the annotation `certs.kamaji.clastix.io/rotate` value.
```
$: kubectl delete secret -l kamaji.clastix.io/certificate_lifecycle_controller=x509
secret "k8s-126-api-server-certificate" deleted
secret "k8s-126-api-server-kubelet-client-certificate" deleted
secret "k8s-126-front-proxy-client-certificate" deleted
secret "k8s-126-konnectivity-certificate" deleted
$: kubectl annotate secret -l kamaji.clastix.io/certificate_lifecycle_controller=x509 certs.kamaji.clastix.io/rotate=""
secret/k8s-133-api-server-certificate annotated
secret/k8s-133-api-server-kubelet-client-certificate annotated
secret/k8s-133-datastore-certificate annotated
secret/k8s-133-front-proxy-client-certificate annotated
secret/k8s-133-konnectivity-certificate annotated
$: kubectl delete secret -l kamaji.clastix.io/certificate_lifecycle_controller=x509
NAME TYPE DATA AGE
k8s-126-admin-kubeconfig Opaque 1 15m
k8s-126-api-server-certificate Opaque 2 12s
k8s-126-api-server-kubelet-client-certificate Opaque 2 12s
k8s-126-ca Opaque 4 3h48m
k8s-126-controller-manager-kubeconfig Opaque 1 3h48m
k8s-126-datastore-certificate Opaque 3 3h48m
k8s-126-datastore-config Opaque 4 3h48m
k8s-126-front-proxy-ca-certificate Opaque 2 3h48m
k8s-126-front-proxy-client-certificate Opaque 2 12s
k8s-126-konnectivity-certificate kubernetes.io/tls 2 11s
k8s-126-konnectivity-kubeconfig Opaque 1 3h48m
k8s-126-sa-certificate Opaque 2 3h48m
k8s-126-scheduler-kubeconfig Opaque 1 3h48m
$: kubectl get secrets -l kamaji.clastix.io/certificate_lifecycle_controller=x509 -ojson | jq -r '.items[] | "\(.metadata.name) rotated at \(.metadata.annotations["certs.kamaji.clastix.io/rotate"])"'
k8s-133-api-server-certificate rotated at 2025-07-15 15:15:08.842191367 +0200 CEST m=+325.785000014
k8s-133-api-server-kubelet-client-certificate rotated at 2025-07-15 15:15:10.468139865 +0200 CEST m=+327.410948506
k8s-133-datastore-certificate rotated at 2025-07-15 15:15:15.454468752 +0200 CEST m=+332.397277417
k8s-133-front-proxy-client-certificate rotated at 2025-07-15 15:15:13.279920467 +0200 CEST m=+330.222729097
k8s-133-konnectivity-certificate rotated at 2025-07-15 15:15:17.361431671 +0200 CEST m=+334.304240277
```
You can notice the secrets have been automatically created back, as well as a TenantControlPlane rollout with the updated certificates.
@@ -70,23 +64,24 @@ You can notice the secrets have been automatically created back, as well as a Te
```
$: kubectl get pods
NAME READY STATUS RESTARTS AGE
k8s-126-76768bdf89-82w8g 4/4 Running 0 58s
k8s-126-76768bdf89-fwltl 4/4 Running 0 58s
k8s-133-67bf496c8c-27bmp 4/4 Running 0 4m52s
k8s-133-67bf496c8c-x4t76 4/4 Running 0 4m52s
```
The same occurs with the `kubeconfig` ones.
```
$: kubectl delete secret -l kamaji.clastix.io/certificate_lifecycle_controller=kubeconfig
secret "k8s-126-admin-kubeconfig" deleted
secret "k8s-126-controller-manager-kubeconfig" deleted
secret "k8s-126-konnectivity-kubeconfig" deleted
secret "k8s-126-scheduler-kubeconfig" deleted
$: kubectl annotate secret -l kamaji.clastix.io/certificate_lifecycle_controller=kubeconfig certs.kamaji.clastix.io/rotate=""
secret/k8s-133-admin-kubeconfig annotated
secret/k8s-133-controller-manager-kubeconfig annotated
secret/k8s-133-konnectivity-kubeconfig annotated
secret/k8s-133-scheduler-kubeconfig annotated
$: kubectl get pods
NAME READY STATUS RESTARTS AGE
k8s-126-576c775b5d-2gr9h 4/4 Running 0 50s
k8s-126-576c775b5d-jmvlm 4/4 Running 0 50s
$: kubectl get secrets -l kamaji.clastix.io/certificate_lifecycle_controller=kubeconfig -ojson | jq -r '.items[] | "\(.metadata.name) rotated at \(.metadata.annotations["certs.kamaji.clastix.io/rotate"])"'
k8s-133-admin-kubeconfig rotated at 2025-07-15 15:20:41.688181782 +0200 CEST m=+658.630990441
k8s-133-controller-manager-kubeconfig rotated at 2025-07-15 15:20:42.712211056 +0200 CEST m=+659.655019677
k8s-133-konnectivity-kubeconfig rotated at 2025-07-15 15:20:46.405567865 +0200 CEST m=+663.348376504
k8s-133-scheduler-kubeconfig rotated at 2025-07-15 15:20:46.333718563 +0200 CEST m=+663.276527216
```
## Automatic certificates rotation
@@ -108,11 +103,11 @@ e.g.: set the value `7d` to trigger the renewal a week before the effective expi
Kamaji is also taking care of your Tenant Clusters Certificate Authority.
This can be rotated manually by deleting the following secret.
This can be rotated manually like other certificates by using the annotation `certs.kamaji.clastix.io/rotate`
```
$: kubectl delete secret k8s-126-ca
secret "k8s-126-ca" deleted
$: kubectl annotate secret k8s-133-ca certs.kamaji.clastix.io/rotate=""
secret/k8s-133-ca annotated
```
Once this occurs the TenantControlPlane will enter in the `CertificateAuthorityRotating` status.
@@ -120,26 +115,26 @@ Once this occurs the TenantControlPlane will enter in the `CertificateAuthorityR
```
$: kubectl get tcp -w
NAME VERSION STATUS CONTROL-PLANE ENDPOINT KUBECONFIG DATASTORE AGE
k8s-126 v1.26.0 Ready 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-126 v1.26.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-126 v1.26.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-126 v1.26.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-126 v1.26.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-126 v1.26.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-126 v1.26.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-126 v1.26.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-126 v1.26.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-126 v1.26.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-126 v1.26.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-126 v1.26.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-126 v1.26.0 Ready 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-126 v1.26.0 Ready 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-126 v1.26.0 Ready 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-126 v1.26.0 Ready 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-126 v1.26.0 Ready 172.18.255.200:6443 k8s-126-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 Ready 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 CertificateAuthorityRotating 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 Ready 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 Ready 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 Ready 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 Ready 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
k8s-133 v1.33.0 Ready 172.18.255.200:6443 k8s-133-admin-kubeconfig default 3h58m
```
This operation is intended to be performed manually since a new Certificate Authority requires the restart of all the components, as well as of the nodes:
in such case, you will need to distribute the new Certificate Authority and the new nodes certificates.
This operation is intended to be performed manually since a new Certificate Authority requires the restart of all the components,
as well as of the nodes: in such a case, you will need to distribute the new Certificate Authority and the new nodes certificates.
Given the sensibility of such operation, the `Secret` controller will not check the _CA_, which is offering validity of 10 years as `kubeadm` default values.

View File

@@ -0,0 +1,33 @@
# Pausing Reconciliations
Kamaji follows the Kubernetes Operator pattern, which includes implementing a reconciliation loop.
This loop continuously reacts to events such as creation, updates, and deletions of resources.
To temporarily disable reconciliation for a resource, you can use the following annotation:
> `kamaji.clastix.io/paused`
!!! info "Annotation value"
The annotation key is sufficient on its own: no value is required.
Its mere presence disables controller reconciliations.
## Pausing `TenantControlPlane` reconciliations
When you add the `kamaji.clastix.io/paused` annotation to a TenantControlPlane object,
Kamaji will halt all reconciliation processes for that object.
This affects **all controllers**, including:
- The primary controller responsible for provisioning resources in the management cluster
- Secondary (soot) controllers responsible for bootstrapping the control plane, deploying addons, and managing any additional resources handled by Kamaji.
## Pausing Secret rotation
Kamaji automatically generates and manages several `Secret` resources, such as:
- `x509` certificates
- `kubeconfig` credentials
These secrets are automatically rotated by Kamaji's built-in **Certificate Lifecycle** feature.
To temporarily disable secret rotation for these resources,
apply the `kamaji.clastix.io/paused` annotation to the corresponding object.

View File

@@ -124,4 +124,4 @@ Clastix Labs no longer provides release artifacts following its own semantic ver
> "Kamaji works exactly as expected: it's 'simple', efficient, scalable, and I especially appreciate how Clastix has always been available for technical discussions and support throughout these two years of collaboration."
>
> — Jeremie Monsinjon, Head of Containers @ OVHCloud
> — Jeremie Monsinjon, Head of Containers @ OVHCloud

View File

@@ -39576,7 +39576,7 @@ Enables the Konnectivity addon in the Tenant Cluster, required if the worker nod
<td>
<br/>
<br/>
<i>Default</i>: map[image:registry.k8s.io/kas-network-proxy/proxy-agent version:v0.28.6]<br/>
<i>Default</i>: map[image:registry.k8s.io/kas-network-proxy/proxy-agent mode:DaemonSet version:v0.28.6]<br/>
</td>
<td>false</td>
</tr><tr>
@@ -39625,6 +39625,26 @@ unxpected ways. Only modify if you know what you are doing.<br/>
<i>Default</i>: registry.k8s.io/kas-network-proxy/proxy-agent<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>mode</b></td>
<td>enum</td>
<td>
Mode allows specifying the Agent deployment mode: Deployment, or DaemonSet (default).<br/>
<br/>
<i>Enum</i>: DaemonSet, Deployment<br/>
<i>Default</i>: DaemonSet<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>replicas</b></td>
<td>integer</td>
<td>
Replicas defines the number of replicas when Mode is Deployment.
Must be 0 if Mode is DaemonSet.<br/>
<br/>
<i>Format</i>: int32<br/>
</td>
<td>false</td>
</tr><tr>
<td><b><a href="#tenantcontrolplanespecaddonskonnectivityagenttolerationsindex">tolerations</a></b></td>
<td>[]object</td>
@@ -40250,6 +40270,13 @@ KonnectivityStatus defines the status of Konnectivity as Addon.
<i>Format</i>: date-time<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>mode</b></td>
<td>string</td>
<td>
<br/>
</td>
<td>false</td>
</tr><tr>
<td><b>name</b></td>
<td>string</td>

View File

@@ -73,6 +73,7 @@ nav:
- guides/alternative-datastore.md
- guides/backup-and-restore.md
- guides/certs-lifecycle.md
- guides/pausing.md
- guides/datastore-migration.md
- guides/gitops.md
- guides/console.md

11
go.mod
View File

@@ -6,7 +6,7 @@ require (
github.com/JamesStewy/go-mysqldump v0.2.2
github.com/blang/semver v3.5.1+incompatible
github.com/clastix/kamaji-telemetry v1.0.0
github.com/docker/docker v28.3.0+incompatible
github.com/docker/docker v28.3.2+incompatible
github.com/go-logr/logr v1.4.3
github.com/go-pg/pg/v10 v10.14.0
github.com/go-sql-driver/mysql v1.9.3
@@ -20,9 +20,9 @@ require (
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.22.0
github.com/spf13/cobra v1.9.1
github.com/spf13/pflag v1.0.6
github.com/spf13/pflag v1.0.7
github.com/spf13/viper v1.20.1
github.com/testcontainers/testcontainers-go v0.37.0
github.com/testcontainers/testcontainers-go v0.38.0
go.etcd.io/etcd/api/v3 v3.5.21
go.etcd.io/etcd/client/v3 v3.5.21
go.uber.org/automaxprocs v1.6.0
@@ -65,7 +65,7 @@ require (
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/ebitengine/purego v0.8.2 // indirect
github.com/ebitengine/purego v0.8.4 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
@@ -107,7 +107,6 @@ require (
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/go-archive v0.1.0 // indirect
github.com/moby/patternmatcher v0.6.0 // indirect
github.com/moby/sys/atomicwriter v0.1.0 // indirect
github.com/moby/sys/sequential v0.6.0 // indirect
github.com/moby/sys/user v0.4.0 // indirect
github.com/moby/sys/userns v0.1.0 // indirect
@@ -129,7 +128,7 @@ require (
github.com/prometheus/common v0.62.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/sagikazarmark/locafero v0.7.0 // indirect
github.com/shirou/gopsutil/v4 v4.25.1 // indirect
github.com/shirou/gopsutil/v4 v4.25.5 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.12.0 // indirect

19
go.sum
View File

@@ -56,16 +56,16 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/docker v28.3.0+incompatible h1:ffS62aKWupCWdvcee7nBU9fhnmknOqDPaJAMtfK0ImQ=
github.com/docker/docker v28.3.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v28.3.2+incompatible h1:wn66NJ6pWB1vBZIilP8G3qQPqHy5XymfYn5vsqeA5oA=
github.com/docker/docker v28.3.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I=
github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw=
github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
@@ -279,8 +279,8 @@ github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsF
github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shirou/gopsutil/v4 v4.25.1 h1:QSWkTc+fu9LTAWfkZwZ6j8MSUk4A2LV7rbH0ZqmLjXs=
github.com/shirou/gopsutil/v4 v4.25.1/go.mod h1:RoUCUpndaJFtT+2zsZzzmhvbfGoDCJ7nFXKJf8GqJbI=
github.com/shirou/gopsutil/v4 v4.25.5 h1:rtd9piuSMGeU8g1RMXjZs9y9luK5BwtnG7dZaQUJAsc=
github.com/shirou/gopsutil/v4 v4.25.5/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
@@ -293,8 +293,9 @@ github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M=
github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=
@@ -315,8 +316,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/testcontainers/testcontainers-go v0.37.0 h1:L2Qc0vkTw2EHWQ08djon0D2uw7Z/PtHS/QzZZ5Ra/hg=
github.com/testcontainers/testcontainers-go v0.37.0/go.mod h1:QPzbxZhQ6Bclip9igjLFj6z0hs01bU8lrl2dHQmgFGM=
github.com/testcontainers/testcontainers-go v0.38.0 h1:d7uEapLcv2P8AvH8ahLqDMMxda2W9gQN1nRbHS28HBw=
github.com/testcontainers/testcontainers-go v0.38.0/go.mod h1:C52c9MoHpWO+C4aqmgSU+hxlR5jlEayWtgYrb8Pzz1w=
github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=

View File

@@ -4,6 +4,7 @@
package kubeadm
import (
"bytes"
"os"
"path"
"path/filepath"
@@ -46,6 +47,21 @@ func CreateKubeconfig(kubeconfigName string, ca CertificatePrivateKeyPair, confi
return os.ReadFile(path)
}
func IsKubeconfigCAValid(in, caCrt []byte) bool {
kc, err := utilities.DecodeKubeconfigYAML(in)
if err != nil {
return false
}
for _, cluster := range kc.Clusters {
if !bytes.Equal(cluster.Cluster.CertificateAuthorityData, caCrt) {
return false
}
}
return true
}
func IsKubeconfigValid(bytes []byte) bool {
kc, err := utilities.DecodeKubeconfigYAML(bytes)
if err != nil {

View File

@@ -117,7 +117,7 @@ func (r *APIServerCertificate) mutate(ctx context.Context, tenantControlPlane *k
r.resource.SetLabels(utilities.MergeMaps(
utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()),
map[string]string{
constants.ControllerLabelResource: "x509",
constants.ControllerLabelResource: utilities.CertificateX509Label,
},
))
@@ -127,7 +127,9 @@ func (r *APIServerCertificate) mutate(ctx context.Context, tenantControlPlane *k
return err
}
if checksum := tenantControlPlane.Status.Certificates.APIServer.Checksum; len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0 {
isRotationRequested := utilities.IsRotationRequested(r.resource)
if checksum := tenantControlPlane.Status.Certificates.APIServer.Checksum; !isRotationRequested && (len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0) {
isCAValid, err := crypto.VerifyCertificate(r.resource.Data[kubeadmconstants.APIServerCertName], secretCA.Data[kubeadmconstants.CACertName], x509.ExtKeyUsageServerAuth)
if err != nil {
logger.Info(fmt.Sprintf("certificate-authority verify failed: %s", err.Error()))
@@ -170,6 +172,10 @@ func (r *APIServerCertificate) mutate(ctx context.Context, tenantControlPlane *k
return err
}
if isRotationRequested {
utilities.SetLastRotationTimestamp(r.resource)
}
r.resource.Data = map[string][]byte{
kubeadmconstants.APIServerCertName: certificateKeyPair.Certificate,
kubeadmconstants.APIServerKeyName: certificateKeyPair.PrivateKey,

View File

@@ -104,7 +104,7 @@ func (r *APIServerKubeletClientCertificate) mutate(ctx context.Context, tenantCo
r.resource.SetLabels(utilities.MergeMaps(
utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()),
map[string]string{
constants.ControllerLabelResource: "x509",
constants.ControllerLabelResource: utilities.CertificateX509Label,
},
))
@@ -114,7 +114,9 @@ func (r *APIServerKubeletClientCertificate) mutate(ctx context.Context, tenantCo
return err
}
if checksum := tenantControlPlane.Status.Certificates.APIServerKubeletClient.Checksum; len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0 {
isRotationRequested := utilities.IsRotationRequested(r.resource)
if checksum := tenantControlPlane.Status.Certificates.APIServerKubeletClient.Checksum; !isRotationRequested && (len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0) {
isCAValid, err := crypto.VerifyCertificate(r.resource.Data[kubeadmconstants.APIServerKubeletClientCertName], secretCA.Data[kubeadmconstants.CACertName], x509.ExtKeyUsageClientAuth)
if err != nil {
logger.Info(fmt.Sprintf("certificate-authority verify failed: %s", err.Error()))
@@ -152,6 +154,10 @@ func (r *APIServerKubeletClientCertificate) mutate(ctx context.Context, tenantCo
return err
}
if isRotationRequested {
utilities.SetLastRotationTimestamp(r.resource)
}
r.resource.Data = map[string][]byte{
kubeadmconstants.APIServerKubeletClientCertName: certificateKeyPair.Certificate,
kubeadmconstants.APIServerKubeletClientKeyName: certificateKeyPair.PrivateKey,

View File

@@ -96,7 +96,9 @@ func (r *CACertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1
return func() error {
logger := log.FromContext(ctx, "resource", r.GetName())
if checksum := tenantControlPlane.Status.Certificates.CA.Checksum; len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0 {
isRotationRequested := utilities.IsRotationRequested(r.resource)
if checksum := tenantControlPlane.Status.Certificates.CA.Checksum; !isRotationRequested && (len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0) {
isValid, err := crypto.CheckCertificateAndPrivateKeyPairValidity(
r.resource.Data[kubeadmconstants.CACertName],
r.resource.Data[kubeadmconstants.CAKeyName],
@@ -116,6 +118,10 @@ func (r *CACertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1
}
}
if isRotationRequested {
utilities.SetLastRotationTimestamp(r.resource)
}
if tenantControlPlane.Status.Kubernetes.Version.Status != nil && *tenantControlPlane.Status.Kubernetes.Version.Status != kamajiv1alpha1.VersionProvisioning {
r.isRotatingCA = true
}

View File

@@ -87,6 +87,8 @@ func (r *Certificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1al
return func() error {
logger := log.FromContext(ctx, "resource", r.GetName())
isRotationRequested := utilities.IsRotationRequested(r.resource)
if r.DataStore.Spec.TLSConfig != nil {
ca, err := r.DataStore.Spec.TLSConfig.CertificateAuthority.Certificate.GetContent(ctx, r.Client)
if err != nil {
@@ -104,7 +106,7 @@ func (r *Certificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1al
r.resource.SetLabels(utilities.MergeMaps(
utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()),
map[string]string{
constants.ControllerLabelResource: "x509",
constants.ControllerLabelResource: utilities.CertificateX509Label,
},
))
@@ -116,7 +118,7 @@ func (r *Certificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1al
if utilities.GetObjectChecksum(r.resource) == utilities.CalculateMapChecksum(r.resource.Data) {
if r.DataStore.Spec.Driver == kamajiv1alpha1.EtcdDriver {
if isValid, _ := crypto.IsValidCertificateKeyPairBytes(r.resource.Data["server.crt"], r.resource.Data["server.key"]); isValid {
if isValid, _ := crypto.IsValidCertificateKeyPairBytes(r.resource.Data["server.crt"], r.resource.Data["server.key"]); isValid && !isRotationRequested {
return nil
}
}
@@ -174,6 +176,10 @@ func (r *Certificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1al
r.resource.Data = map[string][]byte{}
}
if isRotationRequested {
utilities.SetLastRotationTimestamp(r.resource)
}
utilities.SetObjectChecksum(r.resource, r.resource.Data)
return nil

View File

@@ -104,7 +104,7 @@ func (r *FrontProxyClientCertificate) mutate(ctx context.Context, tenantControlP
r.resource.SetLabels(utilities.MergeMaps(
utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()),
map[string]string{
constants.ControllerLabelResource: "x509",
constants.ControllerLabelResource: utilities.CertificateX509Label,
},
))
@@ -114,7 +114,9 @@ func (r *FrontProxyClientCertificate) mutate(ctx context.Context, tenantControlP
return err
}
if checksum := tenantControlPlane.Status.Certificates.FrontProxyClient.Checksum; len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0 {
isRotationRequested := utilities.IsRotationRequested(r.resource)
if checksum := tenantControlPlane.Status.Certificates.FrontProxyClient.Checksum; !isRotationRequested && (len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0) {
isCAValid, err := crypto.VerifyCertificate(r.resource.Data[kubeadmconstants.FrontProxyClientCertName], secretCA.Data[kubeadmconstants.FrontProxyCACertName], x509.ExtKeyUsageClientAuth)
if err != nil {
logger.Info(fmt.Sprintf("certificate-authority verify failed: %s", err.Error()))
@@ -152,6 +154,10 @@ func (r *FrontProxyClientCertificate) mutate(ctx context.Context, tenantControlP
return err
}
if isRotationRequested {
utilities.SetLastRotationTimestamp(r.resource)
}
r.resource.Data = map[string][]byte{
kubeadmconstants.FrontProxyClientCertName: certificateKeyPair.Certificate,
kubeadmconstants.FrontProxyClientKeyName: certificateKeyPair.PrivateKey,

View File

@@ -89,7 +89,9 @@ func (r *FrontProxyCACertificate) mutate(ctx context.Context, tenantControlPlane
return func() error {
logger := log.FromContext(ctx, "resource", r.GetName())
if checksum := tenantControlPlane.Status.Certificates.FrontProxyCA.Checksum; len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0 {
isRotationRequested := utilities.IsRotationRequested(r.resource)
if checksum := tenantControlPlane.Status.Certificates.FrontProxyCA.Checksum; !isRotationRequested && (len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0) {
isValid, err := crypto.CheckCertificateAndPrivateKeyPairValidity(
r.resource.Data[kubeadmconstants.FrontProxyCACertName],
r.resource.Data[kubeadmconstants.FrontProxyCAKeyName],
@@ -123,6 +125,10 @@ func (r *FrontProxyCACertificate) mutate(ctx context.Context, tenantControlPlane
r.resource.SetLabels(utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()))
if isRotationRequested {
utilities.SetLastRotationTimestamp(r.resource)
}
utilities.SetObjectChecksum(r.resource, r.resource.Data)
return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())

View File

@@ -25,7 +25,7 @@ import (
)
type Agent struct {
resource *appsv1.DaemonSet
resource client.Object
Client client.Client
tenantClient client.Client
}
@@ -38,7 +38,8 @@ func (r *Agent) GetHistogram() prometheus.Histogram {
func (r *Agent) ShouldStatusBeUpdated(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) bool {
return tcp.Spec.Addons.Konnectivity == nil && (tcp.Status.Addons.Konnectivity.Agent.Namespace != "" || tcp.Status.Addons.Konnectivity.Agent.Name != "") ||
tcp.Spec.Addons.Konnectivity != nil && (tcp.Status.Addons.Konnectivity.Agent.Namespace != r.resource.Namespace || tcp.Status.Addons.Konnectivity.Agent.Name != r.resource.Name)
tcp.Spec.Addons.Konnectivity != nil && (tcp.Status.Addons.Konnectivity.Agent.Namespace != r.resource.GetNamespace() || tcp.Status.Addons.Konnectivity.Agent.Name != r.resource.GetName()) ||
tcp.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Mode != tcp.Status.Addons.Konnectivity.Agent.Mode
}
func (r *Agent) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
@@ -78,13 +79,20 @@ func (r *Agent) CleanUp(ctx context.Context, _ *kamajiv1alpha1.TenantControlPlan
func (r *Agent) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (err error) {
logger := log.FromContext(ctx, "resource", r.GetName())
r.resource = &appsv1.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Name: AgentName,
Namespace: AgentNamespace,
},
switch tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Mode {
case kamajiv1alpha1.KonnectivityAgentModeDaemonSet:
r.resource = &appsv1.DaemonSet{}
case kamajiv1alpha1.KonnectivityAgentModeDeployment:
r.resource = &appsv1.Deployment{}
default:
logger.Info("TenantControlPlane CRD is not updated, or validation failed, fallback to DaemonSet")
r.resource = &appsv1.DaemonSet{}
}
r.resource.SetNamespace(AgentNamespace)
r.resource.SetName(AgentName)
if r.tenantClient, err = utilities.GetTenantClient(ctx, r.Client, tenantControlPlane); err != nil {
logger.Error(err, "unable to retrieve the Tenant Control Plane client")
@@ -96,7 +104,33 @@ func (r *Agent) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.T
func (r *Agent) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
if tenantControlPlane.Spec.Addons.Konnectivity != nil {
return controllerutil.CreateOrUpdate(ctx, r.tenantClient, r.resource, r.mutate(ctx, tenantControlPlane))
or, err := controllerutil.CreateOrUpdate(ctx, r.tenantClient, r.resource, r.mutate(ctx, tenantControlPlane))
if err != nil {
return controllerutil.OperationResultNone, err
}
switch {
case tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Mode == kamajiv1alpha1.KonnectivityAgentModeDaemonSet &&
tenantControlPlane.Status.Addons.Konnectivity.Agent.Mode != kamajiv1alpha1.KonnectivityAgentModeDaemonSet:
var obj appsv1.Deployment
obj.SetName(r.resource.GetName())
obj.SetNamespace(r.resource.GetNamespace())
if cleanupErr := r.tenantClient.Delete(ctx, &obj); cleanupErr != nil {
log.FromContext(ctx, "resource", r.GetName()).Error(cleanupErr, "cannot cleanup older appsv1.Deployment")
}
case tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Mode == kamajiv1alpha1.KonnectivityAgentModeDeployment &&
tenantControlPlane.Status.Addons.Konnectivity.Agent.Mode != kamajiv1alpha1.KonnectivityAgentModeDeployment:
var obj appsv1.DaemonSet
obj.SetName(r.resource.GetName())
obj.SetNamespace(r.resource.GetNamespace())
if cleanupErr := r.tenantClient.Delete(ctx, &obj); cleanupErr != nil {
log.FromContext(ctx, "resource", r.GetName()).Error(cleanupErr, "cannot cleanup older appsv1.DaemonSet")
}
}
return or, nil
}
return controllerutil.OperationResultNone, nil
@@ -107,13 +141,16 @@ func (r *Agent) GetName() string {
}
func (r *Agent) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
tenantControlPlane.Status.Addons.Konnectivity.Agent = kamajiv1alpha1.ExternalKubernetesObjectStatus{}
tenantControlPlane.Status.Addons.Konnectivity.Agent = kamajiv1alpha1.KonnectivityAgentStatus{}
if tenantControlPlane.Spec.Addons.Konnectivity != nil {
tenantControlPlane.Status.Addons.Konnectivity.Agent = kamajiv1alpha1.ExternalKubernetesObjectStatus{
Name: r.resource.GetName(),
Namespace: r.resource.GetNamespace(),
LastUpdate: metav1.Now(),
tenantControlPlane.Status.Addons.Konnectivity.Agent = kamajiv1alpha1.KonnectivityAgentStatus{
ExternalKubernetesObjectStatus: kamajiv1alpha1.ExternalKubernetesObjectStatus{
Name: r.resource.GetName(),
Namespace: r.resource.GetNamespace(),
LastUpdate: metav1.Now(),
},
Mode: tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Mode,
}
}
@@ -133,27 +170,31 @@ func (r *Agent) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.T
r.resource.SetLabels(utilities.MergeMaps(r.resource.GetLabels(), utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName())))
if r.resource.Spec.Selector == nil {
r.resource.Spec.Selector = &metav1.LabelSelector{}
}
r.resource.Spec.Selector.MatchLabels = map[string]string{
"k8s-app": AgentName,
}
r.resource.Spec.Template.SetLabels(utilities.MergeMaps(
r.resource.Spec.Template.GetLabels(),
map[string]string{
specSelector := &metav1.LabelSelector{
MatchLabels: map[string]string{
"k8s-app": AgentName,
},
))
}
r.resource.Spec.Template.Spec.PriorityClassName = "system-cluster-critical"
r.resource.Spec.Template.Spec.Tolerations = tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Tolerations
r.resource.Spec.Template.Spec.NodeSelector = map[string]string{
var podTemplateSpec *corev1.PodTemplateSpec
switch obj := r.resource.(type) {
case *appsv1.DaemonSet:
obj.Spec.Selector = specSelector
podTemplateSpec = &obj.Spec.Template
case *appsv1.Deployment:
obj.Spec.Selector = specSelector
podTemplateSpec = &obj.Spec.Template
}
podTemplateSpec.SetLabels(utilities.MergeMaps(podTemplateSpec.GetLabels(), specSelector.MatchLabels))
podTemplateSpec.Spec.PriorityClassName = "system-cluster-critical"
podTemplateSpec.Spec.Tolerations = tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Tolerations
podTemplateSpec.Spec.NodeSelector = map[string]string{
"kubernetes.io/os": "linux",
}
r.resource.Spec.Template.Spec.ServiceAccountName = AgentName
r.resource.Spec.Template.Spec.Volumes = []corev1.Volume{
podTemplateSpec.Spec.ServiceAccountName = AgentName
podTemplateSpec.Spec.Volumes = []corev1.Volume{
{
Name: agentTokenName,
VolumeSource: corev1.VolumeSource{
@@ -173,13 +214,13 @@ func (r *Agent) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.T
},
}
if len(r.resource.Spec.Template.Spec.Containers) != 1 {
r.resource.Spec.Template.Spec.Containers = make([]corev1.Container, 1)
if len(podTemplateSpec.Spec.Containers) != 1 {
podTemplateSpec.Spec.Containers = make([]corev1.Container, 1)
}
r.resource.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("%s:%s", tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Image, tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Version)
r.resource.Spec.Template.Spec.Containers[0].Name = AgentName
r.resource.Spec.Template.Spec.Containers[0].Command = []string{"/proxy-agent"}
podTemplateSpec.Spec.Containers[0].Image = fmt.Sprintf("%s:%s", tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Image, tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Version)
podTemplateSpec.Spec.Containers[0].Name = AgentName
podTemplateSpec.Spec.Containers[0].Command = []string{"/proxy-agent"}
args := make(map[string]string)
args["-v"] = "8"
@@ -197,18 +238,18 @@ func (r *Agent) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.T
args[k] = v
}
r.resource.Spec.Template.Spec.Containers[0].Args = utilities.ArgsFromMapToSlice(args)
r.resource.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
podTemplateSpec.Spec.Containers[0].Args = utilities.ArgsFromMapToSlice(args)
podTemplateSpec.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
{
MountPath: "/var/run/secrets/tokens",
Name: agentTokenName,
},
}
r.resource.Spec.Template.Spec.Containers[0].LivenessProbe = &corev1.Probe{
podTemplateSpec.Spec.Containers[0].LivenessProbe = &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/healthz",
Port: intstr.FromInt(8134),
Port: intstr.FromInt32(8134),
Scheme: corev1.URISchemeHTTP,
},
},
@@ -219,6 +260,16 @@ func (r *Agent) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.T
FailureThreshold: 3,
}
switch tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Mode {
case kamajiv1alpha1.KonnectivityAgentModeDaemonSet:
r.resource.(*appsv1.DaemonSet).Spec.Template = *podTemplateSpec //nolint:forcetypeassert
case kamajiv1alpha1.KonnectivityAgentModeDeployment:
//nolint:forcetypeassert
r.resource.(*appsv1.Deployment).Spec.Template = *podTemplateSpec
//nolint:forcetypeassert
r.resource.(*appsv1.Deployment).Spec.Replicas = pointer.To(tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Replicas)
}
return nil
}
}

View File

@@ -104,7 +104,7 @@ func (r *CertificateResource) mutate(ctx context.Context, tenantControlPlane *ka
r.resource.GetLabels(),
utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()),
map[string]string{
constants.ControllerLabelResource: "x509",
constants.ControllerLabelResource: utilities.CertificateX509Label,
},
))
@@ -114,7 +114,9 @@ func (r *CertificateResource) mutate(ctx context.Context, tenantControlPlane *ka
return err
}
if checksum := tenantControlPlane.Status.Addons.Konnectivity.Certificate.Checksum; len(checksum) > 0 && checksum == utilities.CalculateMapChecksum(r.resource.Data) {
isRotationRequested := utilities.IsRotationRequested(r.resource)
if checksum := tenantControlPlane.Status.Addons.Konnectivity.Certificate.Checksum; !isRotationRequested && (len(checksum) > 0 && checksum == utilities.CalculateMapChecksum(r.resource.Data)) {
isValid, err := crypto.IsValidCertificateKeyPairBytes(r.resource.Data[corev1.TLSCertKey], r.resource.Data[corev1.TLSPrivateKeyKey])
if err != nil {
logger.Info(fmt.Sprintf("%s certificate-private_key pair is not valid: %s", konnectivityCertAndKeyBaseName, err.Error()))
@@ -145,6 +147,10 @@ func (r *CertificateResource) mutate(ctx context.Context, tenantControlPlane *ka
return err
}
if isRotationRequested {
utilities.SetLastRotationTimestamp(r.resource)
}
r.resource.Type = corev1.SecretTypeTLS
r.resource.Data = map[string][]byte{
corev1.TLSCertKey: cert.Bytes(),

View File

@@ -103,7 +103,7 @@ func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kam
r.resource.GetLabels(),
utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()),
map[string]string{
constants.ControllerLabelResource: "kubeconfig",
constants.ControllerLabelResource: utilities.CertificateKubeconfigLabel,
},
))
@@ -113,7 +113,10 @@ func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kam
return err
}
if checksum := tenantControlPlane.Status.Addons.Konnectivity.Certificate.Checksum; len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) {
isRotationRequested := utilities.IsRotationRequested(r.resource)
checksum := tenantControlPlane.Status.Addons.Konnectivity.Kubeconfig.Checksum
if len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) && !isRotationRequested {
return nil
}
@@ -181,6 +184,8 @@ func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kam
konnectivityKubeconfigFileName: kubeconfigBytes,
}
utilities.SetLastRotationTimestamp(r.resource)
utilities.SetObjectChecksum(r.resource, r.resource.Data)
return nil

View File

@@ -172,10 +172,10 @@ func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kam
r.resource.SetLabels(utilities.MergeMaps(
utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()),
map[string]string{
constants.ControllerLabelResource: "kubeconfig",
constants.ControllerLabelResource: utilities.CertificateKubeconfigLabel,
},
))
r.resource.SetAnnotations(map[string]string{constants.Checksum: checksum})
r.resource.SetAnnotations(utilities.MergeMaps(r.resource.GetAnnotations(), map[string]string{constants.Checksum: checksum}))
if err = ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil {
logger.Error(err, "cannot set controller reference", "resource", r.GetName())
@@ -185,18 +185,21 @@ func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kam
var shouldCreate bool
shouldCreate = shouldCreate || r.resource.Data == nil // Missing data key
shouldCreate = shouldCreate || len(r.resource.Data) == 0 // Missing data key
shouldCreate = shouldCreate || len(r.resource.Data[r.KubeConfigFileName]) == 0 // Missing kubeconfig file, must be generated
shouldCreate = shouldCreate || r.resource.Data == nil // Missing data key
shouldCreate = shouldCreate || len(r.resource.Data) == 0 // Missing data key
shouldCreate = shouldCreate || len(r.resource.Data[r.KubeConfigFileName]) == 0 // Missing kubeconfig file, must be generated
shouldCreate = shouldCreate || !kubeadm.IsKubeconfigCAValid(r.resource.Data[r.KubeConfigFileName], caCertificatesSecret.Data[kubeadmconstants.CACertName])
shouldCreate = shouldCreate || !kubeadm.IsKubeconfigValid(r.resource.Data[r.KubeConfigFileName]) // invalid kubeconfig, or expired client certificate
shouldCreate = shouldCreate || status.Checksum != checksum || len(r.resource.UID) == 0 // Wrong checksum
shouldRotate := utilities.IsRotationRequested(r.resource)
if !shouldCreate {
v, ok := r.resource.Data[r.KubeConfigFileName]
shouldCreate = len(v) == 0 || !ok
}
//nolint:nestif
if shouldCreate {
if shouldCreate || shouldRotate {
crtKeyPair := kubeadm.CertificatePrivateKeyPair{
Certificate: caCertificatesSecret.Data[kubeadmconstants.CACertName],
PrivateKey: caCertificatesSecret.Data[kubeadmconstants.CAKeyName],
@@ -213,6 +216,10 @@ func (r *KubeconfigResource) mutate(ctx context.Context, tenantControlPlane *kam
return kcErr
}
if shouldRotate {
utilities.SetLastRotationTimestamp(r.resource)
}
r.resource.Data[r.KubeConfigFileName] = kubeconfig
// Adding a kubeconfig useful for the local connections:
// especially for the admin.conf and super-admin.conf, these would use the public IP address.

View File

@@ -91,7 +91,9 @@ func (r *SACertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1
return func() error {
logger := log.FromContext(ctx, "resource", r.GetName())
if checksum := tenantControlPlane.Status.Certificates.SA.Checksum; len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0 {
isRotationRequested := utilities.IsRotationRequested(r.resource)
if checksum := tenantControlPlane.Status.Certificates.SA.Checksum; !isRotationRequested && (len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0) {
isValid, err := crypto.CheckPublicAndPrivateKeyValidity(r.resource.Data[kubeadmconstants.ServiceAccountPublicKeyName], r.resource.Data[kubeadmconstants.ServiceAccountPrivateKeyName])
if err != nil {
logger.Info(fmt.Sprintf("%s public_key-private_key pair is not valid: %s", kubeadmconstants.ServiceAccountKeyBaseName, err.Error()))
@@ -122,6 +124,10 @@ func (r *SACertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1
r.resource.SetLabels(utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()))
if isRotationRequested {
utilities.SetLastRotationTimestamp(r.resource)
}
utilities.SetObjectChecksum(r.resource, r.resource.Data)
return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())

View File

@@ -0,0 +1,40 @@
// Copyright 2022 Clastix Labs
// SPDX-License-Identifier: Apache-2.0
package utilities
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
)
const (
RotateCertificateRequestAnnotation = "certs.kamaji.clastix.io/rotate"
CertificateX509Label = "x509"
CertificateKubeconfigLabel = "kubeconfig"
)
func IsRotationRequested(obj client.Object) bool {
if obj.GetAnnotations() == nil {
return false
}
v, ok := obj.GetAnnotations()[RotateCertificateRequestAnnotation]
if ok && v == "" {
return true
}
return false
}
func SetLastRotationTimestamp(obj client.Object) {
annotations := obj.GetAnnotations()
if annotations == nil {
annotations = map[string]string{}
}
annotations[RotateCertificateRequestAnnotation] = metav1.Now().String()
obj.SetAnnotations(annotations)
}

View File

@@ -4,6 +4,8 @@
package internal
var (
ContainerRepository = "docker.io"
GitRepo = ""
GitTag = "dev"
GitCommit = ""