Compare commits

...

3 Commits

Author SHA1 Message Date
Aleksei Sviridkin
69d62273c2 feat(deployment): make startup probe failure threshold configurable (#1086)
* feat(deployment): make startup probe failure threshold configurable

Add StartupProbeFailureThreshold field to TenantControlPlane CRD
DeploymentSpec, allowing users to configure how many consecutive
startup probe failures are tolerated before a container is considered
failed. The value is applied to all control plane components
(kube-apiserver, controller-manager, and scheduler).

Defaults to 3 (preserving current behavior). With PeriodSeconds=10,
the total startup timeout equals FailureThreshold * 10 seconds.
Setting this to 30 gives 5 minutes, which is useful for
resource-constrained environments.

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>

* chore: regenerate CRD manifests for startupProbeFailureThreshold

Run `make manifests` to update Helm CRD files with the new
startupProbeFailureThreshold field in DeploymentSpec.

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>

* feat(deployment): expand configurable probes to all probe types

Replace StartupProbeFailureThreshold with a full Probes config
supporting liveness, readiness, and startup probes with configurable
TimeoutSeconds, PeriodSeconds, and FailureThreshold parameters.
Use ptr.Deref for safe pointer dereferencing.

Ref: #471

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>

* chore: regenerate CRD manifests and API documentation

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>

* feat(deployment): add per-component probe overrides and expand ProbeSpec

Add cascading probe configuration: global defaults → per-component
overrides (apiServer, controllerManager, scheduler). Expand ProbeSpec
with InitialDelaySeconds and SuccessThreshold fields.

Ref: #471

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>

* chore: regenerate CRD manifests and API documentation

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: Aleksei Sviridkin <f@lex.la>

---------

Signed-off-by: Aleksei Sviridkin <f@lex.la>
Co-authored-by: Claude <noreply@anthropic.com>
2026-02-24 16:20:06 +01:00
Patryk Rostkowski
b3ddfcda27 docs: add externalClusterReference with CAPI usage (#1088)
Signed-off-by: Patryk Rostkowski <patrostkowski@gmail.com>
2026-02-24 10:21:41 +01:00
dependabot[bot]
b13eca045c feat(deps): bump github.com/nats-io/nats.go from 1.48.0 to 1.49.0 (#1091)
Bumps [github.com/nats-io/nats.go](https://github.com/nats-io/nats.go) from 1.48.0 to 1.49.0.
- [Release notes](https://github.com/nats-io/nats.go/releases)
- [Commits](https://github.com/nats-io/nats.go/compare/v1.48.0...v1.49.0)

---
updated-dependencies:
- dependency-name: github.com/nats-io/nats.go
  dependency-version: 1.49.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-02-24 10:21:29 +01:00
13 changed files with 2575 additions and 9 deletions

View File

@@ -68,6 +68,17 @@ all: build
help: ## Display this help.
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
##@ Documentation
.PHONY: docs
docs: ## Serve documentation locally with Docker.
docker run --rm -it \
-p 8000:8000 \
-v "$${PWD}/docs":/docs:Z \
-w /docs \
squidfunk/mkdocs-material \
serve -a 0.0.0.0:8000
##@ Binary
.PHONY: ko

View File

@@ -173,6 +173,54 @@ type ControlPlaneComponentsResources struct {
Kine *corev1.ResourceRequirements `json:"kine,omitempty"`
}
// ProbeSpec defines configurable parameters for a Kubernetes probe.
type ProbeSpec struct {
// InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
//+kubebuilder:validation:Minimum=0
InitialDelaySeconds *int32 `json:"initialDelaySeconds,omitempty"`
// TimeoutSeconds is the number of seconds after which the probe times out.
//+kubebuilder:validation:Minimum=1
TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty"`
// PeriodSeconds is how often (in seconds) to perform the probe.
//+kubebuilder:validation:Minimum=1
PeriodSeconds *int32 `json:"periodSeconds,omitempty"`
// SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
// Must be 1 for liveness and startup probes.
//+kubebuilder:validation:Minimum=1
SuccessThreshold *int32 `json:"successThreshold,omitempty"`
// FailureThreshold is the consecutive failure count required to consider the probe failed.
//+kubebuilder:validation:Minimum=1
FailureThreshold *int32 `json:"failureThreshold,omitempty"`
}
// ProbeSet defines per-probe-type configuration.
type ProbeSet struct {
// Liveness defines parameters for the liveness probe.
Liveness *ProbeSpec `json:"liveness,omitempty"`
// Readiness defines parameters for the readiness probe.
Readiness *ProbeSpec `json:"readiness,omitempty"`
// Startup defines parameters for the startup probe.
Startup *ProbeSpec `json:"startup,omitempty"`
}
// ControlPlaneProbes defines probe configuration for Control Plane components.
// Global probe settings (Liveness, Readiness, Startup) apply to all components.
// Per-component settings (APIServer, ControllerManager, Scheduler) override global settings.
type ControlPlaneProbes struct {
// Liveness defines default parameters for liveness probes of all Control Plane components.
Liveness *ProbeSpec `json:"liveness,omitempty"`
// Readiness defines default parameters for the readiness probe of kube-apiserver.
Readiness *ProbeSpec `json:"readiness,omitempty"`
// Startup defines default parameters for startup probes of all Control Plane components.
Startup *ProbeSpec `json:"startup,omitempty"`
// APIServer defines probe overrides for kube-apiserver, taking precedence over global probe settings.
APIServer *ProbeSet `json:"apiServer,omitempty"`
// ControllerManager defines probe overrides for kube-controller-manager, taking precedence over global probe settings.
ControllerManager *ProbeSet `json:"controllerManager,omitempty"`
// Scheduler defines probe overrides for kube-scheduler, taking precedence over global probe settings.
Scheduler *ProbeSet `json:"scheduler,omitempty"`
}
type DeploymentSpec struct {
// RegistrySettings allows to override the default images for the given Tenant Control Plane instance.
// It could be used to point to a different container registry rather than the public one.
@@ -224,6 +272,10 @@ type DeploymentSpec struct {
// AdditionalVolumeMounts allows to mount an additional volume into each component of the Control Plane
// (kube-apiserver, controller-manager, and scheduler).
AdditionalVolumeMounts *AdditionalVolumeMounts `json:"additionalVolumeMounts,omitempty"`
// Probes defines the probe configuration for the Control Plane components
// (kube-apiserver, controller-manager, and scheduler).
// Override TimeoutSeconds, PeriodSeconds, and FailureThreshold for resource-constrained environments.
Probes *ControlPlaneProbes `json:"probes,omitempty"`
//+kubebuilder:default="default"
// ServiceAccountName allows to specify the service account to be mounted to the pods of the Control plane deployment
ServiceAccountName string `json:"serviceAccountName,omitempty"`

View File

@@ -449,6 +449,51 @@ func (in *ControlPlaneExtraArgs) DeepCopy() *ControlPlaneExtraArgs {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ControlPlaneProbes) DeepCopyInto(out *ControlPlaneProbes) {
*out = *in
if in.Liveness != nil {
in, out := &in.Liveness, &out.Liveness
*out = new(ProbeSpec)
(*in).DeepCopyInto(*out)
}
if in.Readiness != nil {
in, out := &in.Readiness, &out.Readiness
*out = new(ProbeSpec)
(*in).DeepCopyInto(*out)
}
if in.Startup != nil {
in, out := &in.Startup, &out.Startup
*out = new(ProbeSpec)
(*in).DeepCopyInto(*out)
}
if in.APIServer != nil {
in, out := &in.APIServer, &out.APIServer
*out = new(ProbeSet)
(*in).DeepCopyInto(*out)
}
if in.ControllerManager != nil {
in, out := &in.ControllerManager, &out.ControllerManager
*out = new(ProbeSet)
(*in).DeepCopyInto(*out)
}
if in.Scheduler != nil {
in, out := &in.Scheduler, &out.Scheduler
*out = new(ProbeSet)
(*in).DeepCopyInto(*out)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlaneProbes.
func (in *ControlPlaneProbes) DeepCopy() *ControlPlaneProbes {
if in == nil {
return nil
}
out := new(ControlPlaneProbes)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DataStore) DeepCopyInto(out *DataStore) {
*out = *in
@@ -709,6 +754,11 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
*out = new(AdditionalVolumeMounts)
(*in).DeepCopyInto(*out)
}
if in.Probes != nil {
in, out := &in.Probes, &out.Probes
*out = new(ControlPlaneProbes)
(*in).DeepCopyInto(*out)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentSpec.
@@ -1493,6 +1543,76 @@ func (in *Permissions) DeepCopy() *Permissions {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ProbeSet) DeepCopyInto(out *ProbeSet) {
*out = *in
if in.Liveness != nil {
in, out := &in.Liveness, &out.Liveness
*out = new(ProbeSpec)
(*in).DeepCopyInto(*out)
}
if in.Readiness != nil {
in, out := &in.Readiness, &out.Readiness
*out = new(ProbeSpec)
(*in).DeepCopyInto(*out)
}
if in.Startup != nil {
in, out := &in.Startup, &out.Startup
*out = new(ProbeSpec)
(*in).DeepCopyInto(*out)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProbeSet.
func (in *ProbeSet) DeepCopy() *ProbeSet {
if in == nil {
return nil
}
out := new(ProbeSet)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ProbeSpec) DeepCopyInto(out *ProbeSpec) {
*out = *in
if in.InitialDelaySeconds != nil {
in, out := &in.InitialDelaySeconds, &out.InitialDelaySeconds
*out = new(int32)
**out = **in
}
if in.TimeoutSeconds != nil {
in, out := &in.TimeoutSeconds, &out.TimeoutSeconds
*out = new(int32)
**out = **in
}
if in.PeriodSeconds != nil {
in, out := &in.PeriodSeconds, &out.PeriodSeconds
*out = new(int32)
**out = **in
}
if in.SuccessThreshold != nil {
in, out := &in.SuccessThreshold, &out.SuccessThreshold
*out = new(int32)
**out = **in
}
if in.FailureThreshold != nil {
in, out := &in.FailureThreshold, &out.FailureThreshold
*out = new(int32)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProbeSpec.
func (in *ProbeSpec) DeepCopy() *ProbeSpec {
if in == nil {
return nil
}
out := new(ProbeSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *PublicKeyPrivateKeyPairStatus) DeepCopyInto(out *PublicKeyPrivateKeyPairStatus) {
*out = *in

View File

@@ -6165,6 +6165,397 @@ versions:
type: string
type: object
type: object
probes:
description: |-
Probes defines the probe configuration for the Control Plane components
(kube-apiserver, controller-manager, and scheduler).
Override TimeoutSeconds, PeriodSeconds, and FailureThreshold for resource-constrained environments.
properties:
apiServer:
description: APIServer defines probe overrides for kube-apiserver, taking precedence over global probe settings.
properties:
liveness:
description: Liveness defines parameters for the liveness probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
readiness:
description: Readiness defines parameters for the readiness probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
startup:
description: Startup defines parameters for the startup probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
type: object
controllerManager:
description: ControllerManager defines probe overrides for kube-controller-manager, taking precedence over global probe settings.
properties:
liveness:
description: Liveness defines parameters for the liveness probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
readiness:
description: Readiness defines parameters for the readiness probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
startup:
description: Startup defines parameters for the startup probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
type: object
liveness:
description: Liveness defines default parameters for liveness probes of all Control Plane components.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
readiness:
description: Readiness defines default parameters for the readiness probe of kube-apiserver.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
scheduler:
description: Scheduler defines probe overrides for kube-scheduler, taking precedence over global probe settings.
properties:
liveness:
description: Liveness defines parameters for the liveness probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
readiness:
description: Readiness defines parameters for the readiness probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
startup:
description: Startup defines parameters for the startup probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
type: object
startup:
description: Startup defines default parameters for startup probes of all Control Plane components.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
type: object
registrySettings:
default:
apiServerImage: kube-apiserver

View File

@@ -6173,6 +6173,397 @@ spec:
type: string
type: object
type: object
probes:
description: |-
Probes defines the probe configuration for the Control Plane components
(kube-apiserver, controller-manager, and scheduler).
Override TimeoutSeconds, PeriodSeconds, and FailureThreshold for resource-constrained environments.
properties:
apiServer:
description: APIServer defines probe overrides for kube-apiserver, taking precedence over global probe settings.
properties:
liveness:
description: Liveness defines parameters for the liveness probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
readiness:
description: Readiness defines parameters for the readiness probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
startup:
description: Startup defines parameters for the startup probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
type: object
controllerManager:
description: ControllerManager defines probe overrides for kube-controller-manager, taking precedence over global probe settings.
properties:
liveness:
description: Liveness defines parameters for the liveness probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
readiness:
description: Readiness defines parameters for the readiness probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
startup:
description: Startup defines parameters for the startup probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
type: object
liveness:
description: Liveness defines default parameters for liveness probes of all Control Plane components.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
readiness:
description: Readiness defines default parameters for the readiness probe of kube-apiserver.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
scheduler:
description: Scheduler defines probe overrides for kube-scheduler, taking precedence over global probe settings.
properties:
liveness:
description: Liveness defines parameters for the liveness probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
readiness:
description: Readiness defines parameters for the readiness probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
startup:
description: Startup defines parameters for the startup probe.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
type: object
startup:
description: Startup defines default parameters for startup probes of all Control Plane components.
properties:
failureThreshold:
description: FailureThreshold is the consecutive failure count required to consider the probe failed.
format: int32
minimum: 1
type: integer
initialDelaySeconds:
description: InitialDelaySeconds is the number of seconds after the container has started before the probe is initiated.
format: int32
minimum: 0
type: integer
periodSeconds:
description: PeriodSeconds is how often (in seconds) to perform the probe.
format: int32
minimum: 1
type: integer
successThreshold:
description: |-
SuccessThreshold is the minimum consecutive successes for the probe to be considered successful.
Must be 1 for liveness and startup probes.
format: int32
minimum: 1
type: integer
timeoutSeconds:
description: TimeoutSeconds is the number of seconds after which the probe times out.
format: int32
minimum: 1
type: integer
type: object
type: object
registrySettings:
default:
apiServerImage: kube-apiserver

View File

@@ -0,0 +1,154 @@
---
apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
kind: KubeadmConfigTemplate
metadata:
name: worker-external
namespace: default
spec:
template:
spec:
users:
joinConfiguration:
nodeRegistration:
kubeletExtraArgs:
- name: feature-gates
value: "KubeletCrashLoopBackOffMax=true,KubeletEnsureSecretPulledImages=true"
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: KubevirtMachineTemplate
metadata:
name: worker-external
namespace: default
spec:
template:
spec:
virtualMachineBootstrapCheck:
checkStrategy: ssh
virtualMachineTemplate:
metadata:
namespace: default
spec:
runStrategy: Always
template:
spec:
dnsPolicy: None
dnsConfig:
nameservers:
- 1.1.1.1
- 8.8.8.8
searches: []
options:
- name: ndots
value: "1"
domain:
cpu:
cores: 2
devices:
interfaces:
- name: default
masquerade: {}
disks:
- disk:
bus: virtio
name: containervolume
networkInterfaceMultiqueue: true
memory:
guest: 4Gi
evictionStrategy: External
networks:
- name: default
pod: {}
volumes:
- containerDisk:
image: quay.io/capk/ubuntu-2404-container-disk:v1.34.1
name: containervolume
---
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: KubevirtClusterTemplate
metadata:
name: kubevirt-external
namespace: default
spec:
template:
metadata:
annotations:
cluster.x-k8s.io/managed-by: kamaji
spec:
controlPlaneServiceTemplate:
spec:
type: LoadBalancer
---
apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
kind: KamajiControlPlaneTemplate
metadata:
name: kamaji-controlplane-external
namespace: default
spec:
template:
spec:
addons:
coreDNS: {}
konnectivity: {}
kubeProxy: {}
dataStoreName: "default" # reference to DataStore present on external cluster
deployment:
externalClusterReference:
deploymentNamespace: kamaji-tenants
kubeconfigSecretName: kind-external-kubeconfig
kubeconfigSecretKey: kubeconfig
network:
serviceType: LoadBalancer
kubelet:
cgroupfs: systemd
preferredAddressTypes:
- InternalIP
registry: "registry.k8s.io"
---
apiVersion: cluster.x-k8s.io/v1beta2
kind: ClusterClass
metadata:
name: kubevirt-kamaji-kubeadm-external
namespace: default
spec:
controlPlane:
templateRef:
apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
kind: KamajiControlPlaneTemplate
name: kamaji-controlplane-external
infrastructure:
templateRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: KubevirtClusterTemplate
name: kubevirt-external
workers:
machineDeployments:
- class: small
bootstrap:
templateRef:
apiVersion: bootstrap.cluster.x-k8s.io/v1beta2
kind: KubeadmConfigTemplate
name: worker-external
infrastructure:
templateRef:
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
kind: KubevirtMachineTemplate
name: worker-external
---
apiVersion: cluster.x-k8s.io/v1beta2
kind: Cluster
metadata:
name: demo-external
namespace: default
spec:
topology:
classRef:
name: kubevirt-kamaji-kubeadm-external
namespace: default
version: v1.34.0
controlPlane:
replicas: 1
workers:
machineDeployments:
- class: small
name: md-small
replicas: 1

View File

@@ -0,0 +1,305 @@
# Kamaji and `externalClusterReference` usage
This document explains how to use **Kamaji's `externalClusterReference`** together with **Cluster API (CAPI)** to run Kubernetes control planes on an **external cluster**, while managing worker nodes from a management cluster.
It assumes the use of the KubeVirt infrastructure provider for ease of deployment and local testing.
---
## High-level Architecture
The following setup operates on **two Kubernetes clusters**:
- **Management cluster** runs Cluster API controllers and the Kamaji control-plane provider, and manages cluster lifecycle and topology.
- **External cluster** - runs Kamaji, hosts the Kubernetes control plane components and receives control plane workloads via Kamaji
---
## Prerequisites
- `docker`
- `kind`
- `kubectl`
- `clusterctl`
- `helm`
---
## Step 1: Create the KIND clusters
Create the **management** cluster:
```bash
kind create cluster --name management
```
Create the **external** cluster that will host control planes:
```bash
kind create cluster --name external
```
Verify contexts:
```bash
kubectl config get-contexts
```
---
## Step 2: Initialize Cluster API controllers
Switch to the management cluster:
```bash
kubectl config use-context kind-management
```
Enable ClusterClass support and initialize Cluster API with Kamaji and KubeVirt:
```bash
export CLUSTER_TOPOLOGY=true
clusterctl init \
--core cluster-api \
--bootstrap kubeadm \
--infrastructure kubevirt \
--control-plane kamaji
```
---
## Step 3: Enable Kamaji external cluster feature gates
Patch the Kamaji controller to enable `externalClusterReference`:
```bash
kubectl -n kamaji-system patch deployment capi-kamaji-controller-manager \
--type='json' \
-p='[
{
"op": "replace",
"path": "/spec/template/spec/containers/0/args/1",
"value": "--feature-gates=ExternalClusterReference=true,ExternalClusterReferenceCrossNamespace=true"
}
]'
```
---
## Step 4: Install KubeVirt
Fetch the latest stable KubeVirt version and install:
```bash
export VERSION=$(curl -s "https://storage.googleapis.com/kubevirt-prow/release/kubevirt/kubevirt/stable.txt")
kubectl apply -f "https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/kubevirt-operator.yaml"
kubectl apply -f "https://github.com/kubevirt/kubevirt/releases/download/${VERSION}/kubevirt-cr.yaml"
```
Enable emulation (optional, if virtualization is not supported):
```bash
kubectl -n kubevirt patch kubevirt kubevirt \
--type=merge \
--patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}'
```
---
## Step 5: Prepare kubeconfig for the external cluster
Retrieve the external cluster control-plane address:
```bash
EXT_CP_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "external-control-plane")
```
Export and rewrite the kubeconfig:
```bash
kubectl --context kind-external config view --raw --minify --flatten > kind-external.kubeconfig
```
Replace the API endpoint with the cluster IP, required for cross-cluster access from the management cluster:
```bash
bash -c "sed -i -E 's#https://[^:]+:[0-9]+#https://$EXT_CP_IP:6443#g' kind-external.kubeconfig"
```
Create the kubeconfig secret in the management cluster:
```bash
kubectl -n default create secret generic kind-external-kubeconfig \
--from-file=kubeconfig=kind-external.kubeconfig
```
---
## Step 6: Install Kamaji and dependencies on the external cluster
Switch context:
```bash
kubectl config use-context kind-external
```
Install cert-manager:
```bash
helm upgrade --install cert-manager jetstack/cert-manager \
--namespace cert-manager \
--create-namespace \
--set installCRDs=true
```
Install Kamaji:
```bash
helm upgrade --install kamaji clastix/kamaji \
--namespace kamaji-system \
--create-namespace \
--set 'resources=null' \
--version 0.0.0+latest
```
Install MetalLB:
```bash
kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.15.3/config/manifests/metallb-native.yaml
```
Configure MetalLB IP address pool:
```bash
SUBNET=$(docker network inspect kind | jq -r '.[0].IPAM.Config[] | select(.Subnet | test(":") | not) | .Subnet' | head -n1)
NET_PREFIX=$(echo "$SUBNET" | cut -d/ -f1 | awk -F. '{print $1"."$2}')
kubectl apply -f - <<EOF
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
name: default
namespace: metallb-system
spec:
addresses:
- ${NET_PREFIX}.255.200-${NET_PREFIX}.255.250
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
name: default
namespace: metallb-system
EOF
```
Create tenant namespace:
```bash
kubectl create namespace kamaji-tenants
```
---
## Step 7: Definition of KamajiControlPlaneTemplate
The `KamajiControlPlaneTemplate` is defined in [the following manifest](https://raw.githubusercontent.com/clastix/kamaji/master/config/capi/clusterclass-kubevirt-kamaji-external.yaml) and can be applied directly.
This template configures how Kamaji deploys and manages the tenant control plane on an external Kubernetes cluster using Cluster API.
```bash
apiVersion: controlplane.cluster.x-k8s.io/v1alpha1
kind: KamajiControlPlaneTemplate
metadata:
name: kamaji-controlplane-external
namespace: default
spec:
template:
spec:
addons:
coreDNS: {}
konnectivity: {}
kubeProxy: {}
dataStoreName: "default" # reference to DataStore present on external cluster
deployment:
externalClusterReference:
deploymentNamespace: kamaji-tenants
kubeconfigSecretName: kind-external-kubeconfig
kubeconfigSecretKey: kubeconfig
network:
serviceType: LoadBalancer
kubelet:
cgroupfs: systemd
preferredAddressTypes:
- InternalIP
registry: "registry.k8s.io"
```
The `.spec.template.spec.deployment.externalClusterReference` section defines how Kamaji connects to and deploys control plane components into the external cluster:
- `deploymentNamespace` - The namespace on the external cluster where `TenantControlPlane` resources and control plane components are created.
- `kubeconfigSecretName` - The name of the Kubernetes Secret containing a kubeconfig that allows Kamaji to authenticate to the external cluster.
- `kubeconfigSecretKey` - The key inside the secret that holds the kubeconfig data.
The referenced secret must exist in the Kamaji management cluster and provide sufficient permissions to create and manage resources in the target external cluster.
---
## Step 8: Create the Cluster
Switch context back to the management cluster:
```bash
kubectl config use-context kind-management
```
Apply the Cluster manifest:
```bash
kubectl apply -f "https://raw.githubusercontent.com/clastix/kamaji/master/config/capi/clusterclass-kubevirt-kamaji-external.yaml"
```
---
## Validation
Check tenant control plane pods running in the external cluster:
```bash
kubectl --context kind-external -n kamaji-tenants get pods
```
Check cluster status in the management cluster:
```bash
kubectl --context kind-management get clusters
kubectl --context kind-management get kamajicontrolplanes
```
Get cluster kubeconfig and confirm it is working:
```bash
kubectl config use-context kind-management
clusterctl get kubeconfig demo-external > demo-external.kubeconfig
KUBECONFIG=./demo-external.kubeconfig kubectl get nodes
```
---
## Clean up
Delete Kind clusters:
```bash
kind delete cluster --name management
kind delete cluster --name external
```
---
## Summary
Using `externalClusterReference` with Kamaji and Cluster API enables:
- Hosted Kubernetes control planes on remote clusters
- Strong separation of concerns
- Multi-cluster management patterns
- Clean integration with ClusterClass

File diff suppressed because it is too large Load Diff

View File

@@ -68,6 +68,7 @@ nav:
- cluster-api/other-providers.md
- cluster-api/cluster-autoscaler.md
- cluster-api/cluster-class.md
- cluster-api/external-cluster.md
- 'Guides':
- guides/index.md
- guides/alternative-datastore.md

6
go.mod
View File

@@ -15,7 +15,7 @@ require (
github.com/google/uuid v1.6.0
github.com/json-iterator/go v1.1.12
github.com/juju/mutex/v2 v2.0.0
github.com/nats-io/nats.go v1.48.0
github.com/nats-io/nats.go v1.49.0
github.com/onsi/ginkgo/v2 v2.28.1
github.com/onsi/gomega v1.39.1
github.com/prometheus/client_golang v1.23.2
@@ -98,7 +98,7 @@ require (
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/juju/errors v0.0.0-20220203013757-bd733f3c86b9 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/compress v1.18.2 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/lithammer/dedent v1.1.0 // indirect
@@ -117,7 +117,7 @@ require (
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nats-io/nkeys v0.4.11 // indirect
github.com/nats-io/nkeys v0.4.12 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect

12
go.sum
View File

@@ -190,8 +190,8 @@ github.com/juju/version/v2 v2.0.0-20211007103408-2e8da085dc23/go.mod h1:Ljlbryh9
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -243,10 +243,10 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U=
github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE=
github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw=
github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc=
github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=

View File

@@ -51,6 +51,18 @@ const (
kineInitContainerName = "chmod"
)
func applyProbeOverrides(probe *corev1.Probe, spec *kamajiv1alpha1.ProbeSpec) {
if spec == nil {
return
}
probe.InitialDelaySeconds = pointer.Deref(spec.InitialDelaySeconds, probe.InitialDelaySeconds)
probe.TimeoutSeconds = pointer.Deref(spec.TimeoutSeconds, probe.TimeoutSeconds)
probe.PeriodSeconds = pointer.Deref(spec.PeriodSeconds, probe.PeriodSeconds)
probe.SuccessThreshold = pointer.Deref(spec.SuccessThreshold, probe.SuccessThreshold)
probe.FailureThreshold = pointer.Deref(spec.FailureThreshold, probe.FailureThreshold)
}
type DataStoreOverrides struct {
Resource string
DataStore kamajiv1alpha1.DataStore
@@ -384,6 +396,16 @@ func (d Deployment) buildScheduler(podSpec *corev1.PodSpec, tenantControlPlane k
FailureThreshold: 3,
}
if probes := tenantControlPlane.Spec.ControlPlane.Deployment.Probes; probes != nil {
applyProbeOverrides(podSpec.Containers[index].LivenessProbe, probes.Liveness)
applyProbeOverrides(podSpec.Containers[index].StartupProbe, probes.Startup)
if probes.Scheduler != nil {
applyProbeOverrides(podSpec.Containers[index].LivenessProbe, probes.Scheduler.Liveness)
applyProbeOverrides(podSpec.Containers[index].StartupProbe, probes.Scheduler.Startup)
}
}
switch {
case tenantControlPlane.Spec.ControlPlane.Deployment.Resources == nil:
podSpec.Containers[index].Resources = corev1.ResourceRequirements{}
@@ -475,6 +497,17 @@ func (d Deployment) buildControllerManager(podSpec *corev1.PodSpec, tenantContro
SuccessThreshold: 1,
FailureThreshold: 3,
}
if probes := tenantControlPlane.Spec.ControlPlane.Deployment.Probes; probes != nil {
applyProbeOverrides(podSpec.Containers[index].LivenessProbe, probes.Liveness)
applyProbeOverrides(podSpec.Containers[index].StartupProbe, probes.Startup)
if probes.ControllerManager != nil {
applyProbeOverrides(podSpec.Containers[index].LivenessProbe, probes.ControllerManager.Liveness)
applyProbeOverrides(podSpec.Containers[index].StartupProbe, probes.ControllerManager.Startup)
}
}
switch {
case tenantControlPlane.Spec.ControlPlane.Deployment.Resources == nil:
podSpec.Containers[index].Resources = corev1.ResourceRequirements{}
@@ -606,6 +639,19 @@ func (d Deployment) buildKubeAPIServer(podSpec *corev1.PodSpec, tenantControlPla
SuccessThreshold: 1,
FailureThreshold: 3,
}
if probes := tenantControlPlane.Spec.ControlPlane.Deployment.Probes; probes != nil {
applyProbeOverrides(podSpec.Containers[index].LivenessProbe, probes.Liveness)
applyProbeOverrides(podSpec.Containers[index].ReadinessProbe, probes.Readiness)
applyProbeOverrides(podSpec.Containers[index].StartupProbe, probes.Startup)
if probes.APIServer != nil {
applyProbeOverrides(podSpec.Containers[index].LivenessProbe, probes.APIServer.Liveness)
applyProbeOverrides(podSpec.Containers[index].ReadinessProbe, probes.APIServer.Readiness)
applyProbeOverrides(podSpec.Containers[index].StartupProbe, probes.APIServer.Startup)
}
}
podSpec.Containers[index].ImagePullPolicy = corev1.PullAlways
// Volume mounts
var extraVolumeMounts []corev1.VolumeMount

View File

@@ -4,12 +4,21 @@
package controlplane
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
pointer "k8s.io/utils/ptr"
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
)
func TestControlplaneDeployment(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Controlplane Deployment Suite")
}
var _ = Describe("Controlplane Deployment", func() {
var d Deployment
BeforeEach(func() {
@@ -43,4 +52,74 @@ var _ = Describe("Controlplane Deployment", func() {
Expect(etcdSerVersOverrides).To(Equal("/events#https://etcd-0;https://etcd-1;https://etcd-2,/pods#https://etcd-3;https://etcd-4;https://etcd-5"))
})
})
Describe("applyProbeOverrides", func() {
var probe *corev1.Probe
BeforeEach(func() {
probe = &corev1.Probe{
InitialDelaySeconds: 0,
TimeoutSeconds: 1,
PeriodSeconds: 10,
SuccessThreshold: 1,
FailureThreshold: 3,
}
})
It("should not modify probe when spec is nil", func() {
applyProbeOverrides(probe, nil)
Expect(probe.InitialDelaySeconds).To(Equal(int32(0)))
Expect(probe.TimeoutSeconds).To(Equal(int32(1)))
Expect(probe.PeriodSeconds).To(Equal(int32(10)))
Expect(probe.SuccessThreshold).To(Equal(int32(1)))
Expect(probe.FailureThreshold).To(Equal(int32(3)))
})
It("should override only FailureThreshold when only it is set", func() {
spec := &kamajiv1alpha1.ProbeSpec{
FailureThreshold: pointer.To(int32(30)),
}
applyProbeOverrides(probe, spec)
Expect(probe.FailureThreshold).To(Equal(int32(30)))
Expect(probe.InitialDelaySeconds).To(Equal(int32(0)))
Expect(probe.TimeoutSeconds).To(Equal(int32(1)))
Expect(probe.PeriodSeconds).To(Equal(int32(10)))
Expect(probe.SuccessThreshold).To(Equal(int32(1)))
})
It("should override all fields when all are set", func() {
spec := &kamajiv1alpha1.ProbeSpec{
InitialDelaySeconds: pointer.To(int32(15)),
TimeoutSeconds: pointer.To(int32(5)),
PeriodSeconds: pointer.To(int32(30)),
SuccessThreshold: pointer.To(int32(2)),
FailureThreshold: pointer.To(int32(10)),
}
applyProbeOverrides(probe, spec)
Expect(probe.InitialDelaySeconds).To(Equal(int32(15)))
Expect(probe.TimeoutSeconds).To(Equal(int32(5)))
Expect(probe.PeriodSeconds).To(Equal(int32(30)))
Expect(probe.SuccessThreshold).To(Equal(int32(2)))
Expect(probe.FailureThreshold).To(Equal(int32(10)))
})
It("should cascade global then component overrides", func() {
global := &kamajiv1alpha1.ProbeSpec{
FailureThreshold: pointer.To(int32(10)),
PeriodSeconds: pointer.To(int32(20)),
}
applyProbeOverrides(probe, global)
component := &kamajiv1alpha1.ProbeSpec{
FailureThreshold: pointer.To(int32(60)),
}
applyProbeOverrides(probe, component)
Expect(probe.FailureThreshold).To(Equal(int32(60)))
Expect(probe.PeriodSeconds).To(Equal(int32(20)))
Expect(probe.TimeoutSeconds).To(Equal(int32(1)))
Expect(probe.InitialDelaySeconds).To(Equal(int32(0)))
Expect(probe.SuccessThreshold).To(Equal(int32(1)))
})
})
})