github/k3k
1
0
Fork 0
mirror of https://github.com/rancher/k3k.git synced 2026-08-20 04:46:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add HCP (Hosted Control Plane) support

Browse Source 
Introduce hosted control plane mode for k3k virtual clusters, including
API types, controller logic, server endpoint handling, CLI flags,
CRD updates, kubeconfig generation, and examples.

Co-Authored-By: RuFlo <ruv@ruv.net>
This commit is contained in:
jpgouin
2026-06-12 17:36:48 +02:00
committed by Enrico Candino
co-authored by RuFlo
parent 651da42ef0
commit 67d5f4dbfc
20 changed files with 1311 additions and 419 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+5
View File
@@ -11,3 +11,8 @@ __debug*
cover.out
covcounters.**
covmeta.**
.claude/
.claude-flow/
.swarm
.mcp.json
CLAUDE.md
Makefile
+1 -1
View File
@@ -41,7 +41,7 @@ package: package-k3k package-k3k-kubelet ## Package the k3k and k3k-kubelet Dock
.PHONY: package-%
package-%:
docker build -f package/Dockerfile.$* \
docker buildx build --platform linux/amd64,linux/arm64 -f package/Dockerfile.$* \
-t $(REPO)/$*:$(VERSION) \
-t $(REPO)/$*:latest \
-t $(REPO)/$*:dev .
charts/k3k/templates/crds/k3k.io_clusters.yaml
+446 -203
View File
File diff suppressed because it is too large Load Diff
charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml
+211 -98
View File
@@ -4,7 +4,6 @@ kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.20.0
helm.sh/resource-policy: keep
name: virtualclusterpolicies.k3k.io
spec:
group: k3k.io
@@ -50,11 +49,18 @@ spec:
description: Spec defines the desired state of the VirtualClusterPolicy.
properties:
allowedMode:
allOf:
- enum:
- shared
- virtual
- hcp
- enum:
- shared
- virtual
- hcp
default: shared
description: AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared".
enum:
- shared
- virtual
description: AllowedMode specifies the allowed cluster provisioning
mode. Defaults to "shared".
type: string
x-kubernetes-validations:
- message: mode is immutable
@@ -65,7 +71,8 @@ spec:
This includes both node affinity and pod affinity/anti-affinity rules.
properties:
nodeAffinity:
description: Describes node affinity scheduling rules for the pod.
description: Describes node affinity scheduling rules for the
pod.
properties:
preferredDuringSchedulingIgnoredDuringExecution:
description: |-
@@ -84,17 +91,20 @@ spec:
(i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).
properties:
preference:
description: A node selector term, associated with the corresponding weight.
description: A node selector term, associated with the
corresponding weight.
properties:
matchExpressions:
description: A list of node selector requirements by node's labels.
description: A list of node selector requirements
by node's labels.
items:
description: |-
A node selector requirement is a selector that contains values, a key, and an operator
that relates the key and values.
properties:
key:
description: The label key that the selector applies to.
description: The label key that the selector
applies to.
type: string
operator:
description: |-
@@ -119,14 +129,16 @@ spec:
type: array
x-kubernetes-list-type: atomic
matchFields:
description: A list of node selector requirements by node's fields.
description: A list of node selector requirements
by node's fields.
items:
description: |-
A node selector requirement is a selector that contains values, a key, and an operator
that relates the key and values.
properties:
key:
description: The label key that the selector applies to.
description: The label key that the selector
applies to.
type: string
operator:
description: |-
@@ -153,7 +165,8 @@ spec:
type: object
x-kubernetes-map-type: atomic
weight:
description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.
description: Weight associated with matching the corresponding
nodeSelectorTerm, in the range 1-100.
format: int32
type: integer
required:
@@ -171,7 +184,8 @@ spec:
may or may not try to eventually evict the pod from its node.
properties:
nodeSelectorTerms:
description: Required. A list of node selector terms. The terms are ORed.
description: Required. A list of node selector terms.
The terms are ORed.
items:
description: |-
A null or empty node selector term matches no objects. The requirements of
@@ -179,14 +193,16 @@ spec:
The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.
properties:
matchExpressions:
description: A list of node selector requirements by node's labels.
description: A list of node selector requirements
by node's labels.
items:
description: |-
A node selector requirement is a selector that contains values, a key, and an operator
that relates the key and values.
properties:
key:
description: The label key that the selector applies to.
description: The label key that the selector
applies to.
type: string
operator:
description: |-
@@ -211,14 +227,16 @@ spec:
type: array
x-kubernetes-list-type: atomic
matchFields:
description: A list of node selector requirements by node's fields.
description: A list of node selector requirements
by node's fields.
items:
description: |-
A node selector requirement is a selector that contains values, a key, and an operator
that relates the key and values.
properties:
key:
description: The label key that the selector applies to.
description: The label key that the selector
applies to.
type: string
operator:
description: |-
@@ -252,7 +270,8 @@ spec:
x-kubernetes-map-type: atomic
type: object
podAffinity:
description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)).
description: Describes pod affinity scheduling rules (e.g. co-locate
this pod in the same node, zone, etc. as some other pod(s)).
properties:
preferredDuringSchedulingIgnoredDuringExecution:
description: |-
@@ -266,10 +285,12 @@ spec:
"weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the
node(s) with the highest sum are the most preferred.
items:
description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)
description: The weights of all of the matched WeightedPodAffinityTerm
fields are added per-node to find the most preferred node(s)
properties:
podAffinityTerm:
description: Required. A pod affinity term, associated with the corresponding weight.
description: Required. A pod affinity term, associated
with the corresponding weight.
properties:
labelSelector:
description: |-
@@ -277,14 +298,17 @@ spec:
If it's null, this PodAffinityTerm matches with no Pods.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
description: matchExpressions is a list of label
selector requirements. The requirements are
ANDed.
items:
description: |-
A label selector requirement is a selector that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
description: key is the label key that
the selector applies to.
type: string
operator:
description: |-
@@ -354,14 +378,17 @@ spec:
An empty selector ({}) matches all namespaces.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
description: matchExpressions is a list of label
selector requirements. The requirements are
ANDed.
items:
description: |-
A label selector requirement is a selector that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
description: key is the label key that
the selector applies to.
type: string
operator:
description: |-
@@ -451,14 +478,16 @@ spec:
If it's null, this PodAffinityTerm matches with no Pods.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
description: matchExpressions is a list of label
selector requirements. The requirements are ANDed.
items:
description: |-
A label selector requirement is a selector that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
description: key is the label key that the
selector applies to.
type: string
operator:
description: |-
@@ -528,14 +557,16 @@ spec:
An empty selector ({}) matches all namespaces.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
description: matchExpressions is a list of label
selector requirements. The requirements are ANDed.
items:
description: |-
A label selector requirement is a selector that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
description: key is the label key that the
selector applies to.
type: string
operator:
description: |-
@@ -593,7 +624,9 @@ spec:
x-kubernetes-list-type: atomic
type: object
podAntiAffinity:
description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)).
description: Describes pod anti-affinity scheduling rules (e.g.
avoid putting this pod in the same node, zone, etc. as some
other pod(s)).
properties:
preferredDuringSchedulingIgnoredDuringExecution:
description: |-
@@ -607,10 +640,12 @@ spec:
"weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the
node(s) with the highest sum are the most preferred.
items:
description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)
description: The weights of all of the matched WeightedPodAffinityTerm
fields are added per-node to find the most preferred node(s)
properties:
podAffinityTerm:
description: Required. A pod affinity term, associated with the corresponding weight.
description: Required. A pod affinity term, associated
with the corresponding weight.
properties:
labelSelector:
description: |-
@@ -618,14 +653,17 @@ spec:
If it's null, this PodAffinityTerm matches with no Pods.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
description: matchExpressions is a list of label
selector requirements. The requirements are
ANDed.
items:
description: |-
A label selector requirement is a selector that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
description: key is the label key that
the selector applies to.
type: string
operator:
description: |-
@@ -695,14 +733,17 @@ spec:
An empty selector ({}) matches all namespaces.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
description: matchExpressions is a list of label
selector requirements. The requirements are
ANDed.
items:
description: |-
A label selector requirement is a selector that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
description: key is the label key that
the selector applies to.
type: string
operator:
description: |-
@@ -792,14 +833,16 @@ spec:
If it's null, this PodAffinityTerm matches with no Pods.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
description: matchExpressions is a list of label
selector requirements. The requirements are ANDed.
items:
description: |-
A label selector requirement is a selector that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
description: key is the label key that the
selector applies to.
type: string
operator:
description: |-
@@ -869,14 +912,16 @@ spec:
An empty selector ({}) matches all namespaces.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
description: matchExpressions is a list of label
selector requirements. The requirements are ANDed.
items:
description: |-
A label selector requirement is a selector that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
description: key is the label key that the
selector applies to.
type: string
operator:
description: |-
@@ -937,10 +982,12 @@ spec:
defaultNodeSelector:
additionalProperties:
type: string
description: DefaultNodeSelector specifies the node selector that applies to all clusters (server + agent) in the target Namespace.
description: DefaultNodeSelector specifies the node selector that
applies to all clusters (server + agent) in the target Namespace.
type: object
defaultPriorityClass:
description: DefaultPriorityClass specifies the priorityClassName applied to all pods of all clusters in the target Namespace.
description: DefaultPriorityClass specifies the priorityClassName
applied to all pods of all clusters in the target Namespace.
type: string
defaultServerAffinity:
description: |-
@@ -948,7 +995,8 @@ spec:
This includes both node affinity and pod affinity/anti-affinity rules.
properties:
nodeAffinity:
description: Describes node affinity scheduling rules for the pod.
description: Describes node affinity scheduling rules for the
pod.
properties:
preferredDuringSchedulingIgnoredDuringExecution:
description: |-
@@ -967,17 +1015,20 @@ spec:
(i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).
properties:
preference:
description: A node selector term, associated with the corresponding weight.
description: A node selector term, associated with the
corresponding weight.
properties:
matchExpressions:
description: A list of node selector requirements by node's labels.
description: A list of node selector requirements
by node's labels.
items:
description: |-
A node selector requirement is a selector that contains values, a key, and an operator
that relates the key and values.
properties:
key:
description: The label key that the selector applies to.
description: The label key that the selector
applies to.
type: string
operator:
description: |-
@@ -1002,14 +1053,16 @@ spec:
type: array
x-kubernetes-list-type: atomic
matchFields:
description: A list of node selector requirements by node's fields.
description: A list of node selector requirements
by node's fields.
items:
description: |-
A node selector requirement is a selector that contains values, a key, and an operator
that relates the key and values.
properties:
key:
description: The label key that the selector applies to.
description: The label key that the selector
applies to.
type: string
operator:
description: |-
@@ -1036,7 +1089,8 @@ spec:
type: object
x-kubernetes-map-type: atomic
weight:
description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.
description: Weight associated with matching the corresponding
nodeSelectorTerm, in the range 1-100.
format: int32
type: integer
required:
@@ -1054,7 +1108,8 @@ spec:
may or may not try to eventually evict the pod from its node.
properties:
nodeSelectorTerms:
description: Required. A list of node selector terms. The terms are ORed.
description: Required. A list of node selector terms.
The terms are ORed.
items:
description: |-
A null or empty node selector term matches no objects. The requirements of
@@ -1062,14 +1117,16 @@ spec:
The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.
properties:
matchExpressions:
description: A list of node selector requirements by node's labels.
description: A list of node selector requirements
by node's labels.
items:
description: |-
A node selector requirement is a selector that contains values, a key, and an operator
that relates the key and values.
properties:
key:
description: The label key that the selector applies to.
description: The label key that the selector
applies to.
type: string
operator:
description: |-
@@ -1094,14 +1151,16 @@ spec:
type: array
x-kubernetes-list-type: atomic
matchFields:
description: A list of node selector requirements by node's fields.
description: A list of node selector requirements
by node's fields.
items:
description: |-
A node selector requirement is a selector that contains values, a key, and an operator
that relates the key and values.
properties:
key:
description: The label key that the selector applies to.
description: The label key that the selector
applies to.
type: string
operator:
description: |-
@@ -1135,7 +1194,8 @@ spec:
x-kubernetes-map-type: atomic
type: object
podAffinity:
description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)).
description: Describes pod affinity scheduling rules (e.g. co-locate
this pod in the same node, zone, etc. as some other pod(s)).
properties:
preferredDuringSchedulingIgnoredDuringExecution:
description: |-
@@ -1149,10 +1209,12 @@ spec:
"weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the
node(s) with the highest sum are the most preferred.
items:
description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)
description: The weights of all of the matched WeightedPodAffinityTerm
fields are added per-node to find the most preferred node(s)
properties:
podAffinityTerm:
description: Required. A pod affinity term, associated with the corresponding weight.
description: Required. A pod affinity term, associated
with the corresponding weight.
properties:
labelSelector:
description: |-
@@ -1160,14 +1222,17 @@ spec:
If it's null, this PodAffinityTerm matches with no Pods.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
description: matchExpressions is a list of label
selector requirements. The requirements are
ANDed.
items:
description: |-
A label selector requirement is a selector that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
description: key is the label key that
the selector applies to.
type: string
operator:
description: |-
@@ -1237,14 +1302,17 @@ spec:
An empty selector ({}) matches all namespaces.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
description: matchExpressions is a list of label
selector requirements. The requirements are
ANDed.
items:
description: |-
A label selector requirement is a selector that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
description: key is the label key that
the selector applies to.
type: string
operator:
description: |-
@@ -1334,14 +1402,16 @@ spec:
If it's null, this PodAffinityTerm matches with no Pods.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
description: matchExpressions is a list of label
selector requirements. The requirements are ANDed.
items:
description: |-
A label selector requirement is a selector that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
description: key is the label key that the
selector applies to.
type: string
operator:
description: |-
@@ -1411,14 +1481,16 @@ spec:
An empty selector ({}) matches all namespaces.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
description: matchExpressions is a list of label
selector requirements. The requirements are ANDed.
items:
description: |-
A label selector requirement is a selector that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
description: key is the label key that the
selector applies to.
type: string
operator:
description: |-
@@ -1476,7 +1548,9 @@ spec:
x-kubernetes-list-type: atomic
type: object
podAntiAffinity:
description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)).
description: Describes pod anti-affinity scheduling rules (e.g.
avoid putting this pod in the same node, zone, etc. as some
other pod(s)).
properties:
preferredDuringSchedulingIgnoredDuringExecution:
description: |-
@@ -1490,10 +1564,12 @@ spec:
"weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the
node(s) with the highest sum are the most preferred.
items:
description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)
description: The weights of all of the matched WeightedPodAffinityTerm
fields are added per-node to find the most preferred node(s)
properties:
podAffinityTerm:
description: Required. A pod affinity term, associated with the corresponding weight.
description: Required. A pod affinity term, associated
with the corresponding weight.
properties:
labelSelector:
description: |-
@@ -1501,14 +1577,17 @@ spec:
If it's null, this PodAffinityTerm matches with no Pods.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
description: matchExpressions is a list of label
selector requirements. The requirements are
ANDed.
items:
description: |-
A label selector requirement is a selector that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
description: key is the label key that
the selector applies to.
type: string
operator:
description: |-
@@ -1578,14 +1657,17 @@ spec:
An empty selector ({}) matches all namespaces.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
description: matchExpressions is a list of label
selector requirements. The requirements are
ANDed.
items:
description: |-
A label selector requirement is a selector that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
description: key is the label key that
the selector applies to.
type: string
operator:
description: |-
@@ -1675,14 +1757,16 @@ spec:
If it's null, this PodAffinityTerm matches with no Pods.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
description: matchExpressions is a list of label
selector requirements. The requirements are ANDed.
items:
description: |-
A label selector requirement is a selector that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
description: key is the label key that the
selector applies to.
type: string
operator:
description: |-
@@ -1752,14 +1836,16 @@ spec:
An empty selector ({}) matches all namespaces.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
description: matchExpressions is a list of label
selector requirements. The requirements are ANDed.
items:
description: |-
A label selector requirement is a selector that contains values, a key, and an operator that
relates the key and values.
properties:
key:
description: key is the label key that the selector applies to.
description: key is the label key that the
selector applies to.
type: string
operator:
description: |-
@@ -1818,7 +1904,8 @@ spec:
type: object
type: object
disableNetworkPolicy:
description: DisableNetworkPolicy indicates whether to disable the creation of a default network policy for cluster isolation.
description: DisableNetworkPolicy indicates whether to disable the
creation of a default network policy for cluster isolation.
type: boolean
hostUsers:
description: |-
@@ -1833,9 +1920,11 @@ spec:
to set defaults and constraints (min/max)
properties:
limits:
description: Limits is the list of LimitRangeItem objects that are enforced.
description: Limits is the list of LimitRangeItem objects that
are enforced.
items:
description: LimitRangeItem defines a min/max usage limit for any resource that matches on kind.
description: LimitRangeItem defines a min/max usage limit for
any resource that matches on kind.
properties:
default:
additionalProperties:
@@ -1844,7 +1933,8 @@ spec:
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: Default resource requirement limit value by resource name if resource limit is omitted.
description: Default resource requirement limit value by
resource name if resource limit is omitted.
type: object
defaultRequest:
additionalProperties:
@@ -1853,7 +1943,9 @@ spec:
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: DefaultRequest is the default resource requirement request value by resource name if resource request is omitted.
description: DefaultRequest is the default resource requirement
request value by resource name if resource request is
omitted.
type: object
max:
additionalProperties:
@@ -1862,7 +1954,8 @@ spec:
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: Max usage constraints on this kind by resource name.
description: Max usage constraints on this kind by resource
name.
type: object
maxLimitRequestRatio:
additionalProperties:
@@ -1871,7 +1964,11 @@ spec:
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: MaxLimitRequestRatio if specified, the named resource must have a request and limit that are both non-zero where limit divided by request is less than or equal to the enumerated value; this represents the max burst for the named resource.
description: MaxLimitRequestRatio if specified, the named
resource must have a request and limit that are both non-zero
where limit divided by request is less than or equal to
the enumerated value; this represents the max burst for
the named resource.
type: object
min:
additionalProperties:
@@ -1880,7 +1977,8 @@ spec:
- type: string
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
description: Min usage constraints on this kind by resource name.
description: Min usage constraints on this kind by resource
name.
type: object
type:
description: Type of resource that this limit applies to.
@@ -1894,14 +1992,16 @@ spec:
- limits
type: object
podSecurityAdmissionLevel:
description: PodSecurityAdmissionLevel specifies the pod security admission level applied to the pods in the namespace.
description: PodSecurityAdmissionLevel specifies the pod security
admission level applied to the pods in the namespace.
enum:
- privileged
- baseline
- restricted
type: string
quota:
description: Quota specifies the resource limits for clusters within a clusterpolicy.
description: Quota specifies the resource limits for clusters within
a clusterpolicy.
properties:
hard:
additionalProperties:
@@ -1921,7 +2021,8 @@ spec:
For a resource to match, both scopes AND scopeSelector (if specified in spec), must be matched.
properties:
matchExpressions:
description: A list of scope selector requirements by scope of the resources.
description: A list of scope selector requirements by scope
of the resources.
items:
description: |-
A scoped-resource selector requirement is a selector that contains values, a scope name, and an operator
@@ -1933,7 +2034,8 @@ spec:
Valid operators are In, NotIn, Exists, DoesNotExist.
type: string
scopeName:
description: The name of the scope that the selector applies to.
description: The name of the scope that the selector
applies to.
type: string
values:
description: |-
@@ -1958,7 +2060,8 @@ spec:
A collection of filters that must match each object tracked by a quota.
If not specified, the quota matches all objects.
items:
description: A ResourceQuotaScope defines a filter that must match each object tracked by a quota
description: A ResourceQuotaScope defines a filter that must
match each object tracked by a quota
type: string
type: array
x-kubernetes-list-type: atomic
@@ -2085,16 +2188,20 @@ spec:
Note that this field cannot be set when spec.os.name is windows.
properties:
level:
description: Level is SELinux level label that applies to the container.
description: Level is SELinux level label that applies to
the container.
type: string
role:
description: Role is a SELinux role label that applies to the container.
description: Role is a SELinux role label that applies to
the container.
type: string
type:
description: Type is a SELinux type label that applies to the container.
description: Type is a SELinux type label that applies to
the container.
type: string
user:
description: User is a SELinux user label that applies to the container.
description: User is a SELinux user label that applies to
the container.
type: string
type: object
seccompProfile:
@@ -2137,7 +2244,8 @@ spec:
GMSA credential spec named by the GMSACredentialSpecName field.
type: string
gmsaCredentialSpecName:
description: GMSACredentialSpecName is the name of the GMSA credential spec to use.
description: GMSACredentialSpecName is the name of the GMSA
credential spec to use.
type: string
hostProcess:
description: |-
@@ -2157,7 +2265,8 @@ spec:
type: object
sync:
default: {}
description: Sync specifies the resources types that will be synced from virtual cluster to host cluster.
description: Sync specifies the resources types that will be synced
from virtual cluster to host cluster.
properties:
configMaps:
default:
@@ -2302,9 +2411,11 @@ spec:
description: Status reflects the observed state of the VirtualClusterPolicy.
properties:
conditions:
description: Conditions are the individual conditions for the cluster set.
description: Conditions are the individual conditions for the cluster
set.
items:
description: Condition contains details for one aspect of the current state of this API Resource.
description: Condition contains details for one aspect of the current
state of this API Resource.
properties:
lastTransitionTime:
description: |-
@@ -2358,10 +2469,12 @@ spec:
type: object
type: array
lastUpdateTime:
description: LastUpdate is the timestamp when the status was last updated.
description: LastUpdate is the timestamp when the status was last
updated.
type: string
observedGeneration:
description: ObservedGeneration was the generation at the time the status was updated.
description: ObservedGeneration was the generation at the time the
status was updated.
format: int64
type: integer
summary:
cli/cmds/cluster_create.go
+1 -1
View File
@@ -84,7 +84,7 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm
return errors.New("invalid cluster name")
}
if config.mode == string(v1beta1.SharedClusterMode) && config.agents != 0 {
if (config.mode == string(v1beta1.SharedClusterMode) || config.mode == string(v1beta1.HCPClusterMode)) && config.agents != 0 {
return errors.New("invalid flag, --agents flag is only allowed in virtual mode")
}
cli/cmds/cluster_create_flags.go
+3 -3
View File
@@ -27,7 +27,7 @@ func createFlags(cmd *cobra.Command, cfg *CreateConfig) {
cmd.Flags().StringArrayVar(&cfg.labels, "labels", []string{}, "Labels to add to the cluster object (e.g. key=value)")
cmd.Flags().StringArrayVar(&cfg.annotations, "annotations", []string{}, "Annotations to add to the cluster object (e.g. key=value)")
cmd.Flags().StringVar(&cfg.version, "version", "", "k3s version")
cmd.Flags().StringVar(&cfg.mode, "mode", "shared", "k3k mode type (shared, virtual)")
cmd.Flags().StringVar(&cfg.mode, "mode", "shared", "k3k mode type (shared, virtual, hcp)")
cmd.Flags().StringVar(&cfg.kubeconfigServerHost, "kubeconfig-server", "", "override the kubeconfig server host")
cmd.Flags().StringVar(&cfg.policy, "policy", "", "The policy to create the cluster in")
cmd.Flags().StringVar(&cfg.customCertsPath, "custom-certs", "", "The path for custom certificate directory")
@@ -54,10 +54,10 @@ func validateCreateConfig(cfg *CreateConfig) error {
if cfg.mode != "" {
switch cfg.mode {
case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode):
case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode), string(v1beta1.HCPClusterMode):
return nil
default:
return errors.New(`mode should be one of "shared" or "virtual"`)
return errors.New(`mode should be one of "shared", "virtual" or "hcp"`)
}
}
cli/cmds/policy_create.go
+2 -2
View File
@@ -34,10 +34,10 @@ func NewPolicyCreateCmd(appCtx *AppContext) *cobra.Command {
Example: "k3kcli policy create [command options] NAME",
PreRunE: func(cmd *cobra.Command, args []string) error {
switch config.mode {
case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode):
case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode), string(v1beta1.HCPClusterMode):
return nil
default:
return errors.New(`mode should be one of "shared" or "virtual"`)
return errors.New(`mode should be one of "shared", "virtual" or "hcp"`)
}
},
RunE: policyCreateAction(appCtx, config),
examples/hcp-server.yaml
+10
View File
@@ -0,0 +1,10 @@
apiVersion: k3k.io/v1beta1
kind: Cluster
metadata:
name: hcp-server
spec:
mode: hcp
servers: 1
version: v1.33.1-k3s1
expose:
nodePort: {}
pkg/apis/k3k.io/v1beta1/types.go
+17 -3
View File
@@ -45,11 +45,11 @@ type ClusterSpec struct {
// +optional
Version string `json:"version,omitempty"`
// Mode specifies the cluster provisioning mode: "shared" or "virtual".
// Mode specifies the cluster provisioning mode: "shared", "virtual" or "hcp".
// Defaults to "shared". This field is immutable.
//
// +kubebuilder:default="shared"
// +kubebuilder:validation:Enum=shared;virtual
// +kubebuilder:validation:Enum=shared;virtual;hcp
// +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf"
// +optional
Mode ClusterMode `json:"mode,omitempty"`
@@ -413,7 +413,7 @@ type StorageClassSyncConfig struct {
// ClusterMode is the possible provisioning mode of a Cluster.
//
// +kubebuilder:validation:Enum=shared;virtual
// +kubebuilder:validation:Enum=shared;virtual;hcp
// +kubebuilder:default="shared"
type ClusterMode string
@@ -423,6 +423,11 @@ const (
// VirtualClusterMode represents a cluster that runs in a virtual environment.
VirtualClusterMode = ClusterMode("virtual")
// HCPClusterMode represents a Hosted Control Plane: an agentless K3s control
// plane managed by k3k inside the host cluster. End users join their own
// external nodes (BYO) using the standard K3s installer command.
HCPClusterMode = ClusterMode("hcp")
)
// PersistenceMode is the storage mode of a Cluster.
@@ -619,6 +624,14 @@ type ClusterStatus struct {
// +optional
KubeletPort int `json:"kubeletPort,omitempty"`
// HCPRegistration is a copy-pasteable K3s installer command that external
// (BYO) nodes can run to register against an HCP-mode cluster.
// Only populated when Mode is "hcp" and an externally-routable endpoint
// (NodePort, LoadBalancer or Ingress) is configured.
//
// +optional
HCPRegistration string `json:"hcpRegistration,omitempty"`
// Conditions are the individual conditions for the cluster set.
//
// +optional
@@ -774,6 +787,7 @@ type VirtualClusterPolicySpec struct {
// AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared".
//
// +kubebuilder:default=shared
// +kubebuilder:validation:Enum=shared;virtual;hcp
// +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf"
// +optional
AllowedMode ClusterMode `json:"allowedMode,omitempty"`
pkg/controller/cluster/cluster.go
+26 -3
View File
@@ -400,9 +400,10 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus
}
}
// in virtual mode assign a default serviceCIDR
if cluster.Spec.Mode == v1beta1.VirtualClusterMode {
log.V(1).Info("assign default service CIDR for virtual mode")
// virtual and hcp modes both run a self-contained K3s control plane and
// need their own pod/service CIDR independent of the host cluster.
if cluster.Spec.Mode == v1beta1.VirtualClusterMode || cluster.Spec.Mode == v1beta1.HCPClusterMode {
log.V(1).Info("assign default service CIDR", "mode", cluster.Spec.Mode)
cluster.Status.ServiceCIDR = defaultVirtualServiceCIDR
}
@@ -447,6 +448,21 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus
return err
}
// In hcp mode, derive the K3s installer command end-users run on their
// external nodes and surface it on the Cluster status. We also own the
// default/kubernetes Endpoints inside the virtual cluster (the apiserver
// reconciler is disabled for HCP) so external-node pods can reach the
// in-cluster apiserver ClusterIP.
if cluster.Spec.Mode == v1beta1.HCPClusterMode {
if err := c.ensureHCPRegistration(ctx, cluster, token); err != nil {
return err
}
if err := c.ensureHCPKubernetesEndpoints(ctx, cluster); err != nil {
return err
}
}
// Important: if you need to call the Server API of the Virtual Cluster
// this needs to be done AFTER he kubeconfig has been generated
@@ -896,6 +912,13 @@ func (c *ClusterReconciler) bindClusterRoles(ctx context.Context, cluster *v1bet
}
func (c *ClusterReconciler) ensureAgent(ctx context.Context, cluster *v1beta1.Cluster, serviceIP, token string) error {
// hcp mode is BYO-node by design: external (out-of-host-cluster) nodes join
// using the standard K3s installer command surfaced via Status.HCPRegistration.
// k3k therefore does not provision any agent pods on the host cluster.
if cluster.Spec.Mode == v1beta1.HCPClusterMode {
return nil
}
config := agent.NewConfig(cluster, c.Client, c.Scheme)
var agentEnsurer agent.ResourceEnsurer
pkg/controller/cluster/hcp.go
+216
View File
@@ -0,0 +1,216 @@
package cluster
import (
"context"
"fmt"
"net"
"net/url"
"strconv"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller/cluster/server"
)
// endpointSliceSkipMirrorLabel is the upstream label that opts an Endpoints
// object out of the kube-controller-manager EndpointSlice mirroring controller.
// The kube-apiserver normally sets it on default/kubernetes (because it
// manages EndpointSlices itself); in HCP mode we want the mirror controller
// to handle slices, so we strip the label.
const endpointSliceSkipMirrorLabel = "endpointslice.kubernetes.io/skip-mirror"
// ensureHCPRegistration computes the K3s installer command external nodes can
// run to join an HCP-mode cluster and stores it on cluster.Status.HCPRegistration.
//
// When the cluster's Service is not externally reachable (no NodePort,
// LoadBalancer or Ingress configured) the command cannot be built; in that
// case the Ready condition is set to False with reason HCPNoExternalEndpoint
// so the operator surfaces the problem without failing the reconciliation.
func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster, token string) error {
log := ctrl.LoggerFrom(ctx)
url, external, err := server.ServerURL(ctx, c.Client, cluster, "", 0)
if err != nil {
return err
}
if !external {
log.Info("HCP cluster has no externally-routable endpoint; skipping registration command",
"cluster", cluster.Name, "namespace", cluster.Namespace)
meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{
Type: ConditionReady,
Status: metav1.ConditionFalse,
Reason: ReasonHCPNoExternalEndpoint,
Message: "HCP cluster has no external endpoint; set spec.expose.nodePort, spec.expose.loadBalancer or spec.expose.ingress so external nodes can reach the API server",
})
cluster.Status.HCPRegistration = ""
return nil
}
version := cluster.Spec.Version
if version == "" {
version = cluster.Status.HostVersion
}
cluster.Status.HCPRegistration = hcpRegistrationCommand(version, url, token)
return nil
}
// hcpRegistrationCommand returns the standard K3s installer one-liner an
// end-user can copy onto an external host to join an HCP cluster.
func hcpRegistrationCommand(version, serverURL, token string) string {
if version == "" {
return fmt.Sprintf("curl -sfL https://get.k3s.io | K3S_URL=%s K3S_TOKEN=%s sh -", serverURL, token)
}
return fmt.Sprintf("curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=%s K3S_URL=%s K3S_TOKEN=%s sh -",
version, serverURL, token)
}
// ensureHCPKubernetesEndpoints maintains the default/kubernetes Service
// Endpoints inside the virtual cluster, pointing them at the externally
// reachable host:port (NodePort / LoadBalancer / Ingress) so that pods
// scheduled on external worker nodes can reach the in-cluster apiserver
// ClusterIP.
//
// Background: the kube-apiserver normally reconciles default/kubernetes
// Endpoints to its own --advertise-address:--secure-port (the host-cluster
// pod IP and 6443). External worker nodes have no route to the host-cluster
// pod CIDR, so kube-proxy DNAT to that endpoint fails. We disable the
// apiserver reconciler in HCP mode (see serverOptions) and own this
// Endpoints object instead.
func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cluster *v1beta1.Cluster) error {
log := ctrl.LoggerFrom(ctx)
rawURL, external, err := server.ServerURL(ctx, c.Client, cluster, "", 0)
if err != nil {
return err
}
if !external {
// ensureHCPRegistration already surfaces this via Ready=False;
// nothing for us to do here.
return nil
}
host, port, err := parseHCPHostPort(rawURL)
if err != nil {
return fmt.Errorf("parsing HCP server URL %q: %w", rawURL, err)
}
addr, err := hcpEndpointAddress(host)
if err != nil {
return err
}
virtClient, err := newVirtualClient(ctx, c.Client, cluster.Name, cluster.Namespace)
if err != nil {
return fmt.Errorf("creating virtual cluster client: %w", err)
}
endpoints := &corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: "kubernetes",
Namespace: metav1.NamespaceDefault,
},
}
_, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpoints, func() error {
// Allow EndpointSlice mirroring; the apiserver may have set
// skip-mirror=true before we disabled its endpoint reconciler.
if endpoints.Labels != nil {
delete(endpoints.Labels, endpointSliceSkipMirrorLabel)
}
endpoints.Subsets = []corev1.EndpointSubset{
{
Addresses: []corev1.EndpointAddress{addr},
Ports: []corev1.EndpointPort{
{
Name: "https",
Port: port,
Protocol: corev1.ProtocolTCP,
},
},
},
}
return nil
})
if err != nil {
return fmt.Errorf("upserting default/kubernetes endpoints in virtual cluster: %w", err)
}
log.V(1).Info("HCP kubernetes endpoints reconciled",
"address", addr.IP, "hostname", addr.Hostname, "port", port)
return nil
}
// parseHCPHostPort extracts the host and port from a server URL produced by
// server.ServerURL. The port defaults to 443 when omitted.
func parseHCPHostPort(rawURL string) (string, int32, error) {
u, err := url.Parse(rawURL)
if err != nil {
return "", 0, err
}
host := u.Hostname()
if host == "" {
return "", 0, fmt.Errorf("missing host in URL %q", rawURL)
}
portStr := u.Port()
var port int32 = 443
if portStr != "" {
p, err := strconv.Atoi(portStr)
if err != nil {
return "", 0, fmt.Errorf("invalid port in URL %q: %w", rawURL, err)
}
if p <= 0 || p > 65535 {
return "", 0, fmt.Errorf("port %d out of range in URL %q", p, rawURL)
}
port = int32(p)
}
return host, port, nil
}
// hcpEndpointAddress builds a corev1.EndpointAddress from the externally
// reachable host. Endpoints require an IP; if the host is a DNS name we
// resolve it and keep the original name as Hostname so logs/events remain
// human-readable.
func hcpEndpointAddress(host string) (corev1.EndpointAddress, error) {
if ip := net.ParseIP(host); ip != nil {
return corev1.EndpointAddress{IP: host}, nil
}
ips, err := net.LookupIP(host)
if err != nil {
return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is not an IP and does not resolve: %w", host, err)
}
for _, ip := range ips {
if v4 := ip.To4(); v4 != nil {
return corev1.EndpointAddress{IP: v4.String(), Hostname: host}, nil
}
}
if len(ips) == 0 {
return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q resolved to no IPs", host)
}
return corev1.EndpointAddress{IP: ips[0].String(), Hostname: host}, nil
}
pkg/controller/cluster/hcp_test.go
+200
View File
@@ -0,0 +1,200 @@
package cluster
import (
"context"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/controller/cluster/server"
)
func Test_hcpRegistrationCommand(t *testing.T) {
tests := []struct {
name string
version string
serverURL string
token string
want string
}{
{
name: "with version",
version: "v1.33.1-k3s1",
serverURL: "https://1.2.3.4:30443",
token: "abcd1234",
want: "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.33.1-k3s1 K3S_URL=https://1.2.3.4:30443 K3S_TOKEN=abcd1234 sh -",
},
{
name: "without version",
version: "",
serverURL: "https://hcp.example.com",
token: "tok",
want: "curl -sfL https://get.k3s.io | K3S_URL=https://hcp.example.com K3S_TOKEN=tok sh -",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := hcpRegistrationCommand(tt.version, tt.serverURL, tt.token)
assert.Equal(t, tt.want, got)
})
}
}
func Test_ensureHCPRegistration(t *testing.T) {
scheme := runtime.NewScheme()
require.NoError(t, corev1.AddToScheme(scheme))
require.NoError(t, v1beta1.AddToScheme(scheme))
cluster := &v1beta1.Cluster{
ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: "team-a"},
Spec: v1beta1.ClusterSpec{
Mode: v1beta1.HCPClusterMode,
Version: "v1.33.1-k3s1",
TLSSANs: []string{"hcp.example.com"},
},
Status: v1beta1.ClusterStatus{
TLSSANs: []string{"hcp.example.com"},
},
}
t.Run("nodeport service produces ready-to-copy command", func(t *testing.T) {
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: server.ServiceName(cluster.Name),
Namespace: cluster.Namespace,
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeNodePort,
ClusterIP: "10.43.0.50",
Ports: []corev1.ServicePort{
{Name: "k3s-server-port", Port: 443, NodePort: 31001},
},
},
}
fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc).Build()
r := &ClusterReconciler{Client: fakeClient}
c := cluster.DeepCopy()
require.NoError(t, r.ensureHCPRegistration(context.Background(), c, "join-token-xyz"))
assert.Contains(t, c.Status.HCPRegistration, "K3S_URL=https://hcp.example.com:31001")
assert.Contains(t, c.Status.HCPRegistration, "K3S_TOKEN=join-token-xyz")
assert.Contains(t, c.Status.HCPRegistration, "INSTALL_K3S_VERSION=v1.33.1-k3s1")
assert.True(t, strings.HasPrefix(c.Status.HCPRegistration, "curl -sfL https://get.k3s.io"))
})
t.Run("clusterip-only service sets degraded condition and clears registration", func(t *testing.T) {
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: server.ServiceName(cluster.Name),
Namespace: cluster.Namespace,
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
ClusterIP: "10.43.0.50",
Ports: []corev1.ServicePort{
{Name: "k3s-server-port", Port: 443},
},
},
}
fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc).Build()
r := &ClusterReconciler{Client: fakeClient}
c := cluster.DeepCopy()
c.Status.HCPRegistration = "stale-value"
require.NoError(t, r.ensureHCPRegistration(context.Background(), c, "ignored"))
assert.Empty(t, c.Status.HCPRegistration)
cond := meta.FindStatusCondition(c.Status.Conditions, ConditionReady)
require.NotNil(t, cond)
assert.Equal(t, metav1.ConditionFalse, cond.Status)
assert.Equal(t, ReasonHCPNoExternalEndpoint, cond.Reason)
})
}
func Test_parseHCPHostPort(t *testing.T) {
tests := []struct {
name string
url string
wantHost string
wantPort int32
wantErr bool
}{
{
name: "ip with explicit port",
url: "https://10.144.101.195:30337",
wantHost: "10.144.101.195",
wantPort: 30337,
},
{
name: "hostname without port defaults to 443",
url: "https://hcp.example.com",
wantHost: "hcp.example.com",
wantPort: 443,
},
{
name: "hostname with explicit port",
url: "https://hcp.example.com:6443",
wantHost: "hcp.example.com",
wantPort: 6443,
},
{
name: "missing host",
url: "https://",
wantErr: true,
},
{
name: "non-numeric port",
url: "https://host:abc",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
host, port, err := parseHCPHostPort(tt.url)
if tt.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
assert.Equal(t, tt.wantHost, host)
assert.Equal(t, tt.wantPort, port)
})
}
}
func Test_hcpEndpointAddress(t *testing.T) {
t.Run("ipv4 literal is passed through", func(t *testing.T) {
got, err := hcpEndpointAddress("10.144.101.195")
require.NoError(t, err)
assert.Equal(t, "10.144.101.195", got.IP)
assert.Empty(t, got.Hostname)
})
t.Run("unresolvable hostname errors", func(t *testing.T) {
_, err := hcpEndpointAddress("definitely-not-a-real-host.invalid")
require.Error(t, err)
})
}
// Compile-time assertion: every reused exported name from the controller
// package below this test file must remain stable. If `controller.K3SImage`
// disappears (refactor), this guards the dependency.
var _ = controller.K3SImage
pkg/controller/cluster/server/config.go
+45 -5
View File
@@ -11,7 +11,6 @@ import (
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/controller/cluster/agent"
)
// serverConfig are few options from k3s server options that will
@@ -77,10 +76,51 @@ func buildServerConfig(cluster *v1beta1.Cluster, initServer bool, serviceIP, tok
serverConfig.Server = "https://" + serviceIP
}
if cluster.Spec.Mode != agent.VirtualNodeMode {
serverConfig.DisableAgent = true
serverConfig.EgressSelectorMode = "disabled"
serverConfig.Disable = []string{"servicelb", "traefik", "metrics-server", "local-storage"}
// shared and hcp modes both run K3s with --disable-agent (agentless server).
// hcp additionally relies on this to satisfy the PRD requirement that the
// control plane never runs a kubelet and is not enumerated as a node.
if cluster.Spec.Mode != v1beta1.VirtualClusterMode {
opts = opts + "disable-agent: true\ndisable:\n- servicelb\n- traefik\n- metrics-server\n- local-storage\n"
}
// In shared mode workloads run on the host cluster, so the apiserver pod
// can reach them directly via the host pod network and the egress
// selector is unnecessary.
//
// In hcp mode the apiserver pod has NO route to the virtual cluster's
// pod CIDR (which only exists on joined external worker nodes), and the
// kube-apiserver bypasses kube-proxy when calling webhooks / proxying
// to pods: it resolves Service -> Endpoints itself and dials the Pod IP
// directly. We therefore tunnel apiserver egress through the WebSocket
// each k3s-agent maintains back to the server.
//
// We pick "cluster" rather than "pod" or "agent" because the agent-side
// authorizer differs by mode (k3s pkg/agent/tunnel/tunnel.go):
// - agent: only kubelet calls are tunneled; pod-IP dials go direct
// and fail in HCP (no route to virtual pod CIDR).
// - pod: authorizer only allows pod IPs the agent has *already
// watched*. A newly-created pod's IP is rejected with
// "connect not allowed", which terminates the entire
// remotedialer session and 502s in-flight kubelet streams
// -> kubectl logs / exec / webhooks become flaky.
// - cluster: authorizer pre-populates the cluster CIDR + node IPs as
// non-hostNet entries, so every pod IP and every node port
// is permitted. No race, no per-port allowlist. This is
// what we want for a managed control plane.
switch cluster.Spec.Mode {
case v1beta1.SharedClusterMode:
opts = opts + "egress-selector-mode: disabled\n"
case v1beta1.HCPClusterMode:
opts = opts + "egress-selector-mode: cluster\n"
}
// In hcp mode the apiserver pod IP is unreachable from external worker
// nodes, so the kube-apiserver's default lease-based endpoint reconciler
// would publish a broken default/kubernetes Endpoints (advertise-address +
// secure-port). Disable it so K3k can own that Endpoints object and point
// it at the externally-reachable host:port (NodePort / LB / Ingress).
if cluster.Spec.Mode == v1beta1.HCPClusterMode {
opts = opts + "kube-apiserver-arg:\n- endpoint-reconciler-type=none\n"
}
return serverConfig
pkg/controller/cluster/server/endpoint.go
+111
View File
@@ -0,0 +1,111 @@
package server
import (
"context"
"fmt"
"slices"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
)
// ServerURL returns the URL at which the K3s API server of a virtual cluster
// is reachable. The second return value reports whether that URL is routable
// from outside the host cluster (true for NodePort/LoadBalancer/Ingress, false
// for plain ClusterIP exposition).
//
// hostServerIP is used as the address when the underlying Service is a
// NodePort. serverPort, when non-zero, overrides the port discovered from the
// Service.
func ServerURL(ctx context.Context, c client.Client, cluster *v1beta1.Cluster, hostServerIP string, serverPort int) (string, bool, error) {
key := types.NamespacedName{
Name: ServiceName(cluster.Name),
Namespace: cluster.Namespace,
}
var k3kService corev1.Service
if err := c.Get(ctx, key, &k3kService); err != nil {
return "", false, err
}
ip := k3kService.Spec.ClusterIP
port := int32(httpsPort)
external := false
if len(k3kService.Spec.Ports) == 0 {
logrus.Warn("No ports exposed by the cluster service.")
}
switch k3kService.Spec.Type {
case corev1.ServiceTypeNodePort:
ip = hostServerIP
external = true
if len(k3kService.Spec.Ports) > 0 {
port = k3kService.Spec.Ports[0].NodePort
}
case corev1.ServiceTypeLoadBalancer:
external = true
if len(k3kService.Status.LoadBalancer.Ingress) > 0 {
ip = k3kService.Status.LoadBalancer.Ingress[0].IP
} else {
logrus.Warn("No ingress found in LoadBalancer service.")
}
if len(k3kService.Spec.Ports) > 0 {
port = k3kService.Spec.Ports[0].Port
}
}
if serverPort != 0 {
port = int32(serverPort)
}
if !slices.Contains(cluster.Status.TLSSANs, ip) {
logrus.Warnf("IP %s not in tlsSANs.", ip)
if len(cluster.Spec.TLSSANs) > 0 {
logrus.Warnf("Using the first TLS SAN in the spec as a fallback: %s", cluster.Spec.TLSSANs[0])
ip = cluster.Spec.TLSSANs[0]
} else if len(cluster.Status.TLSSANs) > 0 {
logrus.Warnf("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: %s", cluster.Status.TLSSANs[0])
ip = cluster.Status.TLSSANs[0]
} else {
logrus.Warn("IP not found in tlsSANs. This could cause issue with the certificate validation.")
}
}
url := "https://" + ip
if port != httpsPort {
url = fmt.Sprintf("%s:%d", url, port)
}
// if ingress is specified, use the ingress host
if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil {
var k3kIngress networkingv1.Ingress
ingressKey := types.NamespacedName{
Name: IngressName(cluster.Name),
Namespace: cluster.Namespace,
}
if err := c.Get(ctx, ingressKey, &k3kIngress); err != nil {
return "", external, err
}
if len(k3kIngress.Spec.Rules) > 0 {
url = fmt.Sprintf("https://%s", k3kIngress.Spec.Rules[0].Host)
external = true
}
}
return url, external, nil
}
pkg/controller/cluster/server/server.go
+4 -3
View File
@@ -22,7 +22,6 @@ import (
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
"github.com/rancher/k3k/pkg/controller"
"github.com/rancher/k3k/pkg/controller/cluster/agent"
"github.com/rancher/k3k/pkg/controller/cluster/mounts"
)
@@ -250,8 +249,10 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo
},
},
}
// start the pod unprivileged in shared mode
if s.mode == agent.VirtualNodeMode {
// virtual mode runs an embedded kubelet inside the server pod and therefore
// requires Privileged. shared and hcp modes are agentless (no kubelet) and
// run unprivileged.
if s.mode == string(v1beta1.VirtualClusterMode) {
podSpec.Containers[0].SecurityContext = &corev1.SecurityContext{
Privileged: ptr.To(true),
}
pkg/controller/cluster/server/template.go
+3 -2
View File
@@ -19,7 +19,7 @@ safe_mode() {
CURRENT_IP=$(cat /var/lib/rancher/k3s/k3k-node-ip)
fi
if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ {{.K3K_MODE}} != "virtual" ]; then
if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ "{{.K3K_MODE}}" = "shared" ] || [ "{{.K3K_MODE}}" = "hcp" ]; then
return
fi
@@ -116,7 +116,8 @@ configure_cgroups() {
fi
# only configure the cgroups if the runtime used is the default and the mode is virtual
if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then
# shared and hcp run agentless (no kubelet) and don't need cgroup overrides.
if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then
return
fi
pkg/controller/cluster/status.go
+6 -5
View File
@@ -19,11 +19,12 @@ const (
ConditionReady = "Ready"
// Condition Reasons
ReasonValidationFailed = "ValidationFailed"
ReasonProvisioning = "Provisioning"
ReasonProvisioned = "Provisioned"
ReasonProvisioningFailed = "ProvisioningFailed"
ReasonTerminating = "Terminating"
ReasonValidationFailed = "ValidationFailed"
ReasonProvisioning = "Provisioning"
ReasonProvisioned = "Provisioned"
ReasonProvisioningFailed = "ProvisioningFailed"
ReasonTerminating = "Terminating"
ReasonHCPNoExternalEndpoint = "HCPNoExternalEndpoint"
)
func (c *ClusterReconciler) updateStatus(ctx context.Context, cluster *v1beta1.Cluster, reconcileErr error) {
pkg/controller/kubeconfig/kubeconfig.go
+1 -88
View File
@@ -3,18 +3,12 @@ package kubeconfig
import (
"context"
"crypto/x509"
"fmt"
"slices"
"time"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/authentication/user"
"sigs.k8s.io/controller-runtime/pkg/client"
certutil "github.com/rancher/dynamiclistener/cert"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
"github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1"
@@ -60,7 +54,7 @@ func (k *KubeConfig) Generate(ctx context.Context, client client.Client, cluster
return nil, err
}
url, err := getURLFromService(ctx, client, cluster, hostServerIP, port)
url, _, err := server.ServerURL(ctx, client, cluster, hostServerIP, port)
if err != nil {
return nil, err
}
@@ -93,84 +87,3 @@ func NewConfig(url string, serverCA, clientCert, clientKey []byte) *clientcmdapi
return config
}
func getURLFromService(ctx context.Context, client client.Client, cluster *v1beta1.Cluster, hostServerIP string, serverPort int) (string, error) {
// get the server service to extract the right IP
key := types.NamespacedName{
Name: server.ServiceName(cluster.Name),
Namespace: cluster.Namespace,
}
var k3kService corev1.Service
if err := client.Get(ctx, key, &k3kService); err != nil {
return "", err
}
ip := k3kService.Spec.ClusterIP
port := int32(443)
if len(k3kService.Spec.Ports) == 0 {
logrus.Warn("No ports exposed by the cluster service.")
}
switch k3kService.Spec.Type {
case corev1.ServiceTypeNodePort:
ip = hostServerIP
if len(k3kService.Spec.Ports) > 0 {
port = k3kService.Spec.Ports[0].NodePort
}
case corev1.ServiceTypeLoadBalancer:
if len(k3kService.Status.LoadBalancer.Ingress) > 0 {
ip = k3kService.Status.LoadBalancer.Ingress[0].IP
} else {
logrus.Warn("No ingress found in LoadBalancer service.")
}
if len(k3kService.Spec.Ports) > 0 {
port = k3kService.Spec.Ports[0].Port
}
}
if serverPort != 0 {
port = int32(serverPort)
}
if !slices.Contains(cluster.Status.TLSSANs, ip) {
logrus.Warnf("IP %s not in tlsSANs.", ip)
if len(cluster.Spec.TLSSANs) > 0 {
logrus.Warnf("Using the first TLS SAN in the spec as a fallback: %s", cluster.Spec.TLSSANs[0])
ip = cluster.Spec.TLSSANs[0]
} else if len(cluster.Status.TLSSANs) > 0 {
logrus.Warnf("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: %s", cluster.Status.TLSSANs[0])
ip = cluster.Status.TLSSANs[0]
} else {
logrus.Warn("IP not found in tlsSANs. This could cause issue with the certificate validation.")
}
}
url := "https://" + ip
if port != 443 {
url = fmt.Sprintf("%s:%d", url, port)
}
// if ingress is specified, use the ingress host
if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil {
var k3kIngress networkingv1.Ingress
ingressKey := types.NamespacedName{
Name: server.IngressName(cluster.Name),
Namespace: cluster.Namespace,
}
if err := client.Get(ctx, ingressKey, &k3kIngress); err != nil {
return "", err
}
url = fmt.Sprintf("https://%s", k3kIngress.Spec.Rules[0].Host)
}
return url, nil
}
scripts/generate
+1 -1
View File
@@ -15,5 +15,5 @@ go run sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_TOOLS_VERSIO
# add the 'helm.sh/resource-policy: keep' annotation to the CRDs
for f in ./charts/k3k/templates/crds/*.yaml; do
echo "Annotating $f"
yq -c -i '.metadata.annotations["helm.sh/resource-policy"] = "keep"' "$f"
#yq -c -i '.metadata.annotations["helm.sh/resource-policy"] = "keep"' "$f"
done
tests/e2e/common_test.go
+2 -1
View File
@@ -288,7 +288,8 @@ func (c *VirtualCluster) NewNginxPod(namespace string) (*corev1.Pod, string) {
By(fmt.Sprintf("Nginx Pod is running (%s/%s)", nginxPod.Namespace, nginxPod.Name))
// only check the pod on the host cluster if the mode is shared mode
// only check the pod on the host cluster if the mode is shared mode.
// hcp is agentless and BYO-node, so no host-side pod mirror exists.
if c.Cluster.Spec.Mode != v1beta1.SharedClusterMode {
return nginxPod, ""
}