From 67d5f4dbfc4f5dddb538493b4c1b00617b47dae2 Mon Sep 17 00:00:00 2001 From: jpgouin Date: Mon, 18 May 2026 17:08:05 +0200 Subject: [PATCH] Add HCP (Hosted Control Plane) support Introduce hosted control plane mode for k3k virtual clusters, including API types, controller logic, server endpoint handling, CLI flags, CRD updates, kubeconfig generation, and examples. Co-Authored-By: RuFlo --- .gitignore | 5 + Makefile | 2 +- .../k3k/templates/crds/k3k.io_clusters.yaml | 649 ++++++++++++------ .../crds/k3k.io_virtualclusterpolicies.yaml | 309 ++++++--- cli/cmds/cluster_create.go | 2 +- cli/cmds/cluster_create_flags.go | 6 +- cli/cmds/policy_create.go | 4 +- examples/hcp-server.yaml | 10 + pkg/apis/k3k.io/v1beta1/types.go | 20 +- pkg/controller/cluster/cluster.go | 29 +- pkg/controller/cluster/hcp.go | 216 ++++++ pkg/controller/cluster/hcp_test.go | 200 ++++++ pkg/controller/cluster/server/config.go | 50 +- pkg/controller/cluster/server/endpoint.go | 111 +++ pkg/controller/cluster/server/server.go | 7 +- pkg/controller/cluster/server/template.go | 5 +- pkg/controller/cluster/status.go | 11 +- pkg/controller/kubeconfig/kubeconfig.go | 89 +-- scripts/generate | 2 +- tests/e2e/common_test.go | 3 +- 20 files changed, 1311 insertions(+), 419 deletions(-) create mode 100644 examples/hcp-server.yaml create mode 100644 pkg/controller/cluster/hcp.go create mode 100644 pkg/controller/cluster/hcp_test.go create mode 100644 pkg/controller/cluster/server/endpoint.go diff --git a/.gitignore b/.gitignore index b01c05eb..cce85605 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,8 @@ __debug* cover.out covcounters.** covmeta.** +.claude/ +.claude-flow/ +.swarm +.mcp.json +CLAUDE.md \ No newline at end of file diff --git a/Makefile b/Makefile index 64a24330..0b6dd6f9 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ package: package-k3k package-k3k-kubelet ## Package the k3k and k3k-kubelet Dock .PHONY: package-% package-%: - docker build -f package/Dockerfile.$* \ + docker buildx build --platform linux/amd64,linux/arm64 -f package/Dockerfile.$* \ -t $(REPO)/$*:$(VERSION) \ -t $(REPO)/$*:latest \ -t $(REPO)/$*:dev . diff --git a/charts/k3k/templates/crds/k3k.io_clusters.yaml b/charts/k3k/templates/crds/k3k.io_clusters.yaml index 35de1c3e..9eee4c0e 100644 --- a/charts/k3k/templates/crds/k3k.io_clusters.yaml +++ b/charts/k3k/templates/crds/k3k.io_clusters.yaml @@ -4,7 +4,6 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.20.0 - helm.sh/resource-policy: keep name: clusters.k3k.io spec: group: k3k.io @@ -55,9 +54,11 @@ spec: description: Spec defines the desired state of the Cluster. properties: addons: - description: Addons specifies secrets containing raw YAML to deploy on cluster startup. + description: Addons specifies secrets containing raw YAML to deploy + on cluster startup. items: - description: Addon specifies a Secret containing YAML to be deployed on cluster startup. + description: Addon specifies a Secret containing YAML to be deployed + on cluster startup. properties: secretNamespace: description: SecretNamespace is the namespace of the Secret. @@ -73,7 +74,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for the + pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -92,17 +94,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with the + corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -127,14 +132,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -161,7 +168,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -179,7 +187,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -187,14 +196,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -219,14 +230,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -260,7 +273,8 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -274,10 +288,12 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -285,14 +301,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -362,14 +381,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -459,14 +481,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -536,14 +560,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -601,7 +627,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -615,10 +643,12 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -626,14 +656,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -703,14 +736,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -800,14 +836,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -877,14 +915,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -950,9 +990,11 @@ spec: type: string type: array agentEnvs: - description: AgentEnvs specifies list of environment variables to set in the agent pod. + description: AgentEnvs specifies list of environment variables to + set in the agent pod. items: - description: EnvVar represents an environment variable present in a Container. + description: EnvVar represents an environment variable present in + a Container. properties: name: description: |- @@ -972,7 +1014,8 @@ spec: Defaults to "". type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. + description: Source for the environment variable's value. Cannot + be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -990,7 +1033,8 @@ spec: More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap or its key must be defined + description: Specify whether the ConfigMap or its key + must be defined type: boolean required: - key @@ -1002,10 +1046,12 @@ spec: spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to select in the specified + API version. type: string required: - fieldPath @@ -1039,7 +1085,8 @@ spec: Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: - description: The name of the volume mount containing the env file. + description: The name of the volume mount containing + the env file. type: string required: - key @@ -1053,13 +1100,15 @@ spec: (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required for volumes, + optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output format of the exposed + resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: @@ -1073,7 +1122,8 @@ spec: description: Selects a key of a secret in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. + description: The key of the secret to select from. Must + be a valid secret key. type: string name: default: "" @@ -1085,7 +1135,8 @@ spec: More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the Secret or its key must + be defined type: boolean required: - key @@ -1126,14 +1177,16 @@ spec: - message: clusterDNS is immutable rule: self == oldSelf customCAs: - description: CustomCAs specifies the cert/key pairs for custom CA certificates. + description: CustomCAs specifies the cert/key pairs for custom CA + certificates. properties: enabled: default: true description: Enabled toggles this feature on or off. type: boolean sources: - description: Sources defines the sources for all required custom CA certificates. + description: Sources defines the sources for all required custom + CA certificates. properties: clientCA: description: ClientCA specifies the client-ca cert/key pair. @@ -1148,7 +1201,8 @@ spec: - secretName type: object etcdPeerCA: - description: ETCDPeerCA specifies the etcd-peer-ca cert/key pair. + description: ETCDPeerCA specifies the etcd-peer-ca cert/key + pair. properties: secretName: description: |- @@ -1160,7 +1214,8 @@ spec: - secretName type: object etcdServerCA: - description: ETCDServerCA specifies the etcd-server-ca cert/key pair. + description: ETCDServerCA specifies the etcd-server-ca cert/key + pair. properties: secretName: description: |- @@ -1172,7 +1227,8 @@ spec: - secretName type: object requestHeaderCA: - description: RequestHeaderCA specifies the request-header-ca cert/key pair. + description: RequestHeaderCA specifies the request-header-ca + cert/key pair. properties: secretName: description: |- @@ -1196,7 +1252,8 @@ spec: - secretName type: object serviceAccountToken: - description: ServiceAccountToken specifies the service-account-token key. + description: ServiceAccountToken specifies the service-account-token + key. properties: secretName: description: |- @@ -1225,19 +1282,23 @@ spec: By default, it's only exposed as a ClusterIP. properties: ingress: - description: Ingress specifies options for exposing the API server through an Ingress. + description: Ingress specifies options for exposing the API server + through an Ingress. properties: annotations: additionalProperties: type: string - description: Annotations specifies annotations to add to the Ingress. + description: Annotations specifies annotations to add to the + Ingress. type: object ingressClassName: - description: IngressClassName specifies the IngressClass to use for the Ingress. + description: IngressClassName specifies the IngressClass to + use for the Ingress. type: string type: object loadBalancer: - description: LoadBalancer specifies options for exposing the API server through a LoadBalancer service. + description: LoadBalancer specifies options for exposing the API + server through a LoadBalancer service. properties: etcdPort: description: |- @@ -1255,7 +1316,8 @@ spec: type: integer type: object nodePort: - description: NodePort specifies options for exposing the API server through NodePort. + description: NodePort specifies options for exposing the API server + through NodePort. properties: etcdPort: description: |- @@ -1274,8 +1336,10 @@ spec: type: object type: object x-kubernetes-validations: - - message: ingress, loadbalancer and nodePort are mutually exclusive; only one can be set - rule: '[has(self.ingress), has(self.loadBalancer), has(self.nodePort)].filter(x, x).size() <= 1' + - message: ingress, loadbalancer and nodePort are mutually exclusive; + only one can be set + rule: '[has(self.ingress), has(self.loadBalancer), has(self.nodePort)].filter(x, + x).size() <= 1' hostUsers: description: |- HostUsers sets the user namespace for server and agent pods. @@ -1293,12 +1357,14 @@ spec: - enum: - shared - virtual + - hcp - enum: - shared - virtual + - hcp default: shared description: |- - Mode specifies the cluster provisioning mode: "shared" or "virtual". + Mode specifies the cluster provisioning mode: "shared", "virtual" or "hcp". Defaults to "shared". This field is immutable. type: string x-kubernetes-validations: @@ -1414,7 +1480,8 @@ spec: secret contents will be mounted. type: string optional: - description: optional field specify whether the Secret or its keys must be defined + description: optional field specify whether the Secret or its + keys must be defined type: boolean role: description: |- @@ -1556,16 +1623,20 @@ spec: Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label that applies to + the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label that applies to + the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label that applies to + the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label that applies to + the container. type: string type: object seccompProfile: @@ -1608,7 +1679,8 @@ spec: GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is the name of the GMSA + credential spec to use. type: string hostProcess: description: |- @@ -1632,7 +1704,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for the + pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1651,17 +1724,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with the + corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1686,14 +1762,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1720,7 +1798,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -1738,7 +1817,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -1746,14 +1826,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1778,14 +1860,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1819,7 +1903,8 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1833,10 +1918,12 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -1844,14 +1931,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -1921,14 +2011,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -2018,14 +2111,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -2095,14 +2190,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -2160,7 +2257,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -2174,10 +2273,12 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -2185,14 +2286,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -2262,14 +2366,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -2359,14 +2466,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -2436,14 +2545,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -2509,9 +2620,11 @@ spec: type: string type: array serverEnvs: - description: ServerEnvs specifies list of environment variables to set in the server pod. + description: ServerEnvs specifies list of environment variables to + set in the server pod. items: - description: EnvVar represents an environment variable present in a Container. + description: EnvVar represents an environment variable present in + a Container. properties: name: description: |- @@ -2531,7 +2644,8 @@ spec: Defaults to "". type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. + description: Source for the environment variable's value. Cannot + be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -2549,7 +2663,8 @@ spec: More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the ConfigMap or its key must be defined + description: Specify whether the ConfigMap or its key + must be defined type: boolean required: - key @@ -2561,10 +2676,12 @@ spec: spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to select in the specified + API version. type: string required: - fieldPath @@ -2598,7 +2715,8 @@ spec: Must be relative and may not contain the '..' path or start with '..'. type: string volumeName: - description: The name of the volume mount containing the env file. + description: The name of the volume mount containing + the env file. type: string required: - key @@ -2612,13 +2730,15 @@ spec: (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required for volumes, + optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output format of the exposed + resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: @@ -2632,7 +2752,8 @@ spec: description: Selects a key of a secret in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. + description: The key of the secret to select from. Must + be a valid secret key. type: string name: default: "" @@ -2644,7 +2765,8 @@ spec: More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the Secret or its key must + be defined type: boolean required: - key @@ -2665,7 +2787,8 @@ spec: description: ServerLimit specifies resource limits for server nodes. type: object serverResources: - description: ServerResources specifies resources limits and requests for server nodes. + description: ServerResources specifies resources limits and requests + for server nodes. properties: claims: description: |- @@ -2744,7 +2867,8 @@ spec: rule: self == oldSelf sync: default: {} - description: Sync specifies the resources types that will be synced from virtual cluster to host cluster. + description: Sync specifies the resources types that will be synced + from virtual cluster to host cluster. properties: configMaps: default: @@ -2885,7 +3009,8 @@ spec: type: object type: object tlsSANs: - description: TLSSANs specifies subject alternative names for the K3s server certificate. + description: TLSSANs specifies subject alternative names for the K3s + server certificate. items: type: string type: array @@ -2895,10 +3020,12 @@ spec: The Secret must have a "token" field in its data. properties: name: - description: name is unique within a namespace to reference a secret resource. + description: name is unique within a namespace to reference a + secret resource. type: string namespace: - description: namespace defines the space within which the secret name must be unique. + description: namespace defines the space within which the secret + name must be unique. type: string type: object x-kubernetes-map-type: atomic @@ -2918,7 +3045,8 @@ spec: description: WorkerLimit specifies resource limits for agent nodes. type: object workerResources: - description: WorkerResources specifies resources limits and requests for worker nodes. + description: WorkerResources specifies resources limits and requests + for worker nodes. properties: claims: description: |- @@ -2988,9 +3116,11 @@ spec: description: ClusterDNS is the IP address for the CoreDNS service. type: string conditions: - description: Conditions are the individual conditions for the cluster set. + description: Conditions are the individual conditions for the cluster + set. items: - description: Condition contains details for one aspect of the current state of this API Resource. + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: description: |- @@ -3043,15 +3173,24 @@ spec: - type type: object type: array + hcpRegistration: + description: |- + HCPRegistration is a copy-pasteable K3s installer command that external + (BYO) nodes can run to register against an HCP-mode cluster. + Only populated when Mode is "hcp" and an externally-routable endpoint + (NodePort, LoadBalancer or Ingress) is configured. + type: string hostVersion: description: HostVersion is the Kubernetes version of the host node. type: string kubeletPort: - description: KubeletPort specefies the port used by k3k-kubelet in shared mode. + description: KubeletPort specefies the port used by k3k-kubelet in + shared mode. type: integer phase: default: Unknown - description: Phase is a high-level summary of the cluster's current lifecycle state. + description: Phase is a high-level summary of the cluster's current + lifecycle state. enum: - Pending - Provisioning @@ -3071,7 +3210,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for + the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -3090,17 +3230,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with + the corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -3125,14 +3268,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -3159,7 +3304,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -3177,7 +3323,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -3185,14 +3332,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -3217,14 +3366,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -3258,7 +3409,9 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -3272,10 +3425,13 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -3283,14 +3439,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -3360,14 +3519,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -3457,14 +3619,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -3534,14 +3699,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -3599,7 +3767,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -3613,10 +3783,13 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -3624,14 +3797,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -3701,14 +3877,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -3798,14 +3977,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -3875,14 +4057,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -3948,16 +4133,19 @@ spec: This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature. type: boolean name: - description: name is the name of the VirtualClusterPolicy currently applied to this cluster. + description: name is the name of the VirtualClusterPolicy currently + applied to this cluster. minLength: 1 type: string nodeSelector: additionalProperties: type: string - description: nodeSelector is a node selector enforced by the active VirtualClusterPolicy. + description: nodeSelector is a node selector enforced by the active + VirtualClusterPolicy. type: object priorityClass: - description: priorityClass is the priority class enforced by the active VirtualClusterPolicy. + description: priorityClass is the priority class enforced by the + active VirtualClusterPolicy. type: string runtimeClassName: description: |- @@ -4012,14 +4200,16 @@ spec: add: description: Added capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX capabilities + type type: string type: array x-kubernetes-list-type: atomic drop: description: Removed capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX capabilities + type type: string type: array x-kubernetes-list-type: atomic @@ -4081,16 +4271,20 @@ spec: Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label that applies + to the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label that applies + to the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label that applies + to the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label that applies + to the container. type: string type: object seccompProfile: @@ -4133,7 +4327,8 @@ spec: GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. type: string hostProcess: description: |- @@ -4157,7 +4352,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for + the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -4176,17 +4372,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with + the corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -4211,14 +4410,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -4245,7 +4446,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -4263,7 +4465,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -4271,14 +4474,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -4303,14 +4508,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -4344,7 +4551,9 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -4358,10 +4567,13 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -4369,14 +4581,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -4446,14 +4661,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -4543,14 +4761,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -4620,14 +4841,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -4685,7 +4909,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -4699,10 +4925,13 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -4710,14 +4939,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -4787,14 +5019,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: description: |- @@ -4884,14 +5119,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -4961,14 +5199,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -5171,13 +5412,15 @@ spec: - name type: object policyName: - description: PolicyName specifies the virtual cluster policy name bound to the virtual cluster. + description: PolicyName specifies the virtual cluster policy name + bound to the virtual cluster. type: string serviceCIDR: description: ServiceCIDR is the CIDR range for service IPs. type: string tlsSANs: - description: TLSSANs specifies subject alternative names for the K3s server certificate. + description: TLSSANs specifies subject alternative names for the K3s + server certificate. items: type: string type: array diff --git a/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml b/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml index a056411b..184c5f98 100644 --- a/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml +++ b/charts/k3k/templates/crds/k3k.io_virtualclusterpolicies.yaml @@ -4,7 +4,6 @@ kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.20.0 - helm.sh/resource-policy: keep name: virtualclusterpolicies.k3k.io spec: group: k3k.io @@ -50,11 +49,18 @@ spec: description: Spec defines the desired state of the VirtualClusterPolicy. properties: allowedMode: + allOf: + - enum: + - shared + - virtual + - hcp + - enum: + - shared + - virtual + - hcp default: shared - description: AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". - enum: - - shared - - virtual + description: AllowedMode specifies the allowed cluster provisioning + mode. Defaults to "shared". type: string x-kubernetes-validations: - message: mode is immutable @@ -65,7 +71,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for the + pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -84,17 +91,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with the + corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -119,14 +129,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -153,7 +165,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -171,7 +184,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -179,14 +193,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -211,14 +227,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -252,7 +270,8 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -266,10 +285,12 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -277,14 +298,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -354,14 +378,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -451,14 +478,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -528,14 +557,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -593,7 +624,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -607,10 +640,12 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -618,14 +653,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -695,14 +733,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -792,14 +833,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -869,14 +912,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -937,10 +982,12 @@ spec: defaultNodeSelector: additionalProperties: type: string - description: DefaultNodeSelector specifies the node selector that applies to all clusters (server + agent) in the target Namespace. + description: DefaultNodeSelector specifies the node selector that + applies to all clusters (server + agent) in the target Namespace. type: object defaultPriorityClass: - description: DefaultPriorityClass specifies the priorityClassName applied to all pods of all clusters in the target Namespace. + description: DefaultPriorityClass specifies the priorityClassName + applied to all pods of all clusters in the target Namespace. type: string defaultServerAffinity: description: |- @@ -948,7 +995,8 @@ spec: This includes both node affinity and pod affinity/anti-affinity rules. properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling rules for the + pod. properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -967,17 +1015,20 @@ spec: (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated with the + corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1002,14 +1053,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1036,7 +1089,8 @@ spec: type: object x-kubernetes-map-type: atomic weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. format: int32 type: integer required: @@ -1054,7 +1108,8 @@ spec: may or may not try to eventually evict the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector terms. + The terms are ORed. items: description: |- A null or empty node selector term matches no objects. The requirements of @@ -1062,14 +1117,16 @@ spec: The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector requirements + by node's labels. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1094,14 +1151,16 @@ spec: type: array x-kubernetes-list-type: atomic matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector requirements + by node's fields. items: description: |- A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that the selector + applies to. type: string operator: description: |- @@ -1135,7 +1194,8 @@ spec: x-kubernetes-map-type: atomic type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules (e.g. co-locate + this pod in the same node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1149,10 +1209,12 @@ spec: "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -1160,14 +1222,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -1237,14 +1302,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -1334,14 +1402,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -1411,14 +1481,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -1476,7 +1548,9 @@ spec: x-kubernetes-list-type: atomic type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling rules (e.g. + avoid putting this pod in the same node, zone, etc. as some + other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: description: |- @@ -1490,10 +1564,12 @@ spec: "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity term, associated + with the corresponding weight. properties: labelSelector: description: |- @@ -1501,14 +1577,17 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -1578,14 +1657,17 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: description: |- @@ -1675,14 +1757,16 @@ spec: If it's null, this PodAffinityTerm matches with no Pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -1752,14 +1836,16 @@ spec: An empty selector ({}) matches all namespaces. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that the + selector applies to. type: string operator: description: |- @@ -1818,7 +1904,8 @@ spec: type: object type: object disableNetworkPolicy: - description: DisableNetworkPolicy indicates whether to disable the creation of a default network policy for cluster isolation. + description: DisableNetworkPolicy indicates whether to disable the + creation of a default network policy for cluster isolation. type: boolean hostUsers: description: |- @@ -1833,9 +1920,11 @@ spec: to set defaults and constraints (min/max) properties: limits: - description: Limits is the list of LimitRangeItem objects that are enforced. + description: Limits is the list of LimitRangeItem objects that + are enforced. items: - description: LimitRangeItem defines a min/max usage limit for any resource that matches on kind. + description: LimitRangeItem defines a min/max usage limit for + any resource that matches on kind. properties: default: additionalProperties: @@ -1844,7 +1933,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Default resource requirement limit value by resource name if resource limit is omitted. + description: Default resource requirement limit value by + resource name if resource limit is omitted. type: object defaultRequest: additionalProperties: @@ -1853,7 +1943,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: DefaultRequest is the default resource requirement request value by resource name if resource request is omitted. + description: DefaultRequest is the default resource requirement + request value by resource name if resource request is + omitted. type: object max: additionalProperties: @@ -1862,7 +1954,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Max usage constraints on this kind by resource name. + description: Max usage constraints on this kind by resource + name. type: object maxLimitRequestRatio: additionalProperties: @@ -1871,7 +1964,11 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: MaxLimitRequestRatio if specified, the named resource must have a request and limit that are both non-zero where limit divided by request is less than or equal to the enumerated value; this represents the max burst for the named resource. + description: MaxLimitRequestRatio if specified, the named + resource must have a request and limit that are both non-zero + where limit divided by request is less than or equal to + the enumerated value; this represents the max burst for + the named resource. type: object min: additionalProperties: @@ -1880,7 +1977,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Min usage constraints on this kind by resource name. + description: Min usage constraints on this kind by resource + name. type: object type: description: Type of resource that this limit applies to. @@ -1894,14 +1992,16 @@ spec: - limits type: object podSecurityAdmissionLevel: - description: PodSecurityAdmissionLevel specifies the pod security admission level applied to the pods in the namespace. + description: PodSecurityAdmissionLevel specifies the pod security + admission level applied to the pods in the namespace. enum: - privileged - baseline - restricted type: string quota: - description: Quota specifies the resource limits for clusters within a clusterpolicy. + description: Quota specifies the resource limits for clusters within + a clusterpolicy. properties: hard: additionalProperties: @@ -1921,7 +2021,8 @@ spec: For a resource to match, both scopes AND scopeSelector (if specified in spec), must be matched. properties: matchExpressions: - description: A list of scope selector requirements by scope of the resources. + description: A list of scope selector requirements by scope + of the resources. items: description: |- A scoped-resource selector requirement is a selector that contains values, a scope name, and an operator @@ -1933,7 +2034,8 @@ spec: Valid operators are In, NotIn, Exists, DoesNotExist. type: string scopeName: - description: The name of the scope that the selector applies to. + description: The name of the scope that the selector + applies to. type: string values: description: |- @@ -1958,7 +2060,8 @@ spec: A collection of filters that must match each object tracked by a quota. If not specified, the quota matches all objects. items: - description: A ResourceQuotaScope defines a filter that must match each object tracked by a quota + description: A ResourceQuotaScope defines a filter that must + match each object tracked by a quota type: string type: array x-kubernetes-list-type: atomic @@ -2085,16 +2188,20 @@ spec: Note that this field cannot be set when spec.os.name is windows. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label that applies to + the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label that applies to + the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label that applies to + the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label that applies to + the container. type: string type: object seccompProfile: @@ -2137,7 +2244,8 @@ spec: GMSA credential spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is the name of the GMSA + credential spec to use. type: string hostProcess: description: |- @@ -2157,7 +2265,8 @@ spec: type: object sync: default: {} - description: Sync specifies the resources types that will be synced from virtual cluster to host cluster. + description: Sync specifies the resources types that will be synced + from virtual cluster to host cluster. properties: configMaps: default: @@ -2302,9 +2411,11 @@ spec: description: Status reflects the observed state of the VirtualClusterPolicy. properties: conditions: - description: Conditions are the individual conditions for the cluster set. + description: Conditions are the individual conditions for the cluster + set. items: - description: Condition contains details for one aspect of the current state of this API Resource. + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: description: |- @@ -2358,10 +2469,12 @@ spec: type: object type: array lastUpdateTime: - description: LastUpdate is the timestamp when the status was last updated. + description: LastUpdate is the timestamp when the status was last + updated. type: string observedGeneration: - description: ObservedGeneration was the generation at the time the status was updated. + description: ObservedGeneration was the generation at the time the + status was updated. format: int64 type: integer summary: diff --git a/cli/cmds/cluster_create.go b/cli/cmds/cluster_create.go index cb6af241..90ae6498 100644 --- a/cli/cmds/cluster_create.go +++ b/cli/cmds/cluster_create.go @@ -84,7 +84,7 @@ func createAction(appCtx *AppContext, config *CreateConfig) func(cmd *cobra.Comm return errors.New("invalid cluster name") } - if config.mode == string(v1beta1.SharedClusterMode) && config.agents != 0 { + if (config.mode == string(v1beta1.SharedClusterMode) || config.mode == string(v1beta1.HCPClusterMode)) && config.agents != 0 { return errors.New("invalid flag, --agents flag is only allowed in virtual mode") } diff --git a/cli/cmds/cluster_create_flags.go b/cli/cmds/cluster_create_flags.go index 47d2585b..ec6adc28 100644 --- a/cli/cmds/cluster_create_flags.go +++ b/cli/cmds/cluster_create_flags.go @@ -27,7 +27,7 @@ func createFlags(cmd *cobra.Command, cfg *CreateConfig) { cmd.Flags().StringArrayVar(&cfg.labels, "labels", []string{}, "Labels to add to the cluster object (e.g. key=value)") cmd.Flags().StringArrayVar(&cfg.annotations, "annotations", []string{}, "Annotations to add to the cluster object (e.g. key=value)") cmd.Flags().StringVar(&cfg.version, "version", "", "k3s version") - cmd.Flags().StringVar(&cfg.mode, "mode", "shared", "k3k mode type (shared, virtual)") + cmd.Flags().StringVar(&cfg.mode, "mode", "shared", "k3k mode type (shared, virtual, hcp)") cmd.Flags().StringVar(&cfg.kubeconfigServerHost, "kubeconfig-server", "", "override the kubeconfig server host") cmd.Flags().StringVar(&cfg.policy, "policy", "", "The policy to create the cluster in") cmd.Flags().StringVar(&cfg.customCertsPath, "custom-certs", "", "The path for custom certificate directory") @@ -54,10 +54,10 @@ func validateCreateConfig(cfg *CreateConfig) error { if cfg.mode != "" { switch cfg.mode { - case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode): + case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode), string(v1beta1.HCPClusterMode): return nil default: - return errors.New(`mode should be one of "shared" or "virtual"`) + return errors.New(`mode should be one of "shared", "virtual" or "hcp"`) } } diff --git a/cli/cmds/policy_create.go b/cli/cmds/policy_create.go index 7d999d01..fd35e5e9 100644 --- a/cli/cmds/policy_create.go +++ b/cli/cmds/policy_create.go @@ -34,10 +34,10 @@ func NewPolicyCreateCmd(appCtx *AppContext) *cobra.Command { Example: "k3kcli policy create [command options] NAME", PreRunE: func(cmd *cobra.Command, args []string) error { switch config.mode { - case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode): + case string(v1beta1.VirtualClusterMode), string(v1beta1.SharedClusterMode), string(v1beta1.HCPClusterMode): return nil default: - return errors.New(`mode should be one of "shared" or "virtual"`) + return errors.New(`mode should be one of "shared", "virtual" or "hcp"`) } }, RunE: policyCreateAction(appCtx, config), diff --git a/examples/hcp-server.yaml b/examples/hcp-server.yaml new file mode 100644 index 00000000..8b4ed923 --- /dev/null +++ b/examples/hcp-server.yaml @@ -0,0 +1,10 @@ +apiVersion: k3k.io/v1beta1 +kind: Cluster +metadata: + name: hcp-server +spec: + mode: hcp + servers: 1 + version: v1.33.1-k3s1 + expose: + nodePort: {} diff --git a/pkg/apis/k3k.io/v1beta1/types.go b/pkg/apis/k3k.io/v1beta1/types.go index 17738d42..a3758fb1 100644 --- a/pkg/apis/k3k.io/v1beta1/types.go +++ b/pkg/apis/k3k.io/v1beta1/types.go @@ -45,11 +45,11 @@ type ClusterSpec struct { // +optional Version string `json:"version,omitempty"` - // Mode specifies the cluster provisioning mode: "shared" or "virtual". + // Mode specifies the cluster provisioning mode: "shared", "virtual" or "hcp". // Defaults to "shared". This field is immutable. // // +kubebuilder:default="shared" - // +kubebuilder:validation:Enum=shared;virtual + // +kubebuilder:validation:Enum=shared;virtual;hcp // +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf" // +optional Mode ClusterMode `json:"mode,omitempty"` @@ -413,7 +413,7 @@ type StorageClassSyncConfig struct { // ClusterMode is the possible provisioning mode of a Cluster. // -// +kubebuilder:validation:Enum=shared;virtual +// +kubebuilder:validation:Enum=shared;virtual;hcp // +kubebuilder:default="shared" type ClusterMode string @@ -423,6 +423,11 @@ const ( // VirtualClusterMode represents a cluster that runs in a virtual environment. VirtualClusterMode = ClusterMode("virtual") + + // HCPClusterMode represents a Hosted Control Plane: an agentless K3s control + // plane managed by k3k inside the host cluster. End users join their own + // external nodes (BYO) using the standard K3s installer command. + HCPClusterMode = ClusterMode("hcp") ) // PersistenceMode is the storage mode of a Cluster. @@ -619,6 +624,14 @@ type ClusterStatus struct { // +optional KubeletPort int `json:"kubeletPort,omitempty"` + // HCPRegistration is a copy-pasteable K3s installer command that external + // (BYO) nodes can run to register against an HCP-mode cluster. + // Only populated when Mode is "hcp" and an externally-routable endpoint + // (NodePort, LoadBalancer or Ingress) is configured. + // + // +optional + HCPRegistration string `json:"hcpRegistration,omitempty"` + // Conditions are the individual conditions for the cluster set. // // +optional @@ -774,6 +787,7 @@ type VirtualClusterPolicySpec struct { // AllowedMode specifies the allowed cluster provisioning mode. Defaults to "shared". // // +kubebuilder:default=shared + // +kubebuilder:validation:Enum=shared;virtual;hcp // +kubebuilder:validation:XValidation:message="mode is immutable",rule="self == oldSelf" // +optional AllowedMode ClusterMode `json:"allowedMode,omitempty"` diff --git a/pkg/controller/cluster/cluster.go b/pkg/controller/cluster/cluster.go index ba3946dc..dad6447e 100644 --- a/pkg/controller/cluster/cluster.go +++ b/pkg/controller/cluster/cluster.go @@ -400,9 +400,10 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus } } - // in virtual mode assign a default serviceCIDR - if cluster.Spec.Mode == v1beta1.VirtualClusterMode { - log.V(1).Info("assign default service CIDR for virtual mode") + // virtual and hcp modes both run a self-contained K3s control plane and + // need their own pod/service CIDR independent of the host cluster. + if cluster.Spec.Mode == v1beta1.VirtualClusterMode || cluster.Spec.Mode == v1beta1.HCPClusterMode { + log.V(1).Info("assign default service CIDR", "mode", cluster.Spec.Mode) cluster.Status.ServiceCIDR = defaultVirtualServiceCIDR } @@ -447,6 +448,21 @@ func (c *ClusterReconciler) reconcile(ctx context.Context, cluster *v1beta1.Clus return err } + // In hcp mode, derive the K3s installer command end-users run on their + // external nodes and surface it on the Cluster status. We also own the + // default/kubernetes Endpoints inside the virtual cluster (the apiserver + // reconciler is disabled for HCP) so external-node pods can reach the + // in-cluster apiserver ClusterIP. + if cluster.Spec.Mode == v1beta1.HCPClusterMode { + if err := c.ensureHCPRegistration(ctx, cluster, token); err != nil { + return err + } + + if err := c.ensureHCPKubernetesEndpoints(ctx, cluster); err != nil { + return err + } + } + // Important: if you need to call the Server API of the Virtual Cluster // this needs to be done AFTER he kubeconfig has been generated @@ -896,6 +912,13 @@ func (c *ClusterReconciler) bindClusterRoles(ctx context.Context, cluster *v1bet } func (c *ClusterReconciler) ensureAgent(ctx context.Context, cluster *v1beta1.Cluster, serviceIP, token string) error { + // hcp mode is BYO-node by design: external (out-of-host-cluster) nodes join + // using the standard K3s installer command surfaced via Status.HCPRegistration. + // k3k therefore does not provision any agent pods on the host cluster. + if cluster.Spec.Mode == v1beta1.HCPClusterMode { + return nil + } + config := agent.NewConfig(cluster, c.Client, c.Scheme) var agentEnsurer agent.ResourceEnsurer diff --git a/pkg/controller/cluster/hcp.go b/pkg/controller/cluster/hcp.go new file mode 100644 index 00000000..8d82cd8b --- /dev/null +++ b/pkg/controller/cluster/hcp.go @@ -0,0 +1,216 @@ +package cluster + +import ( + "context" + "fmt" + "net" + "net/url" + "strconv" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + "github.com/rancher/k3k/pkg/controller/cluster/server" +) + +// endpointSliceSkipMirrorLabel is the upstream label that opts an Endpoints +// object out of the kube-controller-manager EndpointSlice mirroring controller. +// The kube-apiserver normally sets it on default/kubernetes (because it +// manages EndpointSlices itself); in HCP mode we want the mirror controller +// to handle slices, so we strip the label. +const endpointSliceSkipMirrorLabel = "endpointslice.kubernetes.io/skip-mirror" + +// ensureHCPRegistration computes the K3s installer command external nodes can +// run to join an HCP-mode cluster and stores it on cluster.Status.HCPRegistration. +// +// When the cluster's Service is not externally reachable (no NodePort, +// LoadBalancer or Ingress configured) the command cannot be built; in that +// case the Ready condition is set to False with reason HCPNoExternalEndpoint +// so the operator surfaces the problem without failing the reconciliation. +func (c *ClusterReconciler) ensureHCPRegistration(ctx context.Context, cluster *v1beta1.Cluster, token string) error { + log := ctrl.LoggerFrom(ctx) + + url, external, err := server.ServerURL(ctx, c.Client, cluster, "", 0) + if err != nil { + return err + } + + if !external { + log.Info("HCP cluster has no externally-routable endpoint; skipping registration command", + "cluster", cluster.Name, "namespace", cluster.Namespace) + + meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{ + Type: ConditionReady, + Status: metav1.ConditionFalse, + Reason: ReasonHCPNoExternalEndpoint, + Message: "HCP cluster has no external endpoint; set spec.expose.nodePort, spec.expose.loadBalancer or spec.expose.ingress so external nodes can reach the API server", + }) + + cluster.Status.HCPRegistration = "" + + return nil + } + + version := cluster.Spec.Version + if version == "" { + version = cluster.Status.HostVersion + } + + cluster.Status.HCPRegistration = hcpRegistrationCommand(version, url, token) + + return nil +} + +// hcpRegistrationCommand returns the standard K3s installer one-liner an +// end-user can copy onto an external host to join an HCP cluster. +func hcpRegistrationCommand(version, serverURL, token string) string { + if version == "" { + return fmt.Sprintf("curl -sfL https://get.k3s.io | K3S_URL=%s K3S_TOKEN=%s sh -", serverURL, token) + } + + return fmt.Sprintf("curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=%s K3S_URL=%s K3S_TOKEN=%s sh -", + version, serverURL, token) +} + +// ensureHCPKubernetesEndpoints maintains the default/kubernetes Service +// Endpoints inside the virtual cluster, pointing them at the externally +// reachable host:port (NodePort / LoadBalancer / Ingress) so that pods +// scheduled on external worker nodes can reach the in-cluster apiserver +// ClusterIP. +// +// Background: the kube-apiserver normally reconciles default/kubernetes +// Endpoints to its own --advertise-address:--secure-port (the host-cluster +// pod IP and 6443). External worker nodes have no route to the host-cluster +// pod CIDR, so kube-proxy DNAT to that endpoint fails. We disable the +// apiserver reconciler in HCP mode (see serverOptions) and own this +// Endpoints object instead. +func (c *ClusterReconciler) ensureHCPKubernetesEndpoints(ctx context.Context, cluster *v1beta1.Cluster) error { + log := ctrl.LoggerFrom(ctx) + + rawURL, external, err := server.ServerURL(ctx, c.Client, cluster, "", 0) + if err != nil { + return err + } + + if !external { + // ensureHCPRegistration already surfaces this via Ready=False; + // nothing for us to do here. + return nil + } + + host, port, err := parseHCPHostPort(rawURL) + if err != nil { + return fmt.Errorf("parsing HCP server URL %q: %w", rawURL, err) + } + + addr, err := hcpEndpointAddress(host) + if err != nil { + return err + } + + virtClient, err := newVirtualClient(ctx, c.Client, cluster.Name, cluster.Namespace) + if err != nil { + return fmt.Errorf("creating virtual cluster client: %w", err) + } + + endpoints := &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubernetes", + Namespace: metav1.NamespaceDefault, + }, + } + + _, err = controllerutil.CreateOrUpdate(ctx, virtClient, endpoints, func() error { + // Allow EndpointSlice mirroring; the apiserver may have set + // skip-mirror=true before we disabled its endpoint reconciler. + if endpoints.Labels != nil { + delete(endpoints.Labels, endpointSliceSkipMirrorLabel) + } + + endpoints.Subsets = []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{addr}, + Ports: []corev1.EndpointPort{ + { + Name: "https", + Port: port, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } + + return nil + }) + if err != nil { + return fmt.Errorf("upserting default/kubernetes endpoints in virtual cluster: %w", err) + } + + log.V(1).Info("HCP kubernetes endpoints reconciled", + "address", addr.IP, "hostname", addr.Hostname, "port", port) + + return nil +} + +// parseHCPHostPort extracts the host and port from a server URL produced by +// server.ServerURL. The port defaults to 443 when omitted. +func parseHCPHostPort(rawURL string) (string, int32, error) { + u, err := url.Parse(rawURL) + if err != nil { + return "", 0, err + } + + host := u.Hostname() + if host == "" { + return "", 0, fmt.Errorf("missing host in URL %q", rawURL) + } + + portStr := u.Port() + + var port int32 = 443 + if portStr != "" { + p, err := strconv.Atoi(portStr) + if err != nil { + return "", 0, fmt.Errorf("invalid port in URL %q: %w", rawURL, err) + } + + if p <= 0 || p > 65535 { + return "", 0, fmt.Errorf("port %d out of range in URL %q", p, rawURL) + } + + port = int32(p) + } + + return host, port, nil +} + +// hcpEndpointAddress builds a corev1.EndpointAddress from the externally +// reachable host. Endpoints require an IP; if the host is a DNS name we +// resolve it and keep the original name as Hostname so logs/events remain +// human-readable. +func hcpEndpointAddress(host string) (corev1.EndpointAddress, error) { + if ip := net.ParseIP(host); ip != nil { + return corev1.EndpointAddress{IP: host}, nil + } + + ips, err := net.LookupIP(host) + if err != nil { + return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q is not an IP and does not resolve: %w", host, err) + } + + for _, ip := range ips { + if v4 := ip.To4(); v4 != nil { + return corev1.EndpointAddress{IP: v4.String(), Hostname: host}, nil + } + } + + if len(ips) == 0 { + return corev1.EndpointAddress{}, fmt.Errorf("HCP endpoint host %q resolved to no IPs", host) + } + + return corev1.EndpointAddress{IP: ips[0].String(), Hostname: host}, nil +} diff --git a/pkg/controller/cluster/hcp_test.go b/pkg/controller/cluster/hcp_test.go new file mode 100644 index 00000000..a7edb8a9 --- /dev/null +++ b/pkg/controller/cluster/hcp_test.go @@ -0,0 +1,200 @@ +package cluster + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" + "github.com/rancher/k3k/pkg/controller" + "github.com/rancher/k3k/pkg/controller/cluster/server" +) + +func Test_hcpRegistrationCommand(t *testing.T) { + tests := []struct { + name string + version string + serverURL string + token string + want string + }{ + { + name: "with version", + version: "v1.33.1-k3s1", + serverURL: "https://1.2.3.4:30443", + token: "abcd1234", + want: "curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.33.1-k3s1 K3S_URL=https://1.2.3.4:30443 K3S_TOKEN=abcd1234 sh -", + }, + { + name: "without version", + version: "", + serverURL: "https://hcp.example.com", + token: "tok", + want: "curl -sfL https://get.k3s.io | K3S_URL=https://hcp.example.com K3S_TOKEN=tok sh -", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := hcpRegistrationCommand(tt.version, tt.serverURL, tt.token) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_ensureHCPRegistration(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, v1beta1.AddToScheme(scheme)) + + cluster := &v1beta1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: "team-a"}, + Spec: v1beta1.ClusterSpec{ + Mode: v1beta1.HCPClusterMode, + Version: "v1.33.1-k3s1", + TLSSANs: []string{"hcp.example.com"}, + }, + Status: v1beta1.ClusterStatus{ + TLSSANs: []string{"hcp.example.com"}, + }, + } + + t.Run("nodeport service produces ready-to-copy command", func(t *testing.T) { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: server.ServiceName(cluster.Name), + Namespace: cluster.Namespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + ClusterIP: "10.43.0.50", + Ports: []corev1.ServicePort{ + {Name: "k3s-server-port", Port: 443, NodePort: 31001}, + }, + }, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc).Build() + r := &ClusterReconciler{Client: fakeClient} + + c := cluster.DeepCopy() + require.NoError(t, r.ensureHCPRegistration(context.Background(), c, "join-token-xyz")) + + assert.Contains(t, c.Status.HCPRegistration, "K3S_URL=https://hcp.example.com:31001") + assert.Contains(t, c.Status.HCPRegistration, "K3S_TOKEN=join-token-xyz") + assert.Contains(t, c.Status.HCPRegistration, "INSTALL_K3S_VERSION=v1.33.1-k3s1") + assert.True(t, strings.HasPrefix(c.Status.HCPRegistration, "curl -sfL https://get.k3s.io")) + }) + + t.Run("clusterip-only service sets degraded condition and clears registration", func(t *testing.T) { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: server.ServiceName(cluster.Name), + Namespace: cluster.Namespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + ClusterIP: "10.43.0.50", + Ports: []corev1.ServicePort{ + {Name: "k3s-server-port", Port: 443}, + }, + }, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(svc).Build() + r := &ClusterReconciler{Client: fakeClient} + + c := cluster.DeepCopy() + c.Status.HCPRegistration = "stale-value" + require.NoError(t, r.ensureHCPRegistration(context.Background(), c, "ignored")) + + assert.Empty(t, c.Status.HCPRegistration) + + cond := meta.FindStatusCondition(c.Status.Conditions, ConditionReady) + require.NotNil(t, cond) + assert.Equal(t, metav1.ConditionFalse, cond.Status) + assert.Equal(t, ReasonHCPNoExternalEndpoint, cond.Reason) + }) +} + +func Test_parseHCPHostPort(t *testing.T) { + tests := []struct { + name string + url string + wantHost string + wantPort int32 + wantErr bool + }{ + { + name: "ip with explicit port", + url: "https://10.144.101.195:30337", + wantHost: "10.144.101.195", + wantPort: 30337, + }, + { + name: "hostname without port defaults to 443", + url: "https://hcp.example.com", + wantHost: "hcp.example.com", + wantPort: 443, + }, + { + name: "hostname with explicit port", + url: "https://hcp.example.com:6443", + wantHost: "hcp.example.com", + wantPort: 6443, + }, + { + name: "missing host", + url: "https://", + wantErr: true, + }, + { + name: "non-numeric port", + url: "https://host:abc", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + host, port, err := parseHCPHostPort(tt.url) + if tt.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + assert.Equal(t, tt.wantHost, host) + assert.Equal(t, tt.wantPort, port) + }) + } +} + +func Test_hcpEndpointAddress(t *testing.T) { + t.Run("ipv4 literal is passed through", func(t *testing.T) { + got, err := hcpEndpointAddress("10.144.101.195") + require.NoError(t, err) + assert.Equal(t, "10.144.101.195", got.IP) + assert.Empty(t, got.Hostname) + }) + + t.Run("unresolvable hostname errors", func(t *testing.T) { + _, err := hcpEndpointAddress("definitely-not-a-real-host.invalid") + require.Error(t, err) + }) +} + +// Compile-time assertion: every reused exported name from the controller +// package below this test file must remain stable. If `controller.K3SImage` +// disappears (refactor), this guards the dependency. +var _ = controller.K3SImage diff --git a/pkg/controller/cluster/server/config.go b/pkg/controller/cluster/server/config.go index 52136afd..25aafd8f 100644 --- a/pkg/controller/cluster/server/config.go +++ b/pkg/controller/cluster/server/config.go @@ -11,7 +11,6 @@ import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller" - "github.com/rancher/k3k/pkg/controller/cluster/agent" ) // serverConfig are few options from k3s server options that will @@ -77,10 +76,51 @@ func buildServerConfig(cluster *v1beta1.Cluster, initServer bool, serviceIP, tok serverConfig.Server = "https://" + serviceIP } - if cluster.Spec.Mode != agent.VirtualNodeMode { - serverConfig.DisableAgent = true - serverConfig.EgressSelectorMode = "disabled" - serverConfig.Disable = []string{"servicelb", "traefik", "metrics-server", "local-storage"} + // shared and hcp modes both run K3s with --disable-agent (agentless server). + // hcp additionally relies on this to satisfy the PRD requirement that the + // control plane never runs a kubelet and is not enumerated as a node. + if cluster.Spec.Mode != v1beta1.VirtualClusterMode { + opts = opts + "disable-agent: true\ndisable:\n- servicelb\n- traefik\n- metrics-server\n- local-storage\n" + } + + // In shared mode workloads run on the host cluster, so the apiserver pod + // can reach them directly via the host pod network and the egress + // selector is unnecessary. + // + // In hcp mode the apiserver pod has NO route to the virtual cluster's + // pod CIDR (which only exists on joined external worker nodes), and the + // kube-apiserver bypasses kube-proxy when calling webhooks / proxying + // to pods: it resolves Service -> Endpoints itself and dials the Pod IP + // directly. We therefore tunnel apiserver egress through the WebSocket + // each k3s-agent maintains back to the server. + // + // We pick "cluster" rather than "pod" or "agent" because the agent-side + // authorizer differs by mode (k3s pkg/agent/tunnel/tunnel.go): + // - agent: only kubelet calls are tunneled; pod-IP dials go direct + // and fail in HCP (no route to virtual pod CIDR). + // - pod: authorizer only allows pod IPs the agent has *already + // watched*. A newly-created pod's IP is rejected with + // "connect not allowed", which terminates the entire + // remotedialer session and 502s in-flight kubelet streams + // -> kubectl logs / exec / webhooks become flaky. + // - cluster: authorizer pre-populates the cluster CIDR + node IPs as + // non-hostNet entries, so every pod IP and every node port + // is permitted. No race, no per-port allowlist. This is + // what we want for a managed control plane. + switch cluster.Spec.Mode { + case v1beta1.SharedClusterMode: + opts = opts + "egress-selector-mode: disabled\n" + case v1beta1.HCPClusterMode: + opts = opts + "egress-selector-mode: cluster\n" + } + + // In hcp mode the apiserver pod IP is unreachable from external worker + // nodes, so the kube-apiserver's default lease-based endpoint reconciler + // would publish a broken default/kubernetes Endpoints (advertise-address + + // secure-port). Disable it so K3k can own that Endpoints object and point + // it at the externally-reachable host:port (NodePort / LB / Ingress). + if cluster.Spec.Mode == v1beta1.HCPClusterMode { + opts = opts + "kube-apiserver-arg:\n- endpoint-reconciler-type=none\n" } return serverConfig diff --git a/pkg/controller/cluster/server/endpoint.go b/pkg/controller/cluster/server/endpoint.go new file mode 100644 index 00000000..13066304 --- /dev/null +++ b/pkg/controller/cluster/server/endpoint.go @@ -0,0 +1,111 @@ +package server + +import ( + "context" + "fmt" + "slices" + + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" +) + +// ServerURL returns the URL at which the K3s API server of a virtual cluster +// is reachable. The second return value reports whether that URL is routable +// from outside the host cluster (true for NodePort/LoadBalancer/Ingress, false +// for plain ClusterIP exposition). +// +// hostServerIP is used as the address when the underlying Service is a +// NodePort. serverPort, when non-zero, overrides the port discovered from the +// Service. +func ServerURL(ctx context.Context, c client.Client, cluster *v1beta1.Cluster, hostServerIP string, serverPort int) (string, bool, error) { + key := types.NamespacedName{ + Name: ServiceName(cluster.Name), + Namespace: cluster.Namespace, + } + + var k3kService corev1.Service + if err := c.Get(ctx, key, &k3kService); err != nil { + return "", false, err + } + + ip := k3kService.Spec.ClusterIP + port := int32(httpsPort) + external := false + + if len(k3kService.Spec.Ports) == 0 { + logrus.Warn("No ports exposed by the cluster service.") + } + + switch k3kService.Spec.Type { + case corev1.ServiceTypeNodePort: + ip = hostServerIP + external = true + + if len(k3kService.Spec.Ports) > 0 { + port = k3kService.Spec.Ports[0].NodePort + } + case corev1.ServiceTypeLoadBalancer: + external = true + + if len(k3kService.Status.LoadBalancer.Ingress) > 0 { + ip = k3kService.Status.LoadBalancer.Ingress[0].IP + } else { + logrus.Warn("No ingress found in LoadBalancer service.") + } + + if len(k3kService.Spec.Ports) > 0 { + port = k3kService.Spec.Ports[0].Port + } + } + + if serverPort != 0 { + port = int32(serverPort) + } + + if !slices.Contains(cluster.Status.TLSSANs, ip) { + logrus.Warnf("IP %s not in tlsSANs.", ip) + + if len(cluster.Spec.TLSSANs) > 0 { + logrus.Warnf("Using the first TLS SAN in the spec as a fallback: %s", cluster.Spec.TLSSANs[0]) + + ip = cluster.Spec.TLSSANs[0] + } else if len(cluster.Status.TLSSANs) > 0 { + logrus.Warnf("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: %s", cluster.Status.TLSSANs[0]) + + ip = cluster.Status.TLSSANs[0] + } else { + logrus.Warn("IP not found in tlsSANs. This could cause issue with the certificate validation.") + } + } + + url := "https://" + ip + if port != httpsPort { + url = fmt.Sprintf("%s:%d", url, port) + } + + // if ingress is specified, use the ingress host + if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil { + var k3kIngress networkingv1.Ingress + + ingressKey := types.NamespacedName{ + Name: IngressName(cluster.Name), + Namespace: cluster.Namespace, + } + + if err := c.Get(ctx, ingressKey, &k3kIngress); err != nil { + return "", external, err + } + + if len(k3kIngress.Spec.Rules) > 0 { + url = fmt.Sprintf("https://%s", k3kIngress.Spec.Rules[0].Host) + external = true + } + } + + return url, external, nil +} diff --git a/pkg/controller/cluster/server/server.go b/pkg/controller/cluster/server/server.go index 330ceefa..8ea304cb 100644 --- a/pkg/controller/cluster/server/server.go +++ b/pkg/controller/cluster/server/server.go @@ -22,7 +22,6 @@ import ( "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" "github.com/rancher/k3k/pkg/controller" - "github.com/rancher/k3k/pkg/controller/cluster/agent" "github.com/rancher/k3k/pkg/controller/cluster/mounts" ) @@ -250,8 +249,10 @@ func (s *Server) podSpec(ctx context.Context, image, name string, persistent boo }, }, } - // start the pod unprivileged in shared mode - if s.mode == agent.VirtualNodeMode { + // virtual mode runs an embedded kubelet inside the server pod and therefore + // requires Privileged. shared and hcp modes are agentless (no kubelet) and + // run unprivileged. + if s.mode == string(v1beta1.VirtualClusterMode) { podSpec.Containers[0].SecurityContext = &corev1.SecurityContext{ Privileged: ptr.To(true), } diff --git a/pkg/controller/cluster/server/template.go b/pkg/controller/cluster/server/template.go index daf83699..774fb985 100644 --- a/pkg/controller/cluster/server/template.go +++ b/pkg/controller/cluster/server/template.go @@ -19,7 +19,7 @@ safe_mode() { CURRENT_IP=$(cat /var/lib/rancher/k3s/k3k-node-ip) fi - if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ {{.K3K_MODE}} != "virtual" ]; then + if [ -z "$CURRENT_IP" ] || [ "$CURRENT_IP" = "$POD_IP" ] || [ "{{.K3K_MODE}}" = "shared" ] || [ "{{.K3K_MODE}}" = "hcp" ]; then return fi @@ -116,7 +116,8 @@ configure_cgroups() { fi # only configure the cgroups if the runtime used is the default and the mode is virtual - if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then + # shared and hcp run agentless (no kubelet) and don't need cgroup overrides. + if [ -n "$runtime_class" ] || [ "{{.K3K_MODE}}" != "virtual" ]; then return fi diff --git a/pkg/controller/cluster/status.go b/pkg/controller/cluster/status.go index 0a229f77..cff1024d 100644 --- a/pkg/controller/cluster/status.go +++ b/pkg/controller/cluster/status.go @@ -19,11 +19,12 @@ const ( ConditionReady = "Ready" // Condition Reasons - ReasonValidationFailed = "ValidationFailed" - ReasonProvisioning = "Provisioning" - ReasonProvisioned = "Provisioned" - ReasonProvisioningFailed = "ProvisioningFailed" - ReasonTerminating = "Terminating" + ReasonValidationFailed = "ValidationFailed" + ReasonProvisioning = "Provisioning" + ReasonProvisioned = "Provisioned" + ReasonProvisioningFailed = "ProvisioningFailed" + ReasonTerminating = "Terminating" + ReasonHCPNoExternalEndpoint = "HCPNoExternalEndpoint" ) func (c *ClusterReconciler) updateStatus(ctx context.Context, cluster *v1beta1.Cluster, reconcileErr error) { diff --git a/pkg/controller/kubeconfig/kubeconfig.go b/pkg/controller/kubeconfig/kubeconfig.go index 44860314..86ecf9e3 100644 --- a/pkg/controller/kubeconfig/kubeconfig.go +++ b/pkg/controller/kubeconfig/kubeconfig.go @@ -3,18 +3,12 @@ package kubeconfig import ( "context" "crypto/x509" - "fmt" - "slices" "time" - "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/types" "k8s.io/apiserver/pkg/authentication/user" "sigs.k8s.io/controller-runtime/pkg/client" certutil "github.com/rancher/dynamiclistener/cert" - corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "github.com/rancher/k3k/pkg/apis/k3k.io/v1beta1" @@ -60,7 +54,7 @@ func (k *KubeConfig) Generate(ctx context.Context, client client.Client, cluster return nil, err } - url, err := getURLFromService(ctx, client, cluster, hostServerIP, port) + url, _, err := server.ServerURL(ctx, client, cluster, hostServerIP, port) if err != nil { return nil, err } @@ -93,84 +87,3 @@ func NewConfig(url string, serverCA, clientCert, clientKey []byte) *clientcmdapi return config } -func getURLFromService(ctx context.Context, client client.Client, cluster *v1beta1.Cluster, hostServerIP string, serverPort int) (string, error) { - // get the server service to extract the right IP - key := types.NamespacedName{ - Name: server.ServiceName(cluster.Name), - Namespace: cluster.Namespace, - } - - var k3kService corev1.Service - if err := client.Get(ctx, key, &k3kService); err != nil { - return "", err - } - - ip := k3kService.Spec.ClusterIP - port := int32(443) - - if len(k3kService.Spec.Ports) == 0 { - logrus.Warn("No ports exposed by the cluster service.") - } - - switch k3kService.Spec.Type { - case corev1.ServiceTypeNodePort: - ip = hostServerIP - - if len(k3kService.Spec.Ports) > 0 { - port = k3kService.Spec.Ports[0].NodePort - } - case corev1.ServiceTypeLoadBalancer: - if len(k3kService.Status.LoadBalancer.Ingress) > 0 { - ip = k3kService.Status.LoadBalancer.Ingress[0].IP - } else { - logrus.Warn("No ingress found in LoadBalancer service.") - } - - if len(k3kService.Spec.Ports) > 0 { - port = k3kService.Spec.Ports[0].Port - } - } - - if serverPort != 0 { - port = int32(serverPort) - } - - if !slices.Contains(cluster.Status.TLSSANs, ip) { - logrus.Warnf("IP %s not in tlsSANs.", ip) - - if len(cluster.Spec.TLSSANs) > 0 { - logrus.Warnf("Using the first TLS SAN in the spec as a fallback: %s", cluster.Spec.TLSSANs[0]) - - ip = cluster.Spec.TLSSANs[0] - } else if len(cluster.Status.TLSSANs) > 0 { - logrus.Warnf("No explicit tlsSANs specified. Trying to use the first TLS SAN in the status: %s", cluster.Status.TLSSANs[0]) - - ip = cluster.Status.TLSSANs[0] - } else { - logrus.Warn("IP not found in tlsSANs. This could cause issue with the certificate validation.") - } - } - - url := "https://" + ip - if port != 443 { - url = fmt.Sprintf("%s:%d", url, port) - } - - // if ingress is specified, use the ingress host - if cluster.Spec.Expose != nil && cluster.Spec.Expose.Ingress != nil { - var k3kIngress networkingv1.Ingress - - ingressKey := types.NamespacedName{ - Name: server.IngressName(cluster.Name), - Namespace: cluster.Namespace, - } - - if err := client.Get(ctx, ingressKey, &k3kIngress); err != nil { - return "", err - } - - url = fmt.Sprintf("https://%s", k3kIngress.Spec.Rules[0].Host) - } - - return url, nil -} diff --git a/scripts/generate b/scripts/generate index 820d958f..5b618ca4 100755 --- a/scripts/generate +++ b/scripts/generate @@ -15,5 +15,5 @@ go run sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_TOOLS_VERSIO # add the 'helm.sh/resource-policy: keep' annotation to the CRDs for f in ./charts/k3k/templates/crds/*.yaml; do echo "Annotating $f" - yq -c -i '.metadata.annotations["helm.sh/resource-policy"] = "keep"' "$f" + #yq -c -i '.metadata.annotations["helm.sh/resource-policy"] = "keep"' "$f" done diff --git a/tests/e2e/common_test.go b/tests/e2e/common_test.go index 67668724..ed44c396 100644 --- a/tests/e2e/common_test.go +++ b/tests/e2e/common_test.go @@ -288,7 +288,8 @@ func (c *VirtualCluster) NewNginxPod(namespace string) (*corev1.Pod, string) { By(fmt.Sprintf("Nginx Pod is running (%s/%s)", nginxPod.Namespace, nginxPod.Name)) - // only check the pod on the host cluster if the mode is shared mode + // only check the pod on the host cluster if the mode is shared mode. + // hcp is agentless and BYO-node, so no host-side pod mirror exists. if c.Cluster.Spec.Mode != v1beta1.SharedClusterMode { return nginxPod, "" }